From 813590e7a9081edc5ac3363bb1b97713ca275af4 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 05:08:36 +0000 Subject: [PATCH] feat: Implement Plan-and-Reflect Autonomous Loop This commit introduces a significant architectural enhancement to the Bytebot agent, moving it from a purely reactive model to a proactive, plan-driven autonomous system. The core changes include: 1. **PlannerService**: A new service that uses an LLM to break down high-level user requests into a concrete, step-by-step plan. 2. **ReflectorService**: A new service that evaluates the outcome of each executed step. It determines if a step was successful, failed, or needs to be retried, enabling self-correction. 3. **Plan-and-Execute Loop**: The `AgentProcessor` has been refactored to orchestrate the new autonomous loop. It now generates a plan, executes it step-by-step, and reflects on each outcome to decide the next action. 4. **Database Schema**: The `Task` model in the Prisma schema has been updated to store the `plan` and the current `planStep`, making the agent's state more robust and resumable. 5. **Testing**: * The Jest configuration has been fixed to work correctly in the monorepo environment. * A new unit test for the `PlannerService` has been added, which uncovered and led to a fix for a bug in handling empty LLM responses. This new architecture provides a strong foundation for future autonomous capabilities, such as more advanced planning, memory, and self-optimization. --- packages/bytebot-agent/package-lock.json | 347 +++++++++- packages/bytebot-agent/package.json | 16 +- packages/bytebot-agent/prisma/dev.db | Bin 0 -> 45056 bytes .../migration.sql | 57 -- .../20250413053912_message_role/migration.sql | 15 - .../migration.sql | 55 -- .../migration.sql | 3 - .../migration.sql | 2 - .../migration.sql | 2 - .../migration.sql | 81 --- .../migration.sql | 5 - .../20250706223912_model_picker/migration.sql | 14 - .../20250722041608_files/migration.sql | 16 - .../20250820172813_remove_auth/migration.sql | 40 -- .../migration.sql | 59 ++ .../prisma/migrations/migration_lock.toml | 2 +- packages/bytebot-agent/prisma/schema.prisma | 2 + .../src/agent/agent.constants.ts | 36 ++ .../bytebot-agent/src/agent/agent.module.ts | 6 +- .../src/agent/agent.processor.ts | 601 ++++++++++++------ .../src/agent/planner.service.spec.ts | 108 ++++ .../src/agent/planner.service.ts | 78 +++ .../src/agent/reflector.service.ts | 79 +++ packages/bytebot-agent/tsconfig.json | 5 +- 24 files changed, 1111 insertions(+), 518 deletions(-) create mode 100644 packages/bytebot-agent/prisma/dev.db delete mode 100644 packages/bytebot-agent/prisma/migrations/20250328022708_initial_migration/migration.sql delete mode 100644 packages/bytebot-agent/prisma/migrations/20250413053912_message_role/migration.sql delete mode 100644 packages/bytebot-agent/prisma/migrations/20250522200556_updated_task_structure/migration.sql delete mode 100644 packages/bytebot-agent/prisma/migrations/20250523162632_add_scheduling/migration.sql delete mode 100644 packages/bytebot-agent/prisma/migrations/20250529003255_tasks_control/migration.sql delete mode 100644 packages/bytebot-agent/prisma/migrations/20250530012753_tasks_control/migration.sql delete mode 100644 packages/bytebot-agent/prisma/migrations/20250619013027_add_better_auth_schema/migration.sql delete mode 100644 packages/bytebot-agent/prisma/migrations/20250622195148_add_user_to_task/migration.sql delete mode 100644 packages/bytebot-agent/prisma/migrations/20250706223912_model_picker/migration.sql delete mode 100644 packages/bytebot-agent/prisma/migrations/20250722041608_files/migration.sql delete mode 100644 packages/bytebot-agent/prisma/migrations/20250820172813_remove_auth/migration.sql create mode 100644 packages/bytebot-agent/prisma/migrations/20250905045058_add_task_plan_step/migration.sql create mode 100644 packages/bytebot-agent/src/agent/planner.service.spec.ts create mode 100644 packages/bytebot-agent/src/agent/planner.service.ts create mode 100644 packages/bytebot-agent/src/agent/reflector.service.ts diff --git a/packages/bytebot-agent/package-lock.json b/packages/bytebot-agent/package-lock.json index 04ba36dca..98c8bc056 100644 --- a/packages/bytebot-agent/package-lock.json +++ b/packages/bytebot-agent/package-lock.json @@ -20,7 +20,7 @@ "@nestjs/platform-socket.io": "^11.1.1", "@nestjs/schedule": "^6.0.0", "@nestjs/websockets": "^11.1.1", - "@prisma/client": "^6.6.0", + "@prisma/client": "^6.15.0", "@thallesp/nestjs-better-auth": "^1.0.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.2", @@ -49,6 +49,7 @@ "globals": "^15.14.0", "jest": "^29.7.0", "prettier": "^3.4.2", + "prisma": "^6.15.0", "source-map-support": "^0.5.21", "supertest": "^7.0.0", "ts-jest": "^29.2.5", @@ -2988,9 +2989,9 @@ } }, "node_modules/@prisma/client": { - "version": "6.6.0", - "resolved": "https://registry.npmjs.org/@prisma/client/-/client-6.6.0.tgz", - "integrity": "sha512-vfp73YT/BHsWWOAuthKQ/1lBgESSqYqAWZEYyTdGXyFAHpmewwWL2Iz6ErIzkj4aHbuc6/cGSsE6ZY+pBO04Cg==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-6.15.0.tgz", + "integrity": "sha512-wR2LXUbOH4cL/WToatI/Y2c7uzni76oNFND7+23ypLllBmIS8e3ZHhO+nud9iXSXKFt1SoM3fTZvHawg63emZw==", "hasInstallScript": true, "license": "Apache-2.0", "engines": { @@ -3009,6 +3010,69 @@ } } }, + "node_modules/@prisma/config": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/@prisma/config/-/config-6.15.0.tgz", + "integrity": "sha512-KMEoec9b2u6zX0EbSEx/dRpx1oNLjqJEBZYyK0S3TTIbZ7GEGoVyGyFRk4C72+A38cuPLbfQGQvgOD+gBErKlA==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "c12": "3.1.0", + "deepmerge-ts": "7.1.5", + "effect": "3.16.12", + "empathic": "2.0.0" + } + }, + "node_modules/@prisma/debug": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-6.15.0.tgz", + "integrity": "sha512-y7cSeLuQmyt+A3hstAs6tsuAiVXSnw9T55ra77z0nbNkA8Lcq9rNcQg6PI00by/+WnE/aMRJ/W7sZWn2cgIy1g==", + "devOptional": true, + "license": "Apache-2.0" + }, + "node_modules/@prisma/engines": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-6.15.0.tgz", + "integrity": "sha512-opITiR5ddFJ1N2iqa7mkRlohCZqVSsHhRcc29QXeldMljOf4FSellLT0J5goVb64EzRTKcIDeIsJBgmilNcKxA==", + "devOptional": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@prisma/debug": "6.15.0", + "@prisma/engines-version": "6.15.0-5.85179d7826409ee107a6ba334b5e305ae3fba9fb", + "@prisma/fetch-engine": "6.15.0", + "@prisma/get-platform": "6.15.0" + } + }, + "node_modules/@prisma/engines-version": { + "version": "6.15.0-5.85179d7826409ee107a6ba334b5e305ae3fba9fb", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-6.15.0-5.85179d7826409ee107a6ba334b5e305ae3fba9fb.tgz", + "integrity": "sha512-a/46aK5j6L3ePwilZYEgYDPrhBQ/n4gYjLxT5YncUTJJNRnTCVjPF86QdzUOLRdYjCLfhtZp9aum90W0J+trrg==", + "devOptional": true, + "license": "Apache-2.0" + }, + "node_modules/@prisma/fetch-engine": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-6.15.0.tgz", + "integrity": "sha512-xcT5f6b+OWBq6vTUnRCc7qL+Im570CtwvgSj+0MTSGA1o9UDSKZ/WANvwtiRXdbYWECpyC3CukoG3A04VTAPHw==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "@prisma/debug": "6.15.0", + "@prisma/engines-version": "6.15.0-5.85179d7826409ee107a6ba334b5e305ae3fba9fb", + "@prisma/get-platform": "6.15.0" + } + }, + "node_modules/@prisma/get-platform": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-6.15.0.tgz", + "integrity": "sha512-Jbb+Xbxyp05NSR1x2epabetHiXvpO8tdN2YNoWoA/ZsbYyxxu/CO/ROBauIFuMXs3Ti+W7N7SJtWsHGaWte9Rg==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "@prisma/debug": "6.15.0" + } + }, "node_modules/@sec-ant/readable-stream": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", @@ -3085,6 +3149,13 @@ "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz", "integrity": "sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==" }, + "node_modules/@standard-schema/spec": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.0.0.tgz", + "integrity": "sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA==", + "devOptional": true, + "license": "MIT" + }, "node_modules/@swc/cli": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/@swc/cli/-/cli-0.6.0.tgz", @@ -5598,6 +5669,48 @@ "node": ">= 0.8" } }, + "node_modules/c12": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/c12/-/c12-3.1.0.tgz", + "integrity": "sha512-uWoS8OU1MEIsOv8p/5a82c3H31LsWVR5qiyXVfBNOzfffjUWtPnhAb4BYI2uG2HfGmZmFjCtui5XNWaps+iFuw==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "chokidar": "^4.0.3", + "confbox": "^0.2.2", + "defu": "^6.1.4", + "dotenv": "^16.6.1", + "exsolve": "^1.0.7", + "giget": "^2.0.0", + "jiti": "^2.4.2", + "ohash": "^2.0.11", + "pathe": "^2.0.3", + "perfect-debounce": "^1.0.0", + "pkg-types": "^2.2.0", + "rc9": "^2.1.2" + }, + "peerDependencies": { + "magicast": "^0.3.5" + }, + "peerDependenciesMeta": { + "magicast": { + "optional": true + } + } + }, + "node_modules/c12/node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "devOptional": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/cacheable-lookup": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", @@ -5735,7 +5848,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "readdirp": "^4.0.1" @@ -5773,6 +5886,16 @@ "node": ">=8" } }, + "node_modules/citty": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/citty/-/citty-0.1.6.tgz", + "integrity": "sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "consola": "^3.2.3" + } + }, "node_modules/cjs-module-lexer": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", @@ -6024,6 +6147,13 @@ "typedarray": "^0.0.6" } }, + "node_modules/confbox": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.2.tgz", + "integrity": "sha512-1NB+BKqhtNipMsov4xI/NnhCKp9XG9NamYp5PVm9klAT0fsrNPjaFICsCFhNhwZJKNh7zB/3q8qXz0E9oaMNtQ==", + "devOptional": true, + "license": "MIT" + }, "node_modules/consola": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", @@ -6269,6 +6399,16 @@ "node": ">=0.10.0" } }, + "node_modules/deepmerge-ts": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/deepmerge-ts/-/deepmerge-ts-7.1.5.tgz", + "integrity": "sha512-HOJkrhaYsweh+W+e74Yn7YStZOilkoPb6fycpwNLKzSPtruFs48nYis0zy5yJz1+ktUhHxoRDJ27RQAWLIJVJw==", + "devOptional": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/defaults": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/defaults/-/defaults-3.0.0.tgz", @@ -6295,8 +6435,7 @@ "node_modules/defu": { "version": "6.1.4", "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.4.tgz", - "integrity": "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==", - "peer": true + "integrity": "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==" }, "node_modules/delayed-stream": { "version": "1.0.0", @@ -6316,6 +6455,13 @@ "node": ">= 0.8" } }, + "node_modules/destr": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/destr/-/destr-2.0.5.tgz", + "integrity": "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==", + "devOptional": true, + "license": "MIT" + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -6420,6 +6566,17 @@ "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", "license": "MIT" }, + "node_modules/effect": { + "version": "3.16.12", + "resolved": "https://registry.npmjs.org/effect/-/effect-3.16.12.tgz", + "integrity": "sha512-N39iBk0K71F9nb442TLbTkjl24FLUzuvx2i1I2RsEAQsdAdUTuUoW0vlfUXgkMTUOnYqKnWcFfqw4hK4Pw27hg==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "fast-check": "^3.23.1" + } + }, "node_modules/ejs": { "version": "3.1.10", "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", @@ -6463,6 +6620,16 @@ "dev": true, "license": "MIT" }, + "node_modules/empathic": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/empathic/-/empathic-2.0.0.tgz", + "integrity": "sha512-i6UzDscO/XfAcNYD75CfICkmfLedpyPDdozrLMmQc5ORaQcdMoc21OnlEylMIqI7U8eniKrPMxxtj8k0vhmJhA==", + "devOptional": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -7089,6 +7256,13 @@ "node": ">= 0.6" } }, + "node_modules/exsolve": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/exsolve/-/exsolve-1.0.7.tgz", + "integrity": "sha512-VO5fQUzZtI6C+vx4w/4BWJpg3s/5l+6pRQEHzFRM8WFi4XffSP1Z+4qi7GbjWbvRQEbdIco5mIMq+zX4rPuLrw==", + "devOptional": true, + "license": "MIT" + }, "node_modules/ext-list": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/ext-list/-/ext-list-2.2.2.tgz", @@ -7122,6 +7296,29 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", "license": "MIT" }, + "node_modules/fast-check": { + "version": "3.23.2", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-3.23.2.tgz", + "integrity": "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==", + "devOptional": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^6.1.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -7719,6 +7916,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/giget": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/giget/-/giget-2.0.0.tgz", + "integrity": "sha512-L5bGsVkxJbJgdnwyuheIunkGatUF/zssUoxxjACCseZYAVbaqdh9Tsmmlkl8vYan09H7sbvKt4pS8GqKLBrEzA==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "citty": "^0.1.6", + "consola": "^3.4.0", + "defu": "^6.1.4", + "node-fetch-native": "^1.6.6", + "nypm": "^0.6.0", + "pathe": "^2.0.3" + }, + "bin": { + "giget": "dist/cli.mjs" + } + }, "node_modules/glob": { "version": "11.0.1", "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.1.tgz", @@ -9074,6 +9289,16 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/jiti": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.5.1.tgz", + "integrity": "sha512-twQoecYPiVA5K/h6SxtORw/Bs3ar+mLUtoPSc7iMXzQzK8d7eJ/R09wmTwAjiamETn1cXYPGfNnu7DMoHgu12w==", + "devOptional": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -9781,6 +10006,13 @@ } } }, + "node_modules/node-fetch-native": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz", + "integrity": "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==", + "devOptional": true, + "license": "MIT" + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -9831,6 +10063,26 @@ "node": ">=8" } }, + "node_modules/nypm": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/nypm/-/nypm-0.6.1.tgz", + "integrity": "sha512-hlacBiRiv1k9hZFiphPUkfSQ/ZfQzZDzC+8z0wL3lvDAOUu/2NnChkKuMoMjNur/9OpKuz2QsIeiPVN0xM5Q0w==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "citty": "^0.1.6", + "consola": "^3.4.2", + "pathe": "^2.0.3", + "pkg-types": "^2.2.0", + "tinyexec": "^1.0.1" + }, + "bin": { + "nypm": "dist/cli.mjs" + }, + "engines": { + "node": "^14.16.0 || >=16.10.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -9860,6 +10112,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/ohash": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.11.tgz", + "integrity": "sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==", + "devOptional": true, + "license": "MIT" + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -10166,6 +10425,13 @@ "node": ">=8" } }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "devOptional": true, + "license": "MIT" + }, "node_modules/peek-readable": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-7.0.0.tgz", @@ -10187,6 +10453,13 @@ "dev": true, "license": "MIT" }, + "node_modules/perfect-debounce": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/perfect-debounce/-/perfect-debounce-1.0.0.tgz", + "integrity": "sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==", + "devOptional": true, + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -10296,6 +10569,18 @@ "node": ">=8" } }, + "node_modules/pkg-types": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-2.3.0.tgz", + "integrity": "sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "confbox": "^0.2.2", + "exsolve": "^1.0.7", + "pathe": "^2.0.3" + } + }, "node_modules/pluralize": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", @@ -10373,6 +10658,32 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/prisma": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-6.15.0.tgz", + "integrity": "sha512-E6RCgOt+kUVtjtZgLQDBJ6md2tDItLJNExwI0XJeBc1FKL+Vwb+ovxXxuok9r8oBgsOXBA33fGDuE/0qDdCWqQ==", + "devOptional": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@prisma/config": "6.15.0", + "@prisma/engines": "6.15.0" + }, + "bin": { + "prisma": "build/index.js" + }, + "engines": { + "node": ">=18.18" + }, + "peerDependencies": { + "typescript": ">=5.1.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -10414,7 +10725,7 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", - "dev": true, + "devOptional": true, "funding": [ { "type": "individual", @@ -10528,6 +10839,17 @@ "node": ">= 0.8" } }, + "node_modules/rc9": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/rc9/-/rc9-2.1.2.tgz", + "integrity": "sha512-btXCnMmRIBINM2LDZoEmOogIZU7Qe7zn4BpomSKZ/ykbLObuBdvG+mFq11DL6fjH1DRwHhrlgtYWG96bJiC7Cg==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "defu": "^6.1.4", + "destr": "^2.0.3" + } + }, "node_modules/react-is": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", @@ -10553,7 +10875,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">= 14.18.0" @@ -11928,6 +12250,13 @@ "dev": true, "license": "MIT" }, + "node_modules/tinyexec": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.1.tgz", + "integrity": "sha512-5uC6DDlmeqiOwCPmK9jMSdOuZTh8bU39Ys6yidB+UTt5hfZUPGAypSgFRiEp+jbi9qH40BLDvy85jIU88wKSqw==", + "devOptional": true, + "license": "MIT" + }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", diff --git a/packages/bytebot-agent/package.json b/packages/bytebot-agent/package.json index 390f31c07..25bbb59d4 100644 --- a/packages/bytebot-agent/package.json +++ b/packages/bytebot-agent/package.json @@ -15,7 +15,7 @@ "start:debug": "npm run build --prefix ../shared && nest start --debug --watch", "start:prod": "npm run build --prefix ../shared && npx prisma migrate deploy && npx prisma generate && node dist/main", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", - "test": "jest", + "test": "npm run build --prefix ../shared && jest", "test:watch": "jest --watch", "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", @@ -33,7 +33,7 @@ "@nestjs/platform-socket.io": "^11.1.1", "@nestjs/schedule": "^6.0.0", "@nestjs/websockets": "^11.1.1", - "@prisma/client": "^6.6.0", + "@prisma/client": "^6.15.0", "@thallesp/nestjs-better-auth": "^1.0.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.2", @@ -62,6 +62,7 @@ "globals": "^15.14.0", "jest": "^29.7.0", "prettier": "^3.4.2", + "prisma": "^6.15.0", "source-map-support": "^0.5.21", "supertest": "^7.0.0", "ts-jest": "^29.2.5", @@ -77,16 +78,19 @@ "json", "ts" ], - "rootDir": "src", + "rootDir": ".", "testRegex": ".*\\.spec\\.ts$", "transform": { "^.+\\.(t|j)s$": "ts-jest" }, "collectCoverageFrom": [ - "**/*.(t|j)s" + "src/**/*.(t|j)s" ], - "coverageDirectory": "../coverage", - "testEnvironment": "node" + "coverageDirectory": "./coverage", + "testEnvironment": "node", + "moduleNameMapper": { + "^@bytebot/shared(|/.*)$": "/../shared/src$1" + } }, "overrides": { "openai": { diff --git a/packages/bytebot-agent/prisma/dev.db b/packages/bytebot-agent/prisma/dev.db new file mode 100644 index 0000000000000000000000000000000000000000..da8bc5eeea43972181a21baaa7e0b2f47a27ebf3 GIT binary patch literal 45056 zcmeI&%WvaE90zcxP1812RW4PKda-h#TZurNN0SI4a1*x*3qJ{9an#4OE+cTfv&ofTE>PI!pC1P)6 z4|G?|3a<%~i15BB3WBi5U-ZLYw&;75ud|QHt@Tx_d%|0f#`_0<2=VB1VgI*-pZ22r zKkWPx|26i-&Uex0F?rRFD2D(9{)YljQoGxU!^6n4cU`^PC+F7W6)(3Gxvhw8`K+dh zNmiT`@5M#BBrP*3wv|uXVxvi4omx%2Y*p)W>%RD*a{tXqouNLUJA`WSE$+>D#TLhU zM7LBMZRJ8~O%Kj@Gj&&Aen7(}x=T#iO^OxT?~%8Du`+q6D)Wuzx!vq76%Q>c1bR?BilVI`eQ)~TQ*esUa-CEhuV2-eUf z51j{nI-6*^=Z^e;Yb;Kh#NPLw=C-#JM@Nyb-}3`oCyt|E5&p3g4lghLOA$6kLzfIG z`Uh&WaW)m_X!eLDOep@-SRBi$TBSi~w2yi1R~W*i;|&J7eG^1@F|ZuxACK+oGf$hi zG_j^z=WX)_%DoBp+VSR=2LBDy^Vxj`2A$ zy*@F|NA_$=<%V~dFUdnlTZ1JfuP*N!(xhW87lC+~jG z8d?qwx~5aq*%I7M1xP_}w8_mKuej`h{m~V@au$}`AsqE-hG|{hxW0YXxl__>(5BgP zy9PUGP;1)dX+RCnX2Y`PS_49N9*lY0_3^lG(E~X&9~{jXdF;kToZf488Y;a}Q!0xG zOvg{k8?nU4(yAG8F>so+Shh*OS?sWH7yI-TIp_-X4+{h!009U<00Izz00bZa0SG_< z0)JcJ@lhn+dAz^X6ojo^(@2|zT(OuccC&gamnn5qMZ-v^h}q4T(xrUKFtd``GYhA5 zC7}9TK}y+NCR;G2f{~R<*?g~OmNF8d+lxlAn2CJ*`sw#S zXVRH`x|GhRbNO_>sOhHZ|MaV|-=+MY9sFuP#%aGI2fqvS4+{h!009U<00Izz00bZa z0SG_<0{=0AUer#aR|LK4F#{vNeKmY;|fB*y_009U<00Izzz#0p%=l}Trzs3PZ z*B}4^2tWV=5P$##AOHafKmY<%0MGxC1`vP%1Rwwb2tWV=5P$##AOL~&7r^uX^^Y-n z2muH{00Izz00bZa0SG_<0uaFSf8+oJAOHafKmY;|fB*y_009U m.role === Role.USER); + + if (initialMessage) { + const generatedPlan = await this.plannerService.generatePlan( + task, + initialMessage, + ); + await this.tasksService.update(taskId, { + plan: generatedPlan as any, + }); + this.logger.log(`Plan generated and saved for task ${taskId}.`); + } else { + this.logger.warn( + `Could not find an initial message for task ${taskId} to generate a plan.`, + ); + } + } + // Kick off the first iteration without blocking the caller void this.runIteration(taskId); } @@ -137,7 +165,8 @@ export class AgentProcessor { } try { - const task: Task = await this.tasksService.findById(taskId); + // Use findById to ensure we get the latest task state, including the plan + const task = await this.tasksService.findById(taskId); if (task.status !== TaskStatus.RUNNING) { this.logger.log( @@ -149,237 +178,134 @@ export class AgentProcessor { } this.logger.log(`Processing iteration for task ID: ${taskId}`); - - // Refresh abort controller for this iteration to avoid accumulating - // "abort" listeners on a single AbortSignal across iterations. this.abortController = new AbortController(); - const latestSummary = await this.summariesService.findLatest(taskId); - const unsummarizedMessages = - await this.messagesService.findUnsummarized(taskId); - const messages = [ - ...(latestSummary - ? [ - { - id: '', - createdAt: new Date(), - updatedAt: new Date(), - taskId, - summaryId: null, - userId: null, - role: Role.USER, - content: [ - { - type: MessageContentType.Text, - text: latestSummary.content, - }, - ], - }, - ] - : []), - ...unsummarizedMessages, - ]; - this.logger.debug( - `Sending ${messages.length} messages to LLM for processing`, - ); - - const model = task.model as unknown as BytebotAgentModel; - let agentResponse: BytebotAgentResponse; - - const service = this.services[model.provider]; - if (!service) { - this.logger.warn( - `No service found for model provider: ${model.provider}`, - ); - await this.tasksService.update(taskId, { - status: TaskStatus.FAILED, - }); - this.isProcessing = false; - this.currentTaskId = null; - return; - } - - agentResponse = await service.generateMessage( - AGENT_SYSTEM_PROMPT, - messages, - model.name, - true, - this.abortController.signal, - ); - - const messageContentBlocks = agentResponse.contentBlocks; - - this.logger.debug( - `Received ${messageContentBlocks.length} content blocks from LLM`, - ); - - if (messageContentBlocks.length === 0) { - this.logger.warn( - `Task ID: ${taskId} received no content blocks from LLM, marking as failed`, - ); - await this.tasksService.update(taskId, { - status: TaskStatus.FAILED, - }); - this.isProcessing = false; - this.currentTaskId = null; - return; - } - - await this.messagesService.create({ - content: messageContentBlocks, - role: Role.ASSISTANT, - taskId, - }); - - // Calculate if we need to summarize based on token usage - const contextWindow = model.contextWindow || 200000; // Default to 200k if not specified - const contextThreshold = contextWindow * 0.75; - const shouldSummarize = - agentResponse.tokenUsage.totalTokens >= contextThreshold; - - if (shouldSummarize) { - try { - // After we've successfully generated a response, we can summarize the unsummarized messages - const summaryResponse = await service.generateMessage( - SUMMARIZATION_SYSTEM_PROMPT, - [ - ...messages, - { - id: '', - createdAt: new Date(), - updatedAt: new Date(), - taskId, - summaryId: null, - userId: null, - role: Role.USER, - content: [ - { - type: MessageContentType.Text, - text: 'Respond with a summary of the messages above. Do not include any additional information.', - }, - ], - }, - ], - model.name, - false, - this.abortController.signal, - ); - - const summaryContentBlocks = summaryResponse.contentBlocks; - - this.logger.debug( - `Received ${summaryContentBlocks.length} summary content blocks from LLM`, - ); - const summaryContent = summaryContentBlocks - .filter( - (block: MessageContentBlock) => - block.type === MessageContentType.Text, - ) - .map((block: TextContentBlock) => block.text) - .join('\n'); - - const summary = await this.summariesService.create({ - content: summaryContent, - taskId, + const plan = (task.plan as string[]) || []; + const planStep = task.planStep; + + // If there's a plan, use the plan-driven logic + if (plan && plan.length > 0) { + // Check if the plan is complete + if (planStep >= plan.length) { + this.logger.log(`Plan complete for task ${taskId}. Marking as completed.`); + await this.tasksService.update(taskId, { + status: TaskStatus.COMPLETED, + completedAt: new Date(), }); - - await this.messagesService.attachSummary(taskId, summary.id, [ - ...messages.map((message) => { - return message.id; - }), - ]); - - this.logger.log( - `Generated summary for task ${taskId} due to token usage (${agentResponse.tokenUsage.totalTokens}/${contextWindow})`, - ); - } catch (error: any) { - this.logger.error( - `Error summarizing messages for task ID: ${taskId}`, - error.stack, - ); + this.stopProcessing(); + return; } - } - - this.logger.debug( - `Token usage for task ${taskId}: ${agentResponse.tokenUsage.totalTokens}/${contextWindow} (${Math.round((agentResponse.tokenUsage.totalTokens / contextWindow) * 100)}%)`, - ); - const generatedToolResults: ToolResultContentBlock[] = []; + const messages = await this._getPlanDrivenLLMContext(task); + const agentResponse = await this._callLLM(task, messages); - let setTaskStatusToolUseBlock: SetTaskStatusToolUseBlock | null = null; - - for (const block of messageContentBlocks) { - if (isComputerToolUseContentBlock(block)) { - const result = await handleComputerToolUse(block, this.logger); - generatedToolResults.push(result); - } + await this.messagesService.create({ + content: agentResponse.contentBlocks, + role: Role.ASSISTANT, + taskId, + }); - if (isCreateTaskToolUseBlock(block)) { - const type = block.input.type?.toUpperCase() as TaskType; - const priority = block.input.priority?.toUpperCase() as TaskPriority; - - await this.tasksService.create({ - description: block.input.description, - type, - createdBy: Role.ASSISTANT, - ...(block.input.scheduledFor && { - scheduledFor: new Date(block.input.scheduledFor), - }), - model: task.model, - priority, - }); + const { generatedToolResults, setTaskStatusToolUseBlock } = + await this._executeTools(task, agentResponse.contentBlocks); - generatedToolResults.push({ - type: MessageContentType.ToolResult, - tool_use_id: block.id, - content: [ - { - type: MessageContentType.Text, - text: 'The task has been created', - }, - ], + if (generatedToolResults.length > 0) { + await this.messagesService.create({ + content: generatedToolResults, + role: Role.USER, + taskId, }); } - if (isSetTaskStatusToolUseBlock(block)) { - setTaskStatusToolUseBlock = block; - - generatedToolResults.push({ - type: MessageContentType.ToolResult, - tool_use_id: block.id, - is_error: block.input.status === 'failed', - content: [ - { - type: MessageContentType.Text, - text: block.input.description, - }, - ], - }); - } - } + const reflectionContext = [ + ...messages, + { + role: Role.ASSISTANT, + content: agentResponse.contentBlocks, + } as Message, + { + role: Role.USER, + content: generatedToolResults, + } as Message, + ]; + + const reflection = await this.reflectorService.reflectOnOutcome( + task, + reflectionContext, + ); - if (generatedToolResults.length > 0) { await this.messagesService.create({ - content: generatedToolResults, - role: Role.USER, + content: [ + { + type: MessageContentType.Text, + text: `Reflection: ${reflection.status} - ${reflection.reason}`, + }, + ], + role: Role.ASSISTANT, taskId, }); - } - // Update the task status after all tool results have been generated if we have a set task status tool use block - if (setTaskStatusToolUseBlock) { - switch (setTaskStatusToolUseBlock.input.status) { - case 'completed': + switch (reflection.status) { + case 'success': + this.logger.log( + `Step ${planStep + 1} of plan for task ${taskId} was successful. Advancing to next step.`, + ); await this.tasksService.update(taskId, { - status: TaskStatus.COMPLETED, - completedAt: new Date(), + planStep: planStep + 1, }); break; - case 'needs_help': + case 'failure': + this.logger.warn( + `Step ${planStep + 1} of plan for task ${taskId} failed. Reason: ${reflection.reason}. Marking task as needs help.`, + ); await this.tasksService.update(taskId, { status: TaskStatus.NEEDS_HELP, }); break; + case 'retry': + this.logger.log( + `Step ${planStep + 1} of plan for task ${taskId} needs a retry. Reason: ${reflection.reason}. Retrying step.`, + ); + // Do nothing, the loop will repeat the same step with the new reflection context. + break; + } + + if (setTaskStatusToolUseBlock) { + await this._updateTaskState(taskId, setTaskStatusToolUseBlock); + } + } else { + // Fallback to old logic if there's no plan + this.logger.warn(`No plan found for task ${taskId}. Running in reactive mode.`); + const messages = await this._getLLMContext(taskId); + const agentResponse = await this._callLLM(task, messages); + + if (agentResponse.contentBlocks.length === 0) { + this.logger.warn(`Task ID: ${taskId} received no content blocks from LLM, marking as failed`); + await this.tasksService.update(taskId, { status: TaskStatus.FAILED }); + this.stopProcessing(); + return; + } + + await this.messagesService.create({ + content: agentResponse.contentBlocks, + role: Role.ASSISTANT, + taskId, + }); + + await this._handleSummarization(task, messages, agentResponse); + + const { generatedToolResults, setTaskStatusToolUseBlock } = + await this._executeTools(task, agentResponse.contentBlocks); + + if (generatedToolResults.length > 0) { + await this.messagesService.create({ + content: generatedToolResults, + role: Role.USER, + taskId, + }); + } + + if (setTaskStatusToolUseBlock) { + await this._updateTaskState(taskId, setTaskStatusToolUseBlock); } } @@ -404,6 +330,261 @@ export class AgentProcessor { } } + private async _getLLMContext(taskId: string): Promise { + const latestSummary = await this.summariesService.findLatest(taskId); + const unsummarizedMessages = + await this.messagesService.findUnsummarized(taskId); + const messages = [ + ...(latestSummary + ? [ + ({ + id: '', + createdAt: new Date(), + updatedAt: new Date(), + taskId, + summaryId: null, + userId: null, + role: Role.USER, + content: [ + { + type: MessageContentType.Text, + text: latestSummary.content, + }, + ], + } as unknown) as Message, + ] + : []), + ...unsummarizedMessages, + ]; + this.logger.debug( + `Sending ${messages.length} messages to LLM for processing`, + ); + return messages; + } + + private async _getPlanDrivenLLMContext(task: Task): Promise { + const plan = (task.plan as string[]) || []; + const planStep = task.planStep; + const currentStep = plan[planStep]; + + const planContext = ` +The overall goal is: "${task.description}" + +Here is the plan: +${plan.map((step, index) => `${index + 1}. ${step}`).join('\n')} + +You are currently on step ${planStep + 1}: "${currentStep}" +Please perform the action(s) required to complete this step. Focus only on this step. +`; + + const unsummarizedMessages = await this.messagesService.findUnsummarized( + task.id, + ); + + const messages = [ + { + id: '', + createdAt: new Date(), + updatedAt: new Date(), + taskId: task.id, + summaryId: null, + userId: null, + role: Role.USER, + content: [ + { + type: MessageContentType.Text, + text: planContext, + }, + ], + } as unknown as Message, + ...unsummarizedMessages.slice(-10), // Take the last 10 messages to keep context short + ]; + + return messages; + } + + private async _callLLM( + task: Task, + messages: Message[], + ): Promise { + const model = task.model as unknown as BytebotAgentModel; + const service = this.services[model.provider]; + if (!service) { + this.logger.warn( + `No service found for model provider: ${model.provider}`, + ); + await this.tasksService.update(task.id, { + status: TaskStatus.FAILED, + }); + this.isProcessing = false; + this.currentTaskId = null; + throw new Error(`No service for provider ${model.provider}`); + } + + return service.generateMessage( + AGENT_SYSTEM_PROMPT, + messages, + model.name, + true, + this.abortController.signal, + ); + } + + private async _executeTools( + task: Task, + messageContentBlocks: MessageContentBlock[], + ): Promise<{ + generatedToolResults: ToolResultContentBlock[]; + setTaskStatusToolUseBlock: SetTaskStatusToolUseBlock | null; + }> { + const generatedToolResults: ToolResultContentBlock[] = []; + let setTaskStatusToolUseBlock: SetTaskStatusToolUseBlock | null = null; + + for (const block of messageContentBlocks) { + if (isComputerToolUseContentBlock(block)) { + const result = await handleComputerToolUse(block, this.logger); + generatedToolResults.push(result); + } + + if (isCreateTaskToolUseBlock(block)) { + const type = block.input.type?.toUpperCase() as TaskType; + const priority = block.input.priority?.toUpperCase() as TaskPriority; + + await this.tasksService.create({ + description: block.input.description, + type, + createdBy: Role.ASSISTANT, + ...(block.input.scheduledFor && { + scheduledFor: new Date(block.input.scheduledFor), + }), + model: task.model, + priority, + }); + + generatedToolResults.push({ + type: MessageContentType.ToolResult, + tool_use_id: block.id, + content: [ + { + type: MessageContentType.Text, + text: 'The task has been created', + }, + ], + }); + } + + if (isSetTaskStatusToolUseBlock(block)) { + setTaskStatusToolUseBlock = block; + + generatedToolResults.push({ + type: MessageContentType.ToolResult, + tool_use_id: block.id, + is_error: block.input.status === 'failed', + content: [ + { + type: MessageContentType.Text, + text: block.input.description, + }, + ], + }); + } + } + + return { generatedToolResults, setTaskStatusToolUseBlock }; + } + + private async _updateTaskState( + taskId: string, + setTaskStatusToolUseBlock: SetTaskStatusToolUseBlock, + ) { + switch (setTaskStatusToolUseBlock.input.status) { + case 'completed': + await this.tasksService.update(taskId, { + status: TaskStatus.COMPLETED, + completedAt: new Date(), + }); + break; + case 'needs_help': + await this.tasksService.update(taskId, { + status: TaskStatus.NEEDS_HELP, + }); + break; + } + } + + private async _handleSummarization( + task: Task, + messages: Message[], + agentResponse: BytebotAgentResponse, + ) { + const model = task.model as unknown as BytebotAgentModel; + const service = this.services[model.provider]; + const contextWindow = model.contextWindow || 200000; + const contextThreshold = contextWindow * 0.75; + const shouldSummarize = + agentResponse.tokenUsage.totalTokens >= contextThreshold; + + if (shouldSummarize) { + try { + const summaryResponse = await service.generateMessage( + SUMMARIZATION_SYSTEM_PROMPT, + [ + ...messages, + { + id: '', + createdAt: new Date(), + updatedAt: new Date(), + taskId: task.id, + summaryId: null, + userId: null, + role: Role.USER, + content: [ + { + type: MessageContentType.Text, + text: 'Respond with a summary of the messages above. Do not include any additional information.', + }, + ], + }, + ], + model.name, + false, + this.abortController.signal, + ); + + const summaryContentBlocks = summaryResponse.contentBlocks; + const summaryContent = summaryContentBlocks + .filter( + (block: MessageContentBlock) => + block.type === MessageContentType.Text, + ) + .map((block: TextContentBlock) => block.text) + .join('\n'); + + const summary = await this.summariesService.create({ + content: summaryContent, + taskId: task.id, + }); + + await this.messagesService.attachSummary(task.id, summary.id, [ + ...messages.map((message) => message.id), + ]); + + this.logger.log( + `Generated summary for task ${task.id} due to token usage (${agentResponse.tokenUsage.totalTokens}/${contextWindow})`, + ); + } catch (error: any) { + this.logger.error( + `Error summarizing messages for task ID: ${task.id}`, + error.stack, + ); + } + } + + this.logger.debug( + `Token usage for task ${task.id}: ${agentResponse.tokenUsage.totalTokens}/${contextWindow} (${Math.round((agentResponse.tokenUsage.totalTokens / contextWindow) * 100)}%)`, + ); + } + async stopProcessing(): Promise { if (!this.isProcessing) { return; diff --git a/packages/bytebot-agent/src/agent/planner.service.spec.ts b/packages/bytebot-agent/src/agent/planner.service.spec.ts new file mode 100644 index 000000000..0f8d4d33e --- /dev/null +++ b/packages/bytebot-agent/src/agent/planner.service.spec.ts @@ -0,0 +1,108 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { PlannerService } from './planner.service'; +import { AnthropicService } from '../anthropic/anthropic.service'; +import { OpenAIService } from '../openai/openai.service'; +import { GoogleService } from '../google/google.service'; +import { ProxyService } from '../proxy/proxy.service'; +import { Task, Message, Role } from '@prisma/client'; +import { MessageContentType } from '@bytebot/shared'; + +describe('PlannerService', () => { + let service: PlannerService; + let anthropicService: AnthropicService; + + const mockTask: Task = { + id: 'test-task', + description: 'Test task description', + model: { provider: 'anthropic', name: 'claude-3-opus-20240229' }, + // Add other required Task properties with dummy values + type: 'IMMEDIATE', + status: 'PENDING', + priority: 'MEDIUM', + control: 'ASSISTANT', + createdAt: new Date(), + createdBy: 'USER', + updatedAt: new Date(), + executedAt: null, + completedAt: null, + queuedAt: null, + error: null, + result: null, + plan: null, + planStep: 0, + scheduledFor: null, + }; + + const mockMessage: Message = { + id: 'test-message', + content: [{ type: MessageContentType.Text, text: 'Initial user request' }], + role: Role.USER, + createdAt: new Date(), + updatedAt: new Date(), + taskId: 'test-task', + summaryId: null, + }; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + PlannerService, + { + provide: AnthropicService, + useValue: { + generateMessage: jest.fn(), + }, + }, + { provide: OpenAIService, useValue: { generateMessage: jest.fn() } }, + { provide: GoogleService, useValue: { generateMessage: jest.fn() } }, + { provide: ProxyService, useValue: { generateMessage: jest.fn() } }, + ], + }).compile(); + + service = module.get(PlannerService); + anthropicService = module.get(AnthropicService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + it('should generate a plan from a valid JSON response', async () => { + const mockPlan = ['Step 1', 'Step 2']; + const mockResponse = { + contentBlocks: [{ type: MessageContentType.Text, text: JSON.stringify(mockPlan) }], + tokenUsage: { totalTokens: 100 }, + }; + (anthropicService.generateMessage as jest.Mock).mockResolvedValue(mockResponse); + + const plan = await service.generatePlan(mockTask, mockMessage); + + expect(plan).toEqual(mockPlan); + expect(anthropicService.generateMessage).toHaveBeenCalled(); + }); + + it('should handle a non-JSON response gracefully', async () => { + const mockPlanText = 'This is not JSON'; + const mockResponse = { + contentBlocks: [{ type: MessageContentType.Text, text: mockPlanText }], + tokenUsage: { totalTokens: 100 }, + }; + (anthropicService.generateMessage as jest.Mock).mockResolvedValue(mockResponse); + + const plan = await service.generatePlan(mockTask, mockMessage); + + expect(plan).toEqual([mockPlanText]); + }); + + it('should handle an empty response', async () => { + const mockResponse = { + contentBlocks: [], + tokenUsage: { totalTokens: 100 }, + }; + (anthropicService.generateMessage as jest.Mock).mockResolvedValue(mockResponse); + + const plan = await service.generatePlan(mockTask, mockMessage); + + expect(plan).toEqual([]); + }); +}); diff --git a/packages/bytebot-agent/src/agent/planner.service.ts b/packages/bytebot-agent/src/agent/planner.service.ts new file mode 100644 index 000000000..9f6c3fdda --- /dev/null +++ b/packages/bytebot-agent/src/agent/planner.service.ts @@ -0,0 +1,78 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { AnthropicService } from '../anthropic/anthropic.service'; +import { OpenAIService } from '../openai/openai.service'; +import { GoogleService } from '../google/google.service'; +import { ProxyService } from '../proxy/proxy.service'; +import { BytebotAgentModel, BytebotAgentService } from './agent.types'; +import { PLANNING_SYSTEM_PROMPT } from './agent.constants'; +import { Message, Task } from '@prisma/client'; +import { MessageContentType } from '@bytebot/shared'; + +@Injectable() +export class PlannerService { + private readonly logger = new Logger(PlannerService.name); + private services: Record = {}; + + constructor( + private readonly anthropicService: AnthropicService, + private readonly openaiService: OpenAIService, + private readonly googleService: GoogleService, + private readonly proxyService: ProxyService, + ) { + this.services = { + anthropic: this.anthropicService, + openai: this.openaiService, + google: this.googleService, + proxy: this.proxyService, + }; + } + + async generatePlan(task: Task, initialMessage: Message): Promise { + this.logger.log(`Generating plan for task ID: ${task.id}`); + + const model = task.model as unknown as BytebotAgentModel; + const service = this.services[model.provider]; + + if (!service) { + this.logger.error(`No service found for model provider: ${model.provider}`); + return []; + } + + try { + const response = await service.generateMessage( + PLANNING_SYSTEM_PROMPT, + [initialMessage], + model.name, + false, // No tools needed for planning + ); + + const planText = response.contentBlocks + .filter(block => block.type === MessageContentType.Text) + .map(block => block.text) + .join('\n'); + + if (!planText) { + return []; + } + + // Attempt to parse the plan as JSON, but fall back to a single step if it fails + try { + const plan = JSON.parse(planText); + if (Array.isArray(plan) && plan.every(item => typeof item === 'string')) { + this.logger.log(`Generated plan with ${plan.length} steps for task ID: ${task.id}`); + return plan; + } else { + this.logger.warn(`Parsed plan is not an array of strings for task ID: ${task.id}.`); + return [planText]; + } + } catch (e) { + this.logger.warn(`Failed to parse plan as JSON for task ID: ${task.id}. Using raw text as a single step. Raw output: ${planText}`); + return [planText]; + } + + } catch (error) { + this.logger.error(`Failed to generate plan for task ID: ${task.id}`, error.stack); + return []; + } + } +} diff --git a/packages/bytebot-agent/src/agent/reflector.service.ts b/packages/bytebot-agent/src/agent/reflector.service.ts new file mode 100644 index 000000000..6b5a4476f --- /dev/null +++ b/packages/bytebot-agent/src/agent/reflector.service.ts @@ -0,0 +1,79 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { AnthropicService } from '../anthropic/anthropic.service'; +import { OpenAIService } from '../openai/openai.service'; +import { GoogleService } from '../google/google.service'; +import { ProxyService } from '../proxy/proxy.service'; +import { BytebotAgentModel, BytebotAgentService } from './agent.types'; +import { REFLECTION_SYSTEM_PROMPT } from './agent.constants'; +import { Message, Role, Task } from '@prisma/client'; +import { MessageContentType } from '@bytebot/shared'; + +export interface Reflection { + status: 'success' | 'failure' | 'retry'; + reason: string; +} + +@Injectable() +export class ReflectorService { + private readonly logger = new Logger(ReflectorService.name); + private services: Record = {}; + + constructor( + private readonly anthropicService: AnthropicService, + private readonly openaiService: OpenAIService, + private readonly googleService: GoogleService, + private readonly proxyService: ProxyService, + ) { + this.services = { + anthropic: this.anthropicService, + openai: this.openaiService, + google: this.googleService, + proxy: this.proxyService, + }; + } + + async reflectOnOutcome(task: Task, conversationHistory: Message[]): Promise { + this.logger.log(`Reflecting on outcome for task ID: ${task.id}`); + + const model = task.model as unknown as BytebotAgentModel; + const service = this.services[model.provider]; + + if (!service) { + const errorMessage = `No service found for model provider: ${model.provider}`; + this.logger.error(errorMessage); + return { status: 'failure', reason: errorMessage }; + } + + try { + const response = await service.generateMessage( + REFLECTION_SYSTEM_PROMPT, + conversationHistory, + model.name, + false, // No tools needed for reflection + ); + + const reflectionText = response.contentBlocks + .filter(block => block.type === MessageContentType.Text) + .map(block => block.text) + .join('\n'); + + try { + const reflection: Reflection = JSON.parse(reflectionText); + // Basic validation + if (['success', 'failure', 'retry'].includes(reflection.status) && typeof reflection.reason === 'string') { + this.logger.log(`Reflection result for task ${task.id}: ${reflection.status} - ${reflection.reason}`); + return reflection; + } + this.logger.warn(`Parsed reflection has invalid format for task ID: ${task.id}.`); + return { status: 'retry', reason: 'Reflection output was malformed.' }; + } catch (e) { + this.logger.warn(`Failed to parse reflection as JSON for task ID: ${task.id}. Raw output: ${reflectionText}`); + return { status: 'retry', reason: 'Failed to parse reflection JSON.' }; + } + + } catch (error) { + this.logger.error(`Failed to reflect on outcome for task ID: ${task.id}`, error.stack); + return { status: 'failure', reason: 'An unexpected error occurred during reflection.' }; + } + } +} diff --git a/packages/bytebot-agent/tsconfig.json b/packages/bytebot-agent/tsconfig.json index 216996392..e6ee1dfee 100644 --- a/packages/bytebot-agent/tsconfig.json +++ b/packages/bytebot-agent/tsconfig.json @@ -16,6 +16,9 @@ "forceConsistentCasingInFileNames": true, "noImplicitAny": false, "strictBindCallApply": false, - "noFallthroughCasesInSwitch": false + "noFallthroughCasesInSwitch": false, + "paths": { + "@bytebot/shared/*": ["../shared/src/*"] + } } }