From 591fb7116e52f0ccbb255c9746cdb7cf99a066a4 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 25 Dec 2024 14:32:07 -0500 Subject: [PATCH 01/15] feat: Refactor ModelEndHandler to collect usage metadata only if it exists --- api/server/controllers/agents/callbacks.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js index e0bb52d26cf..de78c07b005 100644 --- a/api/server/controllers/agents/callbacks.js +++ b/api/server/controllers/agents/callbacks.js @@ -58,13 +58,14 @@ class ModelEndHandler { } const usage = data?.output?.usage_metadata; + if (!usage) { + return; + } if (metadata?.model) { usage.model = metadata.model; } - if (usage) { - this.collectedUsage.push(usage); - } + this.collectedUsage.push(usage); } } From 4ee0e01425bf1aca2fadff71c18a0b373ecf06d2 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 25 Dec 2024 19:05:14 -0500 Subject: [PATCH 02/15] feat: google tool end handling, custom anthropic class for better token ux --- api/package.json | 2 +- api/server/controllers/agents/callbacks.js | 26 +- package-lock.json | 405 ++++++--------------- 3 files changed, 128 insertions(+), 305 deletions(-) diff --git a/api/package.json b/api/package.json index 26f8dae9b70..c98e9a3e872 100644 --- a/api/package.json +++ b/api/package.json @@ -44,7 +44,7 @@ "@langchain/google-genai": "^0.1.4", "@langchain/google-vertexai": "^0.1.4", "@langchain/textsplitters": "^0.1.0", - "@librechat/agents": "^1.8.8", + "@librechat/agents": "^1.9.6", "axios": "^1.7.7", "bcryptjs": "^2.4.3", "cheerio": "^1.0.0-rc.12", diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js index de78c07b005..706b9db83d7 100644 --- a/api/server/controllers/agents/callbacks.js +++ b/api/server/controllers/agents/callbacks.js @@ -1,8 +1,10 @@ const { Tools, StepTypes, imageGenTools, FileContext } = require('librechat-data-provider'); const { EnvVar, + Providers, GraphEvents, ToolEndHandler, + handleToolCalls, ChatModelStreamHandler, } = require('@librechat/agents'); const { processCodeOutput } = require('~/server/services/Files/Code/process'); @@ -57,15 +59,23 @@ class ModelEndHandler { return; } - const usage = data?.output?.usage_metadata; - if (!usage) { - return; - } - if (metadata?.model) { - usage.model = metadata.model; - } + try { + if (metadata.provider === Providers.GOOGLE) { + handleToolCalls(data?.output?.tool_calls, metadata, graph); + } - this.collectedUsage.push(usage); + const usage = data?.output?.usage_metadata; + if (!usage) { + return; + } + if (metadata?.model) { + usage.model = metadata.model; + } + + this.collectedUsage.push(usage); + } catch (error) { + logger.error('Error handling model end event:', error); + } } } diff --git a/package-lock.json b/package-lock.json index 34437fbf576..8997e12ca7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,7 +53,7 @@ "@langchain/google-genai": "^0.1.4", "@langchain/google-vertexai": "^0.1.4", "@langchain/textsplitters": "^0.1.0", - "@librechat/agents": "^1.8.8", + "@librechat/agents": "^1.9.6", "axios": "^1.7.7", "bcryptjs": "^2.4.3", "cheerio": "^1.0.0-rc.12", @@ -121,45 +121,6 @@ "supertest": "^6.3.3" } }, - "api/node_modules/@anthropic-ai/sdk": { - "version": "0.32.1", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.32.1.tgz", - "integrity": "sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg==", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - } - }, - "api/node_modules/@google/generative-ai": { - "version": "0.21.0", - "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.21.0.tgz", - "integrity": "sha512-7XhUbtnlkSEZK15kN3t+tzIMxsbKm/dSkKBFalj+20NvPKe1kBY7mR2P7vuijEn+f06z5+A8bVGKO0v39cr6Wg==", - "engines": { - "node": ">=18.0.0" - } - }, - "api/node_modules/@langchain/anthropic": { - "version": "0.3.11", - "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-0.3.11.tgz", - "integrity": "sha512-rYjDZjMwVQ+cYeJd9IoSESdkkG8fc0m3siGRYKNy6qgYMnqCz8sUPKBanXwbZAs6wvspPCGgNK9WONfaCeX97A==", - "dependencies": { - "@anthropic-ai/sdk": "^0.32.1", - "fast-xml-parser": "^4.4.1", - "zod": "^3.22.4", - "zod-to-json-schema": "^3.22.4" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/core": ">=0.2.21 <0.4.0" - } - }, "api/node_modules/@langchain/community": { "version": "0.3.14", "resolved": "https://registry.npmjs.org/@langchain/community/-/community-0.3.14.tgz", @@ -710,34 +671,6 @@ "@langchain/core": ">=0.2.21 <0.4.0" } }, - "api/node_modules/@librechat/agents": { - "version": "1.8.8", - "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-1.8.8.tgz", - "integrity": "sha512-8BM/MeyNMh4wlUIiswN7AfSZZQF2ibVOSiBhmA5PZfo314w/JnkivFNhRnAIh4yjd0ziGIgL2zHB7DRWAPnWSw==", - "dependencies": { - "@aws-crypto/sha256-js": "^5.2.0", - "@aws-sdk/credential-provider-node": "^3.613.0", - "@aws-sdk/types": "^3.609.0", - "@langchain/anthropic": "^0.3.11", - "@langchain/aws": "^0.1.2", - "@langchain/community": "^0.3.14", - "@langchain/core": "^0.3.18", - "@langchain/google-vertexai": "^0.1.2", - "@langchain/langgraph": "^0.2.19", - "@langchain/mistralai": "^0.0.26", - "@langchain/ollama": "^0.1.1", - "@langchain/openai": "^0.3.14", - "@smithy/eventstream-codec": "^2.2.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/signature-v4": "^2.0.10", - "@smithy/util-utf8": "^2.0.0", - "dotenv": "^16.4.5", - "nanoid": "^3.3.7" - }, - "engines": { - "node": ">=14.0.0" - } - }, "api/node_modules/@types/node": { "version": "18.19.14", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.14.tgz", @@ -911,11 +844,6 @@ "url": "https://opencollective.com/mongoose" } }, - "api/node_modules/mongoose/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, "api/node_modules/node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -1142,6 +1070,30 @@ "node": ">=6.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.32.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.32.1.tgz", + "integrity": "sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg==", + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { + "version": "18.19.68", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.68.tgz", + "integrity": "sha512-QGtpFH1vB99ZmTa63K4/FU8twThj4fuVSBkGddTp7uIL/cuoLWIUSL2RcOaigBhfR+hg5pgGkBnkoOxrTVBMKw==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/@ariakit/core": { "version": "0.4.10", "resolved": "https://registry.npmjs.org/@ariakit/core/-/core-0.4.10.tgz", @@ -6043,6 +5995,12 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@cfworker/json-schema": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.0.3.tgz", + "integrity": "sha512-ZykIcDTVv5UNmKWSTLAs3VukO6NDJkkSKxrgUTDPBkAlORVT3H9n5DbRjRl8xIotklscHdbLIa0b9+y3mQq73g==", + "license": "MIT" + }, "node_modules/@codemirror/autocomplete": { "version": "6.18.0", "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.18.0.tgz", @@ -8473,11 +8431,10 @@ "license": "MIT" }, "node_modules/@google/generative-ai": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.1.3.tgz", - "integrity": "sha512-Cm4uJX1sKarpm1mje/MiOIinM7zdUUrQp/5/qGPAgznbdd/B9zup5ehT6c1qGqycFcSopTA1J1HpqHS5kJR8hQ==", - "optional": true, - "peer": true, + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.21.0.tgz", + "integrity": "sha512-7XhUbtnlkSEZK15kN3t+tzIMxsbKm/dSkKBFalj+20NvPKe1kBY7mR2P7vuijEn+f06z5+A8bVGKO0v39cr6Wg==", + "license": "Apache-2.0", "engines": { "node": ">=18.0.0" } @@ -9218,13 +9175,12 @@ } }, "node_modules/@langchain/anthropic": { - "version": "0.3.8", - "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-0.3.8.tgz", - "integrity": "sha512-7qeRDhNnCf1peAbjY825R2HNszobJeGvqi2cfPl+YsduDIYEGUzfoGRRarPI5joIGX5YshCsch6NFtap2bLfmw==", - "optional": true, - "peer": true, + "version": "0.3.11", + "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-0.3.11.tgz", + "integrity": "sha512-rYjDZjMwVQ+cYeJd9IoSESdkkG8fc0m3siGRYKNy6qgYMnqCz8sUPKBanXwbZAs6wvspPCGgNK9WONfaCeX97A==", + "license": "MIT", "dependencies": { - "@anthropic-ai/sdk": "^0.27.3", + "@anthropic-ai/sdk": "^0.32.1", "fast-xml-parser": "^4.4.1", "zod": "^3.22.4", "zod-to-json-schema": "^3.22.4" @@ -9236,32 +9192,6 @@ "@langchain/core": ">=0.2.21 <0.4.0" } }, - "node_modules/@langchain/anthropic/node_modules/@anthropic-ai/sdk": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.27.3.tgz", - "integrity": "sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw==", - "optional": true, - "peer": true, - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - } - }, - "node_modules/@langchain/anthropic/node_modules/@types/node": { - "version": "18.19.64", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.64.tgz", - "integrity": "sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==", - "optional": true, - "peer": true, - "dependencies": { - "undici-types": "~5.26.4" - } - }, "node_modules/@langchain/aws": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/@langchain/aws/-/aws-0.1.2.tgz", @@ -9285,8 +9215,6 @@ "version": "0.3.15", "resolved": "https://registry.npmjs.org/@langchain/community/-/community-0.3.15.tgz", "integrity": "sha512-yG4cv33u7zYar14yqZCI7o2KjwRb+9S7upVzEmVVETimpicm9UjpkMfX4qa4A4IslM1TtC4uy2Ymu9EcINZSpQ==", - "optional": true, - "peer": true, "dependencies": { "@langchain/openai": ">=0.2.0 <0.4.0", "binary-extensions": "^2.2.0", @@ -9798,22 +9726,22 @@ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], - "optional": true, - "peer": true, "bin": { "uuid": "dist/bin/uuid" } }, "node_modules/@langchain/core": { - "version": "0.3.18", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.18.tgz", - "integrity": "sha512-IEZCrFs1Xd0J2FTH1D3Lnm3/Yk2r8LSpwDeLYwcCom3rNAK5k4mKQ2rwIpNq3YuqBdrTNMKRO+PopjkP1SB17A==", + "version": "0.3.26", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.26.tgz", + "integrity": "sha512-6RUQHEp8wv+JwtYIIEBYBzbLlcAQZFc7EDOgAM0ukExjh9HiXoJzoWpgMRRCrr/koIbtwXPJUqBprZK1I1CXHQ==", + "license": "MIT", "dependencies": { + "@cfworker/json-schema": "^4.0.2", "ansi-styles": "^5.0.0", "camelcase": "6", "decamelize": "1.2.0", "js-tiktoken": "^1.0.12", - "langsmith": "^0.2.0", + "langsmith": "^0.2.8", "mustache": "^4.2.0", "p-queue": "^6.6.2", "p-retry": "4", @@ -9850,9 +9778,10 @@ } }, "node_modules/@langchain/google-common": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/@langchain/google-common/-/google-common-0.1.4.tgz", - "integrity": "sha512-EIpJYhat+BpGXRJiLSKKWlbBl88AJLnwGhLNOh85nNPtcqKqWTIJ/WGVNfFNsrAwHZ+f77gZeNfefeRIrChNZw==", + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@langchain/google-common/-/google-common-0.1.5.tgz", + "integrity": "sha512-VgTghTTROBPez8TgAD+GWKzfoJ24EaWMjdo3dB69QCVt23ZyNGM8Hk4rm6LgjMxmLFNnXi6UnhQuoece3ffmAA==", + "license": "MIT", "dependencies": { "uuid": "^10.0.0", "zod-to-json-schema": "^3.22.4" @@ -9872,16 +9801,18 @@ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], + "license": "MIT", "bin": { "uuid": "dist/bin/uuid" } }, "node_modules/@langchain/google-gauth": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/@langchain/google-gauth/-/google-gauth-0.1.4.tgz", - "integrity": "sha512-g/yXfGCgBU5FkH/lW4L0E2HDvQ3JuS5/KZylixWwkV+hk9gSyYcMV3RnhMjp+zEY/68XALiqlwUyvfK5vToz4g==", + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@langchain/google-gauth/-/google-gauth-0.1.5.tgz", + "integrity": "sha512-FYqHtW06gRJdImzlh1nyNHtIgp1F4nf1SiOBN5/ynblze2ho523BruFeKprDRR3Iv+CEJm+3v8fRf0QKf9esPw==", + "license": "MIT", "dependencies": { - "@langchain/google-common": "~0.1.4", + "@langchain/google-common": "~0.1.5", "google-auth-library": "^8.9.0" }, "engines": { @@ -9892,105 +9823,28 @@ } }, "node_modules/@langchain/google-genai": { - "version": "0.0.11", - "resolved": "https://registry.npmjs.org/@langchain/google-genai/-/google-genai-0.0.11.tgz", - "integrity": "sha512-o4+r+ETmcPqcrRTJeJQQ0c796IAx1dvVkZvFsUqLhTIteIQuAc2KenY/UDGQxZVghw6fZf4irN/PvkNHJjfgWw==", - "optional": true, - "peer": true, - "dependencies": { - "@google/generative-ai": "^0.1.3", - "@langchain/core": "~0.1.5" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@langchain/google-genai/node_modules/@langchain/core": { - "version": "0.1.63", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.1.63.tgz", - "integrity": "sha512-+fjyYi8wy6x1P+Ee1RWfIIEyxd9Ee9jksEwvrggPwwI/p45kIDTdYTblXsM13y4mNWTiACyLSdbwnPaxxdoz+w==", + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@langchain/google-genai/-/google-genai-0.1.6.tgz", + "integrity": "sha512-LF3fan9pvgFa1vw2/IYGhi5KjppE0OvPFX3QQBUshBLpXWERP+BSpSD7jcXyqm9Kf7DcFj7w5/2knKeEwih8Xg==", "license": "MIT", - "optional": true, - "peer": true, "dependencies": { - "ansi-styles": "^5.0.0", - "camelcase": "6", - "decamelize": "1.2.0", - "js-tiktoken": "^1.0.12", - "langsmith": "~0.1.7", - "ml-distance": "^4.0.0", - "mustache": "^4.2.0", - "p-queue": "^6.6.2", - "p-retry": "4", - "uuid": "^9.0.0", - "zod": "^3.22.4", - "zod-to-json-schema": "^3.22.3" + "@google/generative-ai": "^0.21.0", + "zod-to-json-schema": "^3.22.4" }, "engines": { "node": ">=18" - } - }, - "node_modules/@langchain/google-genai/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@langchain/google-genai/node_modules/langsmith": { - "version": "0.1.68", - "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.68.tgz", - "integrity": "sha512-otmiysWtVAqzMx3CJ4PrtUBhWRG5Co8Z4o7hSZENPjlit9/j3/vm3TSvbaxpDYakZxtMjhkcJTqrdYFipISEiQ==", - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@types/uuid": "^10.0.0", - "commander": "^10.0.1", - "p-queue": "^6.6.2", - "p-retry": "4", - "semver": "^7.6.3", - "uuid": "^10.0.0" }, "peerDependencies": { - "openai": "*" - }, - "peerDependenciesMeta": { - "openai": { - "optional": true - } - } - }, - "node_modules/@langchain/google-genai/node_modules/langsmith/node_modules/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "optional": true, - "peer": true, - "bin": { - "uuid": "dist/bin/uuid" + "@langchain/core": ">=0.3.17 <0.4.0" } }, "node_modules/@langchain/google-vertexai": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/@langchain/google-vertexai/-/google-vertexai-0.1.2.tgz", - "integrity": "sha512-b8Di2AgSwlyyKl4A5qii+19Wj82I1KvtUXSDvJpDzhucuyrJjmnNb/0ClkaIQv6RyISajtxszxxSGHukPn3PJA==", - "optional": true, - "peer": true, + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@langchain/google-vertexai/-/google-vertexai-0.1.5.tgz", + "integrity": "sha512-wd9mjrQRdlFAnwhCGtHIsOTtuF6Oto8+F8nthU3guU7T/gfuICvg26+MaOpdUmuTVFuIDqVMc0l7kf3ZK4DFlA==", + "license": "MIT", "dependencies": { - "@langchain/google-gauth": "~0.1.2" + "@langchain/google-gauth": "~0.1.5" }, "engines": { "node": ">=18" @@ -10000,9 +9854,10 @@ } }, "node_modules/@langchain/langgraph": { - "version": "0.2.33", - "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-0.2.33.tgz", - "integrity": "sha512-Tx2eU98XicIOoZzRkzQqLxZrF93B9xONYmWSq3kfDUoC0nzQbkydpygF1MTcUM9hKPQsSGMBrxgXht5+sNXzYg==", + "version": "0.2.36", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-0.2.36.tgz", + "integrity": "sha512-zxk7ZCVxP0/Ut9785EiXCS7BE7sXd8cu943mcZUF2aNFUaQRTBbbiKpNdR3nb1+xO/B+HVktrJT2VFdkAywnng==", + "license": "MIT", "dependencies": { "@langchain/langgraph-checkpoint": "~0.0.13", "@langchain/langgraph-sdk": "~0.0.21", @@ -10020,6 +9875,7 @@ "version": "0.0.13", "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-0.0.13.tgz", "integrity": "sha512-amdmBcNT8a9xP2VwcEWxqArng4gtRDcnVyVI4DsQIo1Aaz8e8+hH17zSwrUF3pt1pIYztngIfYnBOim31mtKMg==", + "license": "MIT", "dependencies": { "uuid": "^10.0.0" }, @@ -10038,6 +9894,7 @@ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], + "license": "MIT", "bin": { "uuid": "dist/bin/uuid" } @@ -10046,6 +9903,7 @@ "version": "0.0.32", "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-0.0.32.tgz", "integrity": "sha512-KQyM9kLO7T6AxwNrceajH7JOybP3pYpvUPnhiI2rrVndI1WyZUJ1eVC1e722BVRAPi6o+WcoTT4uMSZVinPOtA==", + "license": "MIT", "dependencies": { "@types/json-schema": "^7.0.15", "p-queue": "^6.6.2", @@ -10061,6 +9919,7 @@ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], + "license": "MIT", "bin": { "uuid": "dist/bin/uuid" } @@ -10352,6 +10211,36 @@ "@lezer/common": "^1.0.0" } }, + "node_modules/@librechat/agents": { + "version": "1.9.6", + "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-1.9.6.tgz", + "integrity": "sha512-NfWKvArsDOlbjxiykIl6uAIGWD9birTpltnVA8KF+3Ak5swb6if2gzaS+tfjaua8GpJGdvQcETbOz0XwkQ7riA==", + "license": "MIT", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-sdk/credential-provider-node": "^3.613.0", + "@aws-sdk/types": "^3.609.0", + "@langchain/anthropic": "^0.3.11", + "@langchain/aws": "^0.1.2", + "@langchain/community": "^0.3.14", + "@langchain/core": "^0.3.26", + "@langchain/google-genai": "^0.1.6", + "@langchain/google-vertexai": "^0.1.5", + "@langchain/langgraph": "^0.2.34", + "@langchain/mistralai": "^0.0.26", + "@langchain/ollama": "^0.1.1", + "@langchain/openai": "^0.3.14", + "@smithy/eventstream-codec": "^2.2.0", + "@smithy/protocol-http": "^3.0.6", + "@smithy/signature-v4": "^2.0.10", + "@smithy/util-utf8": "^2.0.0", + "dotenv": "^16.4.5", + "nanoid": "^3.3.7" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@librechat/backend": { "resolved": "api", "link": true @@ -16373,13 +16262,6 @@ "node": ">=8" } }, - "node_modules/binary-search": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", - "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", - "optional": true, - "peer": true - }, "node_modules/bl": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz", @@ -22442,13 +22324,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/is-any-array": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-2.0.1.tgz", - "integrity": "sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==", - "optional": true, - "peer": true - }, "node_modules/is-arguments": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", @@ -24602,9 +24477,10 @@ } }, "node_modules/langsmith": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.2.5.tgz", - "integrity": "sha512-dA+l7ZEh1Q9Q9FcE39PUSSEMfsFo73R2V81fRo5KSlGNcypOEhoQvv6lbjyZP7MHmt3/9pPcfpuRd5Y4RbFYqQ==", + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.2.14.tgz", + "integrity": "sha512-ClAuAgSf3m9miMYotLEaZKQyKdaWlfjhebCuYco8bc6g72dU2VwTg31Bv4YINBq7EH2i1cMwbOiJxbOXPqjGig==", + "license": "MIT", "dependencies": { "@types/uuid": "^10.0.0", "commander": "^10.0.1", @@ -27146,56 +27022,6 @@ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, - "node_modules/ml-array-mean": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/ml-array-mean/-/ml-array-mean-1.1.6.tgz", - "integrity": "sha512-MIdf7Zc8HznwIisyiJGRH9tRigg3Yf4FldW8DxKxpCCv/g5CafTw0RRu51nojVEOXuCQC7DRVVu5c7XXO/5joQ==", - "optional": true, - "peer": true, - "dependencies": { - "ml-array-sum": "^1.1.6" - } - }, - "node_modules/ml-array-sum": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/ml-array-sum/-/ml-array-sum-1.1.6.tgz", - "integrity": "sha512-29mAh2GwH7ZmiRnup4UyibQZB9+ZLyMShvt4cH4eTK+cL2oEMIZFnSyB3SS8MlsTh6q/w/yh48KmqLxmovN4Dw==", - "optional": true, - "peer": true, - "dependencies": { - "is-any-array": "^2.0.0" - } - }, - "node_modules/ml-distance": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/ml-distance/-/ml-distance-4.0.1.tgz", - "integrity": "sha512-feZ5ziXs01zhyFUUUeZV5hwc0f5JW0Sh0ckU1koZe/wdVkJdGxcP06KNQuF0WBTj8FttQUzcvQcpcrOp/XrlEw==", - "optional": true, - "peer": true, - "dependencies": { - "ml-array-mean": "^1.1.6", - "ml-distance-euclidean": "^2.0.0", - "ml-tree-similarity": "^1.0.0" - } - }, - "node_modules/ml-distance-euclidean": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ml-distance-euclidean/-/ml-distance-euclidean-2.0.0.tgz", - "integrity": "sha512-yC9/2o8QF0A3m/0IXqCTXCzz2pNEzvmcE/9HFKOZGnTjatvBbsn4lWYJkxENkA4Ug2fnYl7PXQxnPi21sgMy/Q==", - "optional": true, - "peer": true - }, - "node_modules/ml-tree-similarity": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/ml-tree-similarity/-/ml-tree-similarity-1.0.0.tgz", - "integrity": "sha512-XJUyYqjSuUQkNQHMscr6tcjldsOoAekxADTplt40QKfwW6nd++1wHWV9AArl0Zvw/TIHgNaZZNvr8QGvE8wLRg==", - "optional": true, - "peer": true, - "dependencies": { - "binary-search": "^1.3.5", - "num-sort": "^2.0.0" - } - }, "node_modules/mnemonist": { "version": "0.39.6", "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.6.tgz", @@ -27979,19 +27805,6 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, - "node_modules/num-sort": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/num-sort/-/num-sort-2.1.0.tgz", - "integrity": "sha512-1MQz1Ed8z2yckoBeSfkQHHO9K1yDRxxtotKSJ9yvcTUUxSvfvzEq5GwBrjjHEpMlq/k5gvXdmJ1SbYxWtpNoVg==", - "optional": true, - "peer": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/nwsapi": { "version": "2.2.7", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.7.tgz", From b72e0acfe018de7ad4546ce30d119996a2c2ae87 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 26 Dec 2024 09:11:57 -0500 Subject: [PATCH 03/15] refactor: differentiate between client <> request options --- api/server/services/Endpoints/anthropic/initialize.js | 8 ++++---- api/server/services/Endpoints/custom/initialize.js | 6 +++--- api/server/services/Endpoints/openAI/initialize.js | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/server/services/Endpoints/anthropic/initialize.js b/api/server/services/Endpoints/anthropic/initialize.js index 3f1336ff31b..ffd61441beb 100644 --- a/api/server/services/Endpoints/anthropic/initialize.js +++ b/api/server/services/Endpoints/anthropic/initialize.js @@ -20,7 +20,7 @@ const initializeClient = async ({ req, res, endpointOption, overrideModel, optio checkUserKeyExpiry(expiresAt, EModelEndpoint.anthropic); } - const clientOptions = {}; + let clientOptions = {}; /** @type {undefined | TBaseEndpoint} */ const anthropicConfig = req.app.locals[EModelEndpoint.anthropic]; @@ -36,7 +36,7 @@ const initializeClient = async ({ req, res, endpointOption, overrideModel, optio } if (optionsOnly) { - const requestOptions = Object.assign( + clientOptions = Object.assign( { reverseProxyUrl: ANTHROPIC_REVERSE_PROXY ?? null, proxy: PROXY ?? null, @@ -45,9 +45,9 @@ const initializeClient = async ({ req, res, endpointOption, overrideModel, optio clientOptions, ); if (overrideModel) { - requestOptions.modelOptions.model = overrideModel; + clientOptions.modelOptions.model = overrideModel; } - return getLLMConfig(anthropicApiKey, requestOptions); + return getLLMConfig(anthropicApiKey, clientOptions); } const client = new AnthropicClient(anthropicApiKey, { diff --git a/api/server/services/Endpoints/custom/initialize.js b/api/server/services/Endpoints/custom/initialize.js index c88e6882f55..fe2beba582b 100644 --- a/api/server/services/Endpoints/custom/initialize.js +++ b/api/server/services/Endpoints/custom/initialize.js @@ -123,7 +123,7 @@ const initializeClient = async ({ req, res, endpointOption, optionsOnly, overrid customOptions.streamRate = allConfig.streamRate; } - const clientOptions = { + let clientOptions = { reverseProxyUrl: baseURL ?? null, proxy: PROXY ?? null, req, @@ -135,13 +135,13 @@ const initializeClient = async ({ req, res, endpointOption, optionsOnly, overrid if (optionsOnly) { const modelOptions = endpointOption.model_parameters; if (endpoint !== Providers.OLLAMA) { - const requestOptions = Object.assign( + clientOptions = Object.assign( { modelOptions, }, clientOptions, ); - const options = getLLMConfig(apiKey, requestOptions); + const options = getLLMConfig(apiKey, clientOptions); if (!customOptions.streamRate) { return options; } diff --git a/api/server/services/Endpoints/openAI/initialize.js b/api/server/services/Endpoints/openAI/initialize.js index 63abbfea9c2..0eb0d566b94 100644 --- a/api/server/services/Endpoints/openAI/initialize.js +++ b/api/server/services/Endpoints/openAI/initialize.js @@ -54,7 +54,7 @@ const initializeClient = async ({ let apiKey = userProvidesKey ? userValues?.apiKey : credentials[endpoint]; let baseURL = userProvidesURL ? userValues?.baseURL : baseURLOptions[endpoint]; - const clientOptions = { + let clientOptions = { contextStrategy, proxy: PROXY ?? null, debug: isEnabled(DEBUG_OPENAI), @@ -134,13 +134,13 @@ const initializeClient = async ({ } if (optionsOnly) { - const requestOptions = Object.assign( + clientOptions = Object.assign( { modelOptions: endpointOption.model_parameters, }, clientOptions, ); - const options = getLLMConfig(apiKey, requestOptions); + const options = getLLMConfig(apiKey, clientOptions); if (!clientOptions.streamRate) { return options; } From c2963d04644f8c1de7458433658b57ef9b6c7741 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 26 Dec 2024 10:01:48 -0500 Subject: [PATCH 04/15] feat: initial support for google agents --- .../services/Endpoints/agents/initialize.js | 6 + .../services/Endpoints/google/initialize.js | 26 ++- api/server/services/Endpoints/google/llm.js | 155 ++++++++++++++++++ 3 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 api/server/services/Endpoints/google/llm.js diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 0e42145a595..28f4d8cdd80 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -12,6 +12,7 @@ const initAnthropic = require('~/server/services/Endpoints/anthropic/initialize' const getBedrockOptions = require('~/server/services/Endpoints/bedrock/options'); const initOpenAI = require('~/server/services/Endpoints/openAI/initialize'); const initCustom = require('~/server/services/Endpoints/custom/initialize'); +const initGoogle = require('~/server/services/Endpoints/google/initialize'); const { getCustomEndpointConfig } = require('~/server/services/Config'); const { loadAgentTools } = require('~/server/services/ToolService'); const AgentClient = require('~/server/controllers/agents/client'); @@ -24,6 +25,7 @@ const providerConfigMap = { [EModelEndpoint.azureOpenAI]: initOpenAI, [EModelEndpoint.anthropic]: initAnthropic, [EModelEndpoint.bedrock]: getBedrockOptions, + [EModelEndpoint.google]: initGoogle, [Providers.OLLAMA]: initCustom, }; @@ -116,6 +118,10 @@ const initializeAgentOptions = async ({ endpointOption: _endpointOption, }); + if (options.provider != null) { + agent.provider = options.provider; + } + agent.model_parameters = Object.assign(model_parameters, options.llmConfig); if (options.configOptions) { agent.model_parameters.configuration = options.configOptions; diff --git a/api/server/services/Endpoints/google/initialize.js b/api/server/services/Endpoints/google/initialize.js index f601c8e4035..c157dd8b28e 100644 --- a/api/server/services/Endpoints/google/initialize.js +++ b/api/server/services/Endpoints/google/initialize.js @@ -1,9 +1,10 @@ const { EModelEndpoint, AuthKeys } = require('librechat-data-provider'); const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService'); -const { GoogleClient } = require('~/app'); +const { getLLMConfig } = require('~/server/services/Endpoints/google/llm'); const { isEnabled } = require('~/server/utils'); +const { GoogleClient } = require('~/app'); -const initializeClient = async ({ req, res, endpointOption }) => { +const initializeClient = async ({ req, res, endpointOption, overrideModel, optionsOnly }) => { const { GOOGLE_KEY, GOOGLE_REVERSE_PROXY, @@ -33,7 +34,7 @@ const initializeClient = async ({ req, res, endpointOption }) => { [AuthKeys.GOOGLE_API_KEY]: GOOGLE_KEY, }; - const clientOptions = {}; + let clientOptions = {}; /** @type {undefined | TBaseEndpoint} */ const allConfig = req.app.locals.all; @@ -48,7 +49,7 @@ const initializeClient = async ({ req, res, endpointOption }) => { clientOptions.streamRate = allConfig.streamRate; } - const client = new GoogleClient(credentials, { + clientOptions = { req, res, reverseProxyUrl: GOOGLE_REVERSE_PROXY ?? null, @@ -56,7 +57,22 @@ const initializeClient = async ({ req, res, endpointOption }) => { proxy: PROXY ?? null, ...clientOptions, ...endpointOption, - }); + }; + + if (optionsOnly) { + clientOptions = Object.assign( + { + modelOptions: endpointOption.model_parameters, + }, + clientOptions, + ); + if (overrideModel) { + clientOptions.modelOptions.model = overrideModel; + } + return getLLMConfig(credentials, clientOptions); + } + + const client = new GoogleClient(credentials, clientOptions); return { client, diff --git a/api/server/services/Endpoints/google/llm.js b/api/server/services/Endpoints/google/llm.js new file mode 100644 index 00000000000..97b7b83b3e4 --- /dev/null +++ b/api/server/services/Endpoints/google/llm.js @@ -0,0 +1,155 @@ +const { Providers } = require('@librechat/agents'); +const { AuthKeys } = require('librechat-data-provider'); + +// Example internal constant from your code +const EXCLUDED_GENAI_MODELS = /gemini-(?:1\.0|1-0|pro)/; + +function getSafetySettings() { + return [ + { + category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', + threshold: + process.env.GOOGLE_SAFETY_SEXUALLY_EXPLICIT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED', + }, + { + category: 'HARM_CATEGORY_HATE_SPEECH', + threshold: process.env.GOOGLE_SAFETY_HATE_SPEECH || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED', + }, + { + category: 'HARM_CATEGORY_HARASSMENT', + threshold: + process.env.GOOGLE_SAFETY_HARASSMENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED', + }, + { + category: 'HARM_CATEGORY_DANGEROUS_CONTENT', + threshold: + process.env.GOOGLE_SAFETY_DANGEROUS_CONTENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED', + }, + { + category: 'HARM_CATEGORY_CIVIC_INTEGRITY', + threshold: process.env.GOOGLE_SAFETY_CIVIC_INTEGRITY || 'BLOCK_NONE', + }, + ]; +} + +/** + * Replicates core logic from GoogleClient's constructor and setOptions, plus client determination. + * Returns an object with the provider label and the final options that would be passed to createLLM. + * + * @param {string | object} credentials - Either a JSON string or an object containing Google keys + * @param {object} [options={}] - The same shape as the "GoogleClient" constructor options + * @returns {{ + * provider: string, + * llmConfig: object, + * requestOptions: object|null, + * project_id: string|null, + * serviceKey: object|null, + * apiKey: string|null, + * safetySettings: object[] + * }} + */ + +function getLLMConfig(credentials, options = {}) { + // 1. Parse credentials + let creds = {}; + if (typeof credentials === 'string') { + try { + creds = JSON.parse(credentials); + } catch (err) { + throw new Error(`Error parsing string credentials: ${err.message}`); + } + } else if (credentials && typeof credentials === 'object') { + creds = credentials; + } + + // Extract from credentials + const serviceKeyRaw = creds[AuthKeys.GOOGLE_SERVICE_KEY] ?? {}; + const serviceKey = + typeof serviceKeyRaw === 'string' ? JSON.parse(serviceKeyRaw) : serviceKeyRaw ?? {}; + + const project_id = serviceKey?.project_id ?? null; + const apiKey = creds[AuthKeys.GOOGLE_API_KEY] ?? null; + + const reverseProxyUrl = options.reverseProxyUrl; + const authHeader = options.authHeader; + + let llmConfig = { + ...(options.modelOptions || {}), + safetySettings: getSafetySettings(), + maxRetries: 2, + }; + + const isGenerativeModel = llmConfig.model.includes('gemini'); + const isChatModel = !isGenerativeModel && llmConfig.model.includes('chat'); + const isTextModel = !isGenerativeModel && !isChatModel && /code|text/.test(llmConfig.model); + + let provider; + + if (project_id && isTextModel) { + provider = Providers.VERTEXAI; + } else if (project_id && isChatModel) { + provider = Providers.VERTEXAI; + } else if (project_id) { + provider = Providers.VERTEXAI; + } else if (!EXCLUDED_GENAI_MODELS.test(llmConfig.model)) { + provider = Providers.GOOGLE; + } else { + provider = Providers.GOOGLE; + } + + let authOptions = {}; + + // If we have a GCP project => Vertex AI + if (project_id && provider === Providers.VERTEXAI) { + authOptions = { + credentials: { ...serviceKey }, + projectId: project_id, + }; + } else if (apiKey && provider === Providers.GOOGLE) { + authOptions = { apiKey }; + } + + llmConfig = { ...llmConfig, ...authOptions }; + + /* + let legacyOptions = {}; + // Filter out any "examples" that are empty + legacyOptions.examples = (legacyOptions.examples ?? []) + .filter(Boolean) + .filter((obj) => obj?.input?.content !== '' && obj?.output?.content !== ''); + + // If user has "examples" from legacyOptions, push them onto llmConfig + if (legacyOptions.examples?.length) { + llmConfig.examples = legacyOptions.examples.map((ex) => { + const { input, output } = ex; + if (!input?.content || !output?.content) {return undefined;} + return { + input: new HumanMessage(input.content), + output: new AIMessage(output.content), + }; + }).filter(Boolean); + } + */ + + // 7. If using a reverse proxy, build requestOptions + let requestOptions = null; + if (reverseProxyUrl) { + requestOptions = { baseUrl: reverseProxyUrl }; + if (authHeader) { + requestOptions.customHeaders = { + Authorization: `Bearer ${apiKey}`, + }; + } + } + + // Return the final shape + return { + provider, + llmConfig, + requestOptions, + }; +} + +module.exports = { + getLLMConfig, +}; From adb7e704e939b7926a27284b0fa485c9ec22178b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 26 Dec 2024 19:41:47 -0500 Subject: [PATCH 05/15] feat: only cache messages with non-empty text --- api/app/clients/BaseClient.js | 20 +++++++++++--------- api/app/clients/PluginsClient.js | 20 +++++++++++--------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 8b9e89860c1..c4ca676ebf8 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -649,15 +649,17 @@ class BaseClient { this.responsePromise = this.saveMessageToDatabase(responseMessage, saveOptions, user); this.savedMessageIds.add(responseMessage.messageId); - const messageCache = getLogStores(CacheKeys.MESSAGES); - messageCache.set( - responseMessageId, - { - text: responseMessage.text, - complete: true, - }, - Time.FIVE_MINUTES, - ); + if (responseMessage.text) { + const messageCache = getLogStores(CacheKeys.MESSAGES); + messageCache.set( + responseMessageId, + { + text: responseMessage.text, + complete: true, + }, + Time.FIVE_MINUTES, + ); + } delete responseMessage.tokenCount; return responseMessage; } diff --git a/api/app/clients/PluginsClient.js b/api/app/clients/PluginsClient.js index 7259eb56f28..0e518bfea02 100644 --- a/api/app/clients/PluginsClient.js +++ b/api/app/clients/PluginsClient.js @@ -256,15 +256,17 @@ class PluginsClient extends OpenAIClient { } this.responsePromise = this.saveMessageToDatabase(responseMessage, saveOptions, user); - const messageCache = getLogStores(CacheKeys.MESSAGES); - messageCache.set( - responseMessage.messageId, - { - text: responseMessage.text, - complete: true, - }, - Time.FIVE_MINUTES, - ); + if (responseMessage.text) { + const messageCache = getLogStores(CacheKeys.MESSAGES); + messageCache.set( + responseMessage.messageId, + { + text: responseMessage.text, + complete: true, + }, + Time.FIVE_MINUTES, + ); + } delete responseMessage.tokenCount; return { ...responseMessage, ...result }; } From cd64fbf9adbf452c04ca08d2c3bc51e6c2c359cd Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 26 Dec 2024 19:43:40 -0500 Subject: [PATCH 06/15] feat: Cache non-empty messages in chatV2 controller --- api/server/controllers/assistants/chatV2.js | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/api/server/controllers/assistants/chatV2.js b/api/server/controllers/assistants/chatV2.js index dd7d5b9125d..047a413433a 100644 --- a/api/server/controllers/assistants/chatV2.js +++ b/api/server/controllers/assistants/chatV2.js @@ -398,15 +398,17 @@ const chatV2 = async (req, res) => { response = streamRunManager; response.text = streamRunManager.intermediateText; - const messageCache = getLogStores(CacheKeys.MESSAGES); - messageCache.set( - responseMessageId, - { - complete: true, - text: response.text, - }, - Time.FIVE_MINUTES, - ); + if (response.text) { + const messageCache = getLogStores(CacheKeys.MESSAGES); + messageCache.set( + responseMessageId, + { + complete: true, + text: response.text, + }, + Time.FIVE_MINUTES, + ); + } }; await processRun(); From 2c1ca07b870fe8b870a806884a2cb1a4d248bdbe Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 28 Dec 2024 14:47:22 -0500 Subject: [PATCH 07/15] fix: anthropic llm client options llmConfig --- .../services/Endpoints/anthropic/llm.js | 22 ++++++++++++------- api/typedefs.js | 6 +++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/api/server/services/Endpoints/anthropic/llm.js b/api/server/services/Endpoints/anthropic/llm.js index 937d66e9264..219e3cc8126 100644 --- a/api/server/services/Endpoints/anthropic/llm.js +++ b/api/server/services/Endpoints/anthropic/llm.js @@ -28,28 +28,34 @@ function getLLMConfig(apiKey, options = {}) { const mergedOptions = Object.assign(defaultOptions, options.modelOptions); + /** @type {AnthropicClientOptions} */ const requestOptions = { apiKey, model: mergedOptions.model, stream: mergedOptions.stream, temperature: mergedOptions.temperature, - top_p: mergedOptions.topP, - top_k: mergedOptions.topK, - stop_sequences: mergedOptions.stop, - max_tokens: + topP: mergedOptions.topP, + topK: mergedOptions.topK, + stopSequences: mergedOptions.stop, + maxTokens: mergedOptions.maxOutputTokens || anthropicSettings.maxOutputTokens.reset(mergedOptions.model), }; - const configOptions = {}; + /** @type {AnthropicClientOptions['clientOptions']} */ + const clientOptions = {}; if (options.proxy) { - configOptions.httpAgent = new HttpsProxyAgent(options.proxy); + clientOptions.httpAgent = new HttpsProxyAgent(options.proxy); } if (options.reverseProxyUrl) { - configOptions.baseURL = options.reverseProxyUrl; + clientOptions.baseURL = options.reverseProxyUrl; } - return { llmConfig: removeNullishValues(requestOptions), configOptions }; + if (Object.keys(clientOptions).length > 0) { + requestOptions.clientOptions = clientOptions; + } + + return { llmConfig: removeNullishValues(requestOptions) }; } module.exports = { getLLMConfig }; diff --git a/api/typedefs.js b/api/typedefs.js index cd6b3ad0c97..83b7f501935 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -38,6 +38,12 @@ * @memberof typedefs */ +/** + * @exports AnthropicClientOptions + * @typedef {import('@librechat/agents').AnthropicClientOptions} AnthropicClientOptions + * @memberof typedefs + */ + /** * @exports BedrockClientOptions * @typedef {import('@librechat/agents').BedrockConverseClientOptions} BedrockClientOptions From aa4f19a592119de36f142ef6cca99edc03aa36f5 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 28 Dec 2024 15:06:37 -0500 Subject: [PATCH 08/15] refactor: streamline client options handling in LLM configuration --- api/server/services/Endpoints/anthropic/llm.js | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/api/server/services/Endpoints/anthropic/llm.js b/api/server/services/Endpoints/anthropic/llm.js index 219e3cc8126..8ec7522e949 100644 --- a/api/server/services/Endpoints/anthropic/llm.js +++ b/api/server/services/Endpoints/anthropic/llm.js @@ -39,20 +39,15 @@ function getLLMConfig(apiKey, options = {}) { stopSequences: mergedOptions.stop, maxTokens: mergedOptions.maxOutputTokens || anthropicSettings.maxOutputTokens.reset(mergedOptions.model), + clientOptions: {}, }; - /** @type {AnthropicClientOptions['clientOptions']} */ - const clientOptions = {}; if (options.proxy) { - clientOptions.httpAgent = new HttpsProxyAgent(options.proxy); + requestOptions.clientOptions.httpAgent = new HttpsProxyAgent(options.proxy); } if (options.reverseProxyUrl) { - clientOptions.baseURL = options.reverseProxyUrl; - } - - if (Object.keys(clientOptions).length > 0) { - requestOptions.clientOptions = clientOptions; + requestOptions.clientOptions.baseURL = options.reverseProxyUrl; } return { llmConfig: removeNullishValues(requestOptions) }; From 4774e8cc574c40cb0be5c9d435182ad5898c48f1 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 28 Dec 2024 15:53:17 -0500 Subject: [PATCH 09/15] fix: VertexAI Agent Auth & Tool Handling --- api/package.json | 2 +- api/server/services/Endpoints/google/llm.js | 17 ++--- package-lock.json | 69 ++++++++++++--------- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/api/package.json b/api/package.json index c98e9a3e872..5ff68951ada 100644 --- a/api/package.json +++ b/api/package.json @@ -44,7 +44,7 @@ "@langchain/google-genai": "^0.1.4", "@langchain/google-vertexai": "^0.1.4", "@langchain/textsplitters": "^0.1.0", - "@librechat/agents": "^1.9.6", + "@librechat/agents": "^1.9.7", "axios": "^1.7.7", "bcryptjs": "^2.4.3", "cheerio": "^1.0.0-rc.12", diff --git a/api/server/services/Endpoints/google/llm.js b/api/server/services/Endpoints/google/llm.js index 97b7b83b3e4..dc4d5cac840 100644 --- a/api/server/services/Endpoints/google/llm.js +++ b/api/server/services/Endpoints/google/llm.js @@ -8,8 +8,7 @@ function getSafetySettings() { return [ { category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', - threshold: - process.env.GOOGLE_SAFETY_SEXUALLY_EXPLICIT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED', + threshold: process.env.GOOGLE_SAFETY_SEXUALLY_EXPLICIT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED', }, { category: 'HARM_CATEGORY_HATE_SPEECH', @@ -17,13 +16,11 @@ function getSafetySettings() { }, { category: 'HARM_CATEGORY_HARASSMENT', - threshold: - process.env.GOOGLE_SAFETY_HARASSMENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED', + threshold: process.env.GOOGLE_SAFETY_HARASSMENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED', }, { category: 'HARM_CATEGORY_DANGEROUS_CONTENT', - threshold: - process.env.GOOGLE_SAFETY_DANGEROUS_CONTENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED', + threshold: process.env.GOOGLE_SAFETY_DANGEROUS_CONTENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED', }, { category: 'HARM_CATEGORY_CIVIC_INTEGRITY', @@ -97,20 +94,16 @@ function getLLMConfig(credentials, options = {}) { provider = Providers.GOOGLE; } - let authOptions = {}; - // If we have a GCP project => Vertex AI if (project_id && provider === Providers.VERTEXAI) { - authOptions = { + llmConfig.authOptions = { credentials: { ...serviceKey }, projectId: project_id, }; } else if (apiKey && provider === Providers.GOOGLE) { - authOptions = { apiKey }; + llmConfig.apiKey = apiKey; } - llmConfig = { ...llmConfig, ...authOptions }; - /* let legacyOptions = {}; // Filter out any "examples" that are empty diff --git a/package-lock.json b/package-lock.json index 8997e12ca7a..0d304bf2967 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,7 +53,7 @@ "@langchain/google-genai": "^0.1.4", "@langchain/google-vertexai": "^0.1.4", "@langchain/textsplitters": "^0.1.0", - "@librechat/agents": "^1.9.6", + "@librechat/agents": "^1.9.7", "axios": "^1.7.7", "bcryptjs": "^2.4.3", "cheerio": "^1.0.0-rc.12", @@ -671,6 +671,35 @@ "@langchain/core": ">=0.2.21 <0.4.0" } }, + "api/node_modules/@librechat/agents": { + "version": "1.9.7", + "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-1.9.7.tgz", + "integrity": "sha512-FprbiuQrjV3ULOmvOl2HpMnWJhV6BC0TqyUt6/mkpqBZVhPxjzzTBAiE/X3tIOOKAHcjORNaZVibGwV3s0U4Mw==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-sdk/credential-provider-node": "^3.613.0", + "@aws-sdk/types": "^3.609.0", + "@langchain/anthropic": "^0.3.11", + "@langchain/aws": "^0.1.2", + "@langchain/community": "^0.3.14", + "@langchain/core": "^0.3.26", + "@langchain/google-genai": "^0.1.6", + "@langchain/google-vertexai": "^0.1.5", + "@langchain/langgraph": "^0.2.34", + "@langchain/mistralai": "^0.0.26", + "@langchain/ollama": "^0.1.1", + "@langchain/openai": "^0.3.14", + "@smithy/eventstream-codec": "^2.2.0", + "@smithy/protocol-http": "^3.0.6", + "@smithy/signature-v4": "^2.0.10", + "@smithy/util-utf8": "^2.0.0", + "dotenv": "^16.4.5", + "nanoid": "^3.3.7" + }, + "engines": { + "node": ">=14.0.0" + } + }, "api/node_modules/@types/node": { "version": "18.19.14", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.14.tgz", @@ -9215,6 +9244,8 @@ "version": "0.3.15", "resolved": "https://registry.npmjs.org/@langchain/community/-/community-0.3.15.tgz", "integrity": "sha512-yG4cv33u7zYar14yqZCI7o2KjwRb+9S7upVzEmVVETimpicm9UjpkMfX4qa4A4IslM1TtC4uy2Ymu9EcINZSpQ==", + "optional": true, + "peer": true, "dependencies": { "@langchain/openai": ">=0.2.0 <0.4.0", "binary-extensions": "^2.2.0", @@ -9726,6 +9757,8 @@ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], + "optional": true, + "peer": true, "bin": { "uuid": "dist/bin/uuid" } @@ -9827,6 +9860,8 @@ "resolved": "https://registry.npmjs.org/@langchain/google-genai/-/google-genai-0.1.6.tgz", "integrity": "sha512-LF3fan9pvgFa1vw2/IYGhi5KjppE0OvPFX3QQBUshBLpXWERP+BSpSD7jcXyqm9Kf7DcFj7w5/2knKeEwih8Xg==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@google/generative-ai": "^0.21.0", "zod-to-json-schema": "^3.22.4" @@ -9843,6 +9878,8 @@ "resolved": "https://registry.npmjs.org/@langchain/google-vertexai/-/google-vertexai-0.1.5.tgz", "integrity": "sha512-wd9mjrQRdlFAnwhCGtHIsOTtuF6Oto8+F8nthU3guU7T/gfuICvg26+MaOpdUmuTVFuIDqVMc0l7kf3ZK4DFlA==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@langchain/google-gauth": "~0.1.5" }, @@ -10211,36 +10248,6 @@ "@lezer/common": "^1.0.0" } }, - "node_modules/@librechat/agents": { - "version": "1.9.6", - "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-1.9.6.tgz", - "integrity": "sha512-NfWKvArsDOlbjxiykIl6uAIGWD9birTpltnVA8KF+3Ak5swb6if2gzaS+tfjaua8GpJGdvQcETbOz0XwkQ7riA==", - "license": "MIT", - "dependencies": { - "@aws-crypto/sha256-js": "^5.2.0", - "@aws-sdk/credential-provider-node": "^3.613.0", - "@aws-sdk/types": "^3.609.0", - "@langchain/anthropic": "^0.3.11", - "@langchain/aws": "^0.1.2", - "@langchain/community": "^0.3.14", - "@langchain/core": "^0.3.26", - "@langchain/google-genai": "^0.1.6", - "@langchain/google-vertexai": "^0.1.5", - "@langchain/langgraph": "^0.2.34", - "@langchain/mistralai": "^0.0.26", - "@langchain/ollama": "^0.1.1", - "@langchain/openai": "^0.3.14", - "@smithy/eventstream-codec": "^2.2.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/signature-v4": "^2.0.10", - "@smithy/util-utf8": "^2.0.0", - "dotenv": "^16.4.5", - "nanoid": "^3.3.7" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/@librechat/backend": { "resolved": "api", "link": true From 2d9ab200cff871fca74e35ac09d1fcfd8b233af8 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 28 Dec 2024 16:09:00 -0500 Subject: [PATCH 10/15] fix: additional fields for llmConfig, however customHeaders are not supported by langchain, requires PR --- api/server/services/Endpoints/google/llm.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/api/server/services/Endpoints/google/llm.js b/api/server/services/Endpoints/google/llm.js index dc4d5cac840..322403ae3f4 100644 --- a/api/server/services/Endpoints/google/llm.js +++ b/api/server/services/Endpoints/google/llm.js @@ -124,22 +124,20 @@ function getLLMConfig(credentials, options = {}) { } */ - // 7. If using a reverse proxy, build requestOptions - let requestOptions = null; if (reverseProxyUrl) { - requestOptions = { baseUrl: reverseProxyUrl }; - if (authHeader) { - requestOptions.customHeaders = { - Authorization: `Bearer ${apiKey}`, - }; - } + llmConfig.baseUrl = reverseProxyUrl; + } + + if (authHeader) { + llmConfig.customHeaders = { + Authorization: `Bearer ${apiKey}`, + }; } // Return the final shape return { provider, llmConfig, - requestOptions, }; } From 0607b9fe5e18f0e11f63816f10eb7e41465376ef Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 28 Dec 2024 16:15:36 -0500 Subject: [PATCH 11/15] feat: set default location for vertexai LLM configuration --- api/server/services/Endpoints/google/llm.js | 1 + 1 file changed, 1 insertion(+) diff --git a/api/server/services/Endpoints/google/llm.js b/api/server/services/Endpoints/google/llm.js index 322403ae3f4..6a7473ca205 100644 --- a/api/server/services/Endpoints/google/llm.js +++ b/api/server/services/Endpoints/google/llm.js @@ -100,6 +100,7 @@ function getLLMConfig(credentials, options = {}) { credentials: { ...serviceKey }, projectId: project_id, }; + llmConfig.location = process.env.GOOGLE_LOC || 'us-central1'; } else if (apiKey && provider === Providers.GOOGLE) { llmConfig.apiKey = apiKey; } From f2caba964aadce91d6ed8746efc2db90afb0578a Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 28 Dec 2024 16:27:06 -0500 Subject: [PATCH 12/15] fix: outdated OpenAI Client options for getLLMConfig --- api/server/services/Endpoints/openAI/llm.js | 30 ++++++++++----------- api/typedefs.js | 6 +++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/api/server/services/Endpoints/openAI/llm.js b/api/server/services/Endpoints/openAI/llm.js index e372c9d7945..676a621bf97 100644 --- a/api/server/services/Endpoints/openAI/llm.js +++ b/api/server/services/Endpoints/openAI/llm.js @@ -38,6 +38,7 @@ function getLLMConfig(apiKey, options = {}) { dropParams, } = options; + /** @type {OpenAIClientOptions} */ let llmConfig = { streaming, }; @@ -54,29 +55,28 @@ function getLLMConfig(apiKey, options = {}) { }); } + /** @type {OpenAIClientOptions['configuration']} */ const configOptions = {}; // Handle OpenRouter or custom reverse proxy if (useOpenRouter || reverseProxyUrl === 'https://openrouter.ai/api/v1') { - configOptions.basePath = 'https://openrouter.ai/api/v1'; - configOptions.baseOptions = { - headers: Object.assign( - { - 'HTTP-Referer': 'https://librechat.ai', - 'X-Title': 'LibreChat', - }, - headers, - ), - }; + configOptions.baseURL = 'https://openrouter.ai/api/v1'; + configOptions.defaultHeaders = Object.assign( + { + 'HTTP-Referer': 'https://librechat.ai', + 'X-Title': 'LibreChat', + }, + headers, + ); } else if (reverseProxyUrl) { - configOptions.basePath = reverseProxyUrl; + configOptions.baseURL = reverseProxyUrl; if (headers) { - configOptions.baseOptions = { headers }; + configOptions.defaultHeaders = headers; } } if (defaultQuery) { - configOptions.baseOptions.defaultQuery = defaultQuery; + configOptions.defaultQuery = defaultQuery; } if (proxy) { @@ -97,9 +97,9 @@ function getLLMConfig(apiKey, options = {}) { llmConfig.model = process.env.AZURE_OPENAI_DEFAULT_MODEL; } - if (configOptions.basePath) { + if (configOptions.baseURL) { const azureURL = constructAzureURL({ - baseURL: configOptions.basePath, + baseURL: configOptions.baseURL, azureOptions: azure, }); azure.azureOpenAIBasePath = azureURL.split(`/${azure.azureOpenAIApiDeploymentName}`)[0]; diff --git a/api/typedefs.js b/api/typedefs.js index 83b7f501935..132e8b468ef 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -38,6 +38,12 @@ * @memberof typedefs */ +/** + * @exports OpenAIClientOptions + * @typedef {import('@librechat/agents').OpenAIClientOptions} OpenAIClientOptions + * @memberof typedefs + */ + /** * @exports AnthropicClientOptions * @typedef {import('@librechat/agents').AnthropicClientOptions} AnthropicClientOptions From 7c13ea92b1fced4e9d68d73f6e95f85df122a72e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 28 Dec 2024 16:37:04 -0500 Subject: [PATCH 13/15] chore: agent provider options typing --- .../services/Endpoints/anthropic/llm.js | 5 +- .../services/Endpoints/bedrock/options.js | 47 +++++++++---------- api/server/services/Endpoints/google/llm.js | 13 ++--- api/server/services/Endpoints/openAI/llm.js | 7 ++- api/typedefs.js | 12 +++++ 5 files changed, 49 insertions(+), 35 deletions(-) diff --git a/api/server/services/Endpoints/anthropic/llm.js b/api/server/services/Endpoints/anthropic/llm.js index 8ec7522e949..301d42712aa 100644 --- a/api/server/services/Endpoints/anthropic/llm.js +++ b/api/server/services/Endpoints/anthropic/llm.js @@ -50,7 +50,10 @@ function getLLMConfig(apiKey, options = {}) { requestOptions.clientOptions.baseURL = options.reverseProxyUrl; } - return { llmConfig: removeNullishValues(requestOptions) }; + return { + /** @type {AnthropicClientOptions} */ + llmConfig: removeNullishValues(requestOptions), + }; } module.exports = { getLLMConfig }; diff --git a/api/server/services/Endpoints/bedrock/options.js b/api/server/services/Endpoints/bedrock/options.js index 7836704e1af..11b33a5357e 100644 --- a/api/server/services/Endpoints/bedrock/options.js +++ b/api/server/services/Endpoints/bedrock/options.js @@ -60,42 +60,41 @@ const getOptions = async ({ req, endpointOption }) => { streamRate = allConfig.streamRate; } - /** @type {import('@librechat/agents').BedrockConverseClientOptions} */ - const requestOptions = Object.assign( - { - model: endpointOption.model, - region: BEDROCK_AWS_DEFAULT_REGION, - streaming: true, - streamUsage: true, - callbacks: [ - { - handleLLMNewToken: async () => { - if (!streamRate) { - return; - } - await sleep(streamRate); - }, + /** @type {BedrockClientOptions} */ + const requestOptions = { + model: endpointOption.model, + region: BEDROCK_AWS_DEFAULT_REGION, + streaming: true, + streamUsage: true, + callbacks: [ + { + handleLLMNewToken: async () => { + if (!streamRate) { + return; + } + await sleep(streamRate); }, - ], - }, - endpointOption.model_parameters, - ); + }, + ], + }; if (credentials) { requestOptions.credentials = credentials; } + if (BEDROCK_REVERSE_PROXY) { + requestOptions.endpointHost = BEDROCK_REVERSE_PROXY; + } + const configOptions = {}; if (PROXY) { + /** NOTE: NOT SUPPORTED BY BEDROCK */ configOptions.httpAgent = new HttpsProxyAgent(PROXY); } - if (BEDROCK_REVERSE_PROXY) { - configOptions.endpointHost = BEDROCK_REVERSE_PROXY; - } - return { - llmConfig: removeNullishValues(requestOptions), + /** @type {BedrockClientOptions} */ + llmConfig: removeNullishValues(Object.assign(requestOptions, endpointOption.model_parameters)), configOptions, }; }; diff --git a/api/server/services/Endpoints/google/llm.js b/api/server/services/Endpoints/google/llm.js index 6a7473ca205..4f0ede283a4 100644 --- a/api/server/services/Endpoints/google/llm.js +++ b/api/server/services/Endpoints/google/llm.js @@ -35,15 +35,6 @@ function getSafetySettings() { * * @param {string | object} credentials - Either a JSON string or an object containing Google keys * @param {object} [options={}] - The same shape as the "GoogleClient" constructor options - * @returns {{ - * provider: string, - * llmConfig: object, - * requestOptions: object|null, - * project_id: string|null, - * serviceKey: object|null, - * apiKey: string|null, - * safetySettings: object[] - * }} */ function getLLMConfig(credentials, options = {}) { @@ -70,6 +61,7 @@ function getLLMConfig(credentials, options = {}) { const reverseProxyUrl = options.reverseProxyUrl; const authHeader = options.authHeader; + /** @type {GoogleClientOptions | VertexAIClientOptions} */ let llmConfig = { ...(options.modelOptions || {}), safetySettings: getSafetySettings(), @@ -96,6 +88,7 @@ function getLLMConfig(credentials, options = {}) { // If we have a GCP project => Vertex AI if (project_id && provider === Providers.VERTEXAI) { + /** @type {VertexAIClientOptions['authOptions']} */ llmConfig.authOptions = { credentials: { ...serviceKey }, projectId: project_id, @@ -137,7 +130,9 @@ function getLLMConfig(credentials, options = {}) { // Return the final shape return { + /** @type {Providers.GOOGLE | Providers.VERTEXAI} */ provider, + /** @type {GoogleClientOptions | VertexAIClientOptions} */ llmConfig, }; } diff --git a/api/server/services/Endpoints/openAI/llm.js b/api/server/services/Endpoints/openAI/llm.js index 676a621bf97..2587b242c99 100644 --- a/api/server/services/Endpoints/openAI/llm.js +++ b/api/server/services/Endpoints/openAI/llm.js @@ -118,7 +118,12 @@ function getLLMConfig(apiKey, options = {}) { llmConfig.organization = process.env.OPENAI_ORGANIZATION; } - return { llmConfig, configOptions }; + return { + /** @type {OpenAIClientOptions} */ + llmConfig, + /** @type {OpenAIClientOptions['configuration']} */ + configOptions, + }; } module.exports = { getLLMConfig }; diff --git a/api/typedefs.js b/api/typedefs.js index 132e8b468ef..186c0e4a528 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -56,6 +56,18 @@ * @memberof typedefs */ +/** + * @exports VertexAIClientOptions + * @typedef {import('@librechat/agents').VertexAIClientOptions} VertexAIClientOptions + * @memberof typedefs + */ + +/** + * @exports GoogleClientOptions + * @typedef {import('@librechat/agents').GoogleClientOptions} GoogleClientOptions + * @memberof typedefs + */ + /** * @exports StreamEventData * @typedef {import('@librechat/agents').StreamEventData} StreamEventData From a9e46ad587b7963db0a331bd77eefcff530e3e14 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 28 Dec 2024 16:46:17 -0500 Subject: [PATCH 14/15] chore: add note about currently unsupported customHeaders in langchain GenAI client --- api/server/services/Endpoints/google/llm.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/server/services/Endpoints/google/llm.js b/api/server/services/Endpoints/google/llm.js index 4f0ede283a4..959e9a494b1 100644 --- a/api/server/services/Endpoints/google/llm.js +++ b/api/server/services/Endpoints/google/llm.js @@ -123,6 +123,10 @@ function getLLMConfig(credentials, options = {}) { } if (authHeader) { + /** + * NOTE: NOT SUPPORTED BY LANGCHAIN GENAI CLIENT, + * REQUIRES PR IN https://github.com/langchain-ai/langchainjs + */ llmConfig.customHeaders = { Authorization: `Bearer ${apiKey}`, }; From 8d6ee7aa9a40af64aac36120f1c9c20888fd8ac6 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 28 Dec 2024 17:05:19 -0500 Subject: [PATCH 15/15] fix: skip transaction creation when rawAmount is NaN --- api/models/Transaction.js | 3 +++ api/models/Transaction.spec.js | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/api/models/Transaction.js b/api/models/Transaction.js index 982f6411d94..8435a812c4a 100644 --- a/api/models/Transaction.js +++ b/api/models/Transaction.js @@ -27,6 +27,9 @@ transactionSchema.methods.calculateTokenValue = function () { */ transactionSchema.statics.create = async function (txData) { const Transaction = this; + if (txData.rawAmount != null && isNaN(txData.rawAmount)) { + return; + } const transaction = new Transaction(txData); transaction.endpointTokenConfig = txData.endpointTokenConfig; diff --git a/api/models/Transaction.spec.js b/api/models/Transaction.spec.js index 87aa541eab8..b8c69e13f47 100644 --- a/api/models/Transaction.spec.js +++ b/api/models/Transaction.spec.js @@ -1,5 +1,6 @@ const mongoose = require('mongoose'); const { MongoMemoryServer } = require('mongodb-memory-server'); +const { Transaction } = require('./Transaction'); const Balance = require('./Balance'); const { spendTokens, spendStructuredTokens } = require('./spendTokens'); const { getMultiplier, getCacheMultiplier } = require('./tx'); @@ -346,3 +347,28 @@ describe('Structured Token Spending Tests', () => { expect(result.completion.completion).toBeCloseTo(-50 * 15 * 1.15, 0); // Assuming multiplier is 15 and cancelRate is 1.15 }); }); + +describe('NaN Handling Tests', () => { + test('should skip transaction creation when rawAmount is NaN', async () => { + const userId = new mongoose.Types.ObjectId(); + const initialBalance = 10000000; + await Balance.create({ user: userId, tokenCredits: initialBalance }); + + const model = 'gpt-3.5-turbo'; + const txData = { + user: userId, + conversationId: 'test-conversation-id', + model, + context: 'test', + endpointTokenConfig: null, + rawAmount: NaN, + tokenType: 'prompt', + }; + + const result = await Transaction.create(txData); + expect(result).toBeUndefined(); + + const balance = await Balance.findOne({ user: userId }); + expect(balance.tokenCredits).toBe(initialBalance); + }); +});