From bb913c3d7f3da790c8c3469b1a3554390ee8bdf0 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 15 Aug 2024 15:46:21 -0700 Subject: [PATCH 01/12] scripts[minor]: Add notebook validation script --- docs/core_docs/package.json | 2 +- libs/langchain-scripts/bin/validate_notebook.js | 1 + libs/langchain-scripts/package.json | 6 ++++-- .../langchain-scripts/src}/validate_notebook.ts | 9 +++------ libs/langchain-scripts/turbo.json | 3 +++ package.json | 2 +- yarn.lock | 1 + 7 files changed, 14 insertions(+), 10 deletions(-) create mode 100755 libs/langchain-scripts/bin/validate_notebook.js rename {docs/core_docs/scripts => libs/langchain-scripts/src}/validate_notebook.ts (94%) diff --git a/docs/core_docs/package.json b/docs/core_docs/package.json index e71b7784e37d..9a74b1014673 100644 --- a/docs/core_docs/package.json +++ b/docs/core_docs/package.json @@ -27,7 +27,7 @@ "broken-links": "node ./scripts/check-broken-links.js", "check:broken-links": "yarn quarto && yarn broken-links", "check:broken-links:ci": "yarn quarto:vercel && yarn broken-links", - "validate": "tsx --experimental-wasm-modules -r dotenv/config ./scripts/validate_notebook.ts" + "validate": "yarn notebook_validate" }, "dependencies": { "@docusaurus/core": "2.4.3", diff --git a/libs/langchain-scripts/bin/validate_notebook.js b/libs/langchain-scripts/bin/validate_notebook.js new file mode 100755 index 000000000000..2ff6aa968a34 --- /dev/null +++ b/libs/langchain-scripts/bin/validate_notebook.js @@ -0,0 +1 @@ +import "../dist/validate_notebook.js"; diff --git a/libs/langchain-scripts/package.json b/libs/langchain-scripts/package.json index 318dca0c76db..436f59d90dc7 100644 --- a/libs/langchain-scripts/package.json +++ b/libs/langchain-scripts/package.json @@ -15,7 +15,8 @@ "homepage": "https://github.com/langchain-ai/langchainjs/tree/main/libs/langchain-scripts/", "bin": { "lc-build": "bin/build.js", - "lc_build_v2": "bin/build_v2.js" + "lc_build_v2": "bin/build_v2.js", + "notebook_validate": "bin/validate_notebook.js" }, "scripts": { "build": "yarn turbo:command build:internal --filter=@langchain/scripts", @@ -35,7 +36,8 @@ "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", "format": "prettier --write \"src\"", "format:check": "prettier --check \"src\"", - "create:integration:doc": "node dist/cli/docs/index.js" + "create:integration:doc": "node dist/cli/docs/index.js", + "notebook:validate": "node dist/validate_notebook.js" }, "author": "LangChain", "license": "MIT", diff --git a/docs/core_docs/scripts/validate_notebook.ts b/libs/langchain-scripts/src/validate_notebook.ts similarity index 94% rename from docs/core_docs/scripts/validate_notebook.ts rename to libs/langchain-scripts/src/validate_notebook.ts index 61ac2439f664..caade263ab36 100644 --- a/docs/core_docs/scripts/validate_notebook.ts +++ b/libs/langchain-scripts/src/validate_notebook.ts @@ -1,5 +1,5 @@ -import * as fs from "node:fs"; -import * as ts from "typescript"; +import fs from "node:fs"; +import ts from "typescript"; import { Project } from "ts-morph"; export function extract(filepath: string) { @@ -62,16 +62,13 @@ export function extract(filepath: string) { return sourceFile.getFullText(); } -let [pathname, ...args] = process.argv.slice(2); +let [pathname] = process.argv.slice(2); if (!pathname) { throw new Error("No pathname provided."); } const run = async () => { - if (pathname.startsWith("docs/core_docs/")) { - pathname = "./" + pathname.slice("docs/core_docs/".length); - } if (!pathname.endsWith(".ipynb")) { throw new Error("Only .ipynb files are supported."); } diff --git a/libs/langchain-scripts/turbo.json b/libs/langchain-scripts/turbo.json index 93ef37be7cec..e175624a0d2d 100644 --- a/libs/langchain-scripts/turbo.json +++ b/libs/langchain-scripts/turbo.json @@ -7,6 +7,9 @@ }, "build:internal": { "outputs": ["**/dist/**"] + }, + "notebook:validate": { + "dependsOn": ["build"] } } } diff --git a/package.json b/package.json index d6161145a605..fdb81c35343e 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "test:standard:int": "turbo test:standard:int", "test:standard": "yarn test:standard:unit && yarn test:standard:int", "example": "yarn workspace examples start", - "notebook:validate": "yarn workspace core_docs validate", + "notebook:validate": "turbo --filter=@langchain/scripts notebook:validate", "precommit": "turbo precommit", "docs": "yarn workspace core_docs start", "docs:api_refs": "yarn workspace api_refs start", diff --git a/yarn.lock b/yarn.lock index 148816971920..bf108aad0c8a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12438,6 +12438,7 @@ __metadata: bin: lc-build: bin/build.js lc_build_v2: bin/build_v2.js + notebook_validate: bin/validate_notebook.js languageName: unknown linkType: soft From ba34cfef6966032ac54c403751f858c9bc4bea77 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 15 Aug 2024 15:46:51 -0700 Subject: [PATCH 02/12] trigger ntbk change --- docs/core_docs/docs/integrations/chat/openai.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/core_docs/docs/integrations/chat/openai.ipynb b/docs/core_docs/docs/integrations/chat/openai.ipynb index eb829b093865..1d70c901c798 100644 --- a/docs/core_docs/docs/integrations/chat/openai.ipynb +++ b/docs/core_docs/docs/integrations/chat/openai.ipynb @@ -10,7 +10,7 @@ }, "source": [ "---\n", - "sidebar_label: OpenAI\n", + "sidebar_label: OpenAI2\n", "---" ] }, From bf4466e38953312bec95a5f1c8e63679ff283851 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 15 Aug 2024 16:01:03 -0700 Subject: [PATCH 03/12] cr --- .github/workflows/validate_new_notebooks.yml | 4 +++- libs/langchain-scripts/src/validate_notebook.ts | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate_new_notebooks.yml b/.github/workflows/validate_new_notebooks.yml index bb6bffebc062..9cb9d1cb8df1 100644 --- a/.github/workflows/validate_new_notebooks.yml +++ b/.github/workflows/validate_new_notebooks.yml @@ -51,7 +51,9 @@ jobs: notebooks=$(echo '${{ steps.changed-files.outputs.all_changed_files }}' | tr ' ' '\n' | grep '^docs/core_docs/.*\.ipynb$' || true) if [ -n "$notebooks" ]; then for notebook in $notebooks; do - yarn notebook:validate "$notebook" + absolute_path="$GITHUB_WORKSPACE/$notebook" + echo "Validating $absolute_path" + yarn notebook:validate "$absolute_path" done else echo "No notebooks in docs/core_docs to validate." diff --git a/libs/langchain-scripts/src/validate_notebook.ts b/libs/langchain-scripts/src/validate_notebook.ts index caade263ab36..bc65e428d46b 100644 --- a/libs/langchain-scripts/src/validate_notebook.ts +++ b/libs/langchain-scripts/src/validate_notebook.ts @@ -69,6 +69,10 @@ if (!pathname) { } const run = async () => { + console.log({ + pathname, + processCwd: process.cwd(), + }); if (!pathname.endsWith(".ipynb")) { throw new Error("Only .ipynb files are supported."); } From 646869a1fe09aceb043446bc54ebd9530994df71 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 15 Aug 2024 16:39:46 -0700 Subject: [PATCH 04/12] format and lint --- libs/langchain-scripts/src/validate_notebook.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libs/langchain-scripts/src/validate_notebook.ts b/libs/langchain-scripts/src/validate_notebook.ts index bc65e428d46b..2ada492555bf 100644 --- a/libs/langchain-scripts/src/validate_notebook.ts +++ b/libs/langchain-scripts/src/validate_notebook.ts @@ -3,13 +3,14 @@ import ts from "typescript"; import { Project } from "ts-morph"; export function extract(filepath: string) { - const cells = JSON.parse(fs.readFileSync(filepath).toString()).cells; + const { cells } = JSON.parse(fs.readFileSync(filepath).toString()); if (cells[0]?.source.includes("lc_docs_skip_validation: true\n")) { return ""; } const project = new Project({ useInMemoryFileSystem: true }); const sourceFile = project.createSourceFile("temp.ts", ""); + // eslint-disable-next-line @typescript-eslint/no-explicit-any cells.forEach((cell: Record) => { const source = cell.source .join("") @@ -62,7 +63,7 @@ export function extract(filepath: string) { return sourceFile.getFullText(); } -let [pathname] = process.argv.slice(2); +const [pathname] = process.argv.slice(2); if (!pathname) { throw new Error("No pathname provided."); @@ -73,6 +74,7 @@ const run = async () => { pathname, processCwd: process.cwd(), }); + if (!pathname.endsWith(".ipynb")) { throw new Error("Only .ipynb files are supported."); } @@ -122,6 +124,7 @@ const run = async () => { const issues = issueStrings.join("\n"); console.error(issues); const err = new Error("Found type errors in new notebook."); + // eslint-disable-next-line @typescript-eslint/no-explicit-any (err as any).details = issues; throw err; } @@ -135,7 +138,7 @@ const run = async () => { }; try { - run(); + void run(); } catch { process.exit(1); } From fe597a4c21353c44316eba5b09802bd5cdd05194 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 15 Aug 2024 16:56:42 -0700 Subject: [PATCH 05/12] add lg=ogs --- .github/workflows/validate_new_notebooks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate_new_notebooks.yml b/.github/workflows/validate_new_notebooks.yml index 9cb9d1cb8df1..a587278642ad 100644 --- a/.github/workflows/validate_new_notebooks.yml +++ b/.github/workflows/validate_new_notebooks.yml @@ -52,6 +52,8 @@ jobs: if [ -n "$notebooks" ]; then for notebook in $notebooks; do absolute_path="$GITHUB_WORKSPACE/$notebook" + echo "$(ls -a)" + echo "$(pwd)" echo "Validating $absolute_path" yarn notebook:validate "$absolute_path" done From 76c3674db37e894b8de07b4a77bd99a5d4b44566 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 15 Aug 2024 17:17:52 -0700 Subject: [PATCH 06/12] verify file exists --- .github/workflows/validate_new_notebooks.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate_new_notebooks.yml b/.github/workflows/validate_new_notebooks.yml index a587278642ad..3692c07c1b46 100644 --- a/.github/workflows/validate_new_notebooks.yml +++ b/.github/workflows/validate_new_notebooks.yml @@ -52,10 +52,14 @@ jobs: if [ -n "$notebooks" ]; then for notebook in $notebooks; do absolute_path="$GITHUB_WORKSPACE/$notebook" - echo "$(ls -a)" - echo "$(pwd)" - echo "Validating $absolute_path" - yarn notebook:validate "$absolute_path" + echo "Checking if $absolute_path exists" + if [ -f "$absolute_path" ]; then + echo "Validating $absolute_path" + yarn notebook:validate "$absolute_path" + else + echo "Error: $absolute_path does not exist" + exit 1 + fi done else echo "No notebooks in docs/core_docs to validate." From afff7f237e54186be11079077cccd358d879f45f Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 15 Aug 2024 17:32:35 -0700 Subject: [PATCH 07/12] fixed --- .github/workflows/validate_new_notebooks.yml | 11 ++--------- libs/langchain-scripts/src/validate_notebook.ts | 5 ----- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/validate_new_notebooks.yml b/.github/workflows/validate_new_notebooks.yml index 3692c07c1b46..4f5d606c2416 100644 --- a/.github/workflows/validate_new_notebooks.yml +++ b/.github/workflows/validate_new_notebooks.yml @@ -52,15 +52,8 @@ jobs: if [ -n "$notebooks" ]; then for notebook in $notebooks; do absolute_path="$GITHUB_WORKSPACE/$notebook" - echo "Checking if $absolute_path exists" - if [ -f "$absolute_path" ]; then - echo "Validating $absolute_path" - yarn notebook:validate "$absolute_path" - else - echo "Error: $absolute_path does not exist" - exit 1 - fi + yarn notebook:validate -- "$absolute_path" done else echo "No notebooks in docs/core_docs to validate." - fi \ No newline at end of file + fi diff --git a/libs/langchain-scripts/src/validate_notebook.ts b/libs/langchain-scripts/src/validate_notebook.ts index 2ada492555bf..65907c159566 100644 --- a/libs/langchain-scripts/src/validate_notebook.ts +++ b/libs/langchain-scripts/src/validate_notebook.ts @@ -70,11 +70,6 @@ if (!pathname) { } const run = async () => { - console.log({ - pathname, - processCwd: process.cwd(), - }); - if (!pathname.endsWith(".ipynb")) { throw new Error("Only .ipynb files are supported."); } From c7173b9e86faae4b0cbf26393933a0e42a8eec4c Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 15 Aug 2024 17:33:59 -0700 Subject: [PATCH 08/12] add -- to script --- .github/workflows/validate_new_notebooks.yml | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_new_notebooks.yml b/.github/workflows/validate_new_notebooks.yml index 4f5d606c2416..aa0911d469b1 100644 --- a/.github/workflows/validate_new_notebooks.yml +++ b/.github/workflows/validate_new_notebooks.yml @@ -52,7 +52,7 @@ jobs: if [ -n "$notebooks" ]; then for notebook in $notebooks; do absolute_path="$GITHUB_WORKSPACE/$notebook" - yarn notebook:validate -- "$absolute_path" + yarn notebook:validate "$absolute_path" done else echo "No notebooks in docs/core_docs to validate." diff --git a/package.json b/package.json index fdb81c35343e..35478c1a5190 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "test:standard:int": "turbo test:standard:int", "test:standard": "yarn test:standard:unit && yarn test:standard:int", "example": "yarn workspace examples start", - "notebook:validate": "turbo --filter=@langchain/scripts notebook:validate", + "notebook:validate": "turbo --filter=@langchain/scripts notebook:validate --", "precommit": "turbo precommit", "docs": "yarn workspace core_docs start", "docs:api_refs": "yarn workspace api_refs start", From d2be388bc58bb4e16c1fd4d654ebfad490a99e41 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 15 Aug 2024 17:54:51 -0700 Subject: [PATCH 09/12] release 0.0.21 --- docs/core_docs/docs/integrations/chat/openai.ipynb | 2 +- libs/langchain-scripts/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/core_docs/docs/integrations/chat/openai.ipynb b/docs/core_docs/docs/integrations/chat/openai.ipynb index 1d70c901c798..eb829b093865 100644 --- a/docs/core_docs/docs/integrations/chat/openai.ipynb +++ b/docs/core_docs/docs/integrations/chat/openai.ipynb @@ -10,7 +10,7 @@ }, "source": [ "---\n", - "sidebar_label: OpenAI2\n", + "sidebar_label: OpenAI\n", "---" ] }, diff --git a/libs/langchain-scripts/package.json b/libs/langchain-scripts/package.json index 436f59d90dc7..85e8d970af0b 100644 --- a/libs/langchain-scripts/package.json +++ b/libs/langchain-scripts/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/scripts", - "version": "0.0.20", + "version": "0.0.21", "description": "Shared scripts for LangChain.js", "type": "module", "engines": { From 8af974629178be411fc8e2a07e2c1e431b3aa929 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 15 Aug 2024 17:56:16 -0700 Subject: [PATCH 10/12] bump all langchain scripts versions --- examples/package.json | 2 +- langchain-core/package.json | 2 +- langchain/package.json | 2 +- .../template/package.json | 2 +- libs/langchain-anthropic/package.json | 2 +- libs/langchain-aws/package.json | 2 +- .../package.json | 2 +- libs/langchain-azure-openai/package.json | 2 +- libs/langchain-baidu-qianfan/package.json | 2 +- libs/langchain-cloudflare/package.json | 2 +- libs/langchain-cohere/package.json | 2 +- libs/langchain-community/package.json | 2 +- libs/langchain-exa/package.json | 2 +- libs/langchain-google-common/package.json | 2 +- libs/langchain-google-gauth/package.json | 2 +- libs/langchain-google-genai/package.json | 2 +- .../package.json | 2 +- libs/langchain-google-vertexai/package.json | 2 +- libs/langchain-google-webauth/package.json | 2 +- libs/langchain-groq/package.json | 2 +- libs/langchain-mistralai/package.json | 2 +- libs/langchain-mixedbread-ai/package.json | 2 +- libs/langchain-mongodb/package.json | 2 +- libs/langchain-nomic/package.json | 2 +- libs/langchain-ollama/package.json | 2 +- libs/langchain-openai/package.json | 2 +- libs/langchain-pinecone/package.json | 2 +- libs/langchain-qdrant/package.json | 2 +- libs/langchain-redis/package.json | 2 +- libs/langchain-textsplitters/package.json | 2 +- libs/langchain-weaviate/package.json | 2 +- libs/langchain-yandex/package.json | 2 +- yarn.lock | 64 +++++++++---------- 33 files changed, 64 insertions(+), 64 deletions(-) diff --git a/examples/package.json b/examples/package.json index 0e0d8df8b6ed..1887b730af06 100644 --- a/examples/package.json +++ b/examples/package.json @@ -58,7 +58,7 @@ "@langchain/pinecone": "workspace:*", "@langchain/qdrant": "workspace:*", "@langchain/redis": "workspace:*", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/textsplitters": "workspace:*", "@langchain/weaviate": "workspace:*", "@langchain/yandex": "workspace:*", diff --git a/langchain-core/package.json b/langchain-core/package.json index 048f0bc4a143..9a2dfae5ac91 100644 --- a/langchain-core/package.json +++ b/langchain-core/package.json @@ -55,7 +55,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@types/decamelize": "^1.2.0", diff --git a/langchain/package.json b/langchain/package.json index bbf93f6eab95..9c256aadca78 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -621,7 +621,7 @@ "@langchain/groq": "^0.0.15", "@langchain/mistralai": "^0.0.26", "@langchain/ollama": "^0.0.2", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@mendable/firecrawl-js": "^0.0.13", "@notionhq/client": "^2.2.10", "@pinecone-database/pinecone": "^1.1.0", diff --git a/libs/create-langchain-integration/template/package.json b/libs/create-langchain-integration/template/package.json index e5598ae4c7da..326580217a7b 100644 --- a/libs/create-langchain-integration/template/package.json +++ b/libs/create-langchain-integration/template/package.json @@ -45,7 +45,7 @@ "@jest/globals": "^29.5.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@tsconfig/recommended": "^1.0.3", "@typescript-eslint/eslint-plugin": "^6.12.0", "@typescript-eslint/parser": "^6.12.0", diff --git a/libs/langchain-anthropic/package.json b/libs/langchain-anthropic/package.json index 314dc51b5b7d..ea843eab9532 100644 --- a/libs/langchain-anthropic/package.json +++ b/libs/langchain-anthropic/package.json @@ -43,7 +43,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-aws/package.json b/libs/langchain-aws/package.json index a82182b6f3a4..0b0251663138 100644 --- a/libs/langchain-aws/package.json +++ b/libs/langchain-aws/package.json @@ -50,7 +50,7 @@ "devDependencies": { "@aws-sdk/types": "^3.609.0", "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@smithy/types": "^3.2.0", "@swc/core": "^1.3.90", diff --git a/libs/langchain-azure-dynamic-sessions/package.json b/libs/langchain-azure-dynamic-sessions/package.json index 0fd02dfb8e8a..b6504cc0be2a 100644 --- a/libs/langchain-azure-dynamic-sessions/package.json +++ b/libs/langchain-azure-dynamic-sessions/package.json @@ -44,7 +44,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-azure-openai/package.json b/libs/langchain-azure-openai/package.json index c4c1157ade8c..a386398a2fc9 100644 --- a/libs/langchain-azure-openai/package.json +++ b/libs/langchain-azure-openai/package.json @@ -41,7 +41,7 @@ "devDependencies": { "@azure/identity": "^4.2.1", "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-baidu-qianfan/package.json b/libs/langchain-baidu-qianfan/package.json index e268a49ba227..ae90a64fdf2e 100644 --- a/libs/langchain-baidu-qianfan/package.json +++ b/libs/langchain-baidu-qianfan/package.json @@ -40,7 +40,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/openai": "~0.1.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-cloudflare/package.json b/libs/langchain-cloudflare/package.json index 249312f78c36..af14fba26d76 100644 --- a/libs/langchain-cloudflare/package.json +++ b/libs/langchain-cloudflare/package.json @@ -43,7 +43,7 @@ "@cloudflare/workers-types": "^4.20231218.0", "@jest/globals": "^29.5.0", "@langchain/langgraph": "~0.0.31", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-cohere/package.json b/libs/langchain-cohere/package.json index de369a8fa775..27ed82d1eadb 100644 --- a/libs/langchain-cohere/package.json +++ b/libs/langchain-cohere/package.json @@ -43,7 +43,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index a958154dffaf..f985f64f710e 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -79,7 +79,7 @@ "@huggingface/inference": "^2.6.4", "@jest/globals": "^29.5.0", "@langchain/langgraph": "<0.1.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@layerup/layerup-security": "^1.5.12", "@mendable/firecrawl-js": "^0.0.36", diff --git a/libs/langchain-exa/package.json b/libs/langchain-exa/package.json index b75eb677e996..b4e74ae0387c 100644 --- a/libs/langchain-exa/package.json +++ b/libs/langchain-exa/package.json @@ -45,7 +45,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-google-common/package.json b/libs/langchain-google-common/package.json index 567fe51b5efb..b5f471d20ada 100644 --- a/libs/langchain-google-common/package.json +++ b/libs/langchain-google-common/package.json @@ -46,7 +46,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-google-gauth/package.json b/libs/langchain-google-gauth/package.json index da8acef40b88..8f6df9bb363c 100644 --- a/libs/langchain-google-gauth/package.json +++ b/libs/langchain-google-gauth/package.json @@ -41,7 +41,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-google-genai/package.json b/libs/langchain-google-genai/package.json index 64cf3fda7e7b..fd1e7545bc03 100644 --- a/libs/langchain-google-genai/package.json +++ b/libs/langchain-google-genai/package.json @@ -41,7 +41,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-google-vertexai-web/package.json b/libs/langchain-google-vertexai-web/package.json index b9000bfbc7ca..48e2058dd9f8 100644 --- a/libs/langchain-google-vertexai-web/package.json +++ b/libs/langchain-google-vertexai-web/package.json @@ -45,7 +45,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-google-vertexai/package.json b/libs/langchain-google-vertexai/package.json index e1ac98e021d4..4d5ef5de85c2 100644 --- a/libs/langchain-google-vertexai/package.json +++ b/libs/langchain-google-vertexai/package.json @@ -49,7 +49,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/google-common": "~0.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-google-webauth/package.json b/libs/langchain-google-webauth/package.json index 316de548ce44..3290a9adf6f2 100644 --- a/libs/langchain-google-webauth/package.json +++ b/libs/langchain-google-webauth/package.json @@ -46,7 +46,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-groq/package.json b/libs/langchain-groq/package.json index dd8ee0678110..0ff8e0db3485 100644 --- a/libs/langchain-groq/package.json +++ b/libs/langchain-groq/package.json @@ -44,7 +44,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/openai": "workspace:^", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-mistralai/package.json b/libs/langchain-mistralai/package.json index 4bd9bd8a7b91..33e611080255 100644 --- a/libs/langchain-mistralai/package.json +++ b/libs/langchain-mistralai/package.json @@ -43,7 +43,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-mixedbread-ai/package.json b/libs/langchain-mixedbread-ai/package.json index 4a072ea27ba6..bdd5c3319687 100644 --- a/libs/langchain-mixedbread-ai/package.json +++ b/libs/langchain-mixedbread-ai/package.json @@ -40,7 +40,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-mongodb/package.json b/libs/langchain-mongodb/package.json index abe5f3850523..ea068045df7f 100644 --- a/libs/langchain-mongodb/package.json +++ b/libs/langchain-mongodb/package.json @@ -46,7 +46,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/openai": "workspace:*", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-nomic/package.json b/libs/langchain-nomic/package.json index 917ea7aac336..8b347543fb53 100644 --- a/libs/langchain-nomic/package.json +++ b/libs/langchain-nomic/package.json @@ -46,7 +46,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/openai": "workspace:^", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-ollama/package.json b/libs/langchain-ollama/package.json index 1f62ba397105..e2b041f03c77 100644 --- a/libs/langchain-ollama/package.json +++ b/libs/langchain-ollama/package.json @@ -41,7 +41,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-openai/package.json b/libs/langchain-openai/package.json index ee1afad9d031..98453121a2f4 100644 --- a/libs/langchain-openai/package.json +++ b/libs/langchain-openai/package.json @@ -44,7 +44,7 @@ "devDependencies": { "@azure/identity": "^4.2.1", "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@langchain/standard-tests": "0.0.0", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", diff --git a/libs/langchain-pinecone/package.json b/libs/langchain-pinecone/package.json index a6590d755e13..519e7f84e357 100644 --- a/libs/langchain-pinecone/package.json +++ b/libs/langchain-pinecone/package.json @@ -48,7 +48,7 @@ "@faker-js/faker": "^8.3.1", "@jest/globals": "^29.5.0", "@langchain/openai": "workspace:*", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-qdrant/package.json b/libs/langchain-qdrant/package.json index 863862272c76..b3a4c254fb9f 100644 --- a/libs/langchain-qdrant/package.json +++ b/libs/langchain-qdrant/package.json @@ -46,7 +46,7 @@ "devDependencies": { "@faker-js/faker": "^8.4.1", "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-redis/package.json b/libs/langchain-redis/package.json index 417ae2c788f8..4d3f13c76531 100644 --- a/libs/langchain-redis/package.json +++ b/libs/langchain-redis/package.json @@ -45,7 +45,7 @@ "devDependencies": { "@faker-js/faker": "^8.4.0", "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-textsplitters/package.json b/libs/langchain-textsplitters/package.json index c13543ccb70d..1efb1da6a906 100644 --- a/libs/langchain-textsplitters/package.json +++ b/libs/langchain-textsplitters/package.json @@ -44,7 +44,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-weaviate/package.json b/libs/langchain-weaviate/package.json index 1a19aed12496..d7939017b918 100644 --- a/libs/langchain-weaviate/package.json +++ b/libs/langchain-weaviate/package.json @@ -47,7 +47,7 @@ "devDependencies": { "@jest/globals": "^29.5.0", "@langchain/openai": "workspace:^", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/libs/langchain-yandex/package.json b/libs/langchain-yandex/package.json index 0e4d92802728..6d4e57ced8c1 100644 --- a/libs/langchain-yandex/package.json +++ b/libs/langchain-yandex/package.json @@ -44,7 +44,7 @@ }, "devDependencies": { "@jest/globals": "^29.5.0", - "@langchain/scripts": "~0.0.20", + "@langchain/scripts": "^0.0.21", "@swc/core": "^1.3.90", "@swc/jest": "^0.2.29", "@tsconfig/recommended": "^1.0.3", diff --git a/yarn.lock b/yarn.lock index bf108aad0c8a..25afa514cb75 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10854,7 +10854,7 @@ __metadata: "@anthropic-ai/sdk": ^0.25.2 "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -10904,7 +10904,7 @@ __metadata: "@aws-sdk/types": ^3.609.0 "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@smithy/types": ^3.2.0 "@swc/core": ^1.3.90 @@ -10973,7 +10973,7 @@ __metadata: "@azure/identity": ^4.2.1 "@jest/globals": ^29.5.0 "@langchain/core": ~0.2 - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -11008,7 +11008,7 @@ __metadata: "@azure/openai": 1.0.0-beta.11 "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -11040,7 +11040,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.56 <0.3.0" "@langchain/openai": ~0.1.0 - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -11076,7 +11076,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" "@langchain/langgraph": ~0.0.31 - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -11114,7 +11114,7 @@ __metadata: dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -11181,7 +11181,7 @@ __metadata: "@langchain/core": ">=0.2.21 <0.3.0" "@langchain/langgraph": <0.1.0 "@langchain/openai": ">=0.2.0 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@layerup/layerup-security": ^1.5.12 "@mendable/firecrawl-js": ^0.0.36 @@ -11692,7 +11692,7 @@ __metadata: resolution: "@langchain/core@workspace:langchain-core" dependencies: "@jest/globals": ^29.5.0 - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@types/decamelize": ^1.2.0 @@ -11734,7 +11734,7 @@ __metadata: dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -11766,7 +11766,7 @@ __metadata: dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -11800,7 +11800,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" "@langchain/google-common": ~0.0.25 - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -11833,7 +11833,7 @@ __metadata: "@google/generative-ai": ^0.7.0 "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -11868,7 +11868,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" "@langchain/google-webauth": ~0.0.25 - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -11901,7 +11901,7 @@ __metadata: "@langchain/core": ">=0.2.21 <0.3.0" "@langchain/google-common": ~0.0 "@langchain/google-gauth": ~0.0.25 - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -11934,7 +11934,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" "@langchain/google-common": ~0.0.25 - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -11979,7 +11979,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" "@langchain/openai": "workspace:^" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -12075,7 +12075,7 @@ __metadata: dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@mistralai/mistralai": ^0.4.0 "@swc/core": ^1.3.90 @@ -12110,7 +12110,7 @@ __metadata: dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.5 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@mixedbread-ai/sdk": ^2.2.3 "@swc/core": ^1.3.90 @@ -12144,7 +12144,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" "@langchain/openai": "workspace:*" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -12178,7 +12178,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" "@langchain/openai": "workspace:^" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@nomic-ai/atlas": ^0.8.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -12221,7 +12221,7 @@ __metadata: dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -12257,7 +12257,7 @@ __metadata: "@azure/identity": ^4.2.1 "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -12304,7 +12304,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.2.0 <0.3.0" "@langchain/openai": "workspace:*" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@pinecone-database/pinecone": ^3.0.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -12339,7 +12339,7 @@ __metadata: "@faker-js/faker": ^8.4.1 "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@qdrant/js-client-rest": ^1.9.0 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -12373,7 +12373,7 @@ __metadata: "@faker-js/faker": ^8.4.0 "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -12400,7 +12400,7 @@ __metadata: languageName: unknown linkType: soft -"@langchain/scripts@workspace:*, @langchain/scripts@workspace:libs/langchain-scripts, @langchain/scripts@~0.0, @langchain/scripts@~0.0.20": +"@langchain/scripts@^0.0.21, @langchain/scripts@workspace:*, @langchain/scripts@workspace:libs/langchain-scripts, @langchain/scripts@~0.0": version: 0.0.0-use.local resolution: "@langchain/scripts@workspace:libs/langchain-scripts" dependencies: @@ -12480,7 +12480,7 @@ __metadata: dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">0.2.0 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -12512,7 +12512,7 @@ __metadata: "@jest/globals": ^29.5.0 "@langchain/core": ">0.2.0 <0.3.0" "@langchain/openai": "workspace:^" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -12546,7 +12546,7 @@ __metadata: dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">0.1.0 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 "@tsconfig/recommended": ^1.0.3 @@ -26691,7 +26691,7 @@ __metadata: "@langchain/pinecone": "workspace:*" "@langchain/qdrant": "workspace:*" "@langchain/redis": "workspace:*" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/textsplitters": "workspace:*" "@langchain/weaviate": "workspace:*" "@langchain/yandex": "workspace:*" @@ -32056,7 +32056,7 @@ __metadata: "@langchain/mistralai": ^0.0.26 "@langchain/ollama": ^0.0.2 "@langchain/openai": ">=0.1.0 <0.3.0" - "@langchain/scripts": ~0.0.20 + "@langchain/scripts": ^0.0.21 "@langchain/textsplitters": ~0.0.0 "@mendable/firecrawl-js": ^0.0.13 "@notionhq/client": ^2.2.10 From ce9cc0dbd1d275804f713c0cfdbf95312f5243d1 Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Thu, 15 Aug 2024 20:25:10 -0700 Subject: [PATCH 11/12] Apply suggestions from code review --- libs/langchain-scripts/package.json | 3 +-- libs/langchain-scripts/turbo.json | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/libs/langchain-scripts/package.json b/libs/langchain-scripts/package.json index 85e8d970af0b..7dcd652ebfa3 100644 --- a/libs/langchain-scripts/package.json +++ b/libs/langchain-scripts/package.json @@ -36,8 +36,7 @@ "test:int": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%", "format": "prettier --write \"src\"", "format:check": "prettier --check \"src\"", - "create:integration:doc": "node dist/cli/docs/index.js", - "notebook:validate": "node dist/validate_notebook.js" + "create:integration:doc": "node dist/cli/docs/index.js" }, "author": "LangChain", "license": "MIT", diff --git a/libs/langchain-scripts/turbo.json b/libs/langchain-scripts/turbo.json index e175624a0d2d..93ef37be7cec 100644 --- a/libs/langchain-scripts/turbo.json +++ b/libs/langchain-scripts/turbo.json @@ -7,9 +7,6 @@ }, "build:internal": { "outputs": ["**/dist/**"] - }, - "notebook:validate": { - "dependsOn": ["build"] } } } From d2bec1c05aac2419c8517ef7c13ccc5d87605664 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Fri, 16 Aug 2024 09:21:48 -0700 Subject: [PATCH 12/12] merge main --- langchain-core/package.json | 2 +- langchain-core/src/tools/index.ts | 8 ++- langchain-core/src/tools/tests/tools.test.ts | 14 +++- langchain/package.json | 4 +- libs/langchain-aws/package.json | 2 +- libs/langchain-aws/src/chat_models.ts | 8 ++- libs/langchain-community/package.json | 4 +- .../src/document_loaders/fs/unstructured.ts | 18 ++++- .../src/vectorstores/elasticsearch.ts | 2 +- .../tests/elasticsearch.int.test.ts | 8 +++ libs/langchain-google-common/package.json | 2 +- .../src/tests/utils.test.ts | 55 ++++++++++++++++ .../src/utils/stream.ts | 4 +- libs/langchain-google-gauth/package.json | 4 +- libs/langchain-google-genai/package.json | 2 +- .../langchain-google-genai/src/chat_models.ts | 10 +++ .../src/tests/chat_models.int.test.ts | 19 ++++++ .../package.json | 4 +- libs/langchain-google-vertexai/package.json | 4 +- libs/langchain-google-webauth/package.json | 4 +- libs/langchain-pinecone/package.json | 2 +- .../src/tests/vectorstores.int.test.ts | 60 ++++++++++++----- libs/langchain-pinecone/src/vectorstores.ts | 66 ++++++++++++------- yarn.lock | 24 +++---- 24 files changed, 248 insertions(+), 82 deletions(-) diff --git a/langchain-core/package.json b/langchain-core/package.json index 9a2dfae5ac91..1fcb7c21d297 100644 --- a/langchain-core/package.json +++ b/langchain-core/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/core", - "version": "0.2.24", + "version": "0.2.25", "description": "Core LangChain.js abstractions and schemas", "type": "module", "engines": { diff --git a/langchain-core/src/tools/index.ts b/langchain-core/src/tools/index.ts index 3a73a0b53dde..a2000a24b21d 100644 --- a/langchain-core/src/tools/index.ts +++ b/langchain-core/src/tools/index.ts @@ -430,7 +430,7 @@ export class DynamicStructuredTool< this.func = fields.func; this.returnDirect = fields.returnDirect ?? this.returnDirect; this.schema = ( - isZodSchema(fields.schema) ? fields.schema : z.object({}) + isZodSchema(fields.schema) ? fields.schema : z.object({}).passthrough() ) as T extends ZodObjectAny ? T : ZodObjectAny; } @@ -557,7 +557,11 @@ export function tool< | DynamicStructuredTool | DynamicTool { // If the schema is not provided, or it's a string schema, create a DynamicTool - if (!fields.schema || !("shape" in fields.schema) || !fields.schema.shape) { + if ( + !fields.schema || + (isZodSchema(fields.schema) && + (!("shape" in fields.schema) || !fields.schema.shape)) + ) { return new DynamicTool({ ...fields, description: diff --git a/langchain-core/src/tools/tests/tools.test.ts b/langchain-core/src/tools/tests/tools.test.ts index 4c38800b3489..2988eb64d9cf 100644 --- a/langchain-core/src/tools/tests/tools.test.ts +++ b/langchain-core/src/tools/tests/tools.test.ts @@ -128,7 +128,11 @@ test("Tool declared with JSON schema", async () => { required: ["location"], }; const weatherTool = tool( - (_) => { + (input) => { + // even without validation expect input to be passed + expect(input).toEqual({ + somethingSilly: true, + }); return "Sunny"; }, { @@ -136,15 +140,21 @@ test("Tool declared with JSON schema", async () => { schema: weatherSchema, } ); + expect(weatherTool).toBeInstanceOf(DynamicStructuredTool); const weatherTool2 = new DynamicStructuredTool({ name: "weather", description: "get the weather", - func: async (_) => { + func: async (input) => { + // even without validation expect input to be passed + expect(input).toEqual({ + somethingSilly: true, + }); return "Sunny"; }, schema: weatherSchema, }); + // No validation on JSON schema tools await weatherTool.invoke({ somethingSilly: true, diff --git a/langchain/package.json b/langchain/package.json index 9c256aadca78..8c5d13a9b751 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -616,8 +616,8 @@ "@langchain/anthropic": "^0.2.8", "@langchain/aws": "^0.0.5", "@langchain/cohere": "^0.2.1", - "@langchain/google-genai": "^0.0.25", - "@langchain/google-vertexai": "^0.0.25", + "@langchain/google-genai": "^0.0.26", + "@langchain/google-vertexai": "^0.0.26", "@langchain/groq": "^0.0.15", "@langchain/mistralai": "^0.0.26", "@langchain/ollama": "^0.0.2", diff --git a/libs/langchain-aws/package.json b/libs/langchain-aws/package.json index 0b0251663138..48e4d63696bb 100644 --- a/libs/langchain-aws/package.json +++ b/libs/langchain-aws/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/aws", - "version": "0.0.9", + "version": "0.0.10", "description": "LangChain AWS integration", "type": "module", "engines": { diff --git a/libs/langchain-aws/src/chat_models.ts b/libs/langchain-aws/src/chat_models.ts index 8de6af7712a9..ee0a4fcc6634 100644 --- a/libs/langchain-aws/src/chat_models.ts +++ b/libs/langchain-aws/src/chat_models.ts @@ -817,7 +817,9 @@ export class ChatBedrockConverse system: converseSystem, ...params, }); - const response = await this.client.send(command); + const response = await this.client.send(command, { + abortSignal: options.signal, + }); const { output, ...responseMetadata } = response; if (!output?.message) { throw new Error("No message found in Bedrock response."); @@ -855,7 +857,9 @@ export class ChatBedrockConverse system: converseSystem, ...params, }); - const response = await this.client.send(command); + const response = await this.client.send(command, { + abortSignal: options.signal, + }); if (response.stream) { for await (const chunk of response.stream) { if (chunk.contentBlockStart) { diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index f985f64f710e..83bd6cecae25 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/community", - "version": "0.2.27", + "version": "0.2.28", "description": "Third-party integrations for LangChain.js", "type": "module", "engines": { @@ -293,7 +293,7 @@ "faiss-node": "^0.5.1", "firebase-admin": "^11.9.0 || ^12.0.0", "google-auth-library": "*", - "googleapis": "^126.0.1", + "googleapis": "*", "hnswlib-node": "^3.0.0", "html-to-text": "^9.0.5", "ignore": "^5.2.0", diff --git a/libs/langchain-community/src/document_loaders/fs/unstructured.ts b/libs/langchain-community/src/document_loaders/fs/unstructured.ts index 48915c89fdc5..2a9bc3271b10 100644 --- a/libs/langchain-community/src/document_loaders/fs/unstructured.ts +++ b/libs/langchain-community/src/document_loaders/fs/unstructured.ts @@ -114,6 +114,8 @@ export type UnstructuredLoaderOptions = { newAfterNChars?: number; maxCharacters?: number; extractImageBlockTypes?: string[]; + overlap?: number; + overlapAll?: boolean; }; export type UnstructuredDirectoryLoaderOptions = UnstructuredLoaderOptions & { @@ -181,6 +183,10 @@ export class UnstructuredLoader extends BaseDocumentLoader { private extractImageBlockTypes?: string[]; + private overlap?: number; + + private overlapAll?: boolean; + constructor( filepathOrBufferOptions: string | UnstructuredMemoryLoaderOptions, unstructuredOptions: UnstructuredLoaderOptions | string = {} @@ -225,6 +231,8 @@ export class UnstructuredLoader extends BaseDocumentLoader { this.newAfterNChars = options.newAfterNChars; this.maxCharacters = options.maxCharacters; this.extractImageBlockTypes = options.extractImageBlockTypes; + this.overlap = options.overlap; + this.overlapAll = options.overlapAll ?? false; } } @@ -299,6 +307,14 @@ export class UnstructuredLoader extends BaseDocumentLoader { ); } + if (this.overlap !== undefined) { + formData.append("overlap", String(this.overlap)); + } + + if (this.overlapAll === true) { + formData.append("overlap_all", "true"); + } + const headers = { "UNSTRUCTURED-API-KEY": this.apiKey ?? "", }; @@ -332,7 +348,7 @@ export class UnstructuredLoader extends BaseDocumentLoader { const documents: Document[] = []; for (const element of elements) { const { metadata, text } = element; - if (typeof text === "string") { + if (typeof text === "string" && text !== "") { documents.push( new Document({ pageContent: text, diff --git a/libs/langchain-community/src/vectorstores/elasticsearch.ts b/libs/langchain-community/src/vectorstores/elasticsearch.ts index 81f480c3f1fc..14a4ee12ac03 100644 --- a/libs/langchain-community/src/vectorstores/elasticsearch.ts +++ b/libs/langchain-community/src/vectorstores/elasticsearch.ts @@ -344,7 +344,7 @@ export class ElasticVectorSearch extends VectorStore { }); } else if (condition.operator === "exclude") { must_not.push({ - terms: { + term: { [metadataField]: condition.value, }, }); diff --git a/libs/langchain-community/src/vectorstores/tests/elasticsearch.int.test.ts b/libs/langchain-community/src/vectorstores/tests/elasticsearch.int.test.ts index cc22163c7763..08ec39ac0f57 100644 --- a/libs/langchain-community/src/vectorstores/tests/elasticsearch.int.test.ts +++ b/libs/langchain-community/src/vectorstores/tests/elasticsearch.int.test.ts @@ -106,6 +106,14 @@ describe("ElasticVectorSearch", () => { ]); const results = await store.similaritySearch("*", 11); expect(results).toHaveLength(11); + const results2 = await store.similaritySearch("*", 11, [ + { + field: "a", + value: createdAt, + operator: "exclude", + }, + ]); + expect(results2).toHaveLength(1); }); test.skip("ElasticVectorSearch integration with text splitting metadata", async () => { diff --git a/libs/langchain-google-common/package.json b/libs/langchain-google-common/package.json index b5f471d20ada..0aba010d59fd 100644 --- a/libs/langchain-google-common/package.json +++ b/libs/langchain-google-common/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-common", - "version": "0.0.25", + "version": "0.0.26", "description": "Core types and classes for Google services.", "type": "module", "engines": { diff --git a/libs/langchain-google-common/src/tests/utils.test.ts b/libs/langchain-google-common/src/tests/utils.test.ts index 84f12dca4dd0..c0398a218c67 100644 --- a/libs/langchain-google-common/src/tests/utils.test.ts +++ b/libs/langchain-google-common/src/tests/utils.test.ts @@ -2,6 +2,7 @@ import { expect, test } from "@jest/globals"; import { z } from "zod"; import { zodToGeminiParameters } from "../utils/zod_to_gemini_parameters.js"; +import { ReadableJsonStream } from "../utils/stream.js"; test("zodToGeminiParameters can convert zod schema to gemini schema", () => { const zodSchema = z @@ -80,3 +81,57 @@ test("zodToGeminiParameters removes additional properties from arrays", () => { expect((arrayItemsSchema as any).additionalProperties).toBeUndefined(); } }); + +function toUint8Array(data: string): Uint8Array { + return new TextEncoder().encode(data); +} + +test("ReadableJsonStream can handle stream", async () => { + const data = [ + toUint8Array("["), + toUint8Array('{"i": 1}'), + toUint8Array('{"i'), + toUint8Array('": 2}'), + toUint8Array("]"), + ]; + + const source = new ReadableStream({ + start(controller) { + data.forEach((chunk) => controller.enqueue(chunk)); + controller.close(); + }, + }); + const stream = new ReadableJsonStream(source); + expect(await stream.nextChunk()).toEqual({ i: 1 }); + expect(await stream.nextChunk()).toEqual({ i: 2 }); + expect(await stream.nextChunk()).toBeNull(); + expect(stream.streamDone).toEqual(true); +}); + +test("ReadableJsonStream can handle multibyte stream", async () => { + const data = [ + toUint8Array("["), + toUint8Array('{"i": 1, "msg":"helloπŸ‘‹"}'), + toUint8Array('{"i": 2,'), + toUint8Array('"msg":"こん'), + new Uint8Array([0xe3]), // 1st byte of "に" + new Uint8Array([0x81, 0xab]), // 2-3rd bytes of "に" + toUint8Array("けは"), + new Uint8Array([0xf0, 0x9f]), // first half bytes of "πŸ‘‹" + new Uint8Array([0x91, 0x8b]), // second half bytes of "πŸ‘‹" + toUint8Array('"}'), + toUint8Array("]"), + ]; + + const source = new ReadableStream({ + start(controller) { + data.forEach((chunk) => controller.enqueue(chunk)); + controller.close(); + }, + }); + const stream = new ReadableJsonStream(source); + expect(await stream.nextChunk()).toEqual({ i: 1, msg: "helloπŸ‘‹" }); + expect(await stream.nextChunk()).toEqual({ i: 2, msg: "γ“γ‚“γ«γ‘γ―πŸ‘‹" }); + expect(await stream.nextChunk()).toBeNull(); + expect(stream.streamDone).toEqual(true); +}); diff --git a/libs/langchain-google-common/src/utils/stream.ts b/libs/langchain-google-common/src/utils/stream.ts index 63ba44edbd4e..226ac49dca10 100644 --- a/libs/langchain-google-common/src/utils/stream.ts +++ b/libs/langchain-google-common/src/utils/stream.ts @@ -252,7 +252,7 @@ export class ReadableJsonStream extends JsonStream { constructor(body: ReadableStream | null) { super(); - this.decoder = new TextDecoder(); + this.decoder = new TextDecoder("utf-8"); if (body) { void this.run(body); } else { @@ -266,7 +266,7 @@ export class ReadableJsonStream extends JsonStream { while (!isDone) { const { value, done } = await reader.read(); if (!done) { - const svalue = this.decoder.decode(value); + const svalue = this.decoder.decode(value, { stream: true }); this.appendBuffer(svalue); } else { isDone = done; diff --git a/libs/langchain-google-gauth/package.json b/libs/langchain-google-gauth/package.json index 8f6df9bb363c..3a9190076726 100644 --- a/libs/langchain-google-gauth/package.json +++ b/libs/langchain-google-gauth/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-gauth", - "version": "0.0.25", + "version": "0.0.26", "description": "Google auth based authentication support for Google services", "type": "module", "engines": { @@ -36,7 +36,7 @@ "license": "MIT", "dependencies": { "@langchain/core": ">=0.2.21 <0.3.0", - "@langchain/google-common": "~0.0.25", + "@langchain/google-common": "~0.0.26", "google-auth-library": "^8.9.0" }, "devDependencies": { diff --git a/libs/langchain-google-genai/package.json b/libs/langchain-google-genai/package.json index fd1e7545bc03..fdf63cc93a04 100644 --- a/libs/langchain-google-genai/package.json +++ b/libs/langchain-google-genai/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-genai", - "version": "0.0.25", + "version": "0.0.26", "description": "Google Generative AI integration for LangChain.js", "type": "module", "engines": { diff --git a/libs/langchain-google-genai/src/chat_models.ts b/libs/langchain-google-genai/src/chat_models.ts index ddd407df6031..197febd63bbb 100644 --- a/libs/langchain-google-genai/src/chat_models.ts +++ b/libs/langchain-google-genai/src/chat_models.ts @@ -75,6 +75,8 @@ export interface GoogleGenerativeAIChatInput extends BaseChatModelParams, Pick { /** + * @deprecated Use "model" instead. + * * Model Name to use * * Alias for `model` @@ -167,6 +169,13 @@ export interface GoogleGenerativeAIChatInput /** Whether to stream the results or not */ streaming?: boolean; + + /** + * Whether or not to force the model to respond with JSON. + * Available for `gemini-1.5` models and later. + * @default false + */ + json?: boolean; } /** @@ -321,6 +330,7 @@ export class ChatGoogleGenerativeAI temperature: this.temperature, topP: this.topP, topK: this.topK, + ...(fields?.json ? { responseMimeType: "application/json" } : {}), }, }, { diff --git a/libs/langchain-google-genai/src/tests/chat_models.int.test.ts b/libs/langchain-google-genai/src/tests/chat_models.int.test.ts index 4e4791495759..ddf5aee6bfc7 100644 --- a/libs/langchain-google-genai/src/tests/chat_models.int.test.ts +++ b/libs/langchain-google-genai/src/tests/chat_models.int.test.ts @@ -519,3 +519,22 @@ test("Invoke token count usage_metadata", async () => { res.usage_metadata.input_tokens + res.usage_metadata.output_tokens ); }); + +test("Invoke with JSON mode", async () => { + const model = new ChatGoogleGenerativeAI({ + model: "gemini-1.5-flash", + temperature: 0, + maxOutputTokens: 10, + json: true, + }); + const res = await model.invoke("Why is the sky blue? Be concise."); + expect(res?.usage_metadata).toBeDefined(); + if (!res?.usage_metadata) { + return; + } + expect(res.usage_metadata.input_tokens).toBeGreaterThan(1); + expect(res.usage_metadata.output_tokens).toBeGreaterThan(1); + expect(res.usage_metadata.total_tokens).toBe( + res.usage_metadata.input_tokens + res.usage_metadata.output_tokens + ); +}); diff --git a/libs/langchain-google-vertexai-web/package.json b/libs/langchain-google-vertexai-web/package.json index 48e2058dd9f8..599ca1c82094 100644 --- a/libs/langchain-google-vertexai-web/package.json +++ b/libs/langchain-google-vertexai-web/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-vertexai-web", - "version": "0.0.25", + "version": "0.0.26", "description": "LangChain.js support for Google Vertex AI Web", "type": "module", "engines": { @@ -41,7 +41,7 @@ "license": "MIT", "dependencies": { "@langchain/core": ">=0.2.21 <0.3.0", - "@langchain/google-webauth": "~0.0.25" + "@langchain/google-webauth": "~0.0.26" }, "devDependencies": { "@jest/globals": "^29.5.0", diff --git a/libs/langchain-google-vertexai/package.json b/libs/langchain-google-vertexai/package.json index 4d5ef5de85c2..e8094e28f5cf 100644 --- a/libs/langchain-google-vertexai/package.json +++ b/libs/langchain-google-vertexai/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-vertexai", - "version": "0.0.25", + "version": "0.0.26", "description": "LangChain.js support for Google Vertex AI", "type": "module", "engines": { @@ -44,7 +44,7 @@ "license": "MIT", "dependencies": { "@langchain/core": ">=0.2.21 <0.3.0", - "@langchain/google-gauth": "~0.0.25" + "@langchain/google-gauth": "~0.0.26" }, "devDependencies": { "@jest/globals": "^29.5.0", diff --git a/libs/langchain-google-webauth/package.json b/libs/langchain-google-webauth/package.json index 3290a9adf6f2..055e54d8e893 100644 --- a/libs/langchain-google-webauth/package.json +++ b/libs/langchain-google-webauth/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/google-webauth", - "version": "0.0.25", + "version": "0.0.26", "description": "Web-based authentication support for Google services", "type": "module", "engines": { @@ -41,7 +41,7 @@ "license": "MIT", "dependencies": { "@langchain/core": ">=0.2.21 <0.3.0", - "@langchain/google-common": "~0.0.25", + "@langchain/google-common": "~0.0.26", "web-auth-library": "^1.0.3" }, "devDependencies": { diff --git a/libs/langchain-pinecone/package.json b/libs/langchain-pinecone/package.json index 519e7f84e357..ff1e1eef2b45 100644 --- a/libs/langchain-pinecone/package.json +++ b/libs/langchain-pinecone/package.json @@ -1,6 +1,6 @@ { "name": "@langchain/pinecone", - "version": "0.0.8", + "version": "0.0.9", "description": "LangChain integration for Pinecone's vector database", "type": "module", "engines": { diff --git a/libs/langchain-pinecone/src/tests/vectorstores.int.test.ts b/libs/langchain-pinecone/src/tests/vectorstores.int.test.ts index b4cb28c50b05..074dccae9004 100644 --- a/libs/langchain-pinecone/src/tests/vectorstores.int.test.ts +++ b/libs/langchain-pinecone/src/tests/vectorstores.int.test.ts @@ -1,6 +1,7 @@ /* eslint-disable no-process-env */ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable no-promise-executor-return */ +/* eslint-disable @typescript-eslint/no-explicit-any */ import { describe, expect, test } from "@jest/globals"; import { faker } from "@faker-js/faker"; import { Pinecone } from "@pinecone-database/pinecone"; @@ -9,12 +10,14 @@ import { SyntheticEmbeddings } from "@langchain/core/utils/testing"; import { Document } from "@langchain/core/documents"; import { PineconeStoreParams, PineconeStore } from "../vectorstores.js"; +const PINECONE_SLEEP_LENGTH = 40000; + function sleep(ms: number) { // eslint-disable-next-line no-promise-executor-return return new Promise((resolve) => setTimeout(resolve, ms)); } -describe.skip("PineconeStore", () => { +describe("PineconeStore", () => { let pineconeStore: PineconeStore; const testIndexName = process.env.PINECONE_INDEX!; let namespaces: string[] = []; @@ -57,22 +60,29 @@ describe.skip("PineconeStore", () => { [{ pageContent, metadata: {} }], [documentId] ); - await sleep(35000); + + await sleep(PINECONE_SLEEP_LENGTH); const results = await pineconeStore.similaritySearch(pageContent, 1); - expect(results).toEqual([new Document({ metadata: {}, pageContent })]); + expect(results).toEqual([ + new Document({ metadata: {}, pageContent, id: documentId }), + ]); await pineconeStore.addDocuments( [{ pageContent: `${pageContent} upserted`, metadata: {} }], [documentId] ); - await sleep(35000); + await sleep(PINECONE_SLEEP_LENGTH); const results2 = await pineconeStore.similaritySearch(pageContent, 1); expect(results2).toEqual([ - new Document({ metadata: {}, pageContent: `${pageContent} upserted` }), + new Document({ + metadata: {}, + pageContent: `${pageContent} upserted`, + id: documentId, + }), ]); }); @@ -83,11 +93,15 @@ describe.skip("PineconeStore", () => { { pageContent, metadata: { foo: "bar" } }, ]); - await sleep(35000); + await sleep(PINECONE_SLEEP_LENGTH); const results = await pineconeStore.similaritySearch(pageContent, 1); expect(results).toEqual([ - new Document({ metadata: { foo: "bar" }, pageContent }), + new Document({ + metadata: { foo: "bar" }, + pageContent, + id: expect.any(String) as any, + }), ]); }); @@ -100,14 +114,18 @@ describe.skip("PineconeStore", () => { { pageContent, metadata: { foo: id } }, { pageContent, metadata: { foo: "qux" } }, ]); - await sleep(35000); + await sleep(PINECONE_SLEEP_LENGTH); // If the filter wasn't working, we'd get all 3 documents back const results = await pineconeStore.similaritySearch(pageContent, 3, { foo: id, }); expect(results).toEqual([ - new Document({ metadata: { foo: id }, pageContent }), + new Document({ + metadata: { foo: id }, + pageContent, + id: expect.any(String) as any, + }), ]); }); @@ -120,7 +138,7 @@ describe.skip("PineconeStore", () => { { pageContent, metadata: { foo: id } }, { pageContent, metadata: { foo: id } }, ]); - await sleep(35000); + await sleep(PINECONE_SLEEP_LENGTH); // If the filter wasn't working, we'd get all 3 documents back const results = await pineconeStore.maxMarginalRelevanceSearch( pageContent, @@ -142,7 +160,7 @@ describe.skip("PineconeStore", () => { { pageContent, metadata: { foo: id } }, { pageContent, metadata: { foo: id } }, ]); - await sleep(35000); + await sleep(PINECONE_SLEEP_LENGTH); const results = await pineconeStore.similaritySearch(pageContent, 2, { foo: id, }); @@ -174,7 +192,7 @@ describe.skip("PineconeStore", () => { ids: [id, id2], } ); - await sleep(40000); + await sleep(PINECONE_SLEEP_LENGTH); const indexStats = await pineconeStore.pineconeIndex.describeIndexStats(); expect(indexStats.namespaces).toHaveProperty(""); expect(indexStats.namespaces?.[""].recordCount).toEqual(2); @@ -184,7 +202,7 @@ describe.skip("PineconeStore", () => { await pineconeStore.delete({ deleteAll: true, }); - await sleep(40000); + await sleep(PINECONE_SLEEP_LENGTH); const indexStats2 = await pineconeStore.pineconeIndex.describeIndexStats(); expect(indexStats2.namespaces).not.toHaveProperty(""); // The new total records should be less than the previous total records @@ -209,7 +227,7 @@ describe.skip("PineconeStore", () => { namespace: namespaces[1], } ); - await sleep(35000); + await sleep(PINECONE_SLEEP_LENGTH); const results = await pineconeStore.similaritySearch(pageContent, 1, { namespace: namespaces[0], }); @@ -234,22 +252,28 @@ describe.skip("PineconeStore", () => { }); await store.addDocuments([{ pageContent, metadata: {} }], [documentId]); - await sleep(35000); + await sleep(PINECONE_SLEEP_LENGTH); const results = await store.similaritySearch(pageContent, 1); - expect(results).toEqual([new Document({ metadata: {}, pageContent })]); + expect(results).toEqual([ + new Document({ metadata: {}, pageContent, id: documentId }), + ]); await store.addDocuments( [{ pageContent: `${pageContent} upserted`, metadata: {} }], [documentId] ); - await sleep(35000); + await sleep(PINECONE_SLEEP_LENGTH); const results2 = await store.similaritySearch(pageContent, 1); expect(results2).toEqual([ - new Document({ metadata: {}, pageContent: `${pageContent} upserted` }), + new Document({ + metadata: {}, + pageContent: `${pageContent} upserted`, + id: documentId, + }), ]); }); }); diff --git a/libs/langchain-pinecone/src/vectorstores.ts b/libs/langchain-pinecone/src/vectorstores.ts index 3a99dbceb057..2228d2456744 100644 --- a/libs/langchain-pinecone/src/vectorstores.ts +++ b/libs/langchain-pinecone/src/vectorstores.ts @@ -5,6 +5,7 @@ import { RecordMetadata, PineconeRecord, Index as PineconeIndex, + ScoredPineconeRecord, } from "@pinecone-database/pinecone"; import type { EmbeddingsInterface } from "@langchain/core/embeddings"; @@ -401,6 +402,40 @@ export class PineconeStore extends VectorStore { return results; } + /** + * Format the matching results from the Pinecone query. + * @param matches Matching results from the Pinecone query. + * @returns An array of arrays, where each inner array contains a document and its score. + */ + private _formatMatches( + matches: ScoredPineconeRecord[] = [] + ): [Document, number][] { + const documentsWithScores: [Document, number][] = []; + + for (const record of matches) { + const { + id, + score, + metadata: { [this.textKey]: pageContent, ...metadata } = { + [this.textKey]: "", + }, + } = record; + + if (score) { + documentsWithScores.push([ + new Document({ + id, + pageContent: pageContent.toString(), + metadata, + }), + score, + ]); + } + } + + return documentsWithScores; + } + /** * Method that performs a similarity search in the Pinecone database and * returns the results along with their scores. @@ -414,20 +449,10 @@ export class PineconeStore extends VectorStore { k: number, filter?: PineconeMetadata ): Promise<[Document, number][]> { - const results = await this._runPineconeQuery(query, k, filter); - const result: [Document, number][] = []; - - if (results.matches) { - for (const res of results.matches) { - const { [this.textKey]: pageContent, ...metadata } = (res.metadata ?? - {}) as PineconeMetadata; - if (res.score) { - result.push([new Document({ metadata, pageContent }), res.score]); - } - } - } + const { matches = [] } = await this._runPineconeQuery(query, k, filter); + const records = this._formatMatches(matches); - return result; + return records; } /** @@ -457,7 +482,7 @@ export class PineconeStore extends VectorStore { { includeValues: true } ); - const matches = results?.matches ?? []; + const { matches = [] } = results; const embeddingList = matches.map((match) => match.values); const mmrIndexes = maximalMarginalRelevance( @@ -468,17 +493,8 @@ export class PineconeStore extends VectorStore { ); const topMmrMatches = mmrIndexes.map((idx) => matches[idx]); - - const finalResult: Document[] = []; - for (const res of topMmrMatches) { - const { [this.textKey]: pageContent, ...metadata } = (res.metadata ?? - {}) as PineconeMetadata; - if (res.score) { - finalResult.push(new Document({ metadata, pageContent })); - } - } - - return finalResult; + const records = this._formatMatches(topMmrMatches); + return records.map(([doc, _score]) => doc); } /** diff --git a/yarn.lock b/yarn.lock index 25afa514cb75..4c737be8498e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11403,7 +11403,7 @@ __metadata: faiss-node: ^0.5.1 firebase-admin: ^11.9.0 || ^12.0.0 google-auth-library: "*" - googleapis: ^126.0.1 + googleapis: "*" hnswlib-node: ^3.0.0 html-to-text: ^9.0.5 ignore: ^5.2.0 @@ -11760,7 +11760,7 @@ __metadata: languageName: unknown linkType: soft -"@langchain/google-common@workspace:*, @langchain/google-common@workspace:libs/langchain-google-common, @langchain/google-common@~0.0, @langchain/google-common@~0.0.25": +"@langchain/google-common@workspace:*, @langchain/google-common@workspace:libs/langchain-google-common, @langchain/google-common@~0.0, @langchain/google-common@~0.0.26": version: 0.0.0-use.local resolution: "@langchain/google-common@workspace:libs/langchain-google-common" dependencies: @@ -11793,13 +11793,13 @@ __metadata: languageName: unknown linkType: soft -"@langchain/google-gauth@workspace:libs/langchain-google-gauth, @langchain/google-gauth@~0.0.25": +"@langchain/google-gauth@workspace:libs/langchain-google-gauth, @langchain/google-gauth@~0.0.26": version: 0.0.0-use.local resolution: "@langchain/google-gauth@workspace:libs/langchain-google-gauth" dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/google-common": ~0.0.25 + "@langchain/google-common": ~0.0.26 "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -11826,7 +11826,7 @@ __metadata: languageName: unknown linkType: soft -"@langchain/google-genai@^0.0.25, @langchain/google-genai@workspace:*, @langchain/google-genai@workspace:libs/langchain-google-genai": +"@langchain/google-genai@^0.0.26, @langchain/google-genai@workspace:*, @langchain/google-genai@workspace:libs/langchain-google-genai": version: 0.0.0-use.local resolution: "@langchain/google-genai@workspace:libs/langchain-google-genai" dependencies: @@ -11867,7 +11867,7 @@ __metadata: dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/google-webauth": ~0.0.25 + "@langchain/google-webauth": ~0.0.26 "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -11893,14 +11893,14 @@ __metadata: languageName: unknown linkType: soft -"@langchain/google-vertexai@^0.0.25, @langchain/google-vertexai@workspace:*, @langchain/google-vertexai@workspace:libs/langchain-google-vertexai": +"@langchain/google-vertexai@^0.0.26, @langchain/google-vertexai@workspace:*, @langchain/google-vertexai@workspace:libs/langchain-google-vertexai": version: 0.0.0-use.local resolution: "@langchain/google-vertexai@workspace:libs/langchain-google-vertexai" dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" "@langchain/google-common": ~0.0 - "@langchain/google-gauth": ~0.0.25 + "@langchain/google-gauth": ~0.0.26 "@langchain/scripts": ^0.0.21 "@langchain/standard-tests": 0.0.0 "@swc/core": ^1.3.90 @@ -11927,13 +11927,13 @@ __metadata: languageName: unknown linkType: soft -"@langchain/google-webauth@workspace:libs/langchain-google-webauth, @langchain/google-webauth@~0.0.25": +"@langchain/google-webauth@workspace:libs/langchain-google-webauth, @langchain/google-webauth@~0.0.26": version: 0.0.0-use.local resolution: "@langchain/google-webauth@workspace:libs/langchain-google-webauth" dependencies: "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/google-common": ~0.0.25 + "@langchain/google-common": ~0.0.26 "@langchain/scripts": ^0.0.21 "@swc/core": ^1.3.90 "@swc/jest": ^0.2.29 @@ -32050,8 +32050,8 @@ __metadata: "@langchain/aws": ^0.0.5 "@langchain/cohere": ^0.2.1 "@langchain/core": ">=0.2.21 <0.3.0" - "@langchain/google-genai": ^0.0.25 - "@langchain/google-vertexai": ^0.0.25 + "@langchain/google-genai": ^0.0.26 + "@langchain/google-vertexai": ^0.0.26 "@langchain/groq": ^0.0.15 "@langchain/mistralai": ^0.0.26 "@langchain/ollama": ^0.0.2