Skip to content

Commit

Permalink
fix(AI Agent Node): Throw better errors for non-tool agents when usin…
Browse files Browse the repository at this point in the history
…g structured tools (#11582)
  • Loading branch information
OlegIvaniv authored Nov 8, 2024
1 parent 658568e commit 9b6123d
Show file tree
Hide file tree
Showing 7 changed files with 141 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {
import { getOptionalOutputParsers } from '../../../../../utils/output_parsers/N8nOutputParser';
import { throwIfToolSchema } from '../../../../../utils/schemaParsing';
import { getTracingConfig } from '../../../../../utils/tracing';
import { extractParsedOutput } from '../utils';
import { checkForStructuredTools, extractParsedOutput } from '../utils';

export async function conversationalAgentExecute(
this: IExecuteFunctions,
Expand All @@ -34,6 +34,8 @@ export async function conversationalAgentExecute(
const tools = await getConnectedTools(this, nodeVersion >= 1.5);
const outputParsers = await getOptionalOutputParsers(this);

await checkForStructuredTools(tools, this.getNode(), 'Conversational Agent');

// TODO: Make it possible in the future to use values for other items than just 0
const options = this.getNodeParameter('options', 0, {}) as {
systemMessage?: string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { getConnectedTools, getPromptInputByType } from '../../../../../utils/he
import { getOptionalOutputParsers } from '../../../../../utils/output_parsers/N8nOutputParser';
import { throwIfToolSchema } from '../../../../../utils/schemaParsing';
import { getTracingConfig } from '../../../../../utils/tracing';
import { extractParsedOutput } from '../utils';
import { checkForStructuredTools, extractParsedOutput } from '../utils';

export async function planAndExecuteAgentExecute(
this: IExecuteFunctions,
Expand All @@ -28,6 +28,7 @@ export async function planAndExecuteAgentExecute(

const tools = await getConnectedTools(this, nodeVersion >= 1.5);

await checkForStructuredTools(tools, this.getNode(), 'Plan & Execute Agent');
const outputParsers = await getOptionalOutputParsers(this);

const options = this.getNodeParameter('options', 0, {}) as {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
import { getOptionalOutputParsers } from '../../../../../utils/output_parsers/N8nOutputParser';
import { throwIfToolSchema } from '../../../../../utils/schemaParsing';
import { getTracingConfig } from '../../../../../utils/tracing';
import { extractParsedOutput } from '../utils';
import { checkForStructuredTools, extractParsedOutput } from '../utils';

export async function reActAgentAgentExecute(
this: IExecuteFunctions,
Expand All @@ -33,6 +33,8 @@ export async function reActAgentAgentExecute(

const tools = await getConnectedTools(this, nodeVersion >= 1.5);

await checkForStructuredTools(tools, this.getNode(), 'ReAct Agent');

const outputParsers = await getOptionalOutputParsers(this);

const options = this.getNodeParameter('options', 0, {}) as {
Expand Down
25 changes: 24 additions & 1 deletion packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/utils.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import type { ZodObjectAny } from '@langchain/core/dist/types/zod';
import type { BaseOutputParser } from '@langchain/core/output_parsers';
import type { IExecuteFunctions } from 'n8n-workflow';
import type { DynamicStructuredTool, Tool } from 'langchain/tools';
import { NodeOperationError, type IExecuteFunctions, type INode } from 'n8n-workflow';

export async function extractParsedOutput(
ctx: IExecuteFunctions,
Expand All @@ -17,3 +19,24 @@ export async function extractParsedOutput(
// with fallback to the original output if it's not present
return parsedOutput?.output ?? parsedOutput;
}

export async function checkForStructuredTools(
tools: Array<Tool | DynamicStructuredTool<ZodObjectAny>>,
node: INode,
currentAgentType: string,
) {
const dynamicStructuredTools = tools.filter(
(tool) => tool.constructor.name === 'DynamicStructuredTool',
);
if (dynamicStructuredTools.length > 0) {
const getToolName = (tool: Tool | DynamicStructuredTool) => `"${tool.name}"`;
throw new NodeOperationError(
node,
`The selected tools are not supported by "${currentAgentType}", please use "Tools Agent" instead`,
{
itemIndex: 0,
description: `Incompatible connected tools: ${dynamicStructuredTools.map(getToolName).join(', ')}`,
},
);
}
}
106 changes: 106 additions & 0 deletions packages/@n8n/nodes-langchain/nodes/agents/Agent/test/utils.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
import type { Tool } from 'langchain/tools';
import { DynamicStructuredTool } from 'langchain/tools';
import { NodeOperationError } from 'n8n-workflow';
import type { INode } from 'n8n-workflow';
import { z } from 'zod';

import { checkForStructuredTools } from '../agents/utils';

describe('checkForStructuredTools', () => {
let mockNode: INode;

beforeEach(() => {
mockNode = {
id: 'test-node',
name: 'Test Node',
type: 'test',
typeVersion: 1,
position: [0, 0],
parameters: {},
};
});

it('should not throw error when no DynamicStructuredTools are present', async () => {
const tools = [
{
name: 'regular-tool',
constructor: { name: 'Tool' },
} as Tool,
];

await expect(
checkForStructuredTools(tools, mockNode, 'Conversation Agent'),
).resolves.not.toThrow();
});

it('should throw NodeOperationError when DynamicStructuredTools are present', async () => {
const dynamicTool = new DynamicStructuredTool({
name: 'dynamic-tool',
description: 'test tool',
schema: z.object({}),
func: async () => 'result',
});

const tools: Array<Tool | DynamicStructuredTool> = [dynamicTool];

await expect(checkForStructuredTools(tools, mockNode, 'Conversation Agent')).rejects.toThrow(
NodeOperationError,
);

await expect(
checkForStructuredTools(tools, mockNode, 'Conversation Agent'),
).rejects.toMatchObject({
message:
'The selected tools are not supported by "Conversation Agent", please use "Tools Agent" instead',
description: 'Incompatible connected tools: "dynamic-tool"',
});
});

it('should list multiple dynamic tools in error message', async () => {
const dynamicTool1 = new DynamicStructuredTool({
name: 'dynamic-tool-1',
description: 'test tool 1',
schema: z.object({}),
func: async () => 'result',
});

const dynamicTool2 = new DynamicStructuredTool({
name: 'dynamic-tool-2',
description: 'test tool 2',
schema: z.object({}),
func: async () => 'result',
});

const tools = [dynamicTool1, dynamicTool2];

await expect(
checkForStructuredTools(tools, mockNode, 'Conversation Agent'),
).rejects.toMatchObject({
description: 'Incompatible connected tools: "dynamic-tool-1", "dynamic-tool-2"',
});
});

it('should throw error with mixed tool types and list only dynamic tools in error message', async () => {
const regularTool = {
name: 'regular-tool',
constructor: { name: 'Tool' },
} as Tool;

const dynamicTool = new DynamicStructuredTool({
name: 'dynamic-tool',
description: 'test tool',
schema: z.object({}),
func: async () => 'result',
});

const tools = [regularTool, dynamicTool];

await expect(
checkForStructuredTools(tools, mockNode, 'Conversation Agent'),
).rejects.toMatchObject({
message:
'The selected tools are not supported by "Conversation Agent", please use "Tools Agent" instead',
description: 'Incompatible connected tools: "dynamic-tool"',
});
});
});
3 changes: 2 additions & 1 deletion packages/workflow/src/NodeHelpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,8 @@ export function convertNodeToAiTool<
};

const noticeProp: INodeProperties = {
displayName: 'Use the expression {{ $fromAI() }} for any data to be filled by the model',
displayName:
"Use the expression {{ $fromAI('placeholder_name') }} for any data to be filled by the model",
name: 'notice',
type: 'notice',
default: '',
Expand Down
2 changes: 1 addition & 1 deletion packages/workflow/src/WorkflowDataProxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -946,7 +946,7 @@ export class WorkflowDataProxy {
defaultValue?: unknown,
) => {
if (!name || name === '') {
throw new ExpressionError('Please provide a key', {
throw new ExpressionError("Add a key, e.g. $fromAI('placeholder_name')", {
runIndex: that.runIndex,
itemIndex: that.itemIndex,
});
Expand Down

0 comments on commit 9b6123d

Please sign in to comment.