From bc958be93bcdb3aa72455ea84d8e8cea097ba47c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 28 Aug 2024 09:32:53 +0200 Subject: [PATCH 1/5] refactor(core): Convert `verbose` to `debug` logs (#10574) --- .../agents/Agent/agents/ConversationalAgent/execute.ts | 2 +- .../agents/Agent/agents/OpenAiFunctionsAgent/execute.ts | 2 +- .../agents/Agent/agents/PlanAndExecuteAgent/execute.ts | 2 +- .../nodes/agents/Agent/agents/ReActAgent/execute.ts | 2 +- .../nodes/agents/Agent/agents/SqlAgent/execute.ts | 2 +- .../nodes/agents/Agent/agents/ToolsAgent/execute.ts | 2 +- .../nodes/chains/ChainLLM/ChainLlm.node.ts | 2 +- .../chains/ChainRetrievalQA/ChainRetrievalQa.node.ts | 2 +- .../ChainSummarization/V1/ChainSummarizationV1.node.ts | 2 +- .../ChainSummarization/V2/ChainSummarizationV2.node.ts | 2 +- .../DocumentBinaryInputLoader.node.ts | 2 +- .../DocumentJsonInputLoader.node.ts | 2 +- .../EmbeddingsAzureOpenAi/EmbeddingsAzureOpenAi.node.ts | 2 +- .../embeddings/EmbeddingsCohere/EmbeddingsCohere.node.ts | 2 +- .../EmbeddingsGoogleGemini/EmbeddingsGoogleGemini.node.ts | 2 +- .../EmbeddingsGooglePalm/EmbeddingsGooglePalm.node.ts | 2 +- .../EmbeddingsHuggingFaceInference.node.ts | 2 +- .../embeddings/EmbeddingsOllama/EmbeddingsOllama.node.ts | 2 +- .../embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts | 2 +- .../MemoryChatRetriever/MemoryChatRetriever.node.ts | 2 +- .../RetrieverContextualCompression.node.ts | 2 +- .../RetrieverMultiQuery/RetrieverMultiQuery.node.ts | 2 +- .../RetrieverVectorStore/RetrieverVectorStore.node.ts | 2 +- .../TextSplitterCharacterTextSplitter.node.ts | 2 +- .../TextSplitterRecursiveCharacterTextSplitter.node.ts | 2 +- .../TextSplitterTokenSplitter.node.ts | 2 +- .../VectorStorePineconeInsert.node.ts | 2 +- .../VectorStorePineconeLoad.node.ts | 2 +- .../VectorStoreSupabaseInsert.node.ts | 2 +- .../VectorStoreSupabaseLoad.node.ts | 2 +- .../VectorStoreZepInsert/VectorStoreZepInsert.node.ts | 2 +- .../VectorStoreZepLoad/VectorStoreZepLoad.node.ts | 2 +- packages/cli/src/active-workflow-manager.ts | 8 ++++---- .../src/controllers/oauth/oauth1-credential.controller.ts | 4 ++-- .../src/controllers/oauth/oauth2-credential.controller.ts | 4 ++-- .../cli/src/controllers/workflow-statistics.controller.ts | 2 +- packages/cli/src/credentials/credentials.controller.ts | 2 +- packages/cli/src/credentials/credentials.service.ts | 2 +- packages/cli/src/logger.ts | 6 +----- packages/cli/src/services/activeWorkflows.service.ts | 2 +- packages/cli/src/services/workflow-statistics.service.ts | 2 +- packages/cli/src/user-management/email/node-mailer.ts | 2 +- packages/cli/src/webhooks/webhook-helpers.ts | 2 +- packages/cli/src/workflow-execute-additional-data.ts | 4 ++-- packages/cli/src/workflow-runner.ts | 2 +- packages/cli/src/workflows/workflow.service.ee.ts | 2 +- packages/cli/src/workflows/workflow.service.ts | 4 ++-- packages/cli/src/workflows/workflows.controller.ts | 4 ++-- packages/core/src/NodeExecuteFunctions.ts | 4 ++-- packages/core/src/WorkflowExecute.ts | 8 ++++---- .../nodes/EmailReadImap/v1/EmailReadImapV1.node.ts | 4 ++-- .../nodes/EmailReadImap/v2/EmailReadImapV2.node.ts | 6 +++--- packages/workflow/src/Constants.ts | 2 +- packages/workflow/src/LoggerProxy.ts | 2 -- 54 files changed, 68 insertions(+), 74 deletions(-) diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ConversationalAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ConversationalAgent/execute.ts index 7693945a70217..899fe57a74306 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ConversationalAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ConversationalAgent/execute.ts @@ -19,7 +19,7 @@ export async function conversationalAgentExecute( this: IExecuteFunctions, nodeVersion: number, ): Promise { - this.logger.verbose('Executing Conversational Agent'); + this.logger.debug('Executing Conversational Agent'); const model = await this.getInputConnectionData(NodeConnectionType.AiLanguageModel, 0); if (!isChatInstance(model)) { diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts index 3c4ff28f06a10..072039259f663 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/OpenAiFunctionsAgent/execute.ts @@ -23,7 +23,7 @@ export async function openAiFunctionsAgentExecute( this: IExecuteFunctions, nodeVersion: number, ): Promise { - this.logger.verbose('Executing OpenAi Functions Agent'); + this.logger.debug('Executing OpenAi Functions Agent'); const model = (await this.getInputConnectionData( NodeConnectionType.AiLanguageModel, 0, diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/PlanAndExecuteAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/PlanAndExecuteAgent/execute.ts index 7aa22d84763cc..4c77a6a0aab79 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/PlanAndExecuteAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/PlanAndExecuteAgent/execute.ts @@ -22,7 +22,7 @@ export async function planAndExecuteAgentExecute( this: IExecuteFunctions, nodeVersion: number, ): Promise { - this.logger.verbose('Executing PlanAndExecute Agent'); + this.logger.debug('Executing PlanAndExecute Agent'); const model = (await this.getInputConnectionData( NodeConnectionType.AiLanguageModel, 0, diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ReActAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ReActAgent/execute.ts index bc012c5201a37..1fdaafc290c7e 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ReActAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ReActAgent/execute.ts @@ -24,7 +24,7 @@ export async function reActAgentAgentExecute( this: IExecuteFunctions, nodeVersion: number, ): Promise { - this.logger.verbose('Executing ReAct Agent'); + this.logger.debug('Executing ReAct Agent'); const model = (await this.getInputConnectionData(NodeConnectionType.AiLanguageModel, 0)) as | BaseLanguageModel diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/execute.ts index 15ef523338464..bd8868ad32a53 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/SqlAgent/execute.ts @@ -29,7 +29,7 @@ const parseTablesString = (tablesString: string) => export async function sqlAgentAgentExecute( this: IExecuteFunctions, ): Promise { - this.logger.verbose('Executing SQL Agent'); + this.logger.debug('Executing SQL Agent'); const model = (await this.getInputConnectionData( NodeConnectionType.AiLanguageModel, diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts index e0da7f1e315f9..fc760cb928518 100644 --- a/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/agents/ToolsAgent/execute.ts @@ -76,7 +76,7 @@ async function extractBinaryMessages(ctx: IExecuteFunctions) { } export async function toolsAgentExecute(this: IExecuteFunctions): Promise { - this.logger.verbose('Executing Tools Agent'); + this.logger.debug('Executing Tools Agent'); const model = await this.getInputConnectionData(NodeConnectionType.AiLanguageModel, 0); if (!isChatInstance(model) || !model.bindTools) { diff --git a/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts index 003c8eb7dd1b9..02b91134e6db2 100644 --- a/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/chains/ChainLLM/ChainLlm.node.ts @@ -517,7 +517,7 @@ export class ChainLlm implements INodeType { }; async execute(this: IExecuteFunctions): Promise { - this.logger.verbose('Executing LLM Chain'); + this.logger.debug('Executing LLM Chain'); const items = this.getInputData(); const returnData: INodeExecutionData[] = []; diff --git a/packages/@n8n/nodes-langchain/nodes/chains/ChainRetrievalQA/ChainRetrievalQa.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/ChainRetrievalQA/ChainRetrievalQa.node.ts index 07bde01a6c8d9..9f79372ebd114 100644 --- a/packages/@n8n/nodes-langchain/nodes/chains/ChainRetrievalQA/ChainRetrievalQa.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/chains/ChainRetrievalQA/ChainRetrievalQa.node.ts @@ -141,7 +141,7 @@ export class ChainRetrievalQa implements INodeType { }; async execute(this: IExecuteFunctions): Promise { - this.logger.verbose('Executing Retrieval QA Chain'); + this.logger.debug('Executing Retrieval QA Chain'); const model = (await this.getInputConnectionData( NodeConnectionType.AiLanguageModel, diff --git a/packages/@n8n/nodes-langchain/nodes/chains/ChainSummarization/V1/ChainSummarizationV1.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/ChainSummarization/V1/ChainSummarizationV1.node.ts index bc18739647dad..3e7b6997d6ccf 100644 --- a/packages/@n8n/nodes-langchain/nodes/chains/ChainSummarization/V1/ChainSummarizationV1.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/chains/ChainSummarization/V1/ChainSummarizationV1.node.ts @@ -162,7 +162,7 @@ export class ChainSummarizationV1 implements INodeType { } async execute(this: IExecuteFunctions): Promise { - this.logger.verbose('Executing Vector Store QA Chain'); + this.logger.debug('Executing Vector Store QA Chain'); const type = this.getNodeParameter('type', 0) as 'map_reduce' | 'stuff' | 'refine'; const model = (await this.getInputConnectionData( diff --git a/packages/@n8n/nodes-langchain/nodes/chains/ChainSummarization/V2/ChainSummarizationV2.node.ts b/packages/@n8n/nodes-langchain/nodes/chains/ChainSummarization/V2/ChainSummarizationV2.node.ts index 3abf749cb0988..7d79ce29afc7b 100644 --- a/packages/@n8n/nodes-langchain/nodes/chains/ChainSummarization/V2/ChainSummarizationV2.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/chains/ChainSummarization/V2/ChainSummarizationV2.node.ts @@ -311,7 +311,7 @@ export class ChainSummarizationV2 implements INodeType { } async execute(this: IExecuteFunctions): Promise { - this.logger.verbose('Executing Summarization Chain V2'); + this.logger.debug('Executing Summarization Chain V2'); const operationMode = this.getNodeParameter('operationMode', 0, 'nodeInputJson') as | 'nodeInputJson' | 'nodeInputBinary' diff --git a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentBinaryInputLoader/DocumentBinaryInputLoader.node.ts b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentBinaryInputLoader/DocumentBinaryInputLoader.node.ts index 6fe6b30db567f..783f12be9da80 100644 --- a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentBinaryInputLoader/DocumentBinaryInputLoader.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentBinaryInputLoader/DocumentBinaryInputLoader.node.ts @@ -178,7 +178,7 @@ export class DocumentBinaryInputLoader implements INodeType { }; async supplyData(this: IExecuteFunctions): Promise { - this.logger.verbose('Supply Data for Binary Input Loader'); + this.logger.debug('Supply Data for Binary Input Loader'); const textSplitter = (await this.getInputConnectionData( NodeConnectionType.AiTextSplitter, 0, diff --git a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentJSONInputLoader/DocumentJsonInputLoader.node.ts b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentJSONInputLoader/DocumentJsonInputLoader.node.ts index 403f215d23d51..3cb2c4bfdb56c 100644 --- a/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentJSONInputLoader/DocumentJsonInputLoader.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/document_loaders/DocumentJSONInputLoader/DocumentJsonInputLoader.node.ts @@ -80,7 +80,7 @@ export class DocumentJsonInputLoader implements INodeType { }; async supplyData(this: IExecuteFunctions): Promise { - this.logger.verbose('Supply Data for JSON Input Loader'); + this.logger.debug('Supply Data for JSON Input Loader'); const textSplitter = (await this.getInputConnectionData( NodeConnectionType.AiTextSplitter, 0, diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAzureOpenAi/EmbeddingsAzureOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAzureOpenAi/EmbeddingsAzureOpenAi.node.ts index 920263a893878..46195be0d389d 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAzureOpenAi/EmbeddingsAzureOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsAzureOpenAi/EmbeddingsAzureOpenAi.node.ts @@ -93,7 +93,7 @@ export class EmbeddingsAzureOpenAi implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply data for embeddings'); + this.logger.debug('Supply data for embeddings'); const credentials = await this.getCredentials<{ apiKey: string; resourceName: string; diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsCohere/EmbeddingsCohere.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsCohere/EmbeddingsCohere.node.ts index aa6d6ab3c3598..a6c246acb5f45 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsCohere/EmbeddingsCohere.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsCohere/EmbeddingsCohere.node.ts @@ -100,7 +100,7 @@ export class EmbeddingsCohere implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply data for embeddings Cohere'); + this.logger.debug('Supply data for embeddings Cohere'); const modelName = this.getNodeParameter('modelName', itemIndex, 'embed-english-v2.0') as string; const credentials = await this.getCredentials<{ apiKey: string }>('cohereApi'); const embeddings = new CohereEmbeddings({ diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGoogleGemini/EmbeddingsGoogleGemini.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGoogleGemini/EmbeddingsGoogleGemini.node.ts index 7249b639f650c..92882dfffaec9 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGoogleGemini/EmbeddingsGoogleGemini.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGoogleGemini/EmbeddingsGoogleGemini.node.ts @@ -117,7 +117,7 @@ export class EmbeddingsGoogleGemini implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply data for embeddings Google Gemini'); + this.logger.debug('Supply data for embeddings Google Gemini'); const modelName = this.getNodeParameter( 'modelName', itemIndex, diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGooglePalm/EmbeddingsGooglePalm.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGooglePalm/EmbeddingsGooglePalm.node.ts index 32cbd8822d065..0e31329f015bc 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGooglePalm/EmbeddingsGooglePalm.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsGooglePalm/EmbeddingsGooglePalm.node.ts @@ -116,7 +116,7 @@ export class EmbeddingsGooglePalm implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply data for embeddings Google PaLM'); + this.logger.debug('Supply data for embeddings Google PaLM'); const modelName = this.getNodeParameter( 'modelName', itemIndex, diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsHuggingFaceInference/EmbeddingsHuggingFaceInference.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsHuggingFaceInference/EmbeddingsHuggingFaceInference.node.ts index 4adc8d6014d6f..93d751b9c4603 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsHuggingFaceInference/EmbeddingsHuggingFaceInference.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsHuggingFaceInference/EmbeddingsHuggingFaceInference.node.ts @@ -82,7 +82,7 @@ export class EmbeddingsHuggingFaceInference implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply data for embeddings HF Inference'); + this.logger.debug('Supply data for embeddings HF Inference'); const model = this.getNodeParameter( 'modelName', itemIndex, diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOllama/EmbeddingsOllama.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOllama/EmbeddingsOllama.node.ts index b0cf8277714ef..c54bc2db68930 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOllama/EmbeddingsOllama.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOllama/EmbeddingsOllama.node.ts @@ -45,7 +45,7 @@ export class EmbeddingsOllama implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply data for embeddings Ollama'); + this.logger.debug('Supply data for embeddings Ollama'); const modelName = this.getNodeParameter('model', itemIndex) as string; const credentials = await this.getCredentials('ollamaApi'); diff --git a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts index 31180cd1e97b5..046f3e4f56683 100644 --- a/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/embeddings/EmbeddingsOpenAI/EmbeddingsOpenAi.node.ts @@ -171,7 +171,7 @@ export class EmbeddingsOpenAi implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply data for embeddings'); + this.logger.debug('Supply data for embeddings'); const credentials = await this.getCredentials('openAiApi'); const options = this.getNodeParameter('options', itemIndex, {}) as { diff --git a/packages/@n8n/nodes-langchain/nodes/memory/MemoryChatRetriever/MemoryChatRetriever.node.ts b/packages/@n8n/nodes-langchain/nodes/memory/MemoryChatRetriever/MemoryChatRetriever.node.ts index 6b9bf6203f964..f2bb0f43b0100 100644 --- a/packages/@n8n/nodes-langchain/nodes/memory/MemoryChatRetriever/MemoryChatRetriever.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/memory/MemoryChatRetriever/MemoryChatRetriever.node.ts @@ -88,7 +88,7 @@ export class MemoryChatRetriever implements INodeType { }; async execute(this: IExecuteFunctions): Promise { - this.logger.verbose('Executing Chat Memory Retriever'); + this.logger.debug('Executing Chat Memory Retriever'); const memory = (await this.getInputConnectionData(NodeConnectionType.AiMemory, 0)) as | BaseChatMemory diff --git a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverContextualCompression/RetrieverContextualCompression.node.ts b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverContextualCompression/RetrieverContextualCompression.node.ts index 2526ecb861e74..5b89a0bf26ce2 100644 --- a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverContextualCompression/RetrieverContextualCompression.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverContextualCompression/RetrieverContextualCompression.node.ts @@ -64,7 +64,7 @@ export class RetrieverContextualCompression implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supplying data for Contextual Compression Retriever'); + this.logger.debug('Supplying data for Contextual Compression Retriever'); const model = (await this.getInputConnectionData( NodeConnectionType.AiLanguageModel, diff --git a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverMultiQuery/RetrieverMultiQuery.node.ts b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverMultiQuery/RetrieverMultiQuery.node.ts index d594c3a28588a..3cb377d65418f 100644 --- a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverMultiQuery/RetrieverMultiQuery.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverMultiQuery/RetrieverMultiQuery.node.ts @@ -83,7 +83,7 @@ export class RetrieverMultiQuery implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supplying data for MultiQuery Retriever'); + this.logger.debug('Supplying data for MultiQuery Retriever'); const options = this.getNodeParameter('options', itemIndex, {}) as { queryCount?: number }; diff --git a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverVectorStore/RetrieverVectorStore.node.ts b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverVectorStore/RetrieverVectorStore.node.ts index 9dd5dff58aa71..6543d061d456f 100644 --- a/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverVectorStore/RetrieverVectorStore.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/retrievers/RetrieverVectorStore/RetrieverVectorStore.node.ts @@ -57,7 +57,7 @@ export class RetrieverVectorStore implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supplying data for Vector Store Retriever'); + this.logger.debug('Supplying data for Vector Store Retriever'); const topK = this.getNodeParameter('topK', itemIndex, 4) as number; const vectorStore = (await this.getInputConnectionData( diff --git a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.ts b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.ts index 9e7aa050f3666..61e62def0f0e1 100644 --- a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.ts @@ -64,7 +64,7 @@ export class TextSplitterCharacterTextSplitter implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply Data for Text Splitter'); + this.logger.debug('Supply Data for Text Splitter'); const separator = this.getNodeParameter('separator', itemIndex) as string; const chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number; diff --git a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.ts b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.ts index ec6f716627657..4d2c5a6ec83db 100644 --- a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.ts @@ -95,7 +95,7 @@ export class TextSplitterRecursiveCharacterTextSplitter implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply Data for Text Splitter'); + this.logger.debug('Supply Data for Text Splitter'); const chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number; const chunkOverlap = this.getNodeParameter('chunkOverlap', itemIndex) as number; diff --git a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts index 65cb7ea213921..c021aa1df756c 100644 --- a/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts @@ -57,7 +57,7 @@ export class TextSplitterTokenSplitter implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply Data for Text Splitter'); + this.logger.debug('Supply Data for Text Splitter'); const chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number; const chunkOverlap = this.getNodeParameter('chunkOverlap', itemIndex) as number; diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeInsert/VectorStorePineconeInsert.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeInsert/VectorStorePineconeInsert.node.ts index 93e3e4d0419fa..023b65be84493 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeInsert/VectorStorePineconeInsert.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeInsert/VectorStorePineconeInsert.node.ts @@ -97,7 +97,7 @@ export class VectorStorePineconeInsert implements INodeType { async execute(this: IExecuteFunctions): Promise { const items = this.getInputData(0); - this.logger.verbose('Executing data for Pinecone Insert Vector Store'); + this.logger.debug('Executing data for Pinecone Insert Vector Store'); const namespace = this.getNodeParameter('pineconeNamespace', 0) as string; const index = this.getNodeParameter('pineconeIndex', 0, '', { extractValue: true }) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeLoad/VectorStorePineconeLoad.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeLoad/VectorStorePineconeLoad.node.ts index c4ac6e3754cb1..7cae9c9d85bde 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeLoad/VectorStorePineconeLoad.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePineconeLoad/VectorStorePineconeLoad.node.ts @@ -85,7 +85,7 @@ export class VectorStorePineconeLoad implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supplying data for Pinecone Load Vector Store'); + this.logger.debug('Supplying data for Pinecone Load Vector Store'); const namespace = this.getNodeParameter('pineconeNamespace', itemIndex) as string; const index = this.getNodeParameter('pineconeIndex', itemIndex, '', { diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseInsert/VectorStoreSupabaseInsert.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseInsert/VectorStoreSupabaseInsert.node.ts index 1eae86971a325..44b3a6a39741c 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseInsert/VectorStoreSupabaseInsert.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseInsert/VectorStoreSupabaseInsert.node.ts @@ -94,7 +94,7 @@ export class VectorStoreSupabaseInsert implements INodeType { methods = { listSearch: { supabaseTableNameSearch } }; async execute(this: IExecuteFunctions): Promise { - this.logger.verbose('Executing data for Supabase Insert Vector Store'); + this.logger.debug('Executing data for Supabase Insert Vector Store'); const items = this.getInputData(0); const tableName = this.getNodeParameter('tableName', 0, '', { extractValue: true }) as string; diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseLoad/VectorStoreSupabaseLoad.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseLoad/VectorStoreSupabaseLoad.node.ts index 267277b36ec89..bae6d0e1a9f79 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseLoad/VectorStoreSupabaseLoad.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabaseLoad/VectorStoreSupabaseLoad.node.ts @@ -82,7 +82,7 @@ export class VectorStoreSupabaseLoad implements INodeType { methods = { listSearch: { supabaseTableNameSearch } }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supply Supabase Load Vector Store'); + this.logger.debug('Supply Supabase Load Vector Store'); const tableName = this.getNodeParameter('tableName', itemIndex, '', { extractValue: true, diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepInsert/VectorStoreZepInsert.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepInsert/VectorStoreZepInsert.node.ts index 71349b586be6a..3b40e07d650ed 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepInsert/VectorStoreZepInsert.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepInsert/VectorStoreZepInsert.node.ts @@ -101,7 +101,7 @@ export class VectorStoreZepInsert implements INodeType { }; async execute(this: IExecuteFunctions): Promise { - this.logger.verbose('Executing data for Zep Insert Vector Store'); + this.logger.debug('Executing data for Zep Insert Vector Store'); const items = this.getInputData(0); const collectionName = this.getNodeParameter('collectionName', 0) as string; const options = diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepLoad/VectorStoreZepLoad.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepLoad/VectorStoreZepLoad.node.ts index ca5a743c15ace..244c0a98434c0 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepLoad/VectorStoreZepLoad.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreZepLoad/VectorStoreZepLoad.node.ts @@ -84,7 +84,7 @@ export class VectorStoreZepLoad implements INodeType { }; async supplyData(this: IExecuteFunctions, itemIndex: number): Promise { - this.logger.verbose('Supplying data for Zep Load Vector Store'); + this.logger.debug('Supplying data for Zep Load Vector Store'); const collectionName = this.getNodeParameter('collectionName', itemIndex) as string; diff --git a/packages/cli/src/active-workflow-manager.ts b/packages/cli/src/active-workflow-manager.ts index 96df40195084a..358d2b514f892 100644 --- a/packages/cli/src/active-workflow-manager.ts +++ b/packages/cli/src/active-workflow-manager.ts @@ -95,7 +95,7 @@ export class ActiveWorkflowManager { */ async removeAll() { let activeWorkflowIds: string[] = []; - this.logger.verbose('Call to remove all active workflows received (removeAll)'); + this.logger.debug('Call to remove all active workflows received (removeAll)'); activeWorkflowIds.push(...this.activeWorkflows.allActiveWorkflows()); @@ -437,7 +437,7 @@ export class ActiveWorkflowManager { }); if (wasActivated) { - this.logger.verbose(`Successfully started workflow ${dbWorkflow.display()}`, { + this.logger.debug(`Successfully started workflow ${dbWorkflow.display()}`, { workflowName: dbWorkflow.name, workflowId: dbWorkflow.id, }); @@ -469,7 +469,7 @@ export class ActiveWorkflowManager { } } - this.logger.verbose('Finished activating workflows (startup)'); + this.logger.debug('Finished activating workflows (startup)'); } async clearAllActivationErrors() { @@ -800,7 +800,7 @@ export class ActiveWorkflowManager { getPollFunctions, ); - this.logger.verbose(`Workflow ${dbWorkflow.display()} activated`, { + this.logger.debug(`Workflow ${dbWorkflow.display()} activated`, { workflowId: dbWorkflow.id, workflowName: dbWorkflow.name, }); diff --git a/packages/cli/src/controllers/oauth/oauth1-credential.controller.ts b/packages/cli/src/controllers/oauth/oauth1-credential.controller.ts index 8707fed7c31c4..5ec036a2aaa98 100644 --- a/packages/cli/src/controllers/oauth/oauth1-credential.controller.ts +++ b/packages/cli/src/controllers/oauth/oauth1-credential.controller.ts @@ -90,7 +90,7 @@ export class OAuth1CredentialController extends AbstractOAuthController { decryptedDataOriginal.csrfSecret = csrfSecret; await this.encryptAndSaveData(credential, decryptedDataOriginal); - this.logger.verbose('OAuth1 authorization successful for new credential', { + this.logger.debug('OAuth1 authorization successful for new credential', { userId: req.user.id, credentialId: credential.id, }); @@ -170,7 +170,7 @@ export class OAuth1CredentialController extends AbstractOAuthController { await this.encryptAndSaveData(credential, decryptedDataOriginal); - this.logger.verbose('OAuth1 callback successful for new credential', { + this.logger.debug('OAuth1 callback successful for new credential', { credentialId, }); return res.render('oauth-callback'); diff --git a/packages/cli/src/controllers/oauth/oauth2-credential.controller.ts b/packages/cli/src/controllers/oauth/oauth2-credential.controller.ts index 3ef4860a6fcb7..5e2d3a0fb2476 100644 --- a/packages/cli/src/controllers/oauth/oauth2-credential.controller.ts +++ b/packages/cli/src/controllers/oauth/oauth2-credential.controller.ts @@ -71,7 +71,7 @@ export class OAuth2CredentialController extends AbstractOAuthController { const oAuthObj = new ClientOAuth2(oAuthOptions); const returnUri = oAuthObj.code.getUri(); - this.logger.verbose('OAuth2 authorization url created for credential', { + this.logger.debug('OAuth2 authorization url created for credential', { userId: req.user.id, credentialId: credential.id, }); @@ -172,7 +172,7 @@ export class OAuth2CredentialController extends AbstractOAuthController { delete decryptedDataOriginal.csrfSecret; await this.encryptAndSaveData(credential, decryptedDataOriginal); - this.logger.verbose('OAuth2 callback successful for credential', { + this.logger.debug('OAuth2 callback successful for credential', { credentialId, }); diff --git a/packages/cli/src/controllers/workflow-statistics.controller.ts b/packages/cli/src/controllers/workflow-statistics.controller.ts index 623a1efe1d5f8..6026b68bb5499 100644 --- a/packages/cli/src/controllers/workflow-statistics.controller.ts +++ b/packages/cli/src/controllers/workflow-statistics.controller.ts @@ -40,7 +40,7 @@ export class WorkflowStatisticsController { if (workflow) { next(); } else { - this.logger.verbose('User attempted to read a workflow without permissions', { + this.logger.warn('User attempted to read a workflow without permissions', { workflowId, userId: user.id, }); diff --git a/packages/cli/src/credentials/credentials.controller.ts b/packages/cli/src/credentials/credentials.controller.ts index 636d804b75ba8..eabeb39a48968 100644 --- a/packages/cli/src/credentials/credentials.controller.ts +++ b/packages/cli/src/credentials/credentials.controller.ts @@ -218,7 +218,7 @@ export class CredentialsController { // Remove the encrypted data as it is not needed in the frontend const { data: _, ...rest } = responseData; - this.logger.verbose('Credential updated', { credentialId }); + this.logger.debug('Credential updated', { credentialId }); this.eventService.emit('credentials-updated', { user: req.user, diff --git a/packages/cli/src/credentials/credentials.service.ts b/packages/cli/src/credentials/credentials.service.ts index 72a6ddd91c0c8..e9a88a915367e 100644 --- a/packages/cli/src/credentials/credentials.service.ts +++ b/packages/cli/src/credentials/credentials.service.ts @@ -388,7 +388,7 @@ export class CredentialsService { return savedCredential; }); - this.logger.verbose('New credential created', { + this.logger.debug('New credential created', { credentialId: newCredential.id, ownerId: user.id, }); diff --git a/packages/cli/src/logger.ts b/packages/cli/src/logger.ts index dc23a6f3477ae..4e105b33043da 100644 --- a/packages/cli/src/logger.ts +++ b/packages/cli/src/logger.ts @@ -36,7 +36,7 @@ export class Logger { if (output.includes('console')) { let format: winston.Logform.Format; - if (['debug', 'verbose'].includes(level)) { + if (level === 'debug') { format = winston.format.combine( winston.format.metadata(), winston.format.timestamp(), @@ -115,8 +115,4 @@ export class Logger { debug(message: string, meta: object = {}): void { this.log('debug', message, meta); } - - verbose(message: string, meta: object = {}): void { - this.log('verbose', message, meta); - } } diff --git a/packages/cli/src/services/activeWorkflows.service.ts b/packages/cli/src/services/activeWorkflows.service.ts index b4312c16b3128..e65ec1e434a87 100644 --- a/packages/cli/src/services/activeWorkflows.service.ts +++ b/packages/cli/src/services/activeWorkflows.service.ts @@ -41,7 +41,7 @@ export class ActiveWorkflowsService { 'workflow:read', ]); if (!workflow) { - this.logger.verbose('User attempted to access workflow errors without permissions', { + this.logger.warn('User attempted to access workflow errors without permissions', { workflowId, userId: user.id, }); diff --git a/packages/cli/src/services/workflow-statistics.service.ts b/packages/cli/src/services/workflow-statistics.service.ts index 4c9e23a578537..37da22d757729 100644 --- a/packages/cli/src/services/workflow-statistics.service.ts +++ b/packages/cli/src/services/workflow-statistics.service.ts @@ -90,7 +90,7 @@ export class WorkflowStatisticsService extends TypedEmitter node.type === errorTriggerType) ) { - logger.verbose('Start internal error workflow', { executionId, workflowId }); + logger.debug('Start internal error workflow', { executionId, workflowId }); void Container.get(OwnershipService) .getWorkflowProjectCached(workflowId) .then((project) => { diff --git a/packages/cli/src/workflow-runner.ts b/packages/cli/src/workflow-runner.ts index 4590bb5c3b20c..77c57397c7f05 100644 --- a/packages/cli/src/workflow-runner.ts +++ b/packages/cli/src/workflow-runner.ts @@ -238,7 +238,7 @@ export class WorkflowRunner { additionalData.executionId = executionId; - this.logger.verbose( + this.logger.debug( `Execution for workflow ${data.workflowData.name} was assigned id ${executionId}`, { executionId }, ); diff --git a/packages/cli/src/workflows/workflow.service.ee.ts b/packages/cli/src/workflows/workflow.service.ee.ts index be3ba3bca6f10..560942064ac0d 100644 --- a/packages/cli/src/workflows/workflow.service.ee.ts +++ b/packages/cli/src/workflows/workflow.service.ee.ts @@ -194,7 +194,7 @@ export class EnterpriseWorkflowService { nodesWithCredentialsUserDoesNotHaveAccessTo.forEach((node) => { if (isTamperingAttempt(node.id)) { - this.logger.verbose('Blocked workflow update due to tampering attempt', { + this.logger.warn('Blocked workflow update due to tampering attempt', { nodeType: node.type, nodeName: node.name, nodeId: node.id, diff --git a/packages/cli/src/workflows/workflow.service.ts b/packages/cli/src/workflows/workflow.service.ts index 46f6438f9b3b0..25943725ab908 100644 --- a/packages/cli/src/workflows/workflow.service.ts +++ b/packages/cli/src/workflows/workflow.service.ts @@ -96,7 +96,7 @@ export class WorkflowService { ]); if (!workflow) { - this.logger.verbose('User attempted to update a workflow without permissions', { + this.logger.warn('User attempted to update a workflow without permissions', { workflowId, userId: user.id, }); @@ -120,7 +120,7 @@ export class WorkflowService { // Update the workflow's version when changing properties such as // `name`, `pinData`, `nodes`, `connections`, `settings` or `tags` workflowUpdateData.versionId = uuid(); - this.logger.verbose( + this.logger.debug( `Updating versionId for workflow ${workflowId} for user ${user.id} after saving`, { previousVersionId: workflow.versionId, diff --git a/packages/cli/src/workflows/workflows.controller.ts b/packages/cli/src/workflows/workflows.controller.ts index b3917e963378a..deeecba04568b 100644 --- a/packages/cli/src/workflows/workflows.controller.ts +++ b/packages/cli/src/workflows/workflows.controller.ts @@ -309,7 +309,7 @@ export class WorkflowsController { ); if (!workflow) { - this.logger.verbose('User attempted to access a workflow without permissions', { + this.logger.warn('User attempted to access a workflow without permissions', { workflowId, userId: req.user.id, }); @@ -362,7 +362,7 @@ export class WorkflowsController { const workflow = await this.workflowService.delete(req.user, workflowId); if (!workflow) { - this.logger.verbose('User attempted to delete a workflow without permissions', { + this.logger.warn('User attempted to delete a workflow without permissions', { workflowId, userId: req.user.id, }); diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 81d5a09f5f468..e1f39c22abdb1 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -1954,7 +1954,7 @@ export function getAdditionalKeys( if (mode === 'manual') { throw e; } - Logger.verbose(e.message); + Logger.debug(e.message); } }, setAll(obj: Record): void { @@ -1964,7 +1964,7 @@ export function getAdditionalKeys( if (mode === 'manual') { throw e; } - Logger.verbose(e.message); + Logger.debug(e.message); } }, get(key: string): string { diff --git a/packages/core/src/WorkflowExecute.ts b/packages/core/src/WorkflowExecute.ts index 303b46b20aa2f..24e4de3395650 100644 --- a/packages/core/src/WorkflowExecute.ts +++ b/packages/core/src/WorkflowExecute.ts @@ -794,7 +794,7 @@ export class WorkflowExecute { // active executions anymore // eslint-disable-next-line @typescript-eslint/promise-function-async processRunExecutionData(workflow: Workflow): PCancelable { - Logger.verbose('Workflow execution started', { workflowId: workflow.id }); + Logger.debug('Workflow execution started', { workflowId: workflow.id }); const startedAt = new Date(); const forceInputNodeExecution = this.forceInputNodeExecution(workflow); @@ -1804,7 +1804,7 @@ export class WorkflowExecute { const fullRunData = this.getFullRunData(startedAt); if (executionError !== undefined) { - Logger.verbose('Workflow execution finished with error', { + Logger.debug('Workflow execution finished with error', { error: executionError, workflowId: workflow.id, }); @@ -1818,13 +1818,13 @@ export class WorkflowExecute { } } else if (this.runExecutionData.waitTill!) { // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - Logger.verbose(`Workflow execution will wait until ${this.runExecutionData.waitTill}`, { + Logger.debug(`Workflow execution will wait until ${this.runExecutionData.waitTill}`, { workflowId: workflow.id, }); fullRunData.waitTill = this.runExecutionData.waitTill; fullRunData.status = 'waiting'; } else { - Logger.verbose('Workflow execution finished successfully', { workflowId: workflow.id }); + Logger.debug('Workflow execution finished successfully', { workflowId: workflow.id }); fullRunData.finished = true; fullRunData.status = 'success'; } diff --git a/packages/nodes-base/nodes/EmailReadImap/v1/EmailReadImapV1.node.ts b/packages/nodes-base/nodes/EmailReadImap/v1/EmailReadImapV1.node.ts index ae26d59caa492..923ffe5832db6 100644 --- a/packages/nodes-base/nodes/EmailReadImap/v1/EmailReadImapV1.node.ts +++ b/packages/nodes-base/nodes/EmailReadImap/v1/EmailReadImapV1.node.ts @@ -589,7 +589,7 @@ export class EmailReadImapV1 implements INodeType { conn.on('error', async (error) => { const errorCode = error.code.toUpperCase(); if (['ECONNRESET', 'EPIPE'].includes(errorCode as string)) { - this.logger.verbose(`IMAP connection was reset (${errorCode}) - reconnecting.`, { + this.logger.debug(`IMAP connection was reset (${errorCode}) - reconnecting.`, { error, }); try { @@ -618,7 +618,7 @@ export class EmailReadImapV1 implements INodeType { if (options.forceReconnect !== undefined) { reconnectionInterval = setInterval( async () => { - this.logger.verbose('Forcing reconnection of IMAP node.'); + this.logger.debug('Forcing reconnection of IMAP node.'); connection.end(); connection = await establishConnection(); await connection.openBox(mailbox); diff --git a/packages/nodes-base/nodes/EmailReadImap/v2/EmailReadImapV2.node.ts b/packages/nodes-base/nodes/EmailReadImap/v2/EmailReadImapV2.node.ts index 5f46f63ce0376..b3796263a7b31 100644 --- a/packages/nodes-base/nodes/EmailReadImap/v2/EmailReadImapV2.node.ts +++ b/packages/nodes-base/nodes/EmailReadImap/v2/EmailReadImapV2.node.ts @@ -598,7 +598,7 @@ export class EmailReadImapV2 implements INodeType { } }, onUpdate: async (seqNo: number, info) => { - this.logger.verbose(`Email Read Imap:update ${seqNo}`, info); + this.logger.debug(`Email Read Imap:update ${seqNo}`, info); }, }; @@ -631,7 +631,7 @@ export class EmailReadImapV2 implements INodeType { }); conn.on('error', async (error) => { const errorCode = ((error as JsonObject).code as string).toUpperCase(); - this.logger.verbose(`IMAP connection experienced an error: (${errorCode})`, { + this.logger.debug(`IMAP connection experienced an error: (${errorCode})`, { error: error as Error, }); this.emitError(error as Error); @@ -647,7 +647,7 @@ export class EmailReadImapV2 implements INodeType { let reconnectionInterval: NodeJS.Timeout | undefined; const handleReconnect = async () => { - this.logger.verbose('Forcing reconnect to IMAP server'); + this.logger.debug('Forcing reconnect to IMAP server'); try { isCurrentlyReconnecting = true; if (connection.closeBox) await connection.closeBox(false); diff --git a/packages/workflow/src/Constants.ts b/packages/workflow/src/Constants.ts index 48bd06e521690..e55ad9fc77301 100644 --- a/packages/workflow/src/Constants.ts +++ b/packages/workflow/src/Constants.ts @@ -8,7 +8,7 @@ export const ALPHABET = [DIGITS, UPPERCASE_LETTERS, LOWERCASE_LETTERS].join(''); export const BINARY_ENCODING = 'base64'; export const WAIT_TIME_UNLIMITED = '3000-01-01T00:00:00.000Z'; -export const LOG_LEVELS = ['silent', 'error', 'warn', 'info', 'debug', 'verbose'] as const; +export const LOG_LEVELS = ['silent', 'error', 'warn', 'info', 'debug'] as const; export const CODE_LANGUAGES = ['javaScript', 'python'] as const; export const CODE_EXECUTION_MODES = ['runOnceForAllItems', 'runOnceForEachItem'] as const; diff --git a/packages/workflow/src/LoggerProxy.ts b/packages/workflow/src/LoggerProxy.ts index cd6f1b1cb98e5..eebfd036b4519 100644 --- a/packages/workflow/src/LoggerProxy.ts +++ b/packages/workflow/src/LoggerProxy.ts @@ -5,12 +5,10 @@ export let error: Logger['error'] = noOp; export let warn: Logger['warn'] = noOp; export let info: Logger['info'] = noOp; export let debug: Logger['debug'] = noOp; -export let verbose: Logger['verbose'] = noOp; export const init = (logger: Logger) => { error = (message, meta) => logger.error(message, meta); warn = (message, meta) => logger.warn(message, meta); info = (message, meta) => logger.info(message, meta); debug = (message, meta) => logger.debug(message, meta); - verbose = (message, meta) => logger.verbose(message, meta); }; From 6017bf5af159149ecea7cad80c7c85e9e2143b49 Mon Sep 17 00:00:00 2001 From: Tomi Turtiainen <10324676+tomi@users.noreply.github.com> Date: Wed, 28 Aug 2024 10:46:17 +0300 Subject: [PATCH 2/5] fix: Fix scenario prefix not being passed (no-changelog) (#10575) --- .../@n8n/benchmark/scripts/runOnVm/runOnVm.mjs | 3 +-- packages/@n8n/benchmark/src/commands/run.ts | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs b/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs index d8d3537dec398..fd853203ac6ef 100755 --- a/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs +++ b/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs @@ -26,14 +26,13 @@ async function main() { N8N_VERSION: n8nTag, BENCHMARK_VERSION: benchmarkTag, K6_API_TOKEN: k6ApiToken, - N8N_BENCHMARK_SCENARIO_NAME_PREFIX: n8nSetupToUse, }, }); try { await $$`docker-compose up -d n8n`; - await $$`docker-compose run benchmark run`; + await $$`docker-compose run benchmark run --scenarioNamePrefix=${n8nSetupToUse} `; } catch (error) { console.error('An error occurred while running the benchmarks:'); console.error(error); diff --git a/packages/@n8n/benchmark/src/commands/run.ts b/packages/@n8n/benchmark/src/commands/run.ts index 4476dbfbe0465..5e2c1d3dc6ba4 100644 --- a/packages/@n8n/benchmark/src/commands/run.ts +++ b/packages/@n8n/benchmark/src/commands/run.ts @@ -16,10 +16,14 @@ export default class RunCommand extends Command { description: 'Comma-separated list of test scenarios to run', required: false, }), + scenarioNamePrefix: Flags.string({ + description: 'Prefix for the scenario name. Defaults to Unnamed', + required: false, + }), }; async run() { - const config = loadConfig(); + const config = await this.loadConfigAndMergeWithFlags(); const scenarioLoader = new ScenarioLoader(); const scenarioRunner = new ScenarioRunner( @@ -41,4 +45,15 @@ export default class RunCommand extends Command { await scenarioRunner.runManyScenarios(allScenarios); } + + private async loadConfigAndMergeWithFlags() { + const config = loadConfig(); + const { flags } = await this.parse(RunCommand); + + if (flags.scenarioNamePrefix) { + config.set('scenarioNamePrefix', flags.scenarioNamePrefix); + } + + return config; + } } From acfd60ac85898d6b6ca24a93effbb969c7f3d542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 28 Aug 2024 10:48:53 +0200 Subject: [PATCH 3/5] refactor(core): Map out pubsub messages (no-changelog) (#10566) --- .../pubsub/pubsub-message-map.ts | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 packages/cli/src/services/orchestration/pubsub/pubsub-message-map.ts diff --git a/packages/cli/src/services/orchestration/pubsub/pubsub-message-map.ts b/packages/cli/src/services/orchestration/pubsub/pubsub-message-map.ts new file mode 100644 index 0000000000000..d379bc93c0bdd --- /dev/null +++ b/packages/cli/src/services/orchestration/pubsub/pubsub-message-map.ts @@ -0,0 +1,104 @@ +import type { IPushDataType, IWorkflowDb } from '@/Interfaces'; +import type { RunningJobSummary } from '@/scaling/scaling.types'; + +export type PubSubMessageMap = { + // #region Lifecycle + + 'reload-license': never; + + 'restart-event-bus': { + result: 'success' | 'error'; + error?: string; + }; + + 'reload-external-secrets-providers': { + result: 'success' | 'error'; + error?: string; + }; + + 'stop-worker': never; + + // #endregion + + // #region Community packages + + 'community-package-install': { + packageName: string; + packageVersion: string; + }; + + 'community-package-update': { + packageName: string; + packageVersion: string; + }; + + 'community-package-uninstall': { + packageName: string; + packageVersion: string; + }; + + // #endregion + + // #region Worker view + + 'get-worker-id': never; + + 'get-worker-status': { + workerId: string; + runningJobsSummary: RunningJobSummary[]; + freeMem: number; + totalMem: number; + uptime: number; + loadAvg: number[]; + cpus: string; + arch: string; + platform: NodeJS.Platform; + hostname: string; + interfaces: Array<{ + family: 'IPv4' | 'IPv6'; + address: string; + internal: boolean; + }>; + version: string; + }; + + // #endregion + + // #region Multi-main setup + + 'add-webhooks-triggers-and-pollers': { + workflowId: string; + }; + + 'remove-triggers-and-pollers': { + workflowId: string; + }; + + 'display-workflow-activation': { + workflowId: string; + }; + + 'display-workflow-deactivation': { + workflowId: string; + }; + + // currently 'workflow-failed-to-activate' + 'display-workflow-activation-error': { + workflowId: string; + errorMessage: string; + }; + + 'relay-execution-lifecycle-event': { + type: IPushDataType; + args: Record; + pushRef: string; + }; + + 'clear-test-webhooks': { + webhookKey: string; + workflowEntity: IWorkflowDb; + pushRef: string; + }; + + // #endregion +}; From 008c510b7623fefb8c60730c7eac54dd9bb2e3fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Wed, 28 Aug 2024 11:36:00 +0200 Subject: [PATCH 4/5] feat(core): Expose queue metrics for Prometheus (#10559) --- .../config/src/configs/endpoints.config.ts | 8 + packages/@n8n/config/test/config.test.ts | 2 + .../__tests__/telemetry-event-relay.test.ts | 2 + packages/cli/src/events/event.service.ts | 5 +- .../cli/src/events/queue-metrics-event-map.ts | 8 + .../cli/src/events/telemetry-event-relay.ts | 1 + .../prometheus-metrics.service.test.ts | 152 ++++++++++++------ .../src/metrics/prometheus-metrics.service.ts | 45 +++++- packages/cli/src/metrics/types.ts | 2 +- .../scaling/__tests__/scaling.service.test.ts | 7 + packages/cli/src/scaling/scaling.service.ts | 61 ++++++- .../integration/prometheus-metrics.test.ts | 63 +++++++- 12 files changed, 304 insertions(+), 52 deletions(-) create mode 100644 packages/cli/src/events/queue-metrics-event-map.ts diff --git a/packages/@n8n/config/src/configs/endpoints.config.ts b/packages/@n8n/config/src/configs/endpoints.config.ts index 88efa01d26bb5..9905d9f48aaf0 100644 --- a/packages/@n8n/config/src/configs/endpoints.config.ts +++ b/packages/@n8n/config/src/configs/endpoints.config.ts @@ -49,6 +49,14 @@ class PrometheusMetricsConfig { /** Whether to include metrics derived from n8n's internal events */ @Env('N8N_METRICS_INCLUDE_MESSAGE_EVENT_BUS_METRICS') includeMessageEventBusMetrics: boolean = false; + + /** Whether to include metrics for jobs in scaling mode. Not supported in multi-main setup. */ + @Env('N8N_METRICS_INCLUDE_QUEUE_METRICS') + includeQueueMetrics: boolean = false; + + /** How often (in seconds) to update queue metrics. */ + @Env('N8N_METRICS_QUEUE_METRICS_INTERVAL') + queueMetricsInterval: number = 20; } @Config diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index 301b99ca567ba..50e5c3b252910 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -165,6 +165,8 @@ describe('GlobalConfig', () => { includeApiMethodLabel: false, includeCredentialTypeLabel: false, includeApiStatusCodeLabel: false, + includeQueueMetrics: false, + queueMetricsInterval: 20, }, additionalNonUIRoutes: '', disableProductionWebhooksOnMainProcess: false, diff --git a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts index a7be9772e4b5d..4c3c1de4b5a62 100644 --- a/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts +++ b/packages/cli/src/events/__tests__/telemetry-event-relay.test.ts @@ -33,6 +33,7 @@ describe('TelemetryEventRelay', () => { includeApiEndpoints: false, includeCacheMetrics: false, includeMessageEventBusMetrics: false, + includeQueueMetrics: false, }, }, }); @@ -948,6 +949,7 @@ describe('TelemetryEventRelay', () => { metrics_category_routes: false, metrics_category_cache: false, metrics_category_logs: false, + metrics_category_queue: false, }, }), ); diff --git a/packages/cli/src/events/event.service.ts b/packages/cli/src/events/event.service.ts index 6744103a07799..8eb61794443b0 100644 --- a/packages/cli/src/events/event.service.ts +++ b/packages/cli/src/events/event.service.ts @@ -1,6 +1,9 @@ import { Service } from 'typedi'; import { TypedEmitter } from '@/TypedEmitter'; import type { RelayEventMap } from './relay-event-map'; +import type { QueueMetricsEventMap } from './queue-metrics-event-map'; + +type EventMap = RelayEventMap & QueueMetricsEventMap; @Service() -export class EventService extends TypedEmitter {} +export class EventService extends TypedEmitter {} diff --git a/packages/cli/src/events/queue-metrics-event-map.ts b/packages/cli/src/events/queue-metrics-event-map.ts new file mode 100644 index 0000000000000..c81360cc64bbb --- /dev/null +++ b/packages/cli/src/events/queue-metrics-event-map.ts @@ -0,0 +1,8 @@ +export type QueueMetricsEventMap = { + 'job-counts-updated': { + active: number; + completed: number; + failed: number; + waiting: number; + }; +}; diff --git a/packages/cli/src/events/telemetry-event-relay.ts b/packages/cli/src/events/telemetry-event-relay.ts index c7ffba499ad3a..e374a4cf7c9f1 100644 --- a/packages/cli/src/events/telemetry-event-relay.ts +++ b/packages/cli/src/events/telemetry-event-relay.ts @@ -768,6 +768,7 @@ export class TelemetryEventRelay extends EventRelay { metrics_category_routes: this.globalConfig.endpoints.metrics.includeApiEndpoints, metrics_category_cache: this.globalConfig.endpoints.metrics.includeCacheMetrics, metrics_category_logs: this.globalConfig.endpoints.metrics.includeMessageEventBusMetrics, + metrics_category_queue: this.globalConfig.endpoints.metrics.includeQueueMetrics, }, }; diff --git a/packages/cli/src/metrics/__tests__/prometheus-metrics.service.test.ts b/packages/cli/src/metrics/__tests__/prometheus-metrics.service.test.ts index 91f99f91b041a..dd79de956ef42 100644 --- a/packages/cli/src/metrics/__tests__/prometheus-metrics.service.test.ts +++ b/packages/cli/src/metrics/__tests__/prometheus-metrics.service.test.ts @@ -7,6 +7,7 @@ import type express from 'express'; import type { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus'; import { mockInstance } from '@test/mocking'; import { GlobalConfig } from '@n8n/config'; +import type { EventService } from '@/events/event.service'; const mockMiddleware = ( _req: express.Request, @@ -22,27 +23,62 @@ describe('PrometheusMetricsService', () => { endpoints: { metrics: { prefix: 'n8n_', - includeDefaultMetrics: true, - includeApiEndpoints: true, - includeCacheMetrics: true, - includeMessageEventBusMetrics: true, + includeDefaultMetrics: false, + includeApiEndpoints: false, + includeCacheMetrics: false, + includeMessageEventBusMetrics: false, includeCredentialTypeLabel: false, includeNodeTypeLabel: false, includeWorkflowIdLabel: false, - includeApiPathLabel: true, - includeApiMethodLabel: true, - includeApiStatusCodeLabel: true, + includeApiPathLabel: false, + includeApiMethodLabel: false, + includeApiStatusCodeLabel: false, + includeQueueMetrics: false, }, }, }); + const app = mock(); + const eventBus = mock(); + const eventService = mock(); + const prometheusMetricsService = new PrometheusMetricsService( + mock(), + eventBus, + globalConfig, + eventService, + ); + + afterEach(() => { + jest.clearAllMocks(); + prometheusMetricsService.disableAllMetrics(); + }); + + describe('constructor', () => { + it('should enable metrics based on global config', async () => { + const customGlobalConfig = { ...globalConfig }; + customGlobalConfig.endpoints.metrics.includeCacheMetrics = true; + const customPrometheusMetricsService = new PrometheusMetricsService( + mock(), + mock(), + customGlobalConfig, + mock(), + ); + + await customPrometheusMetricsService.init(app); + + expect(promClient.Counter).toHaveBeenCalledWith({ + name: 'n8n_cache_hits_total', + help: 'Total number of cache hits.', + labelNames: ['cache'], + }); + }); + }); + describe('init', () => { it('should set up `n8n_version_info`', async () => { - const service = new PrometheusMetricsService(mock(), mock(), globalConfig); + await prometheusMetricsService.init(app); - await service.init(mock()); - - expect(promClient.Gauge).toHaveBeenCalledWith({ + expect(promClient.Gauge).toHaveBeenNthCalledWith(1, { name: 'n8n_version_info', help: 'n8n version info.', labelNames: ['version', 'major', 'minor', 'patch'], @@ -50,48 +86,37 @@ describe('PrometheusMetricsService', () => { }); it('should set up default metrics collection with `prom-client`', async () => { - const service = new PrometheusMetricsService(mock(), mock(), globalConfig); - - await service.init(mock()); + prometheusMetricsService.enableMetric('default'); + await prometheusMetricsService.init(app); expect(promClient.collectDefaultMetrics).toHaveBeenCalled(); }); it('should set up `n8n_cache_hits_total`', async () => { - config.set('endpoints.metrics.includeCacheMetrics', true); - const service = new PrometheusMetricsService(mock(), mock(), globalConfig); - - await service.init(mock()); + prometheusMetricsService.enableMetric('cache'); + await prometheusMetricsService.init(app); expect(promClient.Counter).toHaveBeenCalledWith({ name: 'n8n_cache_hits_total', help: 'Total number of cache hits.', labelNames: ['cache'], }); - // @ts-expect-error private field - expect(service.counters.cacheHitsTotal?.inc).toHaveBeenCalledWith(0); }); it('should set up `n8n_cache_misses_total`', async () => { - config.set('endpoints.metrics.includeCacheMetrics', true); - const service = new PrometheusMetricsService(mock(), mock(), globalConfig); - - await service.init(mock()); + prometheusMetricsService.enableMetric('cache'); + await prometheusMetricsService.init(app); expect(promClient.Counter).toHaveBeenCalledWith({ name: 'n8n_cache_misses_total', help: 'Total number of cache misses.', labelNames: ['cache'], }); - // @ts-expect-error private field - expect(service.counters.cacheMissesTotal?.inc).toHaveBeenCalledWith(0); }); it('should set up `n8n_cache_updates_total`', async () => { - config.set('endpoints.metrics.includeCacheMetrics', true); - const service = new PrometheusMetricsService(mock(), mock(), globalConfig); - - await service.init(mock()); + prometheusMetricsService.enableMetric('cache'); + await prometheusMetricsService.init(app); expect(promClient.Counter).toHaveBeenCalledWith({ name: 'n8n_cache_updates_total', @@ -99,26 +124,19 @@ describe('PrometheusMetricsService', () => { labelNames: ['cache'], }); // @ts-expect-error private field - expect(service.counters.cacheUpdatesTotal?.inc).toHaveBeenCalledWith(0); + expect(prometheusMetricsService.counters.cacheUpdatesTotal?.inc).toHaveBeenCalledWith(0); }); it('should set up route metrics with `express-prom-bundle`', async () => { - config.set('endpoints.metrics.includeApiEndpoints', true); - config.set('endpoints.metrics.includeApiPathLabel', true); - config.set('endpoints.metrics.includeApiMethodLabel', true); - config.set('endpoints.metrics.includeApiStatusCodeLabel', true); - const service = new PrometheusMetricsService(mock(), mock(), globalConfig); - - const app = mock(); - - await service.init(app); + prometheusMetricsService.enableMetric('routes'); + await prometheusMetricsService.init(app); expect(promBundle).toHaveBeenCalledWith({ autoregister: false, includeUp: false, - includePath: true, - includeMethod: true, - includeStatusCode: true, + includePath: false, + includeMethod: false, + includeStatusCode: false, }); expect(app.use).toHaveBeenCalledWith( @@ -137,12 +155,52 @@ describe('PrometheusMetricsService', () => { }); it('should set up event bus metrics', async () => { - const eventBus = mock(); - const service = new PrometheusMetricsService(mock(), eventBus, globalConfig); - - await service.init(mock()); + prometheusMetricsService.enableMetric('logs'); + await prometheusMetricsService.init(app); expect(eventBus.on).toHaveBeenCalledWith('metrics.eventBus.event', expect.any(Function)); }); + + it('should set up queue metrics if enabled', async () => { + config.set('executions.mode', 'queue'); + prometheusMetricsService.enableMetric('queue'); + + await prometheusMetricsService.init(app); + + // call 1 is for `n8n_version_info` (always enabled) + + expect(promClient.Gauge).toHaveBeenNthCalledWith(2, { + name: 'n8n_scaling_mode_queue_jobs_waiting', + help: 'Current number of enqueued jobs waiting for pickup in scaling mode.', + }); + + expect(promClient.Gauge).toHaveBeenNthCalledWith(3, { + name: 'n8n_scaling_mode_queue_jobs_active', + help: 'Current number of jobs being processed across all workers in scaling mode.', + }); + + expect(promClient.Counter).toHaveBeenNthCalledWith(1, { + name: 'n8n_scaling_mode_queue_jobs_completed', + help: 'Total number of jobs completed across all workers in scaling mode since instance start.', + }); + + expect(promClient.Counter).toHaveBeenNthCalledWith(2, { + name: 'n8n_scaling_mode_queue_jobs_failed', + help: 'Total number of jobs failed across all workers in scaling mode since instance start.', + }); + + expect(eventService.on).toHaveBeenCalledWith('job-counts-updated', expect.any(Function)); + }); + + it('should not set up queue metrics if enabled but not on scaling mode', async () => { + config.set('executions.mode', 'regular'); + prometheusMetricsService.enableMetric('queue'); + + await prometheusMetricsService.init(app); + + expect(promClient.Gauge).toHaveBeenCalledTimes(1); // version metric + expect(promClient.Counter).toHaveBeenCalledTimes(0); // cache metrics + expect(eventService.on).not.toHaveBeenCalled(); + }); }); }); diff --git a/packages/cli/src/metrics/prometheus-metrics.service.ts b/packages/cli/src/metrics/prometheus-metrics.service.ts index 32522808fc243..3646f172f183b 100644 --- a/packages/cli/src/metrics/prometheus-metrics.service.ts +++ b/packages/cli/src/metrics/prometheus-metrics.service.ts @@ -1,7 +1,7 @@ import { N8N_VERSION } from '@/constants'; import type express from 'express'; import promBundle from 'express-prom-bundle'; -import promClient, { type Counter } from 'prom-client'; +import promClient, { type Counter, type Gauge } from 'prom-client'; import semverParse from 'semver/functions/parse'; import { Service } from 'typedi'; @@ -11,6 +11,8 @@ import { EventMessageTypeNames } from 'n8n-workflow'; import type { EventMessageTypes } from '@/eventbus'; import type { Includes, MetricCategory, MetricLabel } from './types'; import { GlobalConfig } from '@n8n/config'; +import { EventService } from '@/events/event.service'; +import config from '@/config'; @Service() export class PrometheusMetricsService { @@ -18,10 +20,13 @@ export class PrometheusMetricsService { private readonly cacheService: CacheService, private readonly eventBus: MessageEventBus, private readonly globalConfig: GlobalConfig, + private readonly eventService: EventService, ) {} private readonly counters: { [key: string]: Counter | null } = {}; + private readonly gauges: Record> = {}; + private readonly prefix = this.globalConfig.endpoints.metrics.prefix; private readonly includes: Includes = { @@ -30,6 +35,7 @@ export class PrometheusMetricsService { routes: this.globalConfig.endpoints.metrics.includeApiEndpoints, cache: this.globalConfig.endpoints.metrics.includeCacheMetrics, logs: this.globalConfig.endpoints.metrics.includeMessageEventBusMetrics, + queue: this.globalConfig.endpoints.metrics.includeQueueMetrics, }, labels: { credentialsType: this.globalConfig.endpoints.metrics.includeCredentialTypeLabel, @@ -48,6 +54,7 @@ export class PrometheusMetricsService { this.initCacheMetrics(); this.initEventBusMetrics(); this.initRouteMetrics(app); + this.initQueueMetrics(); this.mountMetricsEndpoint(app); } @@ -218,6 +225,42 @@ export class PrometheusMetricsService { }); } + private initQueueMetrics() { + if (!this.includes.metrics.queue || config.getEnv('executions.mode') !== 'queue') return; + + this.gauges.waiting = new promClient.Gauge({ + name: this.prefix + 'scaling_mode_queue_jobs_waiting', + help: 'Current number of enqueued jobs waiting for pickup in scaling mode.', + }); + + this.gauges.active = new promClient.Gauge({ + name: this.prefix + 'scaling_mode_queue_jobs_active', + help: 'Current number of jobs being processed across all workers in scaling mode.', + }); + + this.counters.completed = new promClient.Counter({ + name: this.prefix + 'scaling_mode_queue_jobs_completed', + help: 'Total number of jobs completed across all workers in scaling mode since instance start.', + }); + + this.counters.failed = new promClient.Counter({ + name: this.prefix + 'scaling_mode_queue_jobs_failed', + help: 'Total number of jobs failed across all workers in scaling mode since instance start.', + }); + + this.gauges.waiting.set(0); + this.gauges.active.set(0); + this.counters.completed.inc(0); + this.counters.failed.inc(0); + + this.eventService.on('job-counts-updated', (jobCounts) => { + this.gauges.waiting.set(jobCounts.waiting); + this.gauges.active.set(jobCounts.active); + this.counters.completed?.inc(jobCounts.completed); + this.counters.failed?.inc(jobCounts.failed); + }); + } + private toLabels(event: EventMessageTypes): Record { const { __type, eventName, payload } = event; diff --git a/packages/cli/src/metrics/types.ts b/packages/cli/src/metrics/types.ts index 6a156d48d7ebc..3b68d5408a122 100644 --- a/packages/cli/src/metrics/types.ts +++ b/packages/cli/src/metrics/types.ts @@ -1,4 +1,4 @@ -export type MetricCategory = 'default' | 'routes' | 'cache' | 'logs'; +export type MetricCategory = 'default' | 'routes' | 'cache' | 'logs' | 'queue'; export type MetricLabel = | 'credentialsType' diff --git a/packages/cli/src/scaling/__tests__/scaling.service.test.ts b/packages/cli/src/scaling/__tests__/scaling.service.test.ts index f87d02b33d0d4..0395ce5577780 100644 --- a/packages/cli/src/scaling/__tests__/scaling.service.test.ts +++ b/packages/cli/src/scaling/__tests__/scaling.service.test.ts @@ -37,6 +37,12 @@ describe('ScalingService', () => { }, }, }, + endpoints: { + metrics: { + includeQueueMetrics: false, + queueMetricsInterval: 20, + }, + }, }); const instanceSettings = Container.get(InstanceSettings); @@ -73,6 +79,7 @@ describe('ScalingService', () => { mock(), instanceSettings, orchestrationService, + mock(), ); getRunningJobsCountSpy = jest.spyOn(scalingService, 'getRunningJobsCount'); diff --git a/packages/cli/src/scaling/scaling.service.ts b/packages/cli/src/scaling/scaling.service.ts index 37436cdb4361e..0803ac1ced796 100644 --- a/packages/cli/src/scaling/scaling.service.ts +++ b/packages/cli/src/scaling/scaling.service.ts @@ -23,6 +23,7 @@ import { GlobalConfig } from '@n8n/config'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; import { InstanceSettings } from 'n8n-core'; import { OrchestrationService } from '@/services/orchestration.service'; +import { EventService } from '@/events/event.service'; @Service() export class ScalingService { @@ -38,6 +39,7 @@ export class ScalingService { private readonly executionRepository: ExecutionRepository, private readonly instanceSettings: InstanceSettings, private readonly orchestrationService: OrchestrationService, + private readonly eventService: EventService, ) {} // #region Lifecycle @@ -66,6 +68,8 @@ export class ScalingService { .on('leader-stepdown', () => this.stopQueueRecovery()); } + this.scheduleQueueMetrics(); + this.logger.debug('[ScalingService] Queue setup completed'); } @@ -89,8 +93,9 @@ export class ScalingService { this.logger.debug('[ScalingService] Queue paused'); this.stopQueueRecovery(); + this.stopQueueMetrics(); - this.logger.debug('[ScalingService] Queue recovery stopped'); + this.logger.debug('[ScalingService] Queue recovery and metrics stopped'); let count = 0; @@ -113,6 +118,12 @@ export class ScalingService { // #region Jobs + async getPendingJobCounts() { + const { active, waiting } = await this.queue.getJobCounts(); + + return { active, waiting }; + } + async addJob(jobData: JobData, jobOptions: JobOptions) { const { executionId } = jobData; @@ -246,6 +257,11 @@ export class ScalingService { this.activeExecutions.resolveResponsePromise(msg.executionId, decodedResponse); } }); + + if (this.isQueueMetricsEnabled) { + this.queue.on('global:completed', () => this.jobCounters.completed++); + this.queue.on('global:failed', () => this.jobCounters.failed++); + } } private isPubSubMessage(candidate: unknown): candidate is PubSubMessage { @@ -282,6 +298,49 @@ export class ScalingService { throw new ApplicationError('This method must be called on a `worker` instance'); } + // #region Queue metrics + + /** Counters for completed and failed jobs, reset on each interval tick. */ + private readonly jobCounters = { completed: 0, failed: 0 }; + + /** Interval for collecting queue metrics to expose via Prometheus. */ + private queueMetricsInterval: NodeJS.Timer | undefined; + + get isQueueMetricsEnabled() { + return ( + this.globalConfig.endpoints.metrics.includeQueueMetrics && + this.instanceType === 'main' && + !this.orchestrationService.isMultiMainSetupEnabled + ); + } + + /** Set up an interval to collect queue metrics and emit them in an event. */ + private scheduleQueueMetrics() { + if (!this.isQueueMetricsEnabled || this.queueMetricsInterval) return; + + this.queueMetricsInterval = setInterval(async () => { + const pendingJobCounts = await this.getPendingJobCounts(); + + this.eventService.emit('job-counts-updated', { + ...pendingJobCounts, // active, waiting + ...this.jobCounters, // completed, failed + }); + + this.jobCounters.completed = 0; + this.jobCounters.failed = 0; + }, this.globalConfig.endpoints.metrics.queueMetricsInterval * Time.seconds.toMilliseconds); + } + + /** Stop collecting queue metrics. */ + private stopQueueMetrics() { + if (this.queueMetricsInterval) { + clearInterval(this.queueMetricsInterval); + this.queueMetricsInterval = undefined; + } + } + + // #endregion + // #region Queue recovery private readonly queueRecoveryContext: QueueRecoveryContext = { diff --git a/packages/cli/test/integration/prometheus-metrics.test.ts b/packages/cli/test/integration/prometheus-metrics.test.ts index f6710ba3a5ec4..c863950e4880b 100644 --- a/packages/cli/test/integration/prometheus-metrics.test.ts +++ b/packages/cli/test/integration/prometheus-metrics.test.ts @@ -6,11 +6,14 @@ import { N8N_VERSION } from '@/constants'; import { PrometheusMetricsService } from '@/metrics/prometheus-metrics.service'; import { setupTestServer } from './shared/utils'; import { GlobalConfig } from '@n8n/config'; +import config from '@/config'; +import { EventService } from '@/events/event.service'; jest.unmock('@/eventbus/message-event-bus/message-event-bus'); const toLines = (response: Response) => response.text.trim().split('\n'); +const eventService = Container.get(EventService); const globalConfig = Container.get(GlobalConfig); globalConfig.endpoints.metrics = { enable: true, @@ -25,6 +28,8 @@ globalConfig.endpoints.metrics = { includeApiPathLabel: true, includeApiMethodLabel: true, includeApiStatusCodeLabel: true, + includeQueueMetrics: true, + queueMetricsInterval: 20, }; const server = setupTestServer({ endpointGroups: ['metrics'] }); @@ -32,7 +37,7 @@ const agent = request.agent(server.app); let prometheusService: PrometheusMetricsService; -describe('Metrics', () => { +describe('PrometheusMetricsService', () => { beforeAll(() => { prometheusService = Container.get(PrometheusMetricsService); }); @@ -222,4 +227,60 @@ describe('Metrics', () => { expect(lines).toContainEqual(expect.stringContaining('path="/webhook-test/some-uuid"')); expect(lines).toContainEqual(expect.stringContaining('status_code="404"')); }); + + it('should return queue metrics if enabled', async () => { + /** + * Arrange + */ + prometheusService.enableMetric('queue'); + config.set('executions.mode', 'queue'); + await prometheusService.init(server.app); + + /** + * Act + */ + const response = await agent.get('/metrics'); + + /** + * Assert + */ + expect(response.status).toEqual(200); + expect(response.type).toEqual('text/plain'); + + const lines = toLines(response); + + expect(lines).toContain('n8n_test_scaling_mode_queue_jobs_waiting 0'); + expect(lines).toContain('n8n_test_scaling_mode_queue_jobs_active 0'); + expect(lines).toContain('n8n_test_scaling_mode_queue_jobs_completed 0'); + expect(lines).toContain('n8n_test_scaling_mode_queue_jobs_failed 0'); + }); + + it('should set queue metrics in response to `job-counts-updated` event', async () => { + /** + * Arrange + */ + prometheusService.enableMetric('queue'); + config.set('executions.mode', 'queue'); + await prometheusService.init(server.app); + + /** + * Act + */ + eventService.emit('job-counts-updated', { waiting: 1, active: 2, completed: 0, failed: 0 }); + + /** + * Assert + */ + const response = await agent.get('/metrics'); + + expect(response.status).toEqual(200); + expect(response.type).toEqual('text/plain'); + + const lines = toLines(response); + + expect(lines).toContain('n8n_test_scaling_mode_queue_jobs_waiting 1'); + expect(lines).toContain('n8n_test_scaling_mode_queue_jobs_active 2'); + expect(lines).toContain('n8n_test_scaling_mode_queue_jobs_completed 0'); + expect(lines).toContain('n8n_test_scaling_mode_queue_jobs_failed 0'); + }); }); From 7522dde3d1f17eb7d69755db93261a5c7431b2dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?= =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?= =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?= Date: Wed, 28 Aug 2024 12:28:12 +0200 Subject: [PATCH 5/5] ci: Fix provenance generation during NPM publish (no-changelog) (#10586) --- .github/workflows/release-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 522d47ff0f52b..7702fbff927b0 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -13,6 +13,8 @@ jobs: runs-on: ubuntu-latest if: github.event.pull_request.merged == true timeout-minutes: 10 + permissions: + id-token: write env: NPM_CONFIG_PROVENANCE: true outputs: