From 7f5776810412c1f3547fc2274f466faa2ed63786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 28 Oct 2024 16:34:02 +0800 Subject: [PATCH 01/19] gather log use threads --- handler/gather/gather_log.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index e48c752c..e79cab42 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -168,17 +168,17 @@ def handle_from_node(node): if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) - - if self.is_ssh: - for node in self.nodes: - handle_from_node(node) - else: - local_ip = NetUtils.get_inner_ip() - node = self.nodes[0] - node["ip"] = local_ip - for node in self.nodes: - handle_from_node(node) - + nodes_threads = [] + for node in self.nodes: + if not self.is_ssh: + local_ip = NetUtils.get_inner_ip() + node = self.nodes[0] + node["ip"] = local_ip + node_threads = threading.Thread(target=handle_from_node, args=(node,)) + node_threads.start() + nodes_threads.append(node_threads) + for node_thread in nodes_threads: + node_thread.join() summary_tuples = self.__get_overall_summary(gather_tuples, self.zip_encrypt) self.stdio.print(summary_tuples) self.pack_dir_this_command = pack_dir_this_command From 138743c2b3b291aa983ed7effca0cdfb43e551ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 28 Oct 2024 18:01:50 +0800 Subject: [PATCH 02/19] gather log use threads --- common/ssh_client/base.py | 2 +- common/ssh_client/remote_client.py | 2 +- handler/gather/gather_log.py | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/common/ssh_client/base.py b/common/ssh_client/base.py index 3d235863..5b6f9a5b 100644 --- a/common/ssh_client/base.py +++ b/common/ssh_client/base.py @@ -54,7 +54,7 @@ def get_ip(self): return self.client.get_ip() def progress_bar(self, transferred, to_be_transferred, suffix=''): - if self.inner_config_manager.get("obdiag", {}).get("logger", {}).get("silent") or False: + if self.stdio.silent: return bar_len = 20 filled_len = int(round(bar_len * transferred / float(to_be_transferred))) diff --git a/common/ssh_client/remote_client.py b/common/ssh_client/remote_client.py index 581544e3..537eebdc 100644 --- a/common/ssh_client/remote_client.py +++ b/common/ssh_client/remote_client.py @@ -119,7 +119,7 @@ def download(self, remote_path, local_path): self._sftp_client.close() def progress_bar(self, transferred, to_be_transferred, suffix=''): - if self.inner_config_manager.get("obdiag", {}).get("logger", {}).get("silent") or False: + if self.stdio.silent: return bar_len = 20 filled_len = int(round(bar_len * transferred / float(to_be_transferred))) diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index e79cab42..f17def21 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -19,6 +19,7 @@ import os import time import tabulate +import threading from handler.base_shell_handler import BaseShellHandler from common.obdiag_exception import OBDIAGFormatException from common.constant import const @@ -169,6 +170,8 @@ def handle_from_node(node): file_size = os.path.getsize(resp["gather_pack_path"]) gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) nodes_threads = [] + self.stdio.print("gather nodes's log start. Please wait a moment...") + self.stdio.set_silent(True) for node in self.nodes: if not self.is_ssh: local_ip = NetUtils.get_inner_ip() @@ -179,6 +182,7 @@ def handle_from_node(node): nodes_threads.append(node_threads) for node_thread in nodes_threads: node_thread.join() + self.stdio.set_silent(False) summary_tuples = self.__get_overall_summary(gather_tuples, self.zip_encrypt) self.stdio.print(summary_tuples) self.pack_dir_this_command = pack_dir_this_command From 7fae0df8662eadafb51e07f5773dfed1b85c7ab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 28 Oct 2024 19:31:04 +0800 Subject: [PATCH 03/19] gather log use threads --- handler/gather/gather_log.py | 1 + 1 file changed, 1 insertion(+) diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index f17def21..7e701a00 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -169,6 +169,7 @@ def handle_from_node(node): if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) + nodes_threads = [] self.stdio.print("gather nodes's log start. Please wait a moment...") self.stdio.set_silent(True) From 39310967fd66274d6599a36c1344b80ff4104b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 28 Oct 2024 20:16:35 +0800 Subject: [PATCH 04/19] analyze_log data silent use json --- handler/analyzer/analyze_log.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index f85eabc3..fb1c0351 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -23,7 +23,7 @@ from handler.base_shell_handler import BaseShellHandler from common.obdiag_exception import OBDIAGFormatException from common.constant import const -from common.command import LocalClient, SshClient +from common.command import SshClient from common.ob_log_level import OBLogLevel from handler.meta.ob_error import OB_RET_DICT from common.command import download_file, get_logfile_name_list, mkdir, delete_file @@ -152,23 +152,31 @@ def handle_from_node(node): self.stdio.start_loading('analyze result start') title, field_names, summary_list, summary_details_list = self.__get_overall_summary(analyze_tuples, self.directly_analyze_files) + analyze_info_nodes = [] + for summary in summary_list: + for summary_data in summary: + analyze_info_node = {} + for m in field_names: + analyze_info_node[m] = summary_data + analyze_info_nodes.append(analyze_info_node) + print("analyze_info_nodes: {0}".format(analyze_info_nodes)) table = tabulate.tabulate(summary_list, headers=field_names, tablefmt="grid", showindex=False) - self.stdio.stop_loading('analyze result sucess') + self.stdio.stop_loading('analyze result success') self.stdio.print(title) self.stdio.print(table) FileUtil.write_append(os.path.join(local_store_parent_dir, "result_details.txt"), title + str(table) + "\n\nDetails:\n\n") - + # build summary details + summary_details_list_data = [] for m in range(len(summary_details_list)): + summary_details_list_data_once = {} for n in range(len(field_names)): extend = "\n\n" if n == len(field_names) - 1 else "\n" FileUtil.write_append(os.path.join(local_store_parent_dir, "result_details.txt"), field_names[n] + ": " + str(summary_details_list[m][n]) + extend) + summary_details_list_data_once[field_names[n]] = str(summary_details_list[m][n]) + summary_details_list_data.append(summary_details_list_data_once) last_info = "For more details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(local_store_parent_dir, "result_details.txt")) self.stdio.print(last_info) - # get info from local_store_parent_dir+/result_details.txt - analyze_info = "" - with open(os.path.join(local_store_parent_dir, "result_details.txt"), "r", encoding="utf-8") as f: - analyze_info = f.read() - return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": analyze_info}) + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": analyze_info_nodes, "summary_details_list": summary_details_list_data}) def __handle_from_node(self, node, local_store_parent_dir): resp = {"skip": False, "error": ""} From 4519c45c14a102ca88af3cd9e4af0406caa6066a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 28 Oct 2024 20:18:14 +0800 Subject: [PATCH 05/19] gather obproxy log use threading --- handler/gather/gather_log.py | 1 - handler/gather/gather_obproxy_log.py | 28 ++++++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index 7e701a00..f17def21 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -169,7 +169,6 @@ def handle_from_node(node): if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) - nodes_threads = [] self.stdio.print("gather nodes's log start. Please wait a moment...") self.stdio.set_silent(True) diff --git a/handler/gather/gather_obproxy_log.py b/handler/gather/gather_obproxy_log.py index e512518f..cd8cc1fa 100644 --- a/handler/gather/gather_obproxy_log.py +++ b/handler/gather/gather_obproxy_log.py @@ -17,13 +17,14 @@ """ import datetime import os +import threading import time import tabulate from handler.base_shell_handler import BaseShellHandler from common.obdiag_exception import OBDIAGFormatException -from common.command import LocalClient, SshClient +from common.command import SshClient from common.constant import const from common.command import get_file_size, download_file, is_empty_dir, get_logfile_name_list, mkdir, delete_empty_file, rm_rf_file, zip_encrypt_dir, zip_dir from common.tool import Util @@ -160,17 +161,20 @@ def handle_from_node(node): if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) - - if self.is_ssh: - for node in self.nodes: - handle_from_node(node) - else: - local_ip = NetUtils.get_inner_ip() - node = self.nodes[0] - node["ip"] = local_ip - for node in self.nodes: - handle_from_node(node) - + nodes_threads = [] + self.stdio.print("gather nodes's log start. Please wait a moment...") + self.stdio.set_silent(True) + for node in self.nodes: + if not self.is_ssh: + local_ip = NetUtils.get_inner_ip() + node = self.nodes[0] + node["ip"] = local_ip + node_threads = threading.Thread(target=handle_from_node, args=(node,)) + node_threads.start() + nodes_threads.append(node_threads) + for node_thread in nodes_threads: + node_thread.join() + self.stdio.set_silent(False) summary_tuples = self.__get_overall_summary(gather_tuples, self.zip_encrypt) self.stdio.print(summary_tuples) self.pack_dir_this_command = pack_dir_this_command From b29e08988f0b9e025f474255f29f631cbb10af20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 28 Oct 2024 20:20:20 +0800 Subject: [PATCH 06/19] gather obproxy log use threading --- handler/gather/gather_log.py | 1 + handler/gather/gather_obproxy_log.py | 1 + 2 files changed, 2 insertions(+) diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index f17def21..7e701a00 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -169,6 +169,7 @@ def handle_from_node(node): if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) + nodes_threads = [] self.stdio.print("gather nodes's log start. Please wait a moment...") self.stdio.set_silent(True) diff --git a/handler/gather/gather_obproxy_log.py b/handler/gather/gather_obproxy_log.py index cd8cc1fa..e61d6cc9 100644 --- a/handler/gather/gather_obproxy_log.py +++ b/handler/gather/gather_obproxy_log.py @@ -161,6 +161,7 @@ def handle_from_node(node): if len(resp["error"]) == 0: file_size = os.path.getsize(resp["gather_pack_path"]) gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) + nodes_threads = [] self.stdio.print("gather nodes's log start. Please wait a moment...") self.stdio.set_silent(True) From 2494dfd5f0e466d0681b11a1d769c0bb176d55a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 28 Oct 2024 20:29:28 +0800 Subject: [PATCH 07/19] gather obproxy log use threading --- handler/analyzer/analyze_log.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index fb1c0351..879e3a50 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -18,6 +18,8 @@ import datetime import os import re +import threading + import tabulate from handler.base_shell_handler import BaseShellHandler @@ -27,7 +29,7 @@ from common.ob_log_level import OBLogLevel from handler.meta.ob_error import OB_RET_DICT from common.command import download_file, get_logfile_name_list, mkdir, delete_file -from common.tool import Util +from common.tool import Util, NetUtils from common.tool import DirectoryUtil from common.tool import FileUtil from common.tool import TimeUtils @@ -141,14 +143,20 @@ def handle_from_node(node): resp, node_results = self.__handle_from_node(node, local_store_parent_dir) analyze_tuples.append((node.get("ip"), False, resp["error"], node_results)) - if self.is_ssh: - for node in self.nodes: - handle_from_node(node) - else: - local_ip = '127.0.0.1' - node = self.nodes[0] - node["ip"] = local_ip - handle_from_node(node) + nodes_threads = [] + self.stdio.print("gather nodes's log start. Please wait a moment...") + self.stdio.set_silent(True) + for node in self.nodes: + if not self.is_ssh: + local_ip = NetUtils.get_inner_ip() + node = self.nodes[0] + node["ip"] = local_ip + node_threads = threading.Thread(target=handle_from_node, args=(node,)) + node_threads.start() + nodes_threads.append(node_threads) + for node_thread in nodes_threads: + node_thread.join() + self.stdio.set_silent(False) self.stdio.start_loading('analyze result start') title, field_names, summary_list, summary_details_list = self.__get_overall_summary(analyze_tuples, self.directly_analyze_files) From 8458cb23010d598a9908325533b26099f1cb6ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 28 Oct 2024 20:59:24 +0800 Subject: [PATCH 08/19] add thread_nums --- conf/inner_config.yml | 3 +++ config.py | 5 ++++- handler/analyzer/analyze_log.py | 10 +++++++--- handler/gather/gather_log.py | 17 +++++++++++------ handler/gather/gather_obproxy_log.py | 17 +++++++++++------ 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/conf/inner_config.yml b/conf/inner_config.yml index 7d84c727..7ee54841 100644 --- a/conf/inner_config.yml +++ b/conf/inner_config.yml @@ -16,6 +16,8 @@ obdiag: silent: false ssh_client: remote_client_sudo: 0 +analyze: + thread_nums: 3 check: ignore_version: false work_path: "~/.obdiag/check" @@ -26,5 +28,6 @@ check: gather: scenes_base_path: "~/.obdiag/gather/tasks" redact_processing_num: 3 + thread_nums: 3 rca: result_path: "./obdiag_rca/" diff --git a/config.py b/config.py index b1809f28..b0f0c02c 100644 --- a/config.py +++ b/config.py @@ -80,6 +80,9 @@ 'remote_client_sudo': False, }, }, + 'analyze': { + "thread_nums": 3 + }, 'check': { 'ignore_version': False, 'work_path': '~/.obdiag/check', @@ -90,7 +93,7 @@ 'package_file': '~/.obdiag/check/check_package.yaml', 'tasks_base_path': '~/.obdiag/check/tasks/', }, - 'gather': {'scenes_base_path': '~/.obdiag/gather/tasks', 'redact_processing_num': 3}, + 'gather': {'scenes_base_path': '~/.obdiag/gather/tasks', 'redact_processing_num': 3, "thread_nums": 3}, 'rca': { 'result_path': './obdiag_rca/', }, diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index 879e3a50..70aae37c 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -138,13 +138,17 @@ def handle(self): local_store_parent_dir = os.path.join(self.gather_pack_dir, "obdiag_analyze_pack_{0}".format(TimeUtils.timestamp_to_filename_time(TimeUtils.get_current_us_timestamp()))) self.stdio.verbose("Use {0} as pack dir.".format(local_store_parent_dir)) analyze_tuples = [] + # analyze_thread default thread nums is 3 + analyze_thread_nums = int(self.context.inner_config_manager.config.get("obdiag", {}).get("analyze", {}).get("thread_nums") or 3) + pool_sema = threading.BoundedSemaphore(value=analyze_thread_nums) def handle_from_node(node): - resp, node_results = self.__handle_from_node(node, local_store_parent_dir) - analyze_tuples.append((node.get("ip"), False, resp["error"], node_results)) + with pool_sema: + resp, node_results = self.__handle_from_node(node, local_store_parent_dir) + analyze_tuples.append((node.get("ip"), False, resp["error"], node_results)) nodes_threads = [] - self.stdio.print("gather nodes's log start. Please wait a moment...") + self.stdio.print("analyze nodes's log start. Please wait a moment...") self.stdio.set_silent(True) for node in self.nodes: if not self.is_ssh: diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index 7e701a00..0483ede6 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -162,13 +162,18 @@ def handle(self): self.stdio.verbose('Use {0} as pack dir.'.format(pack_dir_this_command)) gather_tuples = [] + # gather_thread default thread nums is 3 + gather_thread_nums = int(self.context.inner_config_manager.config.get("obdiag", {}).get("gather", {}).get("thread_nums") or 3) + pool_sema = threading.BoundedSemaphore(value=gather_thread_nums) + def handle_from_node(node): - st = time.time() - resp = self.__handle_from_node(pack_dir_this_command, node) - file_size = "" - if len(resp["error"]) == 0: - file_size = os.path.getsize(resp["gather_pack_path"]) - gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) + with pool_sema: + st = time.time() + resp = self.__handle_from_node(pack_dir_this_command, node) + file_size = "" + if len(resp["error"]) == 0: + file_size = os.path.getsize(resp["gather_pack_path"]) + gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) nodes_threads = [] self.stdio.print("gather nodes's log start. Please wait a moment...") diff --git a/handler/gather/gather_obproxy_log.py b/handler/gather/gather_obproxy_log.py index e61d6cc9..52d09feb 100644 --- a/handler/gather/gather_obproxy_log.py +++ b/handler/gather/gather_obproxy_log.py @@ -154,13 +154,18 @@ def handle(self): self.stdio.verbose("Use {0} as pack dir.".format(pack_dir_this_command)) gather_tuples = [] + # gather_thread default thread nums is 3 + gather_thread_nums = int(self.context.inner_config_manager.config.get("obdiag", {}).get("gather", {}).get("thread_nums") or 3) + pool_sema = threading.BoundedSemaphore(value=gather_thread_nums) + def handle_from_node(node): - st = time.time() - resp = self.__handle_from_node(node, pack_dir_this_command) - file_size = "" - if len(resp["error"]) == 0: - file_size = os.path.getsize(resp["gather_pack_path"]) - gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) + with pool_sema: + st = time.time() + resp = self.__handle_from_node(node, pack_dir_this_command) + file_size = "" + if len(resp["error"]) == 0: + file_size = os.path.getsize(resp["gather_pack_path"]) + gather_tuples.append((node.get("ip"), False, resp["error"], file_size, resp["zip_password"], int(time.time() - st), resp["gather_pack_path"])) nodes_threads = [] self.stdio.print("gather nodes's log start. Please wait a moment...") From 782ac07e9f8bb8d7a9ab16dacd400b78f727b89f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 28 Oct 2024 20:59:37 +0800 Subject: [PATCH 09/19] add thread_nums --- config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/config.py b/config.py index b0f0c02c..4d08ed10 100644 --- a/config.py +++ b/config.py @@ -80,9 +80,7 @@ 'remote_client_sudo': False, }, }, - 'analyze': { - "thread_nums": 3 - }, + 'analyze': {"thread_nums": 3}, 'check': { 'ignore_version': False, 'work_path': '~/.obdiag/check', From 5ca95f5af12351003b7f22fd75e672b955cd8014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Tue, 29 Oct 2024 17:58:12 +0800 Subject: [PATCH 10/19] fix self.context.inner_config --- handler/analyzer/analyze_log.py | 2 +- handler/gather/gather_log.py | 2 +- handler/gather/gather_obproxy_log.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index 70aae37c..f5d1769f 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -139,7 +139,7 @@ def handle(self): self.stdio.verbose("Use {0} as pack dir.".format(local_store_parent_dir)) analyze_tuples = [] # analyze_thread default thread nums is 3 - analyze_thread_nums = int(self.context.inner_config_manager.config.get("obdiag", {}).get("analyze", {}).get("thread_nums") or 3) + analyze_thread_nums = int(self.context.inner_config.get("obdiag", {}).get("analyze", {}).get("thread_nums") or 3) pool_sema = threading.BoundedSemaphore(value=analyze_thread_nums) def handle_from_node(node): diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index 0483ede6..4a8b56c8 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -163,7 +163,7 @@ def handle(self): gather_tuples = [] # gather_thread default thread nums is 3 - gather_thread_nums = int(self.context.inner_config_manager.config.get("obdiag", {}).get("gather", {}).get("thread_nums") or 3) + gather_thread_nums = int(self.context.inner_config.get("obdiag", {}).get("gather", {}).get("thread_nums") or 3) pool_sema = threading.BoundedSemaphore(value=gather_thread_nums) def handle_from_node(node): diff --git a/handler/gather/gather_obproxy_log.py b/handler/gather/gather_obproxy_log.py index 52d09feb..24b5b082 100644 --- a/handler/gather/gather_obproxy_log.py +++ b/handler/gather/gather_obproxy_log.py @@ -155,7 +155,7 @@ def handle(self): gather_tuples = [] # gather_thread default thread nums is 3 - gather_thread_nums = int(self.context.inner_config_manager.config.get("obdiag", {}).get("gather", {}).get("thread_nums") or 3) + gather_thread_nums = int(self.context.inner_config.get("obdiag", {}).get("gather", {}).get("thread_nums") or 3) pool_sema = threading.BoundedSemaphore(value=gather_thread_nums) def handle_from_node(node): From 26b4c2c69c91259f93ebbafdb931c0bf53ce20de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Tue, 29 Oct 2024 17:58:41 +0800 Subject: [PATCH 11/19] fix self.context.inner_config --- handler/analyzer/analyze_log.py | 1 - 1 file changed, 1 deletion(-) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index f5d1769f..7998ccbe 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -171,7 +171,6 @@ def handle_from_node(node): for m in field_names: analyze_info_node[m] = summary_data analyze_info_nodes.append(analyze_info_node) - print("analyze_info_nodes: {0}".format(analyze_info_nodes)) table = tabulate.tabulate(summary_list, headers=field_names, tablefmt="grid", showindex=False) self.stdio.stop_loading('analyze result success') self.stdio.print(title) From 90ad5d9a9c9906877ad43f7cdb0b2d491d99837a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Tue, 29 Oct 2024 19:39:19 +0800 Subject: [PATCH 12/19] build test package --- .github/workflows/build_package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 6287fa6e..499dbe9e 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -7,6 +7,7 @@ on: push: branches: - master + - 2.6.0-update env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true From e7d939e92e5b564b4a4ab89583a606dbc793362d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 4 Nov 2024 10:15:06 +0800 Subject: [PATCH 13/19] fix thread_nums --- handler/analyzer/analyze_log.py | 5 +++-- handler/gather/gather_log.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index 7998ccbe..ff690433 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -139,7 +139,7 @@ def handle(self): self.stdio.verbose("Use {0} as pack dir.".format(local_store_parent_dir)) analyze_tuples = [] # analyze_thread default thread nums is 3 - analyze_thread_nums = int(self.context.inner_config.get("obdiag", {}).get("analyze", {}).get("thread_nums") or 3) + analyze_thread_nums = int(self.context.inner_config.get("analyze", {}).get("thread_nums") or 3) pool_sema = threading.BoundedSemaphore(value=analyze_thread_nums) def handle_from_node(node): @@ -149,6 +149,7 @@ def handle_from_node(node): nodes_threads = [] self.stdio.print("analyze nodes's log start. Please wait a moment...") + old_silent = self.stdio.silent self.stdio.set_silent(True) for node in self.nodes: if not self.is_ssh: @@ -160,7 +161,7 @@ def handle_from_node(node): nodes_threads.append(node_threads) for node_thread in nodes_threads: node_thread.join() - self.stdio.set_silent(False) + self.stdio.set_silent(old_silent) self.stdio.start_loading('analyze result start') title, field_names, summary_list, summary_details_list = self.__get_overall_summary(analyze_tuples, self.directly_analyze_files) diff --git a/handler/gather/gather_log.py b/handler/gather/gather_log.py index 4a8b56c8..87b0c33c 100644 --- a/handler/gather/gather_log.py +++ b/handler/gather/gather_log.py @@ -163,7 +163,7 @@ def handle(self): gather_tuples = [] # gather_thread default thread nums is 3 - gather_thread_nums = int(self.context.inner_config.get("obdiag", {}).get("gather", {}).get("thread_nums") or 3) + gather_thread_nums = int(self.context.inner_config.get("gather", {}).get("thread_nums") or 3) pool_sema = threading.BoundedSemaphore(value=gather_thread_nums) def handle_from_node(node): @@ -177,6 +177,7 @@ def handle_from_node(node): nodes_threads = [] self.stdio.print("gather nodes's log start. Please wait a moment...") + old_silent = self.stdio.silent self.stdio.set_silent(True) for node in self.nodes: if not self.is_ssh: @@ -188,7 +189,7 @@ def handle_from_node(node): nodes_threads.append(node_threads) for node_thread in nodes_threads: node_thread.join() - self.stdio.set_silent(False) + self.stdio.set_silent(old_silent) summary_tuples = self.__get_overall_summary(gather_tuples, self.zip_encrypt) self.stdio.print(summary_tuples) self.pack_dir_this_command = pack_dir_this_command From e2c1aaa940b4b066ecd5e765f101ddaca1f3da7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 4 Nov 2024 14:17:33 +0800 Subject: [PATCH 14/19] fix analyze log obdiag result --- handler/analyzer/analyze_log.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index ff690433..aac706bd 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -167,11 +167,12 @@ def handle_from_node(node): title, field_names, summary_list, summary_details_list = self.__get_overall_summary(analyze_tuples, self.directly_analyze_files) analyze_info_nodes = [] for summary in summary_list: - for summary_data in summary: - analyze_info_node = {} - for m in field_names: - analyze_info_node[m] = summary_data - analyze_info_nodes.append(analyze_info_node) + analyze_info_node = {} + field_names_nu = 0 + for m in field_names: + analyze_info_node[m] = summary[field_names_nu] + field_names_nu += 1 + analyze_info_nodes.append(analyze_info_node) table = tabulate.tabulate(summary_list, headers=field_names, tablefmt="grid", showindex=False) self.stdio.stop_loading('analyze result success') self.stdio.print(title) From 25994f9626051dbfd3ce41238a69f34536440305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 4 Nov 2024 15:20:21 +0800 Subject: [PATCH 15/19] fix analyze log obdiag result --- handler/analyzer/analyze_log.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index aac706bd..7251ba58 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -177,16 +177,18 @@ def handle_from_node(node): self.stdio.stop_loading('analyze result success') self.stdio.print(title) self.stdio.print(table) - FileUtil.write_append(os.path.join(local_store_parent_dir, "result_details.txt"), title + str(table) + "\n\nDetails:\n\n") + with open(os.path.join(local_store_parent_dir, "result_details.txt"), 'a', encoding='utf-8') as fileobj: + fileobj.write(u'{}'.format(title + str(table) + "\n\nDetails:\n\n")) # build summary details summary_details_list_data = [] for m in range(len(summary_details_list)): summary_details_list_data_once = {} for n in range(len(field_names)): extend = "\n\n" if n == len(field_names) - 1 else "\n" - FileUtil.write_append(os.path.join(local_store_parent_dir, "result_details.txt"), field_names[n] + ": " + str(summary_details_list[m][n]) + extend) + with open(os.path.join(local_store_parent_dir, "result_details.txt"), 'a', encoding='utf-8') as fileobj: + fileobj.write(u'{}'.format(field_names[n] + ": " + str(summary_details_list[m][n]) + extend)) summary_details_list_data_once[field_names[n]] = str(summary_details_list[m][n]) - summary_details_list_data.append(summary_details_list_data_once) + summary_details_list_data.append(summary_details_list_data_once) last_info = "For more details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(local_store_parent_dir, "result_details.txt")) self.stdio.print(last_info) return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": analyze_info_nodes, "summary_details_list": summary_details_list_data}) From 05e5a49651fe44f5820548330898d08c6681dadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 4 Nov 2024 16:35:03 +0800 Subject: [PATCH 16/19] fix analyze log obdiag result --- handler/analyzer/analyze_log.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index 7251ba58..d887c4f8 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -172,6 +172,8 @@ def handle_from_node(node): for m in field_names: analyze_info_node[m] = summary[field_names_nu] field_names_nu += 1 + if field_names_nu == len(summary): + break analyze_info_nodes.append(analyze_info_node) table = tabulate.tabulate(summary_list, headers=field_names, tablefmt="grid", showindex=False) self.stdio.stop_loading('analyze result success') From c05376dfad7bd65e57b098c46e93b52fe0430b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 4 Nov 2024 16:40:47 +0800 Subject: [PATCH 17/19] fix analyze log obdiag result --- handler/analyzer/analyze_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index d887c4f8..a7c3889f 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -193,7 +193,7 @@ def handle_from_node(node): summary_details_list_data.append(summary_details_list_data_once) last_info = "For more details, please run cmd \033[32m' cat {0} '\033[0m\n".format(os.path.join(local_store_parent_dir, "result_details.txt")) self.stdio.print(last_info) - return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": analyze_info_nodes, "summary_details_list": summary_details_list_data}) + return ObdiagResult(ObdiagResult.SUCCESS_CODE, data={"result": analyze_info_nodes, "summary_details_list": summary_details_list_data, "store_dir": local_store_parent_dir}) def __handle_from_node(self, node, local_store_parent_dir): resp = {"skip": False, "error": ""} From 64cfea3f6ca49f89cb0b449c8507b8e8d5e0ece9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Mon, 4 Nov 2024 17:08:57 +0800 Subject: [PATCH 18/19] fix analyze log obdiag result --- handler/analyzer/analyze_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handler/analyzer/analyze_log.py b/handler/analyzer/analyze_log.py index a7c3889f..b667e381 100644 --- a/handler/analyzer/analyze_log.py +++ b/handler/analyzer/analyze_log.py @@ -467,8 +467,8 @@ def __get_overall_summary(node_summary_tuples, is_files=False): ] ) if is_empty: - t.append([node, "\033[32mPASS\033[0m", None, None, None, None]) - t_details.append([node, "\033[32mPASS\033[0m", None, None, None, None, None, None, None, None, None]) + t.append([node, "PASS", None, None, None, None]) + t_details.append([node, "PASS", None, None, None, None, None, None, None, None, None]) title = "\nAnalyze OceanBase Offline Log Summary:\n" if is_files else "\nAnalyze OceanBase Online Log Summary:\n" t.sort(key=lambda x: (x[0], x[1], x[2], x[3]), reverse=False) t_details.sort(key=lambda x: (x[0], x[1], x[2], x[3]), reverse=False) From cbcb1ca56b7fba6477c52d964679ed7291a06681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A0=E7=A3=8A?= Date: Tue, 5 Nov 2024 16:53:49 +0800 Subject: [PATCH 19/19] not build package --- .github/workflows/build_package.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 499dbe9e..6287fa6e 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -7,7 +7,6 @@ on: push: branches: - master - - 2.6.0-update env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true