diff --git a/client/node/common/taskdirmgr.py b/client/node/common/taskdirmgr.py index 53e9fe58b..26587bacd 100644 --- a/client/node/common/taskdirmgr.py +++ b/client/node/common/taskdirmgr.py @@ -40,7 +40,7 @@ def acquire_task_dir(self, task_id=None, dirname_prefix="task_"): suffix_no += 1 task_dir = os.path.join(self._task_dirs_root, f"{dirname_prefix}{task_id}_{suffix_no}") else: # 没有传task_id,根据本地task dir名称排序,创建一个新的 - task_id = 0 + task_id = 1 while True: if not os.path.exists(os.path.join(self._task_dirs_root, f"{dirname_prefix}{task_id}")): break diff --git a/client/node/localtask/runlocaltask.py b/client/node/localtask/runlocaltask.py index c3e827e97..dbff25a73 100644 --- a/client/node/localtask/runlocaltask.py +++ b/client/node/localtask/runlocaltask.py @@ -75,6 +75,7 @@ def scan_project(self, execute_request_list, proj_conf): if execute_request_list and not self._remote_task_names: if self._source_dir: task_request = copy.deepcopy(execute_request_list[0]) # 深度copy,以免影响原字典数据 + task_request["task_name"] = "linecount" RequestModify.modify_local_task_request(task_request, self._task_name_id_maps, self._job_id, self._scm_auth_info.ssh_file, self._token, self._server_url, self._source_dir, self._scm_info, diff --git a/client/node/toolloader/httploadtool.py b/client/node/toolloader/httploadtool.py new file mode 100644 index 000000000..42c671029 --- /dev/null +++ b/client/node/toolloader/httploadtool.py @@ -0,0 +1,47 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2022 THL A29 Limited +# +# This source code file is made available under MIT License +# See LICENSE for details +# ============================================================================== + +""" +通过http方式拉取工具 +""" + +import os + +from util.logutil import LogPrinter +from util.api.fileserver import RetryFileServer +from util.exceptions import FileServerError +from util.ziplib import ZipMgr + + +class HttpToolLoader(object): + """通过http方式拉取工具,如果目录已存在,不拉取""" + @staticmethod + def download_tool(tool_url, dest_dir): + if os.path.exists(dest_dir): + # 工具目录存在时,直接复用,不重新拉取(如需更新,先删除工具目录) + LogPrinter.debug(f"tool dir({os.path.basename(dest_dir)}) from zip can be reused.") + return + + tool_root_dir = os.path.dirname(dest_dir) + if not os.path.exists(tool_root_dir): # 如果上层目录不存在,先创建 + os.makedirs(tool_root_dir) + zip_file_name = tool_url.split('/')[-1] + dest_zip_file_path = os.path.join(tool_root_dir, zip_file_name) + + file_server = RetryFileServer(retry_times=2).get_server(server_url=tool_url) + file_server.download_big_file("", dest_zip_file_path) + + if os.path.exists(dest_zip_file_path): + LogPrinter.debug(f"download {tool_url} to {dest_zip_file_path}") + ZipMgr.depress(dest_zip_file_path, tool_root_dir) + LogPrinter.debug(f"unzip {dest_zip_file_path} to {dest_dir}") + else: + raise FileServerError(f"download {tool_url} failed!") + + +if __name__ == '__main__': + pass diff --git a/client/node/toolloader/loadconfig.py b/client/node/toolloader/loadconfig.py index 199a2e979..c9981eaa4 100644 --- a/client/node/toolloader/loadconfig.py +++ b/client/node/toolloader/loadconfig.py @@ -310,7 +310,8 @@ def read_config_from_tool_schemes(self, tool_names=None, task_list=None): new_lib_envs = {} scm_url = tool_lib.get("scm_url") - lib_dir_name = scm_url.split('/')[-1].strip().replace(".git", "") + # 支持git仓库地址和zip包地址两种格式 + lib_dir_name = BaseScmUrlMgr.get_last_dir_name_from_url(scm_url) lib_dir_path = os.path.join(settings.TOOL_BASE_DIR, lib_dir_name) # 将环境变量中的$ROOT_DIR替换为实际路径,重新保存到new_lib_envs diff --git a/client/node/toolloader/loadtool.py b/client/node/toolloader/loadtool.py index fb400b57b..98f32477d 100644 --- a/client/node/toolloader/loadtool.py +++ b/client/node/toolloader/loadtool.py @@ -18,16 +18,25 @@ from node.app import settings from node.toolloader.gitload import GitLoader from node.toolloader.loadconfig import ConfigLoader, ToolConfig, LoadToolTypes +from node.toolloader.httploadtool import HttpToolLoader from util.envset import EnvSet from util.scanlang.callback_queue import CallbackQueue from util.pathlib import PathMgr from util.logutil import LogPrinter from util.scmurlmgr import BaseScmUrlMgr from util.subprocc import SubProcController +from util.textutil import ZIP_EXT class ToolCommonLoader(object): @staticmethod + def is_zip_url(tool_url): + """判断是否是压缩包地址""" + if tool_url.lower().endswith(ZIP_EXT): + return True + else: + return False + @staticmethod def load_tool_type(tool_dirpath, tool_dirname, tool_url=None): """ 判断使用哪种方式加载工具 @@ -38,8 +47,10 @@ def load_tool_type(tool_dirpath, tool_dirname, tool_url=None): """ if settings.USE_LOCAL_TOOL == "True": if os.path.exists(tool_dirpath): - # LogPrinter.info(f"USE_LOCAL_TOOL=True, use local tool dir: {tool_dirpath}") + LogPrinter.info(f"USE_LOCAL_TOOL=True, use local tool dir: {tool_dirpath}") return "Local", None + elif tool_url and ToolCommonLoader.is_zip_url(tool_url): + return "HTTP", None else: return "Git", None else: @@ -54,6 +65,8 @@ def load_tool_type(tool_dirpath, tool_dirname, tool_url=None): return "Copy", tool_dirpath_copy_from else: # 拷贝源目录不存在,从git拉取 return "Git", None + elif tool_url and ToolCommonLoader.is_zip_url(tool_url): + return "HTTP", None else: return "Git", None @@ -69,6 +82,8 @@ def load_tool(load_type, tool_dirpath, tool_dirpath_copy_from, git_url, scm_auth :param print_enable: :return: """ + if settings.DEBUG: + print_enable = True if load_type == "Local": if print_enable: LogPrinter.info(f"Use local tool dir: {tool_dirpath}") @@ -78,6 +93,11 @@ def load_tool(load_type, tool_dirpath, tool_dirpath_copy_from, git_url, scm_auth LogPrinter.info(f"Copy from {tool_dirpath_copy_from} to {tool_dirpath}") PathMgr().retry_copy(tool_dirpath_copy_from, tool_dirpath) return tool_dirpath + elif load_type == "HTTP": + if print_enable: + LogPrinter.info(f"Load from {git_url} to {tool_dirpath}") + HttpToolLoader.download_tool(git_url, tool_dirpath) + return tool_dirpath else: if print_enable: LogPrinter.info(f"Load from git to {tool_dirpath} ...") diff --git a/client/task/basic/downloader.py b/client/task/basic/downloader.py index f23843033..b55a409ac 100644 --- a/client/task/basic/downloader.py +++ b/client/task/basic/downloader.py @@ -14,7 +14,7 @@ import tempfile from node.app import settings -from task.basic.zipmgr import Zip +from util.zipmgr import Zip from util.exceptions import TransferModuleError from util.pathlib import PathMgr from util.api.fileserver import RetryFileServer diff --git a/client/task/transfermgr.py b/client/task/transfermgr.py index 02706003d..d1b8ab383 100644 --- a/client/task/transfermgr.py +++ b/client/task/transfermgr.py @@ -17,7 +17,7 @@ from task.basic.downloader import Downloader from node.app import settings -from task.basic.zipmgr import Zip +from util.zipmgr import Zip from util.exceptions import TransferModuleError from util.api.fileserver import RetryFileServer from util.pathlib import PathMgr diff --git a/client/tool/customtool.cp37-win_amd64.pyd b/client/tool/customtool.cp37-win_amd64.pyd index 846891348..586d40eda 100755 Binary files a/client/tool/customtool.cp37-win_amd64.pyd and b/client/tool/customtool.cp37-win_amd64.pyd differ diff --git a/client/tool/customtool.cpython-37m-aarch64-linux-gnu.so b/client/tool/customtool.cpython-37m-aarch64-linux-gnu.so index abf3039df..a0994be4c 100755 Binary files a/client/tool/customtool.cpython-37m-aarch64-linux-gnu.so and b/client/tool/customtool.cpython-37m-aarch64-linux-gnu.so differ diff --git a/client/tool/customtool.cpython-37m-darwin.so b/client/tool/customtool.cpython-37m-darwin.so index bd2f0b49a..b955f8297 100755 Binary files a/client/tool/customtool.cpython-37m-darwin.so and b/client/tool/customtool.cpython-37m-darwin.so differ diff --git a/client/tool/customtool.cpython-37m-x86_64-linux-gnu.so b/client/tool/customtool.cpython-37m-x86_64-linux-gnu.so index 504a67577..bc799abdf 100755 Binary files a/client/tool/customtool.cpython-37m-x86_64-linux-gnu.so and b/client/tool/customtool.cpython-37m-x86_64-linux-gnu.so differ diff --git a/client/util/api/fileserver.py b/client/util/api/fileserver.py index 4ca2c380d..98e81c463 100644 --- a/client/util/api/fileserver.py +++ b/client/util/api/fileserver.py @@ -12,11 +12,13 @@ import logging import hashlib import os +import ssl from urllib.parse import urljoin from util import wrapper from node.app import settings from util.api.httpclient import HttpClient +from urllib.request import urlopen logger = logging.getLogger(__name__) @@ -46,15 +48,20 @@ def get_sha256(file_path): class FileServer(object): - def __init__(self): + def __init__(self, server_url=None, headers=None): """ 构造函数 :return: """ - # 优先从环境变量读取文件服务器url和token,没有再使用默认值 - self._server_url = os.getenv("FILE_SERVER_URL", settings.FILE_SERVER['URL']) - file_server_token = os.getenv("FILE_SERVER_TOKEN", settings.FILE_SERVER['TOKEN']) - self._headers = {'Authorization': 'Token %s' % file_server_token} + if server_url: + self._server_url = server_url + self._headers = headers if headers else {} + else: + # 优先从环境变量读取文件服务器url和token,没有再使用默认值 + self._server_url = os.getenv("FILE_SERVER_URL", settings.FILE_SERVER['URL']) + file_server_token = os.getenv("FILE_SERVER_TOKEN", settings.FILE_SERVER['TOKEN']) + self._headers = {'Authorization': 'Token %s' % file_server_token} + self._proxies = None def modify_save_time(self, rel_dir, days): @@ -117,6 +124,20 @@ def download_file(self, rel_url, filepath): wf.write(data) return filepath + def download_big_file(self, rel_url, filepath): + """大文件下载""" + context = ssl._create_unverified_context() + download_url = urljoin(self._server_url, rel_url) + resp = urlopen(download_url, context=context) + chunk_size = 16 * 1024 + with open(filepath, 'wb') as wf: + while True: + trunk = resp.read(chunk_size) + if not trunk: + break + wf.write(trunk) + return filepath + class RetryFileServer(object): def __init__(self, retry_times=-1): @@ -136,10 +157,10 @@ def retry_on_error(self, error, method_name): """ return - def get_server(self): + def get_server(self, server_url=None): """ 获取一个server实例 :return: """ - file_server = FileServer() + file_server = FileServer(server_url=server_url) return wrapper.Retry(server=file_server, on_error=self.retry_on_error, total=self._retry_times) diff --git a/client/util/ziplib.py b/client/util/ziplib.py index 549f8544b..81f982f67 100644 --- a/client/util/ziplib.py +++ b/client/util/ziplib.py @@ -16,6 +16,7 @@ logger = logging.getLogger(__name__) + class ZipMgr(object): def zip_dir(self, dir_path, zip_filepath): """ @@ -43,7 +44,6 @@ def zip_dir(self, dir_path, zip_filepath): zf.close() return zip_filepath - def unzip_file(self, zip_filepath, unzip_to_dir): """ 加压缩到指定目录 @@ -71,5 +71,28 @@ def unzip_file(self, zip_filepath, unzip_to_dir): outfile.close() return unzip_to_dir + @staticmethod + def depress(zip_file, target_dir): + """ + 解压到指定目录下, 保留可执行文件权限(仅限UNIX系统创建的文件) + :param zip_file: 压缩文件 + :param target_dir: 解压后的目标目录 + :return: + """ + ZIP_UNIX_SYSTEM = 3 + + def extract_all_with_permission(zf, target_dir): + for info in zf.infolist(): + extracted_path = zf.extract(info, target_dir) + + if info.create_system == ZIP_UNIX_SYSTEM: + unix_attributes = info.external_attr >> 16 + if unix_attributes: + os.chmod(extracted_path, unix_attributes) + + with zipfile.ZipFile(zip_file, 'r') as zf: + extract_all_with_permission(zf, target_dir) + + if __name__ == '__main__': pass diff --git a/client/task/basic/zipmgr.py b/client/util/zipmgr.py similarity index 96% rename from client/task/basic/zipmgr.py rename to client/util/zipmgr.py index db37c4030..a1b44dd9c 100644 --- a/client/task/basic/zipmgr.py +++ b/client/util/zipmgr.py @@ -120,7 +120,7 @@ def decompress_by_7z(self, zip_file, path, print_enable=False): def decompress(self, zip_file, path): logger.info("zip模块执行解压操作...") # 20190927 bug-fixed, 防止在当前目录下删除当前目录,出现权限异常情况 - os.chdir("..") + os.chdir("../task") if os.path.exists(path): PathMgr().rmpath(path) @@ -142,7 +142,7 @@ def zipfile_compress(path, zip_file): zp = zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) for dirpath, dirnames, filenames in os.walk(path): fpath = dirpath.replace(path, "") # 这一句很重要,不replace的话,就从根目录开始复制 - fpath = fpath and fpath + os.sep or "" # 这句话理解我也点郁闷,实现当前文件夹以及包含的所有文件的压缩 + fpath = fpath and fpath + os.sep or "" for filename in filenames: zp.write(os.path.join(dirpath, filename), fpath + filename) zp.close()