Merge pull request #577 from bensonhome/main

工具拉取支持htttp zip包地址

client/node/common/taskdirmgr.py

@@ -40,7 +40,7 @@ def acquire_task_dir(self, task_id=None, dirname_prefix="task_"):
                     suffix_no += 1
                 task_dir = os.path.join(self._task_dirs_root, f"{dirname_prefix}{task_id}_{suffix_no}")
         else:  # 没有传task_id，根据本地task dir名称排序，创建一个新的
-            task_id = 0
+            task_id = 1
             while True:
                 if not os.path.exists(os.path.join(self._task_dirs_root, f"{dirname_prefix}{task_id}")):
                     break

client/node/localtask/runlocaltask.py

@@ -75,6 +75,7 @@ def scan_project(self, execute_request_list, proj_conf):
         if execute_request_list and not self._remote_task_names:
             if self._source_dir:
                 task_request = copy.deepcopy(execute_request_list[0])  # 深度copy,以免影响原字典数据
+                task_request["task_name"] = "linecount"
                 RequestModify.modify_local_task_request(task_request, self._task_name_id_maps, self._job_id,
                                                         self._scm_auth_info.ssh_file,
                                                         self._token, self._server_url, self._source_dir, self._scm_info,

client/node/toolloader/httploadtool.py

@@ -0,0 +1,47 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2022 THL A29 Limited
+#
+# This source code file is made available under MIT License
+# See LICENSE for details
+# ==============================================================================
+
+"""
+通过http方式拉取工具
+"""
+
+import os
+
+from util.logutil import LogPrinter
+from util.api.fileserver import RetryFileServer
+from util.exceptions import FileServerError
+from util.ziplib import ZipMgr
+
+
+class HttpToolLoader(object):
+    """通过http方式拉取工具，如果目录已存在，不拉取"""
+    @staticmethod
+    def download_tool(tool_url, dest_dir):
+        if os.path.exists(dest_dir):
+            # 工具目录存在时，直接复用，不重新拉取（如需更新，先删除工具目录）
+            LogPrinter.debug(f"tool dir({os.path.basename(dest_dir)}) from zip can be reused.")
+            return
+
+        tool_root_dir = os.path.dirname(dest_dir)
+        if not os.path.exists(tool_root_dir):  # 如果上层目录不存在,先创建
+            os.makedirs(tool_root_dir)
+        zip_file_name = tool_url.split('/')[-1]
+        dest_zip_file_path = os.path.join(tool_root_dir, zip_file_name)
+
+        file_server = RetryFileServer(retry_times=2).get_server(server_url=tool_url)
+        file_server.download_big_file("", dest_zip_file_path)
+
+        if os.path.exists(dest_zip_file_path):
+            LogPrinter.debug(f"download {tool_url} to {dest_zip_file_path}")
+            ZipMgr.depress(dest_zip_file_path, tool_root_dir)
+            LogPrinter.debug(f"unzip {dest_zip_file_path} to {dest_dir}")
+        else:
+            raise FileServerError(f"download {tool_url} failed!")
+
+
+if __name__ == '__main__':
+    pass

client/node/toolloader/loadconfig.py

@@ -310,7 +310,8 @@ def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
                                 new_lib_envs = {}
 
                             scm_url = tool_lib.get("scm_url")
-                            lib_dir_name = scm_url.split('/')[-1].strip().replace(".git", "")
+                            # 支持git仓库地址和zip包地址两种格式
+                            lib_dir_name = BaseScmUrlMgr.get_last_dir_name_from_url(scm_url)
                             lib_dir_path = os.path.join(settings.TOOL_BASE_DIR, lib_dir_name)
 
                             # 将环境变量中的$ROOT_DIR替换为实际路径，重新保存到new_lib_envs

client/node/toolloader/loadtool.py

@@ -18,16 +18,25 @@
 from node.app import settings
 from node.toolloader.gitload import GitLoader
 from node.toolloader.loadconfig import ConfigLoader, ToolConfig, LoadToolTypes
+from node.toolloader.httploadtool import HttpToolLoader
 from util.envset import EnvSet
 from util.scanlang.callback_queue import CallbackQueue
 from util.pathlib import PathMgr
 from util.logutil import LogPrinter
 from util.scmurlmgr import BaseScmUrlMgr
 from util.subprocc import SubProcController
+from util.textutil import ZIP_EXT
 
 
 class ToolCommonLoader(object):
     @staticmethod
+    def is_zip_url(tool_url):
+        """判断是否是压缩包地址"""
+        if tool_url.lower().endswith(ZIP_EXT):
+            return True
+        else:
+            return False
+    @staticmethod
     def load_tool_type(tool_dirpath, tool_dirname, tool_url=None):
         """
         判断使用哪种方式加载工具
@@ -38,8 +47,10 @@ def load_tool_type(tool_dirpath, tool_dirname, tool_url=None):
         """
         if settings.USE_LOCAL_TOOL == "True":
             if os.path.exists(tool_dirpath):
-                # LogPrinter.info(f"USE_LOCAL_TOOL=True, use local tool dir: {tool_dirpath}")
+                LogPrinter.info(f"USE_LOCAL_TOOL=True, use local tool dir: {tool_dirpath}")
                 return "Local", None
+            elif tool_url and ToolCommonLoader.is_zip_url(tool_url):
+                return "HTTP", None
             else:
                 return "Git", None
         else:
@@ -54,6 +65,8 @@ def load_tool_type(tool_dirpath, tool_dirname, tool_url=None):
                         return "Copy", tool_dirpath_copy_from
                     else:  # 拷贝源目录不存在，从git拉取
                         return "Git", None
+            elif tool_url and ToolCommonLoader.is_zip_url(tool_url):
+                return "HTTP", None
             else:
                 return "Git", None
 
@@ -69,6 +82,8 @@ def load_tool(load_type, tool_dirpath, tool_dirpath_copy_from, git_url, scm_auth
         :param print_enable:
         :return:
         """
+        if settings.DEBUG:
+            print_enable = True
         if load_type == "Local":
             if print_enable:
                 LogPrinter.info(f"Use local tool dir: {tool_dirpath}")
@@ -78,6 +93,11 @@ def load_tool(load_type, tool_dirpath, tool_dirpath_copy_from, git_url, scm_auth
                 LogPrinter.info(f"Copy from {tool_dirpath_copy_from} to {tool_dirpath}")
             PathMgr().retry_copy(tool_dirpath_copy_from, tool_dirpath)
             return tool_dirpath
+        elif load_type == "HTTP":
+            if print_enable:
+                LogPrinter.info(f"Load from {git_url} to {tool_dirpath}")
+            HttpToolLoader.download_tool(git_url, tool_dirpath)
+            return tool_dirpath
         else:
             if print_enable:
                 LogPrinter.info(f"Load from git to {tool_dirpath} ...")

client/task/basic/downloader.py

@@ -14,7 +14,7 @@
 import tempfile
 
 from node.app import settings
-from task.basic.zipmgr import Zip
+from util.zipmgr import Zip
 from util.exceptions import TransferModuleError
 from util.pathlib import PathMgr
 from util.api.fileserver import RetryFileServer

client/task/transfermgr.py

@@ -17,7 +17,7 @@
 
 from task.basic.downloader import Downloader
 from node.app import settings
-from task.basic.zipmgr import Zip
+from util.zipmgr import Zip
 from util.exceptions import TransferModuleError
 from util.api.fileserver import RetryFileServer
 from util.pathlib import PathMgr

client/util/api/fileserver.py

@@ -12,11 +12,13 @@
 import logging
 import hashlib
 import os
+import ssl
 
 from urllib.parse import urljoin
 from util import wrapper
 from node.app import settings
 from util.api.httpclient import HttpClient
+from urllib.request import urlopen
 
 logger = logging.getLogger(__name__)
 
@@ -46,15 +48,20 @@ def get_sha256(file_path):
 
 
 class FileServer(object):
-    def __init__(self):
+    def __init__(self, server_url=None, headers=None):
         """
         构造函数
         :return:
         """
-        # 优先从环境变量读取文件服务器url和token，没有再使用默认值
-        self._server_url = os.getenv("FILE_SERVER_URL", settings.FILE_SERVER['URL'])
-        file_server_token = os.getenv("FILE_SERVER_TOKEN", settings.FILE_SERVER['TOKEN'])
-        self._headers = {'Authorization': 'Token %s' % file_server_token}
+        if server_url:
+            self._server_url = server_url
+            self._headers = headers if headers else {}
+        else:
+            # 优先从环境变量读取文件服务器url和token，没有再使用默认值
+            self._server_url = os.getenv("FILE_SERVER_URL", settings.FILE_SERVER['URL'])
+            file_server_token = os.getenv("FILE_SERVER_TOKEN", settings.FILE_SERVER['TOKEN'])
+            self._headers = {'Authorization': 'Token %s' % file_server_token}
+
         self._proxies = None
 
     def modify_save_time(self, rel_dir, days):
@@ -117,6 +124,20 @@ def download_file(self, rel_url, filepath):
             wf.write(data)
         return filepath
 
+    def download_big_file(self, rel_url, filepath):
+        """大文件下载"""
+        context = ssl._create_unverified_context()
+        download_url = urljoin(self._server_url, rel_url)
+        resp = urlopen(download_url, context=context)
+        chunk_size = 16 * 1024
+        with open(filepath, 'wb') as wf:
+            while True:
+                trunk = resp.read(chunk_size)
+                if not trunk:
+                    break
+                wf.write(trunk)
+        return filepath
+
 
 class RetryFileServer(object):
     def __init__(self, retry_times=-1):
@@ -136,10 +157,10 @@ def retry_on_error(self, error, method_name):
         """
         return
 
-    def get_server(self):
+    def get_server(self, server_url=None):
         """
         获取一个server实例
         :return:
         """
-        file_server = FileServer()
+        file_server = FileServer(server_url=server_url)
         return wrapper.Retry(server=file_server, on_error=self.retry_on_error, total=self._retry_times)

client/util/ziplib.py

@@ -16,6 +16,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 class ZipMgr(object):
     def zip_dir(self, dir_path, zip_filepath):
         """
@@ -43,7 +44,6 @@ def zip_dir(self, dir_path, zip_filepath):
         zf.close()
         return zip_filepath
 
-
     def unzip_file(self, zip_filepath, unzip_to_dir):
         """
         加压缩到指定目录
@@ -71,5 +71,28 @@ def unzip_file(self, zip_filepath, unzip_to_dir):
                 outfile.close()
         return unzip_to_dir
 
+    @staticmethod
+    def depress(zip_file, target_dir):
+        """
+        解压到指定目录下, 保留可执行文件权限（仅限UNIX系统创建的文件）
+        :param zip_file: 压缩文件
+        :param target_dir: 解压后的目标目录
+        :return:
+        """
+        ZIP_UNIX_SYSTEM = 3
+
+        def extract_all_with_permission(zf, target_dir):
+            for info in zf.infolist():
+                extracted_path = zf.extract(info, target_dir)
+
+                if info.create_system == ZIP_UNIX_SYSTEM:
+                    unix_attributes = info.external_attr >> 16
+                    if unix_attributes:
+                        os.chmod(extracted_path, unix_attributes)
+
+        with zipfile.ZipFile(zip_file, 'r') as zf:
+            extract_all_with_permission(zf, target_dir)
+
+
 if __name__ == '__main__':
     pass

client/task/basic/zipmgr.py → client/util/zipmgr.py

@@ -120,7 +120,7 @@ def decompress_by_7z(self, zip_file, path, print_enable=False):
     def decompress(self, zip_file, path):
         logger.info("zip模块执行解压操作...")
         # 20190927 bug-fixed, 防止在当前目录下删除当前目录，出现权限异常情况
-        os.chdir("..")
+        os.chdir("../task")
         if os.path.exists(path):
             PathMgr().rmpath(path)
 
@@ -142,7 +142,7 @@ def zipfile_compress(path, zip_file):
         zp = zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED)
         for dirpath, dirnames, filenames in os.walk(path):
             fpath = dirpath.replace(path, "")  # 这一句很重要，不replace的话，就从根目录开始复制
-            fpath = fpath and fpath + os.sep or ""  # 这句话理解我也点郁闷，实现当前文件夹以及包含的所有文件的压缩
+            fpath = fpath and fpath + os.sep or ""
             for filename in filenames:
                 zp.write(os.path.join(dirpath, filename), fpath + filename)
         zp.close()