Merge pull request #156 from SeongjunJo/exclude_path

Supports for excluding paths in lint mode

requirements.txt

@@ -3,5 +3,5 @@ binaryornot
 requests
 reuse==1.1.2
 PyYAML
-fosslight_util>=1.4.23
+fosslight_util>=1.4.43
 jinja2

src/fosslight_prechecker/_help.py

@@ -10,6 +10,7 @@
 
     Usage: fosslight_prechecker [Mode] [option1] <arg1> [option2] <arg2>...
      ex) fosslight_prechecker lint -p /home/test/src/
+         fosslight_prechecker lint -p /home/test/src/ -e test.py dep/temp
          fosslight_prechecker add -p /home/test/test.py -c "2019-2021 LG Electronics Inc." -l "GPL-3.0-only"
          fosslight_prechecker convert -p /home/test/sbom_info.yaml
 
@@ -23,6 +24,7 @@
             -h\t\t\t    Print help message
             -v\t\t\t    Print FOSSLight Prechecker version
             -p <path>\t\t    Path to check(Default: current directory)
+            -e <path>\t\t    Path to exclude from checking(only work with 'lint' mode)
             -f <format>\t\t    Result format(yaml, xml, html)
             -o <file_name>\t    Output file name
             -i\t\t\t    Don't both write log file and show progress bar

src/fosslight_prechecker/_precheck.py

@@ -25,6 +25,7 @@
 is_windows = platform.system() == 'Windows'
 REUSE_CONFIG_FILE = ".reuse/dep5"
 DEFAULT_EXCLUDE_EXTENSION_FILES = []  # Exclude files from reuse
+user_exclude_list = []  # Exclude paths from checking
 _turn_on_exclude_config = True
 _check_only_file_mode = False
 error_items = []
@@ -86,7 +87,7 @@ def exclude_git_related_files(path):
         logger.warning(f"Error to get git related files : {ex}")
 
 
-def find_oss_pkg_info_and_exlcude_file(path):
+def find_oss_pkg_info_and_exclude_file(path):
     global DEFAULT_EXCLUDE_EXTENSION_FILES
     oss_pkg_info = []
     git_present = shutil.which("git")
@@ -96,8 +97,14 @@ def find_oss_pkg_info_and_exlcude_file(path):
 
     try:
         for root, dirs, files in os.walk(path):
+            if os.path.abspath(root) in user_exclude_list:
+                continue
             for dir in dirs:
                 # For hidden folders
+                dir_path = os.path.join(root, dir)
+                dir_abs_path = os.path.abspath(dir_path)
+                if any(os.path.commonpath([dir_abs_path, exclude_path]) == exclude_path for exclude_path in user_exclude_list):
+                    continue
                 if dir.startswith("."):
                     all_files_in_dir = [os.path.join(root, dir, file)
                                         for file in os.listdir(os.path.join(root, dir))
@@ -106,17 +113,20 @@ def find_oss_pkg_info_and_exlcude_file(path):
                     DEFAULT_EXCLUDE_EXTENSION_FILES.extend(all_files_rel_path)
 
             for file in files:
+                file_path = os.path.join(root, file)
+                file_abs_path = os.path.abspath(file_path)
+                if any(os.path.commonpath([file_abs_path, exclude_path]) == exclude_path for exclude_path in user_exclude_list):
+                    continue
                 file_lower_case = file.lower()
-                file_abs_path = os.path.join(root, file)
-                file_rel_path = os.path.relpath(file_abs_path, path)
+                file_rel_path = os.path.relpath(file_path, path)
 
                 if any(re.search(re_oss_pkg_pattern, file_lower_case) for re_oss_pkg_pattern in OSS_PKG_INFO_FILES) \
                    or file_lower_case.startswith("module_license_"):
                     oss_pkg_info.append(file_rel_path)
                 # Exclude hidden files
                 elif _turn_on_exclude_config and file.startswith('.'):
                     DEFAULT_EXCLUDE_EXTENSION_FILES.append(file_rel_path)
-                elif is_binary(file_abs_path):
+                elif is_binary(file_path):
                     DEFAULT_EXCLUDE_EXTENSION_FILES.append(file_rel_path)
                 else:
                     extension = file_lower_case.split(".")[-1]
@@ -232,7 +242,7 @@ def precheck_for_project(path_to_find):
     missing_license = []
     missing_copyright = []
 
-    oss_pkg_info_files = find_oss_pkg_info_and_exlcude_file(path_to_find)
+    oss_pkg_info_files = find_oss_pkg_info_and_exclude_file(path_to_find)
     if _turn_on_exclude_config:
         need_rollback, temp_file_name, temp_dir_name = create_reuse_dep5_file(path_to_find)
 
@@ -244,13 +254,16 @@ def precheck_for_project(path_to_find):
         missing_license = [str(sub) for sub in set(report.files_without_licenses)]
         if not path_to_find.endswith(f"{os.sep}"):
             path_to_find += f"{os.sep}"
+        missing_license = filter_missing_list(missing_license)
         missing_license = [sub.replace(path_to_find, '', 1) for sub in missing_license]
 
         # File list that missing copyright text
         missing_copyright = [str(sub) for sub in set(report.files_without_copyright)]
         if not path_to_find.endswith(f"{os.sep}"):
             path_to_find += f"{os.sep}"
+        missing_copyright = filter_missing_list(missing_copyright)
         missing_copyright = [sub.replace(path_to_find, '', 1) for sub in missing_copyright]
+
     except Exception as ex:
         dump_error_msg(f"Error prechecker lint: {ex}", True)
 
@@ -259,6 +272,15 @@ def precheck_for_project(path_to_find):
     return missing_license, missing_copyright, oss_pkg_info_files, project, report
 
 
+def filter_missing_list(missing_list):
+    filtered_list = []
+    for file in missing_list:
+        abs_path = os.path.abspath(file)
+        if not any(os.path.commonpath([abs_path, path]) == path for path in user_exclude_list):
+            filtered_list.append(file)
+    return filtered_list
+
+
 def dump_error_msg(error_msg: str, exit=False):
     global error_items
     error_items.append(error_msg)
@@ -267,13 +289,14 @@ def dump_error_msg(error_msg: str, exit=False):
         sys.exit(1)
 
 
-def init(path_to_find, output_path, file_list, need_log_file=True):
+def init(path_to_find, output_path, file_list, need_log_file=True, exclude_path=[]):
     global logger, _result_log
+
     if file_list:
         _result_log["File list to check"] = file_list
         path_to_find = file_list
     logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_pre_{_start_time}.txt"),
-                                   need_log_file, logging.INFO, logging.DEBUG, PKG_NAME, path_to_find)
+                                   need_log_file, logging.INFO, logging.DEBUG, PKG_NAME, path_to_find, exclude_path)
 
 
 def get_path_to_find(target_path, _check_only_file_mode):
@@ -305,7 +328,15 @@ def get_path_to_find(target_path, _check_only_file_mode):
     return path_to_find, file_to_check_list, _check_only_file_mode
 
 
-def run_lint(target_path, disable, output_file_name, format='', need_log_file=True):
+def set_exclude_list(path_to_find, exclude_path):
+    global user_exclude_list
+
+    for path in exclude_path:
+        if path.strip != "":
+            user_exclude_list.append(os.path.abspath(os.path.join(path_to_find, path)))
+
+
+def run_lint(target_path, disable, output_file_name, format='', need_log_file=True, exclude_path=[]):
     global _turn_on_exclude_config, _check_only_file_mode, _start_time
 
     file_to_check_list = []
@@ -322,9 +353,10 @@ def run_lint(target_path, disable, output_file_name, format='', need_log_file=Tr
         dump_error_msg(f"Error - locale : {ex}")
 
     path_to_find, file_to_check_list, _check_only_file_mode = get_path_to_find(target_path, _check_only_file_mode)
+    set_exclude_list(path_to_find, exclude_path)
 
     result_file, output_path, output_extension = create_result_file(output_file_name, format, _start_time)
-    init(path_to_find, output_path, file_to_check_list, need_log_file)
+    init(path_to_find, output_path, file_to_check_list, need_log_file, exclude_path)
 
     if os.path.isdir(path_to_find):
         oss_pkg_info = []
@@ -353,7 +385,8 @@ def run_lint(target_path, disable, output_file_name, format='', need_log_file=Tr
                                          _check_only_file_mode,
                                          file_to_check_list,
                                          error_items,
-                                         DEFAULT_EXCLUDE_EXTENSION_FILES)
+                                         DEFAULT_EXCLUDE_EXTENSION_FILES,
+                                         user_exclude_list)
 
         success, exit_code = write_result_file(result_file, output_extension, _exit_code,
                                                result_item, _result_log, project, path_to_find)

src/fosslight_prechecker/_result.py

@@ -41,6 +41,7 @@ def __init__(self):
         self._files_without_cop = []
         self._os_info = ""
         self._path_to_analyze = ""
+        self._path_to_exclude = ""
         self._python_ver = ""
         self._fl_prechecker_ver = ""
         self._log_msg = ""
@@ -79,6 +80,8 @@ def get_print_yaml(self):
         result_tool_item = {}
         result_tool_item["OS"] = self._os_info
         result_tool_item["Analyze path"] = self._path_to_analyze
+        if self._path_to_exclude:
+            result_tool_item["Exclude path"] = self._path_to_exclude
         result_tool_item["Python version"] = self._python_ver
         result_tool_item["fosslight_prechecker version"] = self._fl_prechecker_ver
         result_item["Tool Info"] = result_tool_item
@@ -264,10 +267,12 @@ def exclude_file_in_yaml(path_to_find, yaml_files, license_missing_files, copyri
     return license_missing_files, copyright_missing_files, abnormal_yaml_files
 
 
-def get_total_file_list(path_to_find, prj_report, exclude_files):
+def get_total_file_list(path_to_find, prj_report, exclude_files, user_exclude_files=[]):
     if not path_to_find.endswith(f'{os.path.sep}'):
         path_to_find += f'{os.path.sep}'
-    total_files = [str(file_report.path).replace(path_to_find, '', 1) for file_report in prj_report.file_reports]
+    total_files = [str(file_report.path).replace(path_to_find, '', 1) for file_report in prj_report.file_reports
+                   if not any(os.path.commonpath([os.path.abspath(file_report.path), user_exclude_file]) == user_exclude_file
+                              for user_exclude_file in user_exclude_files)]
     total_files_excluded = list(set(total_files) - set(exclude_files))
     return total_files_excluded
 
@@ -292,8 +297,8 @@ def add_reason_to_file_name(oss_pkg_info_files, abnormal_yaml_files, path_to_fin
     return oss_pkg_info_files
 
 
-def result_for_summary(path_to_find, oss_pkg_info_files, license_missing_files, copyright_missing_files,
-                       prj_report, _result_log, _check_only_file_mode, file_to_check_list, error_items, exclude_files):
+def result_for_summary(path_to_find, oss_pkg_info_files, license_missing_files, copyright_missing_files, prj_report,
+                       _result_log, _check_only_file_mode, file_to_check_list, error_items, exclude_files, user_exclude_files):
     prechecker_compliant = False
     detected_lic = []
     missing_both_files = []
@@ -303,7 +308,7 @@ def result_for_summary(path_to_find, oss_pkg_info_files, license_missing_files,
     if _check_only_file_mode:
         file_total_num = len(file_to_check_list)
     else:
-        file_total_num = len(get_total_file_list(path_to_find, prj_report, exclude_files))
+        file_total_num = len(get_total_file_list(path_to_find, prj_report, exclude_files, user_exclude_files))
 
         # Get detected License
         for i, lic in enumerate(sorted(prj_report.used_licenses)):
@@ -345,6 +350,7 @@ def result_for_summary(path_to_find, oss_pkg_info_files, license_missing_files,
     result_item._files_without_cop = sorted(copyright_missing_files)
     result_item._fl_prechecker_ver = _result_log["Tool Info"]
     result_item._path_to_analyze = _result_log["Path to analyze"]
+    result_item._path_to_exclude = _result_log.get("Path to exclude", '')
     result_item._os_info = _result_log["OS"]
     result_item._python_ver = _result_log["Python version"]
     result_item._check_only_file_mode = _check_only_file_mode

src/fosslight_prechecker/cli.py

@@ -13,13 +13,13 @@
 from fosslight_prechecker._precheck import run_lint
 
 
-def run_main(mode, path, output, format, no_log, disable, copyright, license, dl_url, parser):
+def run_main(mode, path, output, format, no_log, disable, copyright, license, dl_url, parser, exclude_path):
     if mode != "add" and (copyright != "" or license != ""):
         parser.print_help()
         sys.exit(1)
 
     if mode == "lint":
-        run_lint(path, disable, output, format, no_log)
+        run_lint(path, disable, output, format, no_log, exclude_path)
     elif mode == "add":
         add_content(path, license, copyright, dl_url, output, no_log)
     elif mode == "convert":
@@ -41,6 +41,7 @@ def main():
     parser.add_argument('-l', '--license', help="License name to add(used in only 'add' mode)", type=str, dest='license', default="")
     parser.add_argument('-c', '--copyright', help="Copyright to add(used in only 'add' mode)", type=str, dest='copyright', default="")
     parser.add_argument('-u', '--dlurl', help="Download URL to add(used in only 'add' mode)", type=str, dest='dlurl', default="")
+    parser.add_argument('-e', '--exclude', help='Path to exclude from checking', nargs='*', dest='exclude_path', default=[])
     parser.add_argument('--notice', help="Show OSS notice", action='store_true', required=False)
     try:
         args = parser.parse_args()
@@ -67,7 +68,7 @@ def main():
             print(f.read())
     else:
         run_main(args.mode, args.path, args.output, args.format,
-                 args.log, args.disable, args.copyright, args.license, args.dlurl, parser)
+                 args.log, args.disable, args.copyright, args.license, args.dlurl, parser, args.exclude_path)
 
 
 if __name__ == "__main__":

tox.ini

@@ -26,6 +26,7 @@ filterwarnings = ignore::DeprecationWarning
 commands = 
     fosslight_prechecker lint -p src -o "test_result\prechecker_result.yaml"
     fosslight_prechecker lint -p src -f yaml -o "test_result2\prechecker_result.yaml"
+    fosslight_prechecker lint -p tests -e convert add\test_no_license.py -o "test_result3\prechecker_result.yaml"
     fosslight_prechecker convert -p "tests\convert"
     cmd /c del /s/q tests\add_result
     cmd /c del /s/q tests\add\LICENSES
@@ -39,6 +40,7 @@ commands =
 commands =
     fosslight_prechecker lint -p src/ -o "test_result/prechecker_result.yaml"
     fosslight_prechecker lint -p src/ -f yaml -o "test_result2/prechecker_result.yaml"
+    fosslight_prechecker lint -p tests/ -e convert add/test_no_license.py -o "test_result3/prechecker_result.yaml"
     fosslight_prechecker convert -p "tests/convert"
     rm -rf tests/add_result
     rm -rf tests/add/LICENSES
@@ -56,6 +58,7 @@ commands =
     fosslight_prechecker -h
     fosslight_prechecker lint -p src/ -o "test_result/prechecker_result.yaml"
     fosslight_prechecker lint -p src/ -f yaml -o "test_result2/prechecker_result.yaml"
+    fosslight_prechecker lint -p tests -e convert add/test_no_license.py -o "test_result3/prechecker_result.yaml"
     fosslight_prechecker convert -p tests/convert
     cp -r tests/add tests/add_result
     fosslight_prechecker add -p tests/add_result -c "2019-2021 LG Electronics Inc." -l "GPL-3.0-only"