Merge pull request #83 from fosslight/fnmatch_new

Apply fnmatch / re match to match file name

requirements.txt

@@ -5,4 +5,4 @@ reuse
 PyYAML
 xlrd==1.2.0
 XlsxWriter==1.2.9
-fosslight_util>=1.4.0
+fosslight_util>=1.4.2

src/fosslight_reuse/_add.py

@@ -419,7 +419,7 @@ def add_content(target_path="", input_license="", input_copyright="", output_pat
         all_files_list = get_allfiles_list(path_to_find)
 
         # Get missing license / copyright file list
-        missing_license, missing_copyright, _, project, _, _, _, _ = reuse_for_project(path_to_find, need_log_file)
+        missing_license, missing_copyright, _, project, _ = reuse_for_project(path_to_find, need_log_file)
 
         # Print Skipped Files
         missing_license_filtered, missing_copyright_filtered = \

src/fosslight_reuse/_constant.py

@@ -5,9 +5,9 @@
 
 DEFAULT_EXCLUDE_EXTENSION = ["jar", "png", "exe", "so", "a", "dll", "jpeg", "jpg", "ttf", "lib", "ttc", "pfb",
                              "pfm", "otf", "afm", "dfont", "json"]
-OSS_PKG_INFO_FILES = ["oss-pkg-info.yaml", "oss-pkg-info.yml", r"oss-package*.info", "requirement.txt",
+OSS_PKG_INFO_FILES = [r"oss-pkg-info[\s\S]*.yaml", "oss-pkg-info.yml", r"oss-package[\s\S]*.info", "requirement.txt",
                       "requirements.txt", "package.json", "pom.xml", "build.gradle", "podfile.lock", "cartfile.resolved",
-                      "pubspec.yaml", "package.resolved", "go.mod", r"fosslight-sbom-info*.yaml"]
+                      "pubspec.yaml", "package.resolved", "go.mod", r"fosslight-sbom-info[\s\S]*.yaml"]
 HTML_RESULT_EXPAND_LIMIT = 10
 HTML_RESULT_PRINT_LIMIT = 100
 

src/fosslight_reuse/_fosslight_reuse.py

@@ -13,7 +13,6 @@
 import fosslight_util.constant as constant
 from fosslight_util.set_log import init_log
 from fosslight_util.timer_thread import TimerThread
-from fosslight_util.parsing_yaml import parsing_yml, find_all_oss_pkg_files
 from reuse import report
 from reuse.project import Project
 from reuse.report import ProjectReport
@@ -149,65 +148,9 @@ def reuse_for_files(path, files):
     return missing_license_list, missing_copyright_list, prj
 
 
-def extract_files_in_path(repository, filepath):
-    extract_files = []
-    for path in filepath:
-        if not repository.endswith("/"):
-            repository += "/"
-        if os.path.isdir(path):
-            files = os.listdir(path)
-            files = [os.path.join(path, file) for file in files]
-            files = [sub.replace(repository, '') for sub in files]
-            extract_files.extend(files)
-        elif os.path.isfile(path):
-            path = path.replace(repository, '')
-            extract_files.append(path)
-    return extract_files
-
-
-def get_rel_path_in_yaml(oss_item):
-    for source_name_or_path in oss_item.source_name_or_path:
-        yield os.path.join(oss_item.relative_path, source_name_or_path)
-
-
-# TODO : Get all file list in 'exclude_filepath' by using Regular expression
-def get_excluded_file_in_yaml(repository, yaml_files):
-    excluded_path = []
-    excluded_list = []
-    lic_present_path = []
-    lic_present_list = []
-    cop_present_path = []
-    cop_present_list = []
-
-    for file in yaml_files:
-        oss_items, _ = parsing_yml(file, repository)
-        for oss_item in oss_items:
-            if oss_item.exclude:
-                excluded_path = get_rel_path_in_yaml(oss_item)
-                excluded_list.extend(extract_files_in_path(repository, excluded_path))
-            if oss_item.license:
-                lic_present_path = get_rel_path_in_yaml(oss_item)
-                lic_present_list.extend(extract_files_in_path(repository, lic_present_path))
-            if oss_item.copyright:
-                cop_present_path = get_rel_path_in_yaml(oss_item)
-                cop_present_list.extend(extract_files_in_path(repository, cop_present_path))
-    return excluded_list, lic_present_list, cop_present_list
-
-
-def get_only_pkg_info_yaml_file(pkg_info_files):
-    for yaml in pkg_info_files:
-        if yaml.split('/')[-1].startswith("oss-pkg-info"):
-            yield yaml
-
-
 def reuse_for_project(path_to_find, need_log_file):
     missing_license = []
     missing_copyright = []
-    pkg_info_yaml_files = []
-    excluded_files = []
-    lic_present_files_in_yaml = []
-    cop_present_files_in_yaml = []
-    yaml_file = []
 
     oss_pkg_info_files = find_oss_pkg_info(path_to_find)
     if _turn_on_default_reuse_config:
@@ -226,13 +169,6 @@ def reuse_for_project(path_to_find, need_log_file):
         if need_log_file:
             timer.stop = True
 
-        if oss_pkg_info_files:
-            pkg_info_yaml_files = find_all_oss_pkg_files(path_to_find, oss_pkg_info_files)
-            yaml_file = get_only_pkg_info_yaml_file(pkg_info_yaml_files)
-
-            # Get path to be excluded in yaml file
-            excluded_files, lic_present_files_in_yaml, cop_present_files_in_yaml = get_excluded_file_in_yaml(path_to_find, yaml_file)
-
         # File list that missing license text
         missing_license = [str(sub) for sub in set(report.files_without_licenses)]
         if not path_to_find.endswith("/"):
@@ -249,8 +185,7 @@ def reuse_for_project(path_to_find, need_log_file):
 
     if _turn_on_default_reuse_config:
         remove_reuse_dep5_file(need_rollback, temp_file_name, temp_dir_name)
-    return missing_license, missing_copyright, oss_pkg_info_files, project, \
-        report, excluded_files, lic_present_files_in_yaml, cop_present_files_in_yaml
+    return missing_license, missing_copyright, oss_pkg_info_files, project, report
 
 
 def dump_error_msg(error_msg: str, exit=False):
@@ -320,29 +255,23 @@ def run_lint(target_path, disable, output_file_name, format='', need_log_file=Tr
     init(path_to_find, output_path, file_to_check_list, need_log_file)
 
     if os.path.isdir(path_to_find):
-        lic_present_files_in_yaml = []
-        cop_present_files_in_yaml = []
-        excluded_files = []
         oss_pkg_info = []
         _turn_on_default_reuse_config = not disable
 
         if _check_only_file_mode:
             license_missing_files, copyright_missing_files, project = reuse_for_files(path_to_find, file_to_check_list)
         else:
-            license_missing_files, copyright_missing_files, oss_pkg_info, project, \
-                report, excluded_files, lic_present_files_in_yaml, cop_present_files_in_yaml = reuse_for_project(path_to_find, need_log_file)
+            license_missing_files, copyright_missing_files, oss_pkg_info, project, report = reuse_for_project(path_to_find, need_log_file)
 
-        result_item = result_for_summary(oss_pkg_info,
+        result_item = result_for_summary(path_to_find,
+                                         oss_pkg_info,
                                          license_missing_files,
                                          copyright_missing_files,
                                          report,
                                          _result_log,
                                          _check_only_file_mode,
                                          file_to_check_list,
-                                         error_items,
-                                         excluded_files,
-                                         lic_present_files_in_yaml,
-                                         cop_present_files_in_yaml)
+                                         error_items)
 
         success, exit_code = write_result_file(result_file, output_extension, _exit_code,
                                                result_item, _result_log, project, path_to_find)

src/fosslight_reuse/_result.py

@@ -6,13 +6,16 @@
 import io
 import sys
 import yaml
+import fnmatch
 import xml.etree.ElementTree as ET
 import logging
 import fosslight_util.constant as constant
 from pathlib import Path
 from reuse.project import Project
 from fosslight_reuse._result_html import result_for_html
+from fosslight_util.parsing_yaml import find_all_oss_pkg_files, parsing_yml
 from fosslight_util.output_format import check_output_format
+import re
 
 
 CUSTOMIZED_FORMAT_FOR_REUSE = {'html': '.html', 'xml': '.xml', 'yaml': '.yaml'}
@@ -212,9 +215,63 @@ def create_result_file(output_file_name, format='', _start_time=""):
     return result_file, output_path, output_extension
 
 
-def result_for_summary(oss_pkg_info_files, license_missing_files, copyright_missing_files,
-                       prj_report, _result_log, _check_only_file_mode, file_to_check_list, error_items,
-                       excluded_files, lic_present_files_in_yaml, cop_present_files_in_yaml):
+def get_only_pkg_info_yaml_file(pkg_info_files):
+    for yaml_file in pkg_info_files:
+        try:
+            if re.search(r"oss-pkg-info[\s\S]*.yaml", os.path.basename(yaml_file).lower()):
+                yield yaml_file
+        except Exception as ex:
+            logger.debug(f"Error to find oss-pkg-info*.yaml: {ex}")
+
+
+def get_path_in_yaml(oss_item):
+    return [os.path.join(oss_item.relative_path, file) for file in oss_item.source_name_or_path]
+
+
+def extract_files_in_path(remove_file_list, base_file_list, return_found=False):
+    extract_files = []
+    remained_file_to_remove = []
+    remained_base_files = []
+
+    if base_file_list:
+        intersection_files = list(set(base_file_list) & set(remove_file_list))
+        extract_files.extend(intersection_files)
+        remained_base_files = list(set(base_file_list) - set(intersection_files))
+        remained_file_to_remove = list(set(remove_file_list) - set(intersection_files))
+
+    for remove_pattern in remained_file_to_remove:
+        for file in remained_base_files[:]:
+            if fnmatch.fnmatch(file, remove_pattern) or re.search(remove_pattern, file):
+                extract_files.append(file)
+                remained_base_files.remove(file)
+    return extract_files if return_found else remained_base_files
+
+
+def exclude_file_in_yaml(path_to_find, yaml_files, license_missing_files, copyright_missing_files):
+    excluded_path = []
+    lic_present_path = []
+    cop_present_path = []
+    for file in yaml_files:
+        oss_items, _ = parsing_yml(file, path_to_find)
+        for oss_item in oss_items:
+            if oss_item.exclude:
+                excluded_path.extend(get_path_in_yaml(oss_item))
+            else:
+                if oss_item.license:
+                    lic_present_path.extend(get_path_in_yaml(oss_item))
+                if oss_item.copyright:
+                    cop_present_path.extend(get_path_in_yaml(oss_item))
+
+    total_missing_files = list(license_missing_files.union(copyright_missing_files))
+    files_with_exclude_removed = extract_files_in_path(excluded_path, total_missing_files, True)
+    license_missing_files = extract_files_in_path(lic_present_path, list(license_missing_files - set(files_with_exclude_removed)))
+    copyright_missing_files = extract_files_in_path(cop_present_path, list(copyright_missing_files - set(files_with_exclude_removed)))
+
+    return license_missing_files, copyright_missing_files
+
+
+def result_for_summary(path_to_find, oss_pkg_info_files, license_missing_files, copyright_missing_files,
+                       prj_report, _result_log, _check_only_file_mode, file_to_check_list, error_items):
     reuse_compliant = False
     detected_lic = []
     missing_both_files = []
@@ -228,15 +285,13 @@ def result_for_summary(oss_pkg_info_files, license_missing_files, copyright_miss
         for i, lic in enumerate(sorted(prj_report.used_licenses)):
             detected_lic.append(lic)
 
-    # Subtract license or copyright presenting file and excluded file
-    license_missing_files = list(set(license_missing_files) -
-                                 set(lic_present_files_in_yaml) -
-                                 set(excluded_files) -
-                                 set(oss_pkg_info_files))
-    copyright_missing_files = list(set(copyright_missing_files) -
-                                   set(cop_present_files_in_yaml) -
-                                   set(excluded_files) -
-                                   set(oss_pkg_info_files))
+    if oss_pkg_info_files:
+        pkg_info_yaml_files = find_all_oss_pkg_files(path_to_find, oss_pkg_info_files)
+        yaml_file = get_only_pkg_info_yaml_file(pkg_info_yaml_files)
+        # Exclude files in yaml
+        license_missing_files, copyright_missing_files = exclude_file_in_yaml(path_to_find, yaml_file,
+                                                                              set(license_missing_files) - set(oss_pkg_info_files),
+                                                                              set(copyright_missing_files) - set(oss_pkg_info_files))
 
     if len(license_missing_files) == 0 and len(copyright_missing_files) == 0:
         reuse_compliant = True