Change the logic to exclude files

src/fosslight_reuse/_constant.py

@@ -5,9 +5,9 @@
 
 DEFAULT_EXCLUDE_EXTENSION = ["jar", "png", "exe", "so", "a", "dll", "jpeg", "jpg", "ttf", "lib", "ttc", "pfb",
                              "pfm", "otf", "afm", "dfont", "json"]
-OSS_PKG_INFO_FILES = ["oss-pkg-info.yaml", "oss-pkg-info.yml", r"oss-package*.info", "requirement.txt",
+OSS_PKG_INFO_FILES = [r"oss-pkg-info[\s\S]*.yaml", "oss-pkg-info.yml", r"oss-package[\s\S]*.info", "requirement.txt",
                       "requirements.txt", "package.json", "pom.xml", "build.gradle", "podfile.lock", "cartfile.resolved",
-                      "pubspec.yaml", "package.resolved", "go.mod", r"fosslight-sbom-info*.yaml"]
+                      "pubspec.yaml", "package.resolved", "go.mod", r"fosslight-sbom-info[\s\S]*.yaml"]
 HTML_RESULT_EXPAND_LIMIT = 10
 HTML_RESULT_PRINT_LIMIT = 100
 

src/fosslight_reuse/_result.py

@@ -15,6 +15,7 @@
 from fosslight_reuse._result_html import result_for_html
 from fosslight_util.parsing_yaml import find_all_oss_pkg_files, parsing_yml
 from fosslight_util.output_format import check_output_format
+import re
 
 
 CUSTOMIZED_FORMAT_FOR_REUSE = {'html': '.html', 'xml': '.xml', 'yaml': '.yaml'}
@@ -216,53 +217,57 @@ def create_result_file(output_file_name, format='', _start_time=""):
 
 def get_only_pkg_info_yaml_file(pkg_info_files):
     for yaml_file in pkg_info_files:
-        if yaml_file.split('/')[-1].startswith("oss-pkg-info"):
-            yield yaml_file
+        try:
+            if re.search(r"oss-pkg-info[\s\S]*.yaml", os.path.basename(yaml_file).lower()):
+                yield yaml_file
+        except Exception as ex:
+            logger.debug(f"Error to find oss-pkg-info*.yaml: {ex}")
 
 
 def get_path_in_yaml(oss_item):
-    for source_name_or_path in oss_item.source_name_or_path:
-        yield os.path.join(oss_item.relative_path, source_name_or_path)
+    return [os.path.join(oss_item.relative_path, file) for file in oss_item.source_name_or_path]
 
 
-def extract_files_in_path(filepath, input_list=None):
+def extract_files_in_path(remove_file_list, base_file_list, return_found=False):
     extract_files = []
-    remained_files = []
-
-    if input_list is None:
-        extract_files.extend(filepath)
-    else:
-        extract_files.extend(list(set(input_list) & set(filepath)))
-        remained_files = list(set(input_list) - set(filepath))
-
-    for file in remained_files:
-        for path in filepath:
-            if fnmatch.fnmatch(file, path):
+    remained_file_to_remove = []
+    remained_base_files = []
+
+    if base_file_list:
+        intersection_files = list(set(base_file_list) & set(remove_file_list))
+        extract_files.extend(intersection_files)
+        remained_base_files = list(set(base_file_list) - set(intersection_files))
+        remained_file_to_remove = list(set(remove_file_list) - set(intersection_files))
+
+    for remove_pattern in remained_file_to_remove:
+        for file in remained_base_files[:]:
+            if fnmatch.fnmatch(file, remove_pattern) or re.search(remove_pattern, file):
                 extract_files.append(file)
-    return extract_files
+                remained_base_files.remove(file)
+    return extract_files if return_found else remained_base_files
 
 
-def get_excluded_file_in_yaml(repository, yaml_files, license_missing_files, copyright_missing_files):
+def exclude_file_in_yaml(path_to_find, yaml_files, license_missing_files, copyright_missing_files):
     excluded_path = []
-    excluded_list = []
     lic_present_path = []
-    lic_present_list = []
     cop_present_path = []
-    cop_present_list = []
-
     for file in yaml_files:
-        oss_items, _ = parsing_yml(file, repository)
+        oss_items, _ = parsing_yml(file, path_to_find)
         for oss_item in oss_items:
             if oss_item.exclude:
-                excluded_path = list(get_path_in_yaml(oss_item))
-                excluded_list.extend(extract_files_in_path(excluded_path))
-            if oss_item.license:
-                lic_present_path = list(get_path_in_yaml(oss_item))
-                lic_present_list.extend(extract_files_in_path(lic_present_path, license_missing_files))
-            if oss_item.copyright:
-                cop_present_path = list(get_path_in_yaml(oss_item))
-                cop_present_list.extend(extract_files_in_path(cop_present_path, copyright_missing_files))
-    return excluded_list, lic_present_list, cop_present_list
+                excluded_path.extend(get_path_in_yaml(oss_item))
+            else:
+                if oss_item.license:
+                    lic_present_path.extend(get_path_in_yaml(oss_item))
+                if oss_item.copyright:
+                    cop_present_path.extend(get_path_in_yaml(oss_item))
+
+    total_missing_files = list(license_missing_files.union(copyright_missing_files))
+    files_with_exclude_removed = extract_files_in_path(excluded_path, total_missing_files, True)
+    license_missing_files = extract_files_in_path(lic_present_path, list(license_missing_files - set(files_with_exclude_removed)))
+    copyright_missing_files = extract_files_in_path(cop_present_path, list(copyright_missing_files - set(files_with_exclude_removed)))
+
+    return license_missing_files, copyright_missing_files
 
 
 def result_for_summary(path_to_find, oss_pkg_info_files, license_missing_files, copyright_missing_files,
@@ -283,20 +288,10 @@ def result_for_summary(path_to_find, oss_pkg_info_files, license_missing_files,
     if oss_pkg_info_files:
         pkg_info_yaml_files = find_all_oss_pkg_files(path_to_find, oss_pkg_info_files)
         yaml_file = get_only_pkg_info_yaml_file(pkg_info_yaml_files)
-        # Get path to be excluded in yaml file
-        excluded_files, lic_present_files_in_yaml, cop_present_files_in_yaml = get_excluded_file_in_yaml(path_to_find, yaml_file,
-                                                                                                         license_missing_files,
-                                                                                                         copyright_missing_files)
-
-    # Subtract license or copyright presenting file and excluded file
-    license_missing_files = list(set(license_missing_files) -
-                                 set(lic_present_files_in_yaml) -
-                                 set(excluded_files) -
-                                 set(oss_pkg_info_files))
-    copyright_missing_files = list(set(copyright_missing_files) -
-                                   set(cop_present_files_in_yaml) -
-                                   set(excluded_files) -
-                                   set(oss_pkg_info_files))
+        # Exclude files in yaml
+        license_missing_files, copyright_missing_files = exclude_file_in_yaml(path_to_find, yaml_file,
+                                                                              set(license_missing_files) - set(oss_pkg_info_files),
+                                                                              set(copyright_missing_files) - set(oss_pkg_info_files))
 
     if len(license_missing_files) == 0 and len(copyright_missing_files) == 0:
         reuse_compliant = True