Fix bug with FBC YAML Operators

When an operator has catalog in yaml insted of json, some requests
(like add) fails on "duplicate package" as `merge_catalogs_dirs`
function generates FBC from hidden index.db created as json files
and then copy over its content to final location where are all yaml
files.

To prevent this issue this simple fix will walk through the config
directory and convert any YAML files into JSON to ensure that everything
uses the same format in the index image.

Refers to CLOUDDST-21432

iib/workers/tasks/fbc_utils.py

@@ -3,13 +3,17 @@
 import os
 import logging
 import shutil
+import json
+from pathlib import Path
 
+import ruamel.yaml
 from typing import Tuple
 from iib.exceptions import IIBError
 from iib.workers.config import get_worker_config
 from iib.common.tracing import instrument_tracing
 
 log = logging.getLogger(__name__)
+yaml = ruamel.yaml.YAML()
 
 
 def is_image_fbc(image: str) -> bool:
@@ -86,6 +90,7 @@ def merge_catalogs_dirs(src_config: str, dest_config: str):
             msg = f"config directory does not exist: {conf_dir}"
             log.error(msg)
             raise IIBError(msg)
+        enforce_json_config_dir(conf_dir)
 
     log.info("Merging config folders: %s to %s", src_config, dest_config)
     shutil.copytree(src_config, dest_config, dirs_exist_ok=True)
@@ -118,3 +123,24 @@ def extract_fbc_fragment(temp_dir: str, fbc_fragment: str) -> Tuple[str, str]:
         raise IIBError("More than 1 package is present in fbc_fragment %s", fbc_fragment)
 
     return fbc_fragment_path, operator_packages[0]
+
+
+def enforce_json_config_dir(config_dir: str) -> None:
+    """
+    Ensure the files from config dir are in JSON format.
+
+    It will walk recursively and convert any YAML files to the JSON format.
+
+    :param str config_dir: The config dir to walk recursively converting any YAML to JSON.
+    """
+    log.info("Enforcing JSON content on config_dir: %s", config_dir)
+    for dirpath, _, filenames in os.walk(config_dir):
+        for file in filenames:
+            in_file = os.path.join(dirpath, file)
+            if in_file.lower().endswith(".yaml"):
+                out_file = os.path.join(dirpath, f"{Path(in_file).stem}.json")
+                log.debug(f"Converting {in_file} to {out_file}.")
+                with open(in_file, 'r') as yaml_in, open(out_file, 'w') as json_out:
+                    data = yaml.load(yaml_in)
+                    json.dump(data, json_out)
+                os.remove(in_file)

tests/test_workers/test_tasks/test_fbc_utils.py

@@ -1,13 +1,23 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
+import json
 import os
 import tempfile
 from unittest import mock
 
 import pytest
+import ruamel.yaml
 
 from iib.exceptions import IIBError
 from iib.workers.config import get_worker_config
-from iib.workers.tasks.fbc_utils import is_image_fbc, merge_catalogs_dirs, extract_fbc_fragment
+from iib.workers.tasks.fbc_utils import (
+    is_image_fbc,
+    merge_catalogs_dirs,
+    enforce_json_config_dir,
+    extract_fbc_fragment,
+)
+
+
+yaml = ruamel.yaml.YAML()
 
 
 @pytest.mark.parametrize(
@@ -92,7 +102,8 @@ def test_is_image_fbc(mock_si, skopeo_output, is_fbc):
     assert is_image_fbc(image) is is_fbc
 
 
-def test_merge_catalogs_dirs(tmpdir):
+@mock.patch("iib.workers.tasks.fbc_utils.enforce_json_config_dir")
+def test_merge_catalogs_dirs(mock_enforce_json, tmpdir):
     source_dir = os.path.join(tmpdir, 'src')
     destination_dir = os.path.join(tmpdir, 'dst')
     os.makedirs(destination_dir, exist_ok=True)
@@ -104,6 +115,12 @@ def test_merge_catalogs_dirs(tmpdir):
         tempfile.NamedTemporaryFile(dir=operator_dir, delete=False)
 
     merge_catalogs_dirs(src_config=source_dir, dest_config=destination_dir)
+    mock_enforce_json.assert_has_calls(
+        [
+            mock.call(source_dir),
+            mock.call(destination_dir),
+        ]
+    )
 
     for r, d, f in os.walk(source_dir):
 
@@ -141,6 +158,23 @@ def test_merge_catalogs_dirs_raise(mock_isdir, mock_cpt, tmpdir):
     mock_cpt.not_called()
 
 
+def test_enforce_json_config_dir(tmpdir):
+    file_prefix = "test_file"
+    data = {"foo": "bar"}
+    test_file = os.path.join(tmpdir, f"{file_prefix}.yaml")
+    expected_file = os.path.join(tmpdir, f"{file_prefix}.json")
+    with open(test_file, 'w') as w:
+        yaml.dump(data, w)
+
+    enforce_json_config_dir(tmpdir)
+
+    assert os.path.isfile(expected_file)
+    assert not os.path.isfile(test_file)
+
+    with open(expected_file, 'r') as f:
+        assert json.load(f) == data
+
+
 @pytest.mark.parametrize('ldr_output', [['testoperator'], ['test1', 'test2'], []])
 @mock.patch('os.listdir')
 @mock.patch('iib.workers.tasks.build._copy_files_from_image')