Add a bit of code

deep_utils/utils/dir_utils/dir_utils.py

@@ -258,7 +258,7 @@ def remove_create(dir_: str, remove=True, logger=None, verbose=1):
     raise ValueError("dir_ should be provided!")
 
 
-def mkdir_incremental(dir_path: str, base_name="exp", fix_name=None, overwrite=False) -> Path:
+def mkdir_incremental(dir_path: str | list[str], base_name="exp", fix_name=None, overwrite=False) -> Path:
     """
     makes new directories, if it exists increment it and makes another one. Good for hyperparameter tuning!
     Args:
@@ -279,7 +279,7 @@ def mkdir_incremental(dir_path: str, base_name="exp", fix_name=None, overwrite=F
         os.makedirs(final_path, exist_ok=True)
     else:
         folders = []
-        for dir_ in os.listdir(dir_path):
+        for dir_ in (os.listdir(dir_path) if isinstance(dir_path, str) else dir_path):
             if base_name in dir_:
                 counter = dir_.split(base_name + "_")[-1]
                 if counter.isdigit():
@@ -295,7 +295,7 @@ def mkdir_incremental(dir_path: str, base_name="exp", fix_name=None, overwrite=F
     return Path(final_path)
 
 
-def file_incremental(file_path, artifact_type="prefix", artifact_value=0, extra_punctuation="_",
+def file_incremental(file_path: str, artifact_type="prefix", artifact_value=0, extra_punctuation="_",
                      add_artifact_value=False):
     """
     This function is used to increment a file's address with prefix or suffix values until it becomes unique
@@ -306,18 +306,22 @@ def file_incremental(file_path, artifact_type="prefix", artifact_value=0, extra_
     :param add_artifact_value: If set to True, adds the artifact_value then checks its existence.
     :return:
     """
-    dir_, n = os.path.split(file_path)
+    dir_, filename = os.path.split(file_path)
     artifact_value = int(artifact_value)
     while True:
         if add_artifact_value:
             # Maybe someone requires their file to have the first artifact
-            file_path = join(dir_, split_extension(n, artifact_type=artifact_type, artifact_value=artifact_value,
+            file_path = join(dir_, split_extension(filename,
+                                                   artifact_type=artifact_type,
+                                                   artifact_value=artifact_value,
                                                    extra_punctuation=extra_punctuation))
         if not os.path.exists(file_path):
             break
 
         if not add_artifact_value:
-            file_path = join(dir_, split_extension(n, artifact_type=artifact_type, artifact_value=artifact_value,
+            file_path = join(dir_, split_extension(filename,
+                                                   artifact_type=artifact_type,
+                                                   artifact_value=artifact_value,
                                                    extra_punctuation=extra_punctuation))
         artifact_value += 1
     return file_path
@@ -782,7 +786,7 @@ def get_filename(file_path: str, remove_extension: Union[bool, str] = False) ->
     @staticmethod
     def crawl_directory_dataset(
             dir_: str,
-            ext_filter: list | str= None,
+            ext_filter: list | str = None,
             map_labels=False,
             label_map_dict: dict = None,
             logger=None,
@@ -911,3 +915,41 @@ def endswith(filepath: str, ext: list[str] | str):
             return False
         else:
             return filepath.endswith(ext)
+
+    @staticmethod
+    def file_incremental(file_path: str | None, artifact_type="prefix", artifact_value=0, extra_punctuation="_",
+                         add_artifact_value=False, dir_items: list[str] | None = None):
+        """
+        This function is used to increment a file's address with prefix or suffix values until it becomes unique
+        :param file_path:
+        :param artifact_type:
+        :param artifact_value:
+        :param extra_punctuation:
+        :param add_artifact_value: If set to True, adds the artifact_value then checks its existence.
+        :param dir_items: list of items
+        :return:
+        """
+        dir_, filename = os.path.split(file_path)
+        artifact_value = int(artifact_value)
+        while True:
+            if add_artifact_value:
+                # Maybe someone requires their file to have the first artifact
+
+                file_path = split_extension(filename,
+                                            artifact_type=artifact_type,
+                                            artifact_value=artifact_value,
+                                            extra_punctuation=extra_punctuation)
+                if not dir_items:
+                    file_path = join(dir_, file_path)
+            if (dir_items and not (file_path in dir_items)) or (not dir_items and not os.path.exists(file_path)):
+                break
+
+            if not add_artifact_value:
+                file_path = split_extension(filename,
+                                            artifact_type=artifact_type,
+                                            artifact_value=artifact_value,
+                                            extra_punctuation=extra_punctuation)
+                if not dir_items:
+                    file_path = join(dir_, file_path)
+            artifact_value += 1
+        return file_path

deep_utils/utils/minio_lib/main.py

@@ -2,6 +2,7 @@
 from typing import Union, Dict
 from deep_utils.utils.logging_utils.logging_utils import log_print, value_error_log
 import minio
+from os.path import split
 
 
 class MinIOUtils:
@@ -174,7 +175,7 @@ def make_bucket_public(client, bucket_name, logger=None, verbose=1):
 
     @staticmethod
     def exists(client: minio.Minio, bucket_name: str, object_name: str):
-        from os.path import split
+
         prefix, name = split(object_name)
         if prefix:
             prefix = prefix + "/"
@@ -186,3 +187,20 @@ def exists(client: minio.Minio, bucket_name: str, object_name: str):
             return True
         else:
             return False
+
+    @staticmethod
+    def list(client: minio.Minio, bucket_name: str, object_name: str = "", directory: str = "") -> list[str]:
+        if directory:
+            prefix = directory
+        elif object_name:
+            prefix, name = split(object_name)
+        else:
+            raise ValueError("object_name or directory should be provided!")
+
+        if prefix:
+            prefix = (prefix + "/") if not prefix.endswith("/") else prefix
+            list_of_objects = [item._object_name.replace(prefix, "") for item in
+                               client.list_objects(bucket_name, prefix=prefix, recursive=True)]
+        else:
+            list_of_objects = [item._object_name for item in client.list_objects(bucket_name, recursive=True)]
+        return list_of_objects