From 5380cd7916691a7160e1a3c66889903a5b81d837 Mon Sep 17 00:00:00 2001 From: Julien Date: Fri, 25 Nov 2022 20:30:27 +0100 Subject: [PATCH 1/2] Make the remote git handler much more robust to fix #147 --- .../profiles/remote_git_handler.py | 121 ++++++++++++++++-- 1 file changed, 107 insertions(+), 14 deletions(-) diff --git a/qgis_deployment_toolbelt/profiles/remote_git_handler.py b/qgis_deployment_toolbelt/profiles/remote_git_handler.py index e31f6cc8..9ffa826d 100644 --- a/qgis_deployment_toolbelt/profiles/remote_git_handler.py +++ b/qgis_deployment_toolbelt/profiles/remote_git_handler.py @@ -16,10 +16,12 @@ # Standard library import logging from pathlib import Path +from shutil import rmtree from typing import Union # 3rd party from dulwich import porcelain +from dulwich.errors import GitProtocolError from dulwich.repo import Repo from giturlparse import GitUrlParsed from giturlparse import parse as git_parse @@ -46,19 +48,25 @@ def __init__(self, url: str, branch: str = None) -> None: raise ValueError(f"Invalid git URL: {url}") self.url = url self.branch = branch or self.url_parsed.branch - print(self.branch) @property def is_url_git_repository(self) -> bool: - """Flag if a repository is a git repository.""" + """Flag if a repository is a git repository. + + :return bool: True if the URL is a valid remote git repository. + """ return git_validate(self.url) @property def url_parsed(self) -> GitUrlParsed: + """Return URL parsed to extract git information. + + :return GitUrlParsed: parsed URL object + """ return git_parse(self.url) def download(self, local_path: Union[str, Path]) -> Repo: - """Just a wrapper around the specific logic of this handler. + """Generic wrapper around the specific logic of this handler. :param Union[str, Path] local_path: path to the local folder where to download :return Repo: the local repository object @@ -70,10 +78,12 @@ def is_local_path_git_repository(self, local_path: Union[str, Path]) -> bool: return Path(local_path / ".git").is_dir() def clone_or_pull(self, local_path: Union[str, Path]) -> Repo: - """Clone or pull remote repository to local path. - If this one doesn't exist, it's created. + """Clone or pull remote repository to local path. If this one doesn't exist, + it's created. If fetch or pull action fail, it removes the existing folder and + clone the remote again. :param Union[str, Path] local_path: path to the folder where to clone (or pull) + :return Repo: the local repository object """ # convert to path @@ -83,16 +93,38 @@ def clone_or_pull(self, local_path: Union[str, Path]) -> Repo: # clone if local_path.exists() and not self.is_local_path_git_repository(local_path): logger.info(f"Cloning repository {self.url} to {local_path}") - return porcelain.clone( - source=self.url, - target=str(local_path.resolve()), - branch=self.branch, - depth=5, - ) + try: + return self._clone(local_path=local_path) + except Exception as err: + logger.error( + f"Error cloning the remote repository{self.url} " + f"(branch {self.branch}) to {local_path}." + f"Trace: {err}." + ) + raise err elif local_path.exists() and self.is_local_path_git_repository(local_path): - logger.info(f"Pulling repository {self.url} to {local_path}") - porcelain.pull(str(local_path.resolve()), force=True) - return Repo(root=str(local_path.resolve())) + # FETCH + try: + self._fetch(local_path=local_path) + except GitProtocolError as error: + logger.error( + f"Error fetching {self.url} repository to " + f"{local_path.resolve()}. Trace: {error}." + "Trying to remove the local folder and cloning again..." + ) + rmtree(path=local_path, ignore_errors=True) + return self.clone_or_pull(local_path=local_path) + # PULL + try: + return self._pull(local_path=local_path) + except GitProtocolError as error: + logger.error( + f"Error fetching {self.url} repository to " + f"{local_path.resolve()}. Trace: {error}." + "Trying to remove the local folder and cloning again..." + ) + rmtree(path=local_path, ignore_errors=True) + return self.clone_or_pull(local_path=local_path) elif not local_path.exists(): logger.debug( f"Local path does not exists: {local_path.as_uri()}. " @@ -101,6 +133,67 @@ def clone_or_pull(self, local_path: Union[str, Path]) -> Repo: local_path.mkdir(parents=True, exist_ok=True) return self.clone_or_pull(local_path) + def _clone(self, local_path: Union[str, Path]) -> Repo: + """Clone the remote repository to local path. + + :param Union[str, Path] local_path: path to the folder where to clone + + :return Repo: the local repository object + """ + + # clone + if local_path.exists() and not self.is_local_path_git_repository(local_path): + logger.info(f"Cloning repository {self.url} to {local_path}") + with porcelain.open_repo_closing(str(local_path.resolve())) as local_repo: + repo = porcelain.clone( + source=self.url, + target=local_repo, + branch=self.branch, + depth=5, + ) + gobj = local_repo.get_object(repo.head()) + logger.debug( + f"Latest commit cloned: {gobj.sha().hexdigest()} by {gobj.author}" + f" at {gobj.commit_time}" + ) + return local_repo + + def _fetch(self, local_path: Union[str, Path]) -> Repo: + """Fetch the remote repository from the existing local repository. + + :param Union[str, Path] local_path: path to the folder where to fetch + + :return Repo: the local repository object + """ + with porcelain.open_repo_closing(str(local_path.resolve())) as local_repo: + logger.info( + f"Fetching repository {self.url} to {local_path}", + ) + porcelain.fetch( + repo=local_repo, + remote_location=self.url, + force=True, + prune=True, + depth=5, + ) + + def _pull(self, local_path: Union[str, Path]) -> Repo: + """Pull the remote repository from the existing local repository. + + :param Union[str, Path] local_path: path to the folder where to pull + + :return Repo: the local repository object + """ + with porcelain.open_repo_closing(str(local_path.resolve())) as local_repo: + logger.info(f"Pulling repository {self.url} to {local_path}") + porcelain.pull(repo=local_repo, remote_location=self.url, force=True) + gobj = local_repo.get_object(local_repo.head()) + logger.debug( + f"Latest commit cloned: {gobj.sha().hexdigest()} by {gobj.author}" + f" at {gobj.commit_time}" + ) + return local_repo + # ############################################################################# # ##### Stand alone program ######## From 7bad3ed72f0fbb83a59cfd77e6879def98ff42aa Mon Sep 17 00:00:00 2001 From: Julien Date: Fri, 25 Nov 2022 20:41:27 +0100 Subject: [PATCH 2/2] open_repo_closing is reserved to existing git repo --- .../profiles/remote_git_handler.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/qgis_deployment_toolbelt/profiles/remote_git_handler.py b/qgis_deployment_toolbelt/profiles/remote_git_handler.py index 9ffa826d..5df2cde0 100644 --- a/qgis_deployment_toolbelt/profiles/remote_git_handler.py +++ b/qgis_deployment_toolbelt/profiles/remote_git_handler.py @@ -92,13 +92,12 @@ def clone_or_pull(self, local_path: Union[str, Path]) -> Repo: # clone if local_path.exists() and not self.is_local_path_git_repository(local_path): - logger.info(f"Cloning repository {self.url} to {local_path}") try: return self._clone(local_path=local_path) except Exception as err: logger.error( - f"Error cloning the remote repository{self.url} " - f"(branch {self.branch}) to {local_path}." + f"Error cloning the remote repository {self.url} " + f"(branch {self.branch}) to {local_path}. " f"Trace: {err}." ) raise err @@ -144,18 +143,17 @@ def _clone(self, local_path: Union[str, Path]) -> Repo: # clone if local_path.exists() and not self.is_local_path_git_repository(local_path): logger.info(f"Cloning repository {self.url} to {local_path}") - with porcelain.open_repo_closing(str(local_path.resolve())) as local_repo: - repo = porcelain.clone( - source=self.url, - target=local_repo, - branch=self.branch, - depth=5, - ) - gobj = local_repo.get_object(repo.head()) - logger.debug( - f"Latest commit cloned: {gobj.sha().hexdigest()} by {gobj.author}" - f" at {gobj.commit_time}" - ) + local_repo = porcelain.clone( + source=self.url, + target=str(local_path.resolve()), + branch=self.branch, + depth=5, + ) + gobj = local_repo.get_object(local_repo.head()) + logger.debug( + f"Latest commit cloned: {gobj.sha().hexdigest()} by {gobj.author}" + f" at {gobj.commit_time}" + ) return local_repo def _fetch(self, local_path: Union[str, Path]) -> Repo: