From 9d0be5860a9535b9984ec68cd16ca841e6320b29 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 7 Nov 2024 17:06:34 +0100 Subject: [PATCH 1/7] Implement module list-backup-repositories Return the list of backup destinations with a Restic repository for the current module. --- .../50list_backup_repositories | 103 +++++++++++++++++ .../validate-output.json | 107 ++++++++++++++++++ .../validate-output.json | 2 +- 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100755 core/imageroot/usr/local/agent/actions/list-backup-repositories/50list_backup_repositories create mode 100644 core/imageroot/usr/local/agent/actions/list-backup-repositories/validate-output.json diff --git a/core/imageroot/usr/local/agent/actions/list-backup-repositories/50list_backup_repositories b/core/imageroot/usr/local/agent/actions/list-backup-repositories/50list_backup_repositories new file mode 100755 index 000000000..15199f81a --- /dev/null +++ b/core/imageroot/usr/local/agent/actions/list-backup-repositories/50list_backup_repositories @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import sys +import json +import agent +import asyncio +import os +import time +from datetime import datetime, timezone + +rdb = agent.redis_connect(privileged=False) +module_id = os.environ['MODULE_ID'] +module_uuid = os.environ['MODULE_UUID'] +module_ui_name = rdb.get(f'module/{module_id}/ui_name') or "" +image_name = agent.get_image_name_from_url(os.environ["IMAGE_URL"]) +cluster_uuid = rdb.get("cluster/uuid") or "" +odests = {} +for krepo in rdb.scan_iter('cluster/backup_repository/*'): + dest_uuid = krepo.removeprefix('cluster/backup_repository/') + odests[dest_uuid] = rdb.hgetall(krepo) +rdb.close() + +# +# Fetch data from all backup destinations +# + +async def read_destination_repo(dest_uuid, dest_path): + proc = await asyncio.create_subprocess_exec('rclone-wrapper', dest_uuid, 'lsjson', f'REMOTE_PATH/{dest_path}/config', stdout=asyncio.subprocess.PIPE) + # Return the first and only element of the expected JSON array + out, _ = await proc.communicate() + if out == b'[\n]\n' or not out: + data = {} + else: + data = json.loads(out)[0] + return data + +async def read_destination_meta(dest_uuid, dest_path): + proc = await asyncio.create_subprocess_exec('rclone-wrapper', dest_uuid, 'cat', f'REMOTE_PATH/{dest_path}.json', stdout=asyncio.subprocess.PIPE) + out, _ = await proc.communicate() + if out: + data = json.loads(out) + else: + data = {} + return data + + +async def get_destination_info(dest_uuid, odest): + global cluster_uuid, module_id, module_uuid, module_ui_name, image_name + + dest_path = f"{image_name}/{module_uuid}" + + async with asyncio.TaskGroup() as tg: + task_repo = tg.create_task(read_destination_repo(dest_uuid, dest_path)) + task_meta = tg.create_task(read_destination_meta(dest_uuid, dest_path)) + + info = { + "module_id": module_id, + "module_ui_name": module_ui_name, + "node_fqdn": "", + "path": dest_path, + "name": image_name, + "uuid": module_uuid, + "timestamp": 0, + "repository_id" : dest_uuid, + "repository_name": odest["name"], + "repository_provider": odest["provider"], + "repository_url": odest["url"], + "installed_instance": module_id, + "installed_instance_ui_name": module_ui_name, + "is_generated_locally": False, + } + + result_repo = task_repo.result() + if not result_repo: + return None + + try: + # Obtain from lsjson the repository creation timestamp + info['timestamp'] = int(time.mktime(datetime.fromisoformat(result_repo["ModTime"]).timetuple())) + except: + info['timestamp'] = int(time.time()) + + result_meta = task_meta.result() + if "cluster_uuid" in result_meta and result_meta["cluster_uuid"] == cluster_uuid: + info['is_generated_locally'] = True + info.update(result_meta) # merge two dictionaries + + return info + +async def print_destinations(odests): + tasks = [] + async with asyncio.TaskGroup() as tg: + for dest_uuid, odest in odests.items(): + tasks.append(tg.create_task(get_destination_info(dest_uuid, odest))) + destinations = list(filter(lambda r: r, [task.result() for task in tasks])) + json.dump(destinations, fp=sys.stdout) + +asyncio.run(print_destinations(odests)) diff --git a/core/imageroot/usr/local/agent/actions/list-backup-repositories/validate-output.json b/core/imageroot/usr/local/agent/actions/list-backup-repositories/validate-output.json new file mode 100644 index 000000000..0602e1cad --- /dev/null +++ b/core/imageroot/usr/local/agent/actions/list-backup-repositories/validate-output.json @@ -0,0 +1,107 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "list-backup-repositories output", + "$id": "http://schema.nethserver.org/module/list-backup-repositories-output.json", + "description": "Return a list of the module's Restic backup repositories. The format is the same of cluster/read-backup-repositories.", + "examples": [ + [ + { + "module_id": "loki1", + "module_ui_name": "My Loki", + "node_fqdn": "rl1.dp.nethserver.net", + "path": "loki/35f45b73-f81e-467b-b622-96ec3b7fec19", + "name": "loki", + "uuid": "35f45b73-f81e-467b-b622-96ec3b7fec19", + "timestamp": 1721405723, + "repository_id": "14030a59-a4e6-54cc-b8ea-cd5f97fe44c8", + "repository_name": "BackBlaze repo1", + "repository_provider": "backblaze", + "repository_url": "b2:ns8-davidep", + "installed_instance": "loki1", + "installed_instance_ui_name": "My Loki", + "is_generated_locally": true + } + ] + ], + "type": "array", + "items": { + "type": "object", + "properties": { + "module_id": { + "type": "string", + "description": "Original module ID value." + }, + "module_ui_name": { + "type": "string", + "description": "Original module label, assigned by the user." + }, + "node_fqdn": { + "type": "string", + "description": "The FQDN of the node where the module of the backup is hosted." + }, + "path": { + "type": "string", + "description": "Path of the repository, relative to the backup destination." + }, + "name": { + "type": "string", + "description": "Name of the module. It is equal to the module image name." + }, + "uuid": { + "type": "string", + "description": "Universal, unique identifier of the module instance." + }, + "timestamp": { + "type": "integer", + "description": "Unix timestamp of the last backup run." + }, + "repository_id": { + "type": "string", + "description": "UUID of the backup destination." + }, + "repository_name": { + "type": "string", + "description": "Human readable name of the backup destination." + }, + "repository_provider": { + "type": "string", + "description": "Type of backup destination provider, e.g. SMB, S3..." + }, + "repository_url": { + "type": "string", + "description": "Restic URL of the backup destination." + }, + "installed_instance": { + "type": "string", + "description": "If the backup belongs to an installed module instance this is its module ID." + }, + "installed_instance_ui_name": { + "type": "string", + "description": "If the backup belongs to an installed module instance this is its module friendly name." + }, + "is_generated_locally": { + "type": [ + "boolean", + "null" + ], + "description": "Tells if the backup originates from the local cluster or from another cluster. The null value is returned if this information is missing completely, as it happens in old backups." + } + }, + "required": [ + "module_id", + "module_ui_name", + "node_fqdn", + "path", + "name", + "uuid", + "timestamp", + "repository_id", + "repository_name", + "repository_provider", + "repository_url", + "installed_instance", + "installed_instance_ui_name", + "is_generated_locally" + ] + } +} diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-backup-repositories/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/list-backup-repositories/validate-output.json index ed3f734ea..c716c5fcf 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-backup-repositories/validate-output.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-backup-repositories/validate-output.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "list-backup-repositories output", "$id": "http://schema.nethserver.org/cluster/list-backup-repositories-output.json", - "description": "Get the list of available backup repositories and the status of cluster backup password", + "description": "Get the list of available backup destinations and the status of cluster backup password", "examples": [ { "repositories": [ From 7945529bebd4491585c283fc968a4562285f67e5 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 8 Nov 2024 11:30:08 +0100 Subject: [PATCH 2/7] Agent prepare_restic_command() function Split the run_restic() function to ease custom subprocess invocation. --- core/imageroot/usr/local/agent/pypkg/agent/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/imageroot/usr/local/agent/pypkg/agent/__init__.py b/core/imageroot/usr/local/agent/pypkg/agent/__init__.py index 56e33be39..9fba51656 100644 --- a/core/imageroot/usr/local/agent/pypkg/agent/__init__.py +++ b/core/imageroot/usr/local/agent/pypkg/agent/__init__.py @@ -187,7 +187,7 @@ def run_helper(*args, log_command=True, **kwargs): return subprocess.CompletedProcess(args, proc.returncode) -def run_restic(rdb, repository, repo_path, podman_args, restic_args, **kwargs): +def prepare_restic_command(rdb, repository, repo_path, podman_args, restic_args): core_env = read_envfile('/etc/nethserver/core.env') # Import URLs of core images orepo = rdb.hgetall(f"cluster/backup_repository/{repository}") assert_exp(len(orepo) > 0) # Check the repository exists @@ -240,6 +240,11 @@ def run_restic(rdb, repository, repo_path, podman_args, restic_args, **kwargs): podman_cmd.append(core_env["RESTIC_IMAGE"]) podman_cmd.extend(restic_args) + return (podman_cmd, restic_env) + +def run_restic(rdb, repository, repo_path, podman_args, restic_args, **kwargs): + podman_cmd, restic_env = prepare_restic_command(rdb, repository, repo_path, podman_args, restic_args) + penv = os.environ.copy() penv.update(restic_env) if os.getenv('DEBUG', False): @@ -251,7 +256,6 @@ def run_restic(rdb, repository, repo_path, podman_args, restic_args, **kwargs): kwargs.setdefault('env', penv) kwargs.setdefault('stdout', sys.stdout) kwargs.setdefault('stderr', sys.stderr) - return subprocess.run(podman_cmd, **kwargs) def get_existing_volume_args(): From d794f9c6965f19a8dd1561226c2ecf6eff98c0b6 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 15 Nov 2024 09:53:41 +0100 Subject: [PATCH 3/7] Disable backup progress estimation Run Restic backup faster with "--no-scan". This option avoid calculating the backup size in advance. This is a time consuming operation, unnecessary for a background job. --- core/imageroot/usr/local/agent/bin/module-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/imageroot/usr/local/agent/bin/module-backup b/core/imageroot/usr/local/agent/bin/module-backup index a5298cafa..1d81d13d6 100755 --- a/core/imageroot/usr/local/agent/bin/module-backup +++ b/core/imageroot/usr/local/agent/bin/module-backup @@ -108,7 +108,7 @@ time_start = int(time.time()) errors = 0 try: # Run the backup - agent.run_restic(rdb, repository, repopath, podman_args, ["backup"] + backup_args).check_returncode() + agent.run_restic(rdb, repository, repopath, podman_args, ["backup", "--no-scan"] + backup_args).check_returncode() # Apply retention policy agent.run_restic(rdb, repository, repopath, [], ["forget", "--prune", "--keep-last=" + obackup['retention']]).check_returncode() From a89ad4da4e8cdcb3a6f2b7213e5946035546cd09 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 15 Nov 2024 10:02:09 +0100 Subject: [PATCH 4/7] Capture and track Restic restore progress messages Add an optional "progress_callback" argument to agent.run_restic() function, to provide UI feedback of Restic's progress. Restic's output must switched to JSON format with "--json" argument. --- .../usr/local/agent/pypkg/agent/__init__.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/core/imageroot/usr/local/agent/pypkg/agent/__init__.py b/core/imageroot/usr/local/agent/pypkg/agent/__init__.py index 9fba51656..ecfbc8f02 100644 --- a/core/imageroot/usr/local/agent/pypkg/agent/__init__.py +++ b/core/imageroot/usr/local/agent/pypkg/agent/__init__.py @@ -242,7 +242,7 @@ def prepare_restic_command(rdb, repository, repo_path, podman_args, restic_args) return (podman_cmd, restic_env) -def run_restic(rdb, repository, repo_path, podman_args, restic_args, **kwargs): +def run_restic(rdb, repository, repo_path, podman_args, restic_args, progress_callback=None, **kwargs): podman_cmd, restic_env = prepare_restic_command(rdb, repository, repo_path, podman_args, restic_args) penv = os.environ.copy() @@ -256,7 +256,22 @@ def run_restic(rdb, repository, repo_path, podman_args, restic_args, **kwargs): kwargs.setdefault('env', penv) kwargs.setdefault('stdout', sys.stdout) kwargs.setdefault('stderr', sys.stderr) - return subprocess.run(podman_cmd, **kwargs) + if progress_callback and '--json' in restic_args: + kwargs['stdout'] = subprocess.PIPE + kwargs.setdefault('errors', 'replace') + kwargs.setdefault('text', True) + with subprocess.Popen(podman_cmd, **kwargs) as prestic: + while True: + line = prestic.stdout.readline() + if not line: + break + try: + progress_callback(json.loads(line)) + except Exception as ex: + print(SD_DEBUG + "Error decoding Restic status message", ex, file=kwargs['stderr']) + else: + prestic = subprocess.run(podman_cmd, **kwargs) + return prestic def get_existing_volume_args(): """Return a list of --volume arguments for Podman run and similar. The argument values From 74f6e1238d811a047ebac733d555c7c4ff731de7 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 15 Nov 2024 12:35:41 +0100 Subject: [PATCH 5/7] Adjust progress tracking of restore-module action Capture Restic progress during the restoration of a module instance. Tune progress intervals of cluster/restore-module action. --- .../agent/actions/restore-module/00progress | 13 +++++++++++ .../agent/actions/restore-module/10restore | 23 ++++++++++++++++--- .../actions/restore-module/50restore_module | 6 ++--- 3 files changed, 36 insertions(+), 6 deletions(-) create mode 100755 core/imageroot/usr/local/agent/actions/restore-module/00progress diff --git a/core/imageroot/usr/local/agent/actions/restore-module/00progress b/core/imageroot/usr/local/agent/actions/restore-module/00progress new file mode 100755 index 000000000..ec6c74a97 --- /dev/null +++ b/core/imageroot/usr/local/agent/actions/restore-module/00progress @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import agent + +agent.set_weight('00progress', '0') +agent.set_weight('05replace', '0') +agent.set_weight('10restore', '4') +agent.set_weight('20label', '0') diff --git a/core/imageroot/usr/local/agent/actions/restore-module/10restore b/core/imageroot/usr/local/agent/actions/restore-module/10restore index 605ef3dff..f34d81219 100755 --- a/core/imageroot/usr/local/agent/actions/restore-module/10restore +++ b/core/imageroot/usr/local/agent/actions/restore-module/10restore @@ -23,7 +23,7 @@ import agent import json import sys -import os, os.path +import os request = json.load(sys.stdin) @@ -37,8 +37,25 @@ rdb = agent.redis_connect(host='127.0.0.1') # Connect to local replica podman_args = ["--workdir=/srv"] podman_args.extend(agent.get_state_volume_args()) # get volumes from state-include.conf -restic_args = ["restore", snapshot, +restic_args = ["restore", "--json", snapshot, "--target", ".", # workdir should be /srv "--exclude", "state/environment", # special core file exception ] -agent.run_restic(rdb, repository, repopath, podman_args, restic_args, stdout=sys.stderr).check_returncode() + +# Prepare progress callback function that captures non-progress messages too: +last_restic_message = {} +def build_restore_progress_callback(): + restore_progress = agent.get_progress_callback(1, 100) + def fprog(omessage): + global last_restic_message + last_restic_message = omessage + if omessage['message_type'] == 'status': + fpercent = float(omessage['percent_done']) + restore_progress(int(fpercent * 100)) + return fprog + +prestore = agent.run_restic(rdb, repository, repopath, podman_args, restic_args, progress_callback=build_restore_progress_callback()) +json.dump(last_restic_message, fp=open("restic_restore.json", "w")) +if prestore.returncode != 0: + print(agent.SD_ERR + "Restic restore failed", last_restic_message, file=sys.stderr) + sys.exit(1) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/restore-module/50restore_module b/core/imageroot/var/lib/nethserver/cluster/actions/restore-module/50restore_module index 2178ee9fe..70bb302d4 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/restore-module/50restore_module +++ b/core/imageroot/var/lib/nethserver/cluster/actions/restore-module/50restore_module @@ -55,7 +55,7 @@ add_module_result = agent.tasks.run("cluster", "add-module", "node": node_id, }, endpoint="redis://cluster-leader", - progress_callback=agent.get_progress_callback(2, 50) + progress_callback=agent.get_progress_callback(2, 15) ) agent.assert_exp(add_module_result['exit_code'] == 0) omodule = add_module_result['output'] @@ -87,7 +87,7 @@ restore_task_result = agent.tasks.run("module/" + module_id, "restore-module", "replace": replace_requested or len(remove_tasks) == 0 }, endpoint="redis://cluster-leader", - progress_callback=agent.get_progress_callback(51, 85), + progress_callback=agent.get_progress_callback(16, 99 if len(remove_tasks) == 0 else 94), ) agent.assert_exp(restore_task_result['exit_code'] == 0) @@ -97,7 +97,7 @@ if len(remove_tasks) > 0 and replace_requested: remove_modules_errors = agent.tasks.runp_brief( remove_tasks, endpoint="redis://cluster-leader", - progress_callback=agent.get_progress_callback(86, 98) + progress_callback=agent.get_progress_callback(95, 99) ) agent.assert_exp(remove_modules_errors == 0) From 0b11f479df1bf8c2c8c9b5e1f4289814a279f656 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 15 Nov 2024 18:51:44 +0100 Subject: [PATCH 6/7] Track Restic backup progress in UI While Restic backup is invoked in background most of times, the UI still allows to run the backup immediately. In that case, a detailed backup progress feedback is needed. - Add fine-grained backup progess trace to module-backup command - Run the backup from UI directly with module-backup command - If module-backup runs interactively, skip progress calculation --- .../agent/actions/run-backup/50run_backup | 9 ++++++ .../actions/run-backup/50start_service_unit | 29 ------------------- .../usr/local/agent/bin/module-backup | 15 +++++++++- 3 files changed, 23 insertions(+), 30 deletions(-) create mode 100755 core/imageroot/usr/local/agent/actions/run-backup/50run_backup delete mode 100755 core/imageroot/usr/local/agent/actions/run-backup/50start_service_unit diff --git a/core/imageroot/usr/local/agent/actions/run-backup/50run_backup b/core/imageroot/usr/local/agent/actions/run-backup/50run_backup new file mode 100755 index 000000000..ff59e4fdb --- /dev/null +++ b/core/imageroot/usr/local/agent/actions/run-backup/50run_backup @@ -0,0 +1,9 @@ +#!/bin/bash + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +BACKUP_ID=$(jq -r .id) +exec module-backup "${BACKUP_ID}" diff --git a/core/imageroot/usr/local/agent/actions/run-backup/50start_service_unit b/core/imageroot/usr/local/agent/actions/run-backup/50start_service_unit deleted file mode 100755 index 324d16b3b..000000000 --- a/core/imageroot/usr/local/agent/actions/run-backup/50start_service_unit +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# -# Copyright (C) 2021 Nethesis S.r.l. -# http://www.nethesis.it - nethserver@nethesis.it -# -# This script is part of NethServer. -# -# NethServer is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, -# or any later version. -# -# NethServer is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with NethServer. If not, see COPYING. -# - -BACKUP_ID=$(jq -r .id) - -if [[ $EUID == "0" ]]; then - systemctl start "backup${BACKUP_ID}-${MODULE_ID}.service" -else - systemctl start --user "backup${BACKUP_ID}.service" -fi diff --git a/core/imageroot/usr/local/agent/bin/module-backup b/core/imageroot/usr/local/agent/bin/module-backup index 1d81d13d6..4c91e7853 100755 --- a/core/imageroot/usr/local/agent/bin/module-backup +++ b/core/imageroot/usr/local/agent/bin/module-backup @@ -104,11 +104,24 @@ else: print(f"Initializing repository {repository} at path {repopath}", file=sys.stderr) agent.run_restic(rdb, repository, repopath, [], ["init"]).check_returncode() +agent_progress_callback = agent.get_progress_callback(1, 95) +def backup_progress_callback(omessage): + global agent_progress_callback + if omessage['message_type'] == 'status': + fpercent = float(omessage['percent_done']) + agent_progress_callback(int(fpercent * 100)) + time_start = int(time.time()) errors = 0 try: # Run the backup - agent.run_restic(rdb, repository, repopath, podman_args, ["backup", "--no-scan"] + backup_args).check_returncode() + if os.getenv('AGENT_TASK_ID'): + pbackup = agent.run_restic(rdb, repository, repopath, podman_args, ["backup", "--json"] + backup_args, progress_callback=backup_progress_callback) + if pbackup.returncode != 0: + print(agent.SD_ERR + f"Restic restore command failed with exit code {pbackup.returncode}.", file=sys.stderr) + sys.exit(1) + else: + agent.run_restic(rdb, repository, repopath, podman_args, ["backup", "--no-scan"] + backup_args).check_returncode() # Apply retention policy agent.run_restic(rdb, repository, repopath, [], ["forget", "--prune", "--keep-last=" + obackup['retention']]).check_returncode() From b0e54d6428953d8bbab26a819f45ea446ee86fb0 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 5 Dec 2024 09:35:19 +0100 Subject: [PATCH 7/7] Handle missing repository errors If the module has no backup information in some destination, rclone commands fail with invalid JSON output. --- .../50list_backup_repositories | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/core/imageroot/usr/local/agent/actions/list-backup-repositories/50list_backup_repositories b/core/imageroot/usr/local/agent/actions/list-backup-repositories/50list_backup_repositories index 15199f81a..145bb28c4 100755 --- a/core/imageroot/usr/local/agent/actions/list-backup-repositories/50list_backup_repositories +++ b/core/imageroot/usr/local/agent/actions/list-backup-repositories/50list_backup_repositories @@ -36,14 +36,22 @@ async def read_destination_repo(dest_uuid, dest_path): if out == b'[\n]\n' or not out: data = {} else: - data = json.loads(out)[0] + try: + data = json.loads(out)[0] + except Exception as ex: + print(agent.SD_DEBUG + f"Ignored output from rclone-wrapper. Does the Restic repository configuration file, {dest_path}/config, exist in destination {dest_uuid}?", repr(ex), 'Data read:', out, file=sys.stderr) + data = {} return data async def read_destination_meta(dest_uuid, dest_path): proc = await asyncio.create_subprocess_exec('rclone-wrapper', dest_uuid, 'cat', f'REMOTE_PATH/{dest_path}.json', stdout=asyncio.subprocess.PIPE) out, _ = await proc.communicate() if out: - data = json.loads(out) + try: + data = json.loads(out) + except Exception as ex: + print(agent.SD_DEBUG + f"Ignored output from rclone-wrapper. Does {dest_path}.json file exist in destination {dest_uuid}?", repr(ex), 'Data read:', out, file=sys.stderr) + data = {} else: data = {} return data