From d8ea2f51e93c02e908e3491984b80a870271981a Mon Sep 17 00:00:00 2001 From: Ken Kundert Date: Sat, 2 Nov 2024 13:10:19 -0700 Subject: [PATCH] enhance emporg-overdue to support remote repositories --- doc/index.rst | 7 ++ doc/monitoring.rst | 10 ++- doc/releases.rst | 5 ++ emborg/command.py | 7 +- emborg/hooks.py | 2 + emborg/main.py | 3 + emborg/overdue.py | 165 +++++++++++++++++++++++++++--------------- emborg/patterns.py | 2 + emborg/preferences.py | 3 +- 9 files changed, 143 insertions(+), 61 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index 3ddc2ab..ba55a7c 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -326,6 +326,13 @@ flood, that claims your original files. One option is RSync_. Another is BorgBase_. I have experience with both, and both seem quite good. One I have not tried is Hetzner_. +*Borg* supports many different ways of excluding files and directories from your +backup. Thus it is always possible that a small mistake results essential files +from being excluded from your backups. Once you have performed your first +backup you should :ref:`mount ` the most recent archive and then +carefully examine the resulting snapshot and make sure it contains all the +expected files. + Finally, it is a good idea to practice a recovery. Pretend that you have lost all your files and then see if you can do a restore from backup. Doing this and working out the kinks before you lose your files can save you if you ever do diff --git a/doc/monitoring.rst b/doc/monitoring.rst index 9bde605..c2172dc 100644 --- a/doc/monitoring.rst +++ b/doc/monitoring.rst @@ -94,6 +94,14 @@ The dictionaries in *repositories* can contain the following fields: *host*, modification time of the target of this path is used as the time of the last backup. If *path* is an absolute path, it is used, otherwise it is added to the end of *root*. + + If the path contains a colon (‘:’), then everything before the colon is + taken to be an SSH hostname and everything after the colon is assumed to be + the name of the *emborg-overdue* command on that local machine without + arguments. In most cases the colon will be the last character of the path, + in which case the command name is assumed to be ‘emborg-overdue’. This + command is run on the remote host and the results reported locally. The + version of *emborg* on the remote host must be 1.41 or greater. *maintainer*: An email address, an email is sent to this address if there is an issue. *max_age* is the number of hours that may pass before an archive is @@ -160,7 +168,7 @@ There are some additional settings available: - strings than include field width and justification, ex. {host:>20} - floats can include width, precision and form, ex. {hours:0.1f} - - datetime can include Arrow formats, ex: {mdime:DD MMM YY @ H:mm A} + - datetime can include Arrow formats, ex: {mtime:DD MMM YY @ H:mm A} - overdue can include true/false strings: {overdue:PAST DUE!/current} To run the program interactively, just make sure *emborg-overdue* has been diff --git a/doc/releases.rst b/doc/releases.rst index d2ecd8a..facf32a 100644 --- a/doc/releases.rst +++ b/doc/releases.rst @@ -18,9 +18,14 @@ Latest development release a Linux system you will now have to set `XDG_CONFIG_HOME` to `$HOME/.config`. +1.41 (2024-11-??) +----------------- + - When *Emborg* encounters an error when operating on a composite configuration it will terminate the problematic configuration and move to the next. Previously it would exit without attempting the remaining configs. +- :ref:`emborg-overdue ` can now run an *emborg-overdue* process + on a remote host and include the result in its report. 1.40 (2024-08-05) diff --git a/emborg/command.py b/emborg/command.py index 92c496d..db593bb 100644 --- a/emborg/command.py +++ b/emborg/command.py @@ -1,6 +1,8 @@ # Commands # License {{{1 +# Copyright (C) 2016-2024 Kenneth S. Kundert +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -774,7 +776,6 @@ def run(cls, command, args, settings, options): activity = "checking" check_status = 0 if settings.check_after_create: - announce("Checking repository ...") if settings.check_after_create == "latest": args = [] elif settings.check_after_create in [True, "all"]: @@ -789,6 +790,10 @@ def run(cls, command, args, settings, options): cuplrit = "check_after_create", ) args = [] + if '--all' in args: + announce("Checking repository ...") + else: + announce("Checking archive ...") check = CheckCommand() try: check.run("check", args, settings, options) diff --git a/emborg/hooks.py b/emborg/hooks.py index e701076..224ff3f 100644 --- a/emborg/hooks.py +++ b/emborg/hooks.py @@ -1,6 +1,8 @@ # Hooks # License {{{1 +# Copyright (C) 2018-2024 Kenneth S. Kundert +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/emborg/main.py b/emborg/main.py index 6647952..a098f13 100644 --- a/emborg/main.py +++ b/emborg/main.py @@ -21,6 +21,8 @@ """ # License {{{1 +# Copyright (C) 2018-2024 Kenneth S. Kundert +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -126,6 +128,7 @@ def main(): if exit_status and exit_status > worst_exit_status: worst_exit_status = exit_status + inform.errors_accrued(reset=True) # execute the command termination exit_status = cmd.execute_late(cmd_name, args, None, emborg_opts) diff --git a/emborg/overdue.py b/emborg/overdue.py index 5e9f689..c02400e 100644 --- a/emborg/overdue.py +++ b/emborg/overdue.py @@ -15,6 +15,7 @@ -h, --help Output basic usage information -m, --mail Send mail message if backup is overdue -n, --notify Send notification if backup is overdue + -N, --nt Output summary in NestedText format -p, --no-passes Do not show hosts that are not overdue -q, --quiet Suppress output to stdout -v, --verbose Give more information about each repository @@ -39,11 +40,13 @@ formatting directives. For example: - strings than include field width and justification, ex. {host:>20} - floats can include width, precision and form, ex. {hours:0.1f} -- datetime can include Arrow formats, ex: {mdime:DD MMM YY @ H:mm A} +- datetime can include Arrow formats, ex: {mtime:DD MMM YY @ H:mm A} - overdue can include true/false strings: {overdue:PAST DUE!/current} """ # License {{{1 +# Copyright (C) 2018-2024 Kenneth S. Kundert +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -62,7 +65,6 @@ import os import pwd import socket -from textwrap import dedent import arrow from docopt import docopt from inform import ( @@ -71,17 +73,19 @@ Inform, InformantFactory, conjoin, + dedent, display, error, fatal, - fmt, get_prog_name, is_str, os_error, + output, terminate, truth, warn, ) +import nestedtext as nt from . import __released__, __version__ from .preferences import CONFIG_DIR, DATA_DIR, OVERDUE_FILE, OVERDUE_LOG_FILE @@ -95,10 +99,12 @@ hostname = socket.gethostname() now = arrow.now() +# colors {{{2 default_colorscheme = "dark" current_color = "green" overdue_color = "red" +# message templates {{{2 verbose_status_message = dedent("""\ HOST: {host} sentinel file: {path!s} @@ -106,25 +112,69 @@ since last change: {hours:0.1f} hours maximum age: {max_age} hours overdue: {overdue} -""") -terse_status_message = "{host}: {age} ago" -mail_status_message = dedent( - f""" - Backup of {{host}} is overdue: - from: {username}@{hostname} at {now} - message: the backup sentinel file has not changed in {{hours:0.1f}} hours. - sentinel file: {{path!s}} - """ -).strip() - -error_message = dedent( - f""" +""", strip_nl='l') + +terse_status_message = "{host}: {age} ago{overdue: — PAST DUE}" + +mail_status_message = dedent(""" + Backup of {host} is overdue: + the backup sentinel file has not changed in {hours:0.1f} hours. +""", strip_nl='b') + +error_message = dedent(f""" {get_prog_name()} generated the following error: from: {username}@{hostname} at {now} message: {{}} - """ -) +""", strip_nl='b') +# Utilities {{{1 +# get_local_data {{{2 +def get_local_data(path, host, max_age): + if path.is_dir(): + paths = list(path.glob("index.*")) + if not paths: + raise Error("no sentinel file found.", culprit=path) + if len(paths) > 1: + raise Error("too many sentinel files.", *paths, sep="\n ") + path = paths[0] + mtime = arrow.get(path.stat().st_mtime) + if path.suffix == '.nt': + latest = read_latest(path) + mtime = latest.get('create last run') + if not mtime: + raise Error('backup time is not available.', culprit=path) + delta = now - mtime + hours = 24 * delta.days + delta.seconds / 3600 + overdue = truth(hours > max_age) + yield dict( + host=host, path=path, mtime=mtime, + hours=hours, max_age=max_age, overdue=overdue + ) + +# get_remote_data {{{2 +def get_remote_data(name, path): + host, _, cmd = path.partition(':') + cmd = cmd or "emborg-overdue" + display(f"\n{name}:") + try: + ssh = Run(['ssh', host, cmd, '--nt'], 'sOEW1') + for repo_data in nt.loads(ssh.stdout, top=list): + if 'mtime' in repo_data: + repo_data['mtime'] = arrow.get(repo_data['mtime']) + if 'overdue' in repo_data: + repo_data['overdue'] = truth(repo_data['overdue'] == 'yes') + if 'hours' in repo_data: + repo_data['hours'] = float(repo_data['hours']) + if 'max_age' in repo_data: + repo_data['max_age'] = float(repo_data['max_age']) + yield repo_data + except Error as e: + e.report(culprit=host) + +# fixed() {{{2 +# formats float using fixed point notation while removing trailing zeros +def fixed(num, prec=2): + return format(num, f".{prec}f").strip('0').strip('.') # Main {{{1 def main(): @@ -175,9 +225,11 @@ def main(): log = False with Inform( - flush=True, quiet=quiet, logfile=log, + flush=True, quiet=quiet or cmdline["--nt"], logfile=log, colorscheme=colorscheme, version=version ): + overdue_hosts = {} + # process repositories table backups = [] if is_str(repositories): @@ -207,46 +259,34 @@ def send_mail(recipient, subject, message): # check age of repositories for host, path, maintainer, max_age in backups: maintainer = default_maintainer if not maintainer else maintainer - max_age = float(max_age) if max_age else default_max_age + max_age = float(max_age if max_age else default_max_age) try: - path = to_path(root, path) - if path.is_dir(): - paths = list(path.glob("index.*")) - if not paths: - raise Error("no sentinel file found.", culprit=path) - if len(paths) > 1: - raise Error("too many sentinel files.", *paths, sep="\n ") - path = paths[0] - mtime = arrow.get(path.stat().st_mtime) - if path.suffix == '.nt': - latest = read_latest(path) - mtime = latest.get('create last run') - if not mtime: - raise Error('backup time is not available.', culprit=path) - delta = now - mtime - hours = 24 * delta.days + delta.seconds / 3600 - age = mtime.humanize(only_distance=True) - overdue = truth(hours > max_age) - report = report_as_overdue if overdue else report_as_current - if overdue or not cmdline["--no-passes"]: - replacements = dict( - host=host, path=path, mtime=mtime, age=age, - hours=hours, max_age=max_age, overdue=overdue - ) - try: - report(status_message.format(**replacements)) - except KeyError as e: - fatal( - f"‘{e.args[0]}’ is an unknown key.", - culprit='--message', - codicil=f"Choose from: {conjoin(replacements.keys())}.", - ) - - if overdue: - problem = True - subject = f"backup of {host} is overdue" - msg = fmt(mail_status_message) - send_mail(maintainer, subject, msg) + if ':' in str(path): + repos_data = get_remote_data(host, str(path)) + else: + repos_data = get_local_data(to_path(root, path), host, max_age) + for repo_data in repos_data: + repo_data['age'] = repo_data['mtime'].humanize(only_distance=True) + overdue = repo_data['overdue'] + report = report_as_overdue if overdue else report_as_current + + if overdue or not cmdline["--no-passes"]: + if cmdline["--nt"]: + output(nt.dumps([repo_data], converters={float:fixed}, default=str)) + else: + try: + report(status_message.format(**repo_data)) + except KeyError as e: + fatal( + f"‘{e.args[0]}’ is an unknown key.", + culprit='--message', + codicil=f"Choose from: {conjoin(repo_data.keys())}.", + ) + + if overdue: + problem = True + overdue_hosts[host] = mail_status_message.format(**repo_data) + except OSError as e: problem = True msg = os_error(e) @@ -266,4 +306,13 @@ def send_mail(recipient, subject, message): f"{get_prog_name()} error", error_message.format(str(e)), ) + + if overdue_hosts: + if len(overdue_hosts) > 1: + subject = "backups are overdue" + else: + subject = "backup is overdue" + messages = '\n\n'.join(overdue_hosts.values()) + send_mail(maintainer, subject, messages) + terminate(problem) diff --git a/emborg/patterns.py b/emborg/patterns.py index 667f770..23ba94a 100644 --- a/emborg/patterns.py +++ b/emborg/patterns.py @@ -116,6 +116,8 @@ def check_patterns( codicil = repr(pattern) kind = pattern[0:1] arg = pattern[1:].lstrip() + if not kind or not arg: + raise Error(f"invalid pattern: ‘{pattern}’") if kind in ["", "#"]: continue # is comment if kind not in known_kinds: diff --git a/emborg/preferences.py b/emborg/preferences.py index 4791b9a..c91a01d 100644 --- a/emborg/preferences.py +++ b/emborg/preferences.py @@ -1,8 +1,9 @@ # Emborg Preferences # -# Copyright (C) 2018-2024 Kenneth S. Kundert # License {{{1 +# Copyright (C) 2018-2024 Kenneth S. Kundert +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or