diff --git a/.gitignore b/.gitignore index 10f536a..cdd8295 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ __pycache__/ -.pytest_cache/ -.mypy_cache/ venv diff --git a/Makefile b/Makefile index d804805..9ccf7e3 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,10 @@ install-dev: # Run all configured tasks in main VAR targets dir. run: - unicron/unicron.py --verbose + python3 -m unicron.unicron --verbose + +run-quiet: + python3 -m unicron.unicron # View configured tasks. @@ -33,11 +36,11 @@ ls-test-runs: # Make all task scripts executable. -permission: +perms: chmod +x unicron/var/targets/* -# Tail the app log. +# Tail the app logs. log-app: cd unicron/var && tail -F app.log # Same as above but with longer history. @@ -48,18 +51,15 @@ log-app-long: # Tail the task logs. log-tasks: cd unicron/var && tail -F output/*.log - -# Same as above but with longer history. log-tasks-long: cd unicron/var && tail -n50 -F output/*.log # Tail both the app and task logs. log: cd unicron/var && tail -F output/*.log app.log - -# As above, for test tasks. We make the _test_var path shown here for clarity. +# Tail logs created by `debug` target. log-tests: - cd unicron && tail -n20 -F _test_var/output/*.log _test_var/app.log + cd unicron/_test_var && tail -n20 -F output/*.log app.log format: @@ -68,8 +68,8 @@ format-check: black . --diff --check pylint: - # Exit on error code if needed. - pylint unicron/unicron.py || pylint-exit $$? + # Exit on error code on a fail. Expand failure to all non-fatal messages too. + pylint unicron tests || pylint-exit -efail -wfail -rfail -cfail $$? lint: pylint @@ -80,13 +80,17 @@ typecheck: mypy unicron tests +clean: + find . -name '*.pyc' -delete + + # Reset tasks and logs in the TEST VAR dir. reset: bin/reset.sh # Run unit tests. unit: reset - pytest + TEST=true pytest # Integration test. debug: reset diff --git a/bin/cron_target.sh b/bin/cron_target.sh index 7c5f833..f092227 100755 --- a/bin/cron_target.sh +++ b/bin/cron_target.sh @@ -1,24 +1,26 @@ #!/bin/bash -# Run Unicron as a cron command. -# -# On success, be silent. -# On error, send all output to the user's local mailbox. +# Run Unicron as a scheduled cron command. # # This script can be run from anywhere. +# +# On success, run silently - nothing printed at all. +# On error, capture all stdout and stderr so it can be sent to user's local mailbox using cron's mechanism or the mail command. +# Either way, you won't see anything printed if run this command. set -e -SCRIPT_DIR=$(dirname $(realpath $0)) -SCRIPT_FILEPATH="$SCRIPT_DIR/../unicron/unicron.py" +SCRIPT_DIR=$(dirname $(dirname $(realpath "$0"))) +cd "$SCRIPT_DIR" set +e -OUTPUT="$($SCRIPT_FILEPATH 2>&1)" + +CMD_OUTPUT="$(make run-quiet 2>&1)" if [[ $? -ne 0 ]]; then - set -e - echo "$OUTPUT" | mail -s 'Unicron task failed!' $USER + set -e + echo "$CMD_OUTPUT" | mail -s 'Unicron task failed!' $USER - exit 1 + exit 1 fi exit 0 diff --git a/bin/reset.sh b/bin/reset.sh index 9f8a10b..ae48a85 100755 --- a/bin/reset.sh +++ b/bin/reset.sh @@ -10,14 +10,15 @@ echo 'Entering _test_var directory.' cd unicron/_test_var +echo 'Remove logs.' +rm app.log >/dev/null 2>&1 || true +rm output/*.log >/dev/null 2>&1 || true + echo 'Create last-run file fixtures.' cd last_run/ +# Note that the today.sh task will never actually run when it unicron checks it, +# so there today.sh.log will never get created and this is okay. echo $(date +%Y-%m-%d) >today.sh.txt echo "2020-01-01" >old.sh.txt rm never_run_before.sh.txt >/dev/null 2>&1 || true rm fail.sh.txt >/dev/null 2>&1 || true -cd .. - -echo 'Remove logs.' -rm app.log >/dev/null 2>&1 || true -rm output/*.log >/dev/null 2>&1 || true diff --git a/bin/test.sh b/bin/test.sh index ca62bd2..2e563f0 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -7,12 +7,12 @@ echo "Main - first run" echo "===" -TEST=true unicron/unicron.py -v +TEST=true python -m unicron.unicron -v echo echo echo "Main - second run" echo "===" -TEST=true unicron/unicron.py -v +TEST=true python -m unicron.unicron -v true diff --git a/docs/features.md b/docs/features.md index 2db2c1f..3c431c9 100644 --- a/docs/features.md +++ b/docs/features.md @@ -27,28 +27,67 @@ * Application log - *var/app.log* - There is also `_test_var` directory for testing the application without affecting the main `var` directory. + ## Sample -Given a configured script `hello.sh` in the targets directory. +Given an executable script `hello.sh` added to the targets directory. + +- `hello.sh` + ```sh + echo 'Hello, word' + ``` + +Running directly: + +```sh +$ ./unicron/var/targets/hello.sh +Hello, world! +``` + - +Run with Unicron - note the verbose flag is implied in the `run` target of the `Makefile`. 1. First run today - the script executes. ```bash - $ ./unicron.py --verbose - 2020-01-05 19:23:05 INFO:unicron hello.sh - Success. + $ make run + 2020-10-24 08:02:18,516 INFO:unicron.py unicron - Task count: 1 + 2020-10-24 08:02:18,532 INFO:run.py hello.sh - Success. + 2020-10-24 08:02:18,537 INFO:unicron.py unicron - Succeeded: 1; Failed: 0; Skipped: 0 ``` -2. Second run today - the script is skipped. +2. A repeat run today - the script is skipped. ```bash - $ ./unicron.py --verbose - 2020-01-05 19:23:56 INFO:unicron hello.sh - Skipping, since already ran today. + $ make run + 2020-10-24 08:03:36,313 INFO:unicron.py unicron - Task count: 7 + 2020-10-24 08:03:36,317 INFO:history.py hello.sh - Skipping, since already ran today. + 2020-10-24 08:02:18,537 INFO:unicron.py unicron - Succeeded: 0; Failed: 0; Skipped: 1 ``` -3. First run tomorrow - the script executes. +3. On the first run tomorrow - the script executes. ```bash - $ ./unicron.py --verbose - 2020-01-06 12:22:00 INFO:unicron hello.sh - Success. + $ make run + 2020-10-25 08:02:18,516 INFO:unicron.py unicron - Task count: 1 + 2020-10-25 08:02:18,532 INFO:run.py hello.sh - Success. + 2020-10-25 08:02:18,537 INFO:unicron.py unicron - Succeeded: 1; Failed: 0; Skipped: 0 ``` -4. Scheduling - add a single command to your _crontab_ configuration. For example, run every 30 minutes and only send mail if at least one job fails. + +Scheduling - add a single command to your _crontab_ configuration. + +For example, run every 30 minutes and only send mail if at least one job fails. + +```sh +$ crontab -e +``` +``` +*/30 * * * * ~/repos/unicron/bin/cron_target.sh +``` + +Then access runs using: + +```sh +$ mail +Mail version 8.1 6/6/93. Type ? for help. +"/var/mail/mcurrin": 1 message 1 new +>N 1 mcurrin@C02WL0Y2HV2T Sat Oct 24 20:00 17/836 "Unicron task failed!" +``` ## What is the point of running once but retrying? diff --git a/docs/usage.md b/docs/usage.md index a72d718..5923d56 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -15,7 +15,7 @@ The main things you to do with _Unicron_ are:
- + Click to expand. ```bash @@ -64,7 +64,7 @@ The example output below is for the demo script which was setup using [Installat $ make run ``` ``` - unicron/unicron.py -v + python3 -m unicron.unicron -v 2020-04-05 10:00:00,414 INFO:unicron.py unicron - Task count: 1 2020-04-05 10:00:00,429 DEBUG:unicron.py hello.sh - Executing, since last run date is old. 2020-04-05 10:00:02,224 INFO:unicron.py hello.sh - Success. @@ -75,7 +75,7 @@ The example output below is for the demo script which was setup using [Installat $ make run ``` ``` - unicron/unicron.py -v + python3 -m unicron.unicron -v 2020-04-05 10:10:00,414 INFO:unicron.py unicron - Task count: 1 2020-04-05 10:10:00,429 INFO:unicron.py hello.sh - Skipping, since already ran today. 2020-04-05 10:10:00,500 INFO:unicron.py unicron - Suceeded: 0; Failed: 0; Skipped: 1 @@ -94,7 +94,7 @@ make: *** [run] Error 1 Without any custom tasks setup, you start test _Unicron_ immediately by running the versioned test tasks. ```bash -$ make run-test +$ make debug ``` @@ -222,6 +222,7 @@ cd unicron && tail -n20 -F _test_var/output/*.log _test_var/app.log Instead of going through `make`, you can run the Python script directly: ```bash -$ cd ~/repos/unicron/unicron -$ ./unicron.py --help +$ python3 -m unicron.unicron -v ``` + +You have to use the `-m` syntax for the relative imports in the script to work. If they are not relative imports and just `import logger`, then the tests can't run properly. diff --git a/tests/__init__.py b/tests/__init__.py index 20d82cd..876da88 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,4 +2,10 @@ Tests initialization module. This file is required for pytest to correctly do imports. + +Make sure TEST=true is set when running tests. That will ensure that main app +var file references are in the test var directory, to keep the main one clean. + +You may have to run reset.sh before test. Note that this will delete all test +logs. """ diff --git a/tests/test_history.py b/tests/test_history.py new file mode 100644 index 0000000..c22f7f6 --- /dev/null +++ b/tests/test_history.py @@ -0,0 +1,27 @@ +""" +History module tests. +""" +# pylint: disable=missing-function-docstring +import datetime + +from unicron import history + + +def test_get_last_run_date(): + assert history.get_last_run_date("never_run_before.sh") is None + assert history.get_last_run_date("fail.sh") is None + + assert history.get_last_run_date("old.sh") == datetime.date( + year=2020, month=1, day=1 + ) + assert history.get_last_run_date("today.sh") == datetime.date.today() + + +def test_check_need_to_run(): + # Expect to run. + assert history.check_need_to_run("never_run_before.sh") is True + assert history.check_need_to_run("fail.sh") is True + assert history.check_need_to_run("old.sh") is True + + # Expect not to run. + assert history.check_need_to_run("today.sh") is False diff --git a/tests/test_logger.py b/tests/test_logger.py new file mode 100644 index 0000000..129ca55 --- /dev/null +++ b/tests/test_logger.py @@ -0,0 +1,20 @@ +""" +Logger module tests. +""" +# pylint: disable=missing-function-docstring +from pathlib import Path + +from unicron import logger + + +APP_DIR = Path("unicron") +VAR_DIR = APP_DIR / Path("_test_var") +LOG_DIR = VAR_DIR / "last_run" + + +def test_setup_logger(): + app_logger = logger.setup_logger(__name__, VAR_DIR / "app.log", is_task=False) + app_logger.debug("test_setup_logger", extra={"task": "pytest"}) + + task_logger = logger.setup_logger(__name__, LOG_DIR / "unit_task.log", is_task=True) + task_logger.debug("test_setup_logger") diff --git a/tests/test_paths.py b/tests/test_paths.py new file mode 100644 index 0000000..fb0880e --- /dev/null +++ b/tests/test_paths.py @@ -0,0 +1,17 @@ +""" +Paths module tests. +""" +# pylint: disable=missing-function-docstring +from unicron import paths + + +def test_mk_last_run_path(): + path = paths.mk_last_run_path("foo") + + assert str(path).endswith("_test_var/last_run/foo.txt") + + +def test_mk_output_path(): + path = paths.mk_output_path("foo") + + assert str(path).endswith("_test_var/output/foo.log") diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 0000000..34a6490 --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,21 @@ +""" +Run module tests. +""" +# pylint: disable=missing-function-docstring +from unicron import run + + +def test_run_in_shell_success(): + cmd = 'echo "Test output"' + success, output = run.run_in_shell(cmd) + + assert success + assert output == "Test output" + + +def test_run_in_shell_fail(): + cmd = "echo Fail! ; exit 1" + success, output = run.run_in_shell(cmd) + + assert not success + assert output == "Fail!" diff --git a/tests/test_unicron.py b/tests/test_unicron.py deleted file mode 100644 index 7d060ca..0000000 --- a/tests/test_unicron.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Test Unicron application. - -You may have to run reset.sh before this test. Note that will delete all test -logs. -""" -import datetime -import os -from pathlib import Path - -# Ensure that main app var file references are in the test var directory, to -# keep the main one clean. -# NB. This must be done BEFORE unicron imports. -os.environ["TEST"] = "true" - - -from unicron.unicron import ( - setup_logger, - run_in_shell, - mk_last_run_path, - mk_output_path, - get_last_run_date, - check_need_to_run, -) - - -APP_DIR = Path("unicron") -VAR_DIR = APP_DIR / Path("_test_var") -LOG_DIR = VAR_DIR / "last_run" - - -def test_setup_logger(): - app_logger = setup_logger(__name__, VAR_DIR / "app.log", is_task=False) - app_logger.debug("test_setup_logger", extra={"task": "pytest"}) - - task_logger = setup_logger(__name__, LOG_DIR / "unit_task.log", is_task=True) - task_logger.debug("test_setup_logger") - - -def test_run_in_shell_success(): - cmd = 'echo "Test output"' - success, output = run_in_shell(cmd) - assert success - assert output == "Test output" - - -def test_run_in_shell_fail(): - cmd = "echo Fail! ; exit 1" - success, output = run_in_shell(cmd) - assert not success - assert output == "Fail!" - - -def test_mk_last_run_path(): - path = mk_last_run_path("foo") - assert str(path).endswith("_test_var/last_run/foo.txt") - - -def test_mk_output_path(): - path = mk_output_path("foo") - assert str(path).endswith("_test_var/output/foo.log") - - -def test_get_last_run_date(): - assert get_last_run_date("never_run_before.sh") is None - assert get_last_run_date("fail.sh") is None - - assert get_last_run_date("old.sh") == datetime.date(year=2020, month=1, day=1) - assert get_last_run_date("today.sh") == datetime.date.today() - - -def test_check_need_to_run(): - assert check_need_to_run("never_run_before.sh") is True - assert check_need_to_run("fail.sh") is True - assert check_need_to_run("old.sh") is True - - assert check_need_to_run("today.sh") is False diff --git a/unicron/constants.py b/unicron/constants.py new file mode 100644 index 0000000..3c9f662 --- /dev/null +++ b/unicron/constants.py @@ -0,0 +1,19 @@ +""" +Constants module. +""" +import os +from pathlib import Path + + +USE_TEST_MODE = os.environ.get("TEST") is not None + +APP_DIR = Path(__file__).resolve().parent +VAR_DIR = APP_DIR / ("_test_var" if USE_TEST_MODE else "var") + +TASKS_DIR = VAR_DIR / "targets" +LAST_RUN_DIR = VAR_DIR / "last_run" +RUN_EXT = ".txt" + +APP_LOG_PATH = VAR_DIR / "app.log" +OUTPUT_DIR = VAR_DIR / "output" +OUTPUT_EXT = ".log" diff --git a/unicron/history.py b/unicron/history.py new file mode 100644 index 0000000..a8746d6 --- /dev/null +++ b/unicron/history.py @@ -0,0 +1,50 @@ +""" +History module. +""" +import datetime + +from . import constants, logger, paths + + +def get_last_run_date(task_name: str): + """ + Get data of task's last run file and return as datetime object if set. + """ + last_run_path = paths.mk_last_run_path(task_name) + + if last_run_path.exists(): + last_run = last_run_path.read_text().strip() + if last_run: + return datetime.datetime.strptime(last_run, "%Y-%m-%d").date() + + return None + + +def check_need_to_run(task_name: str) -> bool: + """ + Check whether the given task needs to run today. + + If a last run file exists for the task, the file is non-empty and contains + a valid YYYY-MM-DD date which matches today's date, then the task can be + skipped. Otherwise it needs to be run today. + + The debug-level messages here are useful for in development for checking + on the reason for executing, but otherwise they can be ignored. + """ + app_logger = logger.setup_logger("unicron", constants.APP_LOG_PATH) + extra = {"task": task_name} + + last_run_date = get_last_run_date(task_name) + + if last_run_date: + if last_run_date == datetime.date.today(): + app_logger.info("Skipping, since already ran today.", extra=extra) + status = False + else: + app_logger.debug("Executing, since last run date is old.", extra=extra) + status = True + else: + app_logger.debug("Executing, since no run record found.", extra=extra) + status = True + + return status diff --git a/unicron/logger.py b/unicron/logger.py new file mode 100644 index 0000000..1ea115f --- /dev/null +++ b/unicron/logger.py @@ -0,0 +1,47 @@ +""" +Logger module. +""" +import logging +from pathlib import Path + + +APP_FORMATTER = logging.Formatter( + "%(asctime)s %(levelname)s:%(filename)s %(task)s - %(message)s" +) +TASK_FORMATTER = logging.Formatter( + "%(asctime)s %(levelname)s:%(filename)s - %(message)s" +) + +VERBOSE = False + + +def setup_logger(name: str, log_file: Path, is_task: bool = False): + """ + Configure a logger object and return it. + + It is safe to run this setup multiple in a script as the log filehandler + will only be added if it not there already. + + >>> app_logger = setup_logger('foo', 'foo.log') + >>> app_logger.info('This is just an info message.') + + >>> task_logger = setup_logger('bar', 'bar.log', is_task=True) + >>> task_logger.info('This is just an info message.') + """ + formatter = TASK_FORMATTER if is_task else APP_FORMATTER + + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + + if not logger.handlers: + file_handler = logging.FileHandler(log_file) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + if not is_task: + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + stream_handler.setLevel(logging.DEBUG if VERBOSE else logging.ERROR) + logger.addHandler(stream_handler) + + return logger diff --git a/unicron/paths.py b/unicron/paths.py new file mode 100644 index 0000000..a6ae090 --- /dev/null +++ b/unicron/paths.py @@ -0,0 +1,20 @@ +""" +Paths module. +""" +from pathlib import Path + +from . import constants + + +def mk_last_run_path(task_name: str) -> Path: + """ + Return full path to a task's last run file. + """ + return constants.LAST_RUN_DIR / "".join((task_name, constants.RUN_EXT)) + + +def mk_output_path(task_name: str) -> Path: + """ + Return output file path for a task. + """ + return constants.OUTPUT_DIR / "".join((task_name, constants.OUTPUT_EXT)) diff --git a/unicron/run.py b/unicron/run.py new file mode 100644 index 0000000..b44ed7a --- /dev/null +++ b/unicron/run.py @@ -0,0 +1,106 @@ +""" +Run module. +""" +import datetime +import subprocess +import textwrap +from pathlib import Path +from typing import Tuple + +from . import constants, logger, history, paths + + +def run_in_shell(cmd: str) -> Tuple[bool, str]: + """ + Run given command in the shell and return result of the command. + + Usually a malformed command or error in the executed code will result in + the CalledProcessError and then that message is shown. During development + of this project, the OSError was experienced so this is covered below too. + + :return success: True if it ran without error, False otherwise. + :return output: Text result of the command. If there was an error, this + will be the error message. + """ + try: + exitcode, output = subprocess.getstatusoutput(cmd) + except OSError as os_err: + success = False + output = str(os_err) + else: + success = exitcode == 0 + + return success, output + + +def proccess_cmd_result( + task_name: str, task_log_path: Path, last_run_path: Path, status: bool, output: str +) -> None: + """ + Process the result of running a command. + + Log activity for the task and update the last run date if the task was run + successfully. + """ + app_logger = logger.setup_logger("unicron", constants.APP_LOG_PATH) + task_logger = logger.setup_logger(task_name, task_log_path, is_task=True) + + extra = {"task": task_name} + output_log_msg = f"Output:\n{textwrap.indent(output, ' '*4)}" + + if status: + app_logger.info("Success.", extra=extra) + task_logger.info(output_log_msg) + + today = datetime.date.today() + last_run_path.write_text(str(today)) + else: + app_logger.error( + "Task exited with error status! Check this task's log: %s", + task_log_path, + extra=extra, + ) + task_logger.error(output_log_msg) + + +def execute(task_name: str) -> bool: + """ + On a succesful run, set today's date in the last run event file for the + executable, so that on subsequent runs today this executable will be + ignored. On a failing run, do not update the file so we leave it marked as + need to run today still. + + Regardless of the executed task's output, capture all output and send it to + a log file dedicated to that task. This makes it easy to view the + executable's history later. + + :return status: True if ran without error, False otherwise. + """ + last_run_path = paths.mk_last_run_path(task_name) + + task_log_path = paths.mk_output_path(task_name) + task_logger = logger.setup_logger(task_name, task_log_path, is_task=True) + + task_logger.info("Executing...") + cmd = constants.TASKS_DIR / task_name + status, output = run_in_shell(str(cmd)) + + proccess_cmd_result(task_name, task_log_path, last_run_path, status, output) + + return status + + +def handle_task(task_name): + """ + Run a task, if it needs to run now. + + :return status: True on task success, False on failure and None on skipped. + """ + should_run = history.check_need_to_run(task_name) + + if should_run: + status = execute(task_name) + else: + status = None + + return status diff --git a/unicron/unicron.py b/unicron/unicron.py index 10340b8..c3f11ab 100755 --- a/unicron/unicron.py +++ b/unicron/unicron.py @@ -34,239 +34,23 @@ symlink. """ import argparse -import datetime -import logging -import subprocess -import os import sys -import textwrap from pathlib import Path from typing import List, Tuple +from . import constants, logger, run -USE_TEST_MODE = os.environ.get("TEST") is not None -APP_DIR = Path(__file__).resolve().parent -VAR_DIR = APP_DIR / ("_test_var" if USE_TEST_MODE else "var") - -TASKS_DIR = VAR_DIR / "targets" -LAST_RUN_DIR = VAR_DIR / "last_run" -RUN_EXT = ".txt" - -APP_LOG_PATH = VAR_DIR / "app.log" -OUTPUT_DIR = VAR_DIR / "output" -OUTPUT_EXT = ".log" - -APP_FORMATTER = logging.Formatter( - "%(asctime)s %(levelname)s:%(filename)s %(task)s - %(message)s" -) -TASK_FORMATTER = logging.Formatter( - "%(asctime)s %(levelname)s:%(filename)s - %(message)s" -) - -VERBOSE = False - - -def setup_logger(name: str, log_file: Path, is_task: bool = False): - """ - Configure a logger object and return it. - - It is safer to run this setup multiple in a script as the log file handler - will only be added if it not there already. - - >>> app_logger = setup_logger('foo', 'foo.log') - >>> app_logger.info('This is just an info message.') - - >>> task_logger = setup_logger('bar', 'bar.log', is_task=True) - >>> task_logger.info('This is just an info message.') +def get_tasks(tasks_dir: Path) -> List[str]: """ - formatter = TASK_FORMATTER if is_task else APP_FORMATTER - - logger = logging.getLogger(name) - logger.setLevel(logging.DEBUG) - - if not logger.handlers: - file_handler = logging.FileHandler(log_file) - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - - if not is_task: - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(formatter) - stream_lvl = logging.DEBUG if VERBOSE else logging.ERROR - stream_handler.setLevel(stream_lvl) - logger.addHandler(stream_handler) - - return logger - - -def run_in_shell(cmd: str) -> Tuple[bool, str]: + Get Path objects for tasks in a given tasks diectory. """ - Run given command in the shell and return result of the command. - - Usually a malformed command or error in the executed code will result in - the CalledProcessError and then that message is shown. During development - of this project, the OSError was experienced so this is covered below too. - - :return success: True if ran without error, False otherwise. - :return output: Text result of the command. If there was an error, this - will be the error message. - """ - try: - exitcode, output = subprocess.getstatusoutput(cmd) - except OSError as os_err: - success = False - output = str(os_err) - else: - success = exitcode == 0 - - return success, output - - -def mk_last_run_path(task_name: str) -> Path: - """ - Return full path to a task's last run file. - """ - return LAST_RUN_DIR / "".join((task_name, RUN_EXT)) - - -def mk_output_path(task_name: str) -> Path: - """ - Return output file path for a task. - """ - return OUTPUT_DIR / "".join((task_name, OUTPUT_EXT)) - - -def get_last_run_date(task_name: str): - """ - Get data of task's last run file and return as datetime object if set. - """ - last_run_path = mk_last_run_path(task_name) - - if last_run_path.exists(): - last_run = last_run_path.read_text().strip() - if last_run: - return datetime.datetime.strptime(last_run, "%Y-%m-%d").date() - - return None - - -def check_need_to_run(task_name: str) -> bool: - """ - Check whether the given task needs to run today. - - If a last run file exists for the task, the file is non-empty and contains - a valid YYYY-MM-DD date which matches today's date, then the task can be - skipped. Otherwise it needs to be run today. - - The debug-level messages here are useful for in development for checking - on the reason for executing, but otherwise they can be ignored. - """ - app_logger = setup_logger("unicron", APP_LOG_PATH) - extra = {"task": task_name} - - last_run_date = get_last_run_date(task_name) - - if last_run_date: - if last_run_date == datetime.date.today(): - app_logger.info("Skipping, since already ran today.", extra=extra) - status = False - else: - app_logger.debug("Executing, since last run date is old.", extra=extra) - status = True - else: - app_logger.debug("Executing, since no run record found.", extra=extra) - status = True - - return status - - -def proccess_cmd_result( - task_name: str, task_log_path: Path, last_run_path: Path, status: bool, output: str -) -> None: - """ - Process the result of running a command. - - Log activity for the task and update the last run date if the task was - successful. - """ - assert status is not None, "Status must indicate success (True) or fail (False)." - - app_logger = setup_logger("unicron", APP_LOG_PATH) - task_logger = setup_logger(task_name, task_log_path, is_task=True) - - extra = {"task": task_name} - output_log_msg = f"Output:\n{textwrap.indent(output, ' '*4)}" - - if status: - app_logger.info("Success.", extra=extra) - task_logger.info(output_log_msg) - - today = datetime.date.today() - last_run_path.write_text(str(today)) - else: - app_logger.error( - "Task exited with error status! Check this task's log: %s", - task_log_path, - extra=extra, - ) - task_logger.error(output_log_msg) - - -def execute(task_name: str) -> bool: - """ - On a succesful run, set today's date in the last run event file for the - executable, so that on subsequent runs today this executable will be - ignored. On a failing run, do not update the file so we leave it marked as - need to run today still. - - Regardless of the executed task's output, capture all output and send it to - a log file dedicated to that task. This makes it easy to view the - executable's history later. - - :return status: True if ran without error, False otherwise. - """ - last_run_path = mk_last_run_path(task_name) - - task_log_path = mk_output_path(task_name) - task_logger = setup_logger(task_name, task_log_path, is_task=True) - - task_logger.info("Executing...") - cmd = TASKS_DIR / task_name - status, output = run_in_shell(str(cmd)) - - proccess_cmd_result(task_name, task_log_path, last_run_path, status, output) - - return status - - -def handle_task(task_name): - """ - Run a task, if it needs to run now. - - :return status: True on task success, False on failure and None on not - running. - """ - should_run = check_need_to_run(task_name) - - if should_run: - status = execute(task_name) - else: - status = None - - return status - - -def get_tasks() -> List[str]: - """ - Get Path objects for tasks in the configured tasks diectory. - """ - globbed_tasks = sorted(TASKS_DIR.iterdir()) + globbed_tasks = sorted(tasks_dir.iterdir()) return [p.name for p in globbed_tasks if not p.name.startswith(".")] -def handle_tasks() -> Tuple[int, int, int]: +def handle_tasks(tasks_dir: Path, app_log_path: Path) -> Tuple[int, int, int]: """ Find tasks, check their run status for today and run any if needed. @@ -274,15 +58,15 @@ def handle_tasks() -> Tuple[int, int, int]: """ success = fail = skipped = 0 - app_logger = setup_logger("unicron", APP_LOG_PATH, is_task=False) + app_logger = logger.setup_logger("unicron", app_log_path, is_task=False) extra = {"task": "unicron"} - tasks = get_tasks() + tasks = get_tasks(tasks_dir) msg = f"Task count: {len(tasks)}" app_logger.info(msg, extra=extra) for task_name in tasks: - status = handle_task(task_name) + status = run.handle_task(task_name) if status is True: success += 1 @@ -301,13 +85,11 @@ def main() -> None: """ Main command-line argument parser. - :raises: Exit script on error code if there are any failures. + Exit script on error code if there are any failures. """ - global VERBOSE # pylint: disable=global-statement - parser = argparse.ArgumentParser( - description="Uniron task scheduler.", - epilog="Run against the test var directory, using TEST=1 as script prefix.", + description="Unicron task scheduler.", + epilog="Run against the test var directory by using TEST=true as script prefix.", ) parser.add_argument( "-v", @@ -317,10 +99,12 @@ def main() -> None: action="store_true", ) args = parser.parse_args() - if args.verbose: - VERBOSE = True - _, fail, _ = handle_tasks() + logger.VERBOSE = args.verbose + + _, fail, _ = handle_tasks( + tasks_dir=constants.TASKS_DIR, app_log_path=constants.APP_LOG_PATH + ) if fail != 0: sys.exit(1)