Merge pull request #54 from MichaelCurrin/refactor-split-main-script

refactor: Split main script into modules

.gitignore

@@ -1,6 +1,4 @@
 __pycache__/
-.pytest_cache/
-.mypy_cache/
 
 venv
 

Makefile

@@ -16,7 +16,10 @@ install-dev:
 
 # Run all configured tasks in main VAR targets dir.
 run:
-	unicron/unicron.py --verbose
+	python3 -m unicron.unicron --verbose
+
+run-quiet:
+	python3 -m unicron.unicron
 
 
 # View configured tasks.
@@ -33,11 +36,11 @@ ls-test-runs:
 
 
 # Make all task scripts executable.
-permission:
+perms:
 	chmod +x unicron/var/targets/*
 
 
-# Tail the app log.
+# Tail the app logs.
 log-app:
 	cd unicron/var && tail -F app.log
 # Same as above but with longer history.
@@ -48,18 +51,15 @@ log-app-long:
 # Tail the task logs.
 log-tasks:
 	cd unicron/var && tail -F output/*.log
-
-# Same as above but with longer history.
 log-tasks-long:
 	cd unicron/var && tail -n50 -F output/*.log
 
 # Tail both the app and task logs.
 log:
 	cd unicron/var && tail -F output/*.log app.log
-
-# As above, for test tasks. We make the _test_var path shown here for clarity.
+# Tail logs created by `debug` target.
 log-tests:
-	cd unicron && tail -n20 -F _test_var/output/*.log _test_var/app.log
+	cd unicron/_test_var && tail -n20 -F output/*.log app.log
 
 
 format:
@@ -68,8 +68,8 @@ format-check:
 	black . --diff --check
 
 pylint:
-	# Exit on error code if needed.
-	pylint unicron/unicron.py || pylint-exit $$?
+	# Exit on error code on a fail. Expand failure to all non-fatal messages too.
+	pylint unicron tests || pylint-exit -efail -wfail -rfail -cfail $$?
 
 lint: pylint
 
@@ -80,13 +80,17 @@ typecheck:
 	mypy unicron tests
 
 
+clean:
+	find . -name '*.pyc' -delete
+
+
 # Reset tasks and logs in the TEST VAR dir.
 reset:
 	bin/reset.sh
 
 # Run unit tests.
 unit: reset
-	pytest
+	TEST=true pytest
 
 # Integration test.
 debug: reset

bin/cron_target.sh

@@ -1,24 +1,26 @@
 #!/bin/bash
-# Run Unicron as a cron command.
-#
-# On success, be silent.
-# On error, send all output to the user's local mailbox.
+# Run Unicron as a scheduled cron command.
 #
 # This script can be run from anywhere.
+#
+# On success, run silently - nothing printed at all.
+# On error, capture all stdout and stderr so it can be sent to user's local mailbox using cron's mechanism or the mail command.
+# Either way, you won't see anything printed if run this command.
 
 set -e
 
-SCRIPT_DIR=$(dirname $(realpath $0))
-SCRIPT_FILEPATH="$SCRIPT_DIR/../unicron/unicron.py"
+SCRIPT_DIR=$(dirname $(dirname $(realpath "$0")))
+cd "$SCRIPT_DIR"
 
 set +e
-OUTPUT="$($SCRIPT_FILEPATH 2>&1)"
+
+CMD_OUTPUT="$(make run-quiet 2>&1)"
 
 if [[ $? -ne 0 ]]; then
-    set -e
-    echo "$OUTPUT" | mail -s 'Unicron task failed!' $USER
+  set -e
+  echo "$CMD_OUTPUT" | mail -s 'Unicron task failed!' $USER
 
-    exit 1
+  exit 1
 fi
 
 exit 0

bin/reset.sh

@@ -10,14 +10,15 @@
 echo 'Entering _test_var directory.'
 cd unicron/_test_var
 
+echo 'Remove logs.'
+rm app.log >/dev/null 2>&1 || true
+rm output/*.log >/dev/null 2>&1 || true
+
 echo 'Create last-run file fixtures.'
 cd last_run/
+# Note that the today.sh task will never actually run when it unicron checks it,
+# so there today.sh.log will never get created and this is okay.
 echo $(date +%Y-%m-%d) >today.sh.txt
 echo "2020-01-01" >old.sh.txt
 rm never_run_before.sh.txt >/dev/null 2>&1 || true
 rm fail.sh.txt >/dev/null 2>&1 || true
-cd ..
-
-echo 'Remove logs.'
-rm app.log >/dev/null 2>&1 || true
-rm output/*.log >/dev/null 2>&1 || true

bin/test.sh

@@ -7,12 +7,12 @@
 
 echo "Main - first run"
 echo "==="
-TEST=true unicron/unicron.py -v
+TEST=true python -m unicron.unicron -v
 echo
 echo
 
 echo "Main - second run"
 echo "==="
-TEST=true unicron/unicron.py -v
+TEST=true python -m unicron.unicron -v
 
 true

docs/features.md

@@ -27,28 +27,67 @@
     * Application log - *var/app.log*
 - There is also `_test_var` directory for testing the application without affecting the main `var` directory.
 
+
 ## Sample
 
-Given a configured script `hello.sh` in the targets directory.
+Given an executable script `hello.sh` added to the targets directory.
+
+- `hello.sh`
+    ```sh
+    echo 'Hello, word'
+    ```
+
+Running directly:
+
+```sh
+$ ./unicron/var/targets/hello.sh
+Hello, world!
+```
+
 
-<!-- TODO: Update with new output -->
+Run with Unicron - note the verbose flag is implied in the `run` target of the `Makefile`.
 
 1. First run today - the script executes.
     ```bash
-    $ ./unicron.py --verbose
-    2020-01-05 19:23:05 INFO:unicron hello.sh - Success.
+    $ make run
+    2020-10-24 08:02:18,516 INFO:unicron.py unicron - Task count: 1
+    2020-10-24 08:02:18,532 INFO:run.py hello.sh - Success.
+    2020-10-24 08:02:18,537 INFO:unicron.py unicron - Succeeded: 1; Failed: 0; Skipped: 0
     ```
-2. Second run today - the script is skipped.
+2. A repeat run today - the script is skipped.
     ```bash
-    $ ./unicron.py --verbose
-    2020-01-05 19:23:56 INFO:unicron hello.sh - Skipping, since already ran today.
+    $ make run
+    2020-10-24 08:03:36,313 INFO:unicron.py unicron - Task count: 7
+    2020-10-24 08:03:36,317 INFO:history.py hello.sh - Skipping, since already ran today.
+    2020-10-24 08:02:18,537 INFO:unicron.py unicron - Succeeded: 0; Failed: 0; Skipped: 1
     ```
-3. First run tomorrow - the script executes.
+3. On the first run tomorrow - the script executes.
     ```bash
-    $ ./unicron.py --verbose
-    2020-01-06 12:22:00 INFO:unicron hello.sh - Success.
+    $ make run
+    2020-10-25 08:02:18,516 INFO:unicron.py unicron - Task count: 1
+    2020-10-25 08:02:18,532 INFO:run.py hello.sh - Success.
+    2020-10-25 08:02:18,537 INFO:unicron.py unicron - Succeeded: 1; Failed: 0; Skipped: 0
     ```
-4. Scheduling - add a single command to your _crontab_ configuration. For example, run every 30 minutes and only send mail if at least one job fails.
+
+Scheduling - add a single command to your _crontab_ configuration.
+
+For example, run every 30 minutes and only send mail if at least one job fails.
+
+```sh
+$ crontab -e
+```
+```
+*/30    *       *       *       *       ~/repos/unicron/bin/cron_target.sh
+```
+
+Then access runs using:
+
+```sh
+$ mail
+Mail version 8.1 6/6/93.  Type ? for help.
+"/var/mail/mcurrin": 1 message 1 new
+>N  1 mcurrin@C02WL0Y2HV2T  Sat Oct 24 20:00  17/836   "Unicron task failed!"
+```
 
 
 ## What is the point of running once but retrying?

docs/usage.md

@@ -15,7 +15,7 @@ The main things you to do with _Unicron_ are:
 
 <details>
 <summary>
-    
+
 Click to expand.
 
 ```bash
@@ -64,7 +64,7 @@ The example output below is for the demo script which was setup using [Installat
     $ make run
     ```
     ```
-    unicron/unicron.py -v
+    python3 -m unicron.unicron -v
     2020-04-05 10:00:00,414 INFO:unicron.py unicron - Task count: 1
     2020-04-05 10:00:00,429 DEBUG:unicron.py hello.sh - Executing, since last run date is old.
     2020-04-05 10:00:02,224 INFO:unicron.py hello.sh - Success.
@@ -75,7 +75,7 @@ The example output below is for the demo script which was setup using [Installat
     $ make run
     ```
     ```
-    unicron/unicron.py -v
+    python3 -m unicron.unicron -v
     2020-04-05 10:10:00,414 INFO:unicron.py unicron - Task count: 1
     2020-04-05 10:10:00,429 INFO:unicron.py hello.sh - Skipping, since already ran today.
     2020-04-05 10:10:00,500 INFO:unicron.py unicron - Suceeded: 0; Failed: 0; Skipped: 1
@@ -94,7 +94,7 @@ make: *** [run] Error 1
 Without any custom tasks setup, you start test _Unicron_ immediately by running the versioned test tasks.
 
 ```bash
-$ make run-test
+$ make debug
 ```
 
 
@@ -222,6 +222,7 @@ cd unicron && tail -n20 -F _test_var/output/*.log _test_var/app.log
 Instead of going through `make`, you can run the Python script directly:
 
 ```bash
-$ cd ~/repos/unicron/unicron
-$ ./unicron.py --help
+$ python3 -m unicron.unicron -v
 ```
+
+You have to use the `-m` syntax for the relative imports in the script to work. If they are not relative imports and just `import logger`, then the tests can't run properly.

tests/__init__.py

@@ -2,4 +2,10 @@
 Tests initialization module.
 
 This file is required for pytest to correctly do imports.
+
+Make sure TEST=true is set when running tests. That will ensure that main app
+var file references are in the test var directory, to keep the main one clean.
+
+You may have to run reset.sh before test. Note that this will delete all test
+logs.
 """

tests/test_history.py

@@ -0,0 +1,27 @@
+"""
+History module tests.
+"""
+# pylint: disable=missing-function-docstring
+import datetime
+
+from unicron import history
+
+
+def test_get_last_run_date():
+    assert history.get_last_run_date("never_run_before.sh") is None
+    assert history.get_last_run_date("fail.sh") is None
+
+    assert history.get_last_run_date("old.sh") == datetime.date(
+        year=2020, month=1, day=1
+    )
+    assert history.get_last_run_date("today.sh") == datetime.date.today()
+
+
+def test_check_need_to_run():
+    # Expect to run.
+    assert history.check_need_to_run("never_run_before.sh") is True
+    assert history.check_need_to_run("fail.sh") is True
+    assert history.check_need_to_run("old.sh") is True
+
+    # Expect not to run.
+    assert history.check_need_to_run("today.sh") is False

tests/test_logger.py

@@ -0,0 +1,20 @@
+"""
+Logger module tests.
+"""
+# pylint: disable=missing-function-docstring
+from pathlib import Path
+
+from unicron import logger
+
+
+APP_DIR = Path("unicron")
+VAR_DIR = APP_DIR / Path("_test_var")
+LOG_DIR = VAR_DIR / "last_run"
+
+
+def test_setup_logger():
+    app_logger = logger.setup_logger(__name__, VAR_DIR / "app.log", is_task=False)
+    app_logger.debug("test_setup_logger", extra={"task": "pytest"})
+
+    task_logger = logger.setup_logger(__name__, LOG_DIR / "unit_task.log", is_task=True)
+    task_logger.debug("test_setup_logger")

tests/test_paths.py

@@ -0,0 +1,17 @@
+"""
+Paths module tests.
+"""
+# pylint: disable=missing-function-docstring
+from unicron import paths
+
+
+def test_mk_last_run_path():
+    path = paths.mk_last_run_path("foo")
+
+    assert str(path).endswith("_test_var/last_run/foo.txt")
+
+
+def test_mk_output_path():
+    path = paths.mk_output_path("foo")
+
+    assert str(path).endswith("_test_var/output/foo.log")

tests/test_run.py

@@ -0,0 +1,21 @@
+"""
+Run module tests.
+"""
+# pylint: disable=missing-function-docstring
+from unicron import run
+
+
+def test_run_in_shell_success():
+    cmd = 'echo "Test output"'
+    success, output = run.run_in_shell(cmd)
+
+    assert success
+    assert output == "Test output"
+
+
+def test_run_in_shell_fail():
+    cmd = "echo Fail! ; exit 1"
+    success, output = run.run_in_shell(cmd)
+
+    assert not success
+    assert output == "Fail!"