Skip to content

Commit

Permalink
Feature: better logging (#233)
Browse files Browse the repository at this point in the history
  • Loading branch information
Guts authored Mar 10, 2023
2 parents 47d0c87 + 9d832ed commit bb1ee6a
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 52 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ jobs:
- name: Unit tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
QDT_LOGS_LEVEL: 4
run: pytest

- name: Upload coverage to Codecov
Expand Down Expand Up @@ -89,3 +90,6 @@ jobs:

- name: QDT - Sample scenario
run: qgis-deployment-toolbelt --verbose

- name: QDT - Sample scenario
run: qgis-deployment-toolbelt --verbose --no-logfile
23 changes: 0 additions & 23 deletions docs/usage/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,26 +9,3 @@
:prog: qgis-deployment-toolbelt
:title: Commands and options
```

----

## Environment variables

### CLI arguments

Some options and arguments can be set with environment variables.

| Variable name | Corresponding CLI argument | Default value |
| :------------------ | :------------------------: | :----------------: |
| `QDT_UPGRADE_CHECK_ONLY` | `-c`, `--check-only` in `upgrade` | `False` |
| `QDT_UPGRADE_DISPLAY_RELEASE_NOTES` | `-n`, `--dont-show-release-notes` in `upgrade` | `True` |
| `QDT_UPGRADE_DOWNLOAD_FOLDER` | `-w`, `--where` in `upgrade` | `./` (current folder) |
| `QDT_SCENARIO_PATH` | `--scenario` in `deploy` | `scenario.qdt.yml` |

### Others

Some others parameters can be set using environment variables.

| Variable name | Description | Default value |
| :------------------ | :----------------------: | :----------------: |
| `QDT_LOCAL_WORK_DIR` | Local folder where QDT download remote resources (profiles, plugins, etc.) | `~/.cache/qgis-deployment-toolbelt/default/` |
23 changes: 17 additions & 6 deletions docs/usage/settings.md
Original file line number Diff line number Diff line change
@@ -1,13 +1,24 @@
# Configuration

> TO DOC
## Using environment variables

> TO DOC
### CLI arguments

Some options and arguments can be set with environment variables.

| Variable name | Corresponding CLI argument | Default value |
| :------------------ | :------------------------: | :----------------: |
| `QDT_LOGS_LEVEL` | `-v`, `--verbose` | `1` (= `logging.WARNING`). Must be an integer. |
| `QDT_UPGRADE_CHECK_ONLY` | `-c`, `--check-only` in `upgrade` | `False` |
| `QDT_UPGRADE_DISPLAY_RELEASE_NOTES` | `-n`, `--dont-show-release-notes` in `upgrade` | `True` |
| `QDT_UPGRADE_DOWNLOAD_FOLDER` | `-w`, `--where` in `upgrade` | `./` (current folder) |
| `QDT_SCENARIO_PATH` | `--scenario` in `deploy` | `scenario.qdt.yml` |

----
### Others

## Settings
Some others parameters can be set using environment variables.

> TO DOC
| Variable name | Description | Default value |
| :------------------ | :----------------------: | :----------------: |
| `QDT_LOCAL_WORK_DIR` | Local folder where QDT download remote resources (profiles, plugins, etc.) | `~/.cache/qgis-deployment-toolbelt/default/` |
| `QDT_LOGS_DIR` | Folder where QDT writes the log files, which are automatically rotated. | `~/.cache/qgis-deployment-toolbelt/logs/` |
48 changes: 25 additions & 23 deletions qgis_deployment_toolbelt/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
__version__,
)
from qgis_deployment_toolbelt.commands import parser_main_deployment, parser_upgrade
from qgis_deployment_toolbelt.utils.journalizer import configure_logger

# #############################################################################
# ########## Globals ###############
Expand Down Expand Up @@ -70,10 +71,14 @@ def set_default_subparser(
"""
subparser_found = False
for arg in sys.argv[1:]:
if arg in ["-h", "--help"]: # global help if no subparser
break
elif arg in ["--version"]: # global version if no subparser
if arg in [
"-h",
"--help",
"--version",
"--no-logfile",
]: # ignore main parser args
break

else:
for x in parser_to_update._subparsers._actions:
if not isinstance(x, argparse._SubParsersAction):
Expand Down Expand Up @@ -116,7 +121,17 @@ def main(in_args: list[str] = None):
action="count",
default=1,
dest="verbosity",
help="Verbosity level. None = WARNING, -v = INFO, -vv = DEBUG",
help="Verbosity level. None = WARNING, -v = INFO, -vv = DEBUG. Can be set with "
"QDT_LOGS_LEVEL environment variable and logs location with QDT_LOGS_DIR.",
)

main_parser.add_argument(
"--no-logfile",
default=True,
action="store_false",
dest="opt_logfile_disabled",
help="Disable log file. Log files are usually created, rotated and stored in the"
"folder set by QDT_LOGS_DIR.",
)

main_parser.add_argument(
Expand Down Expand Up @@ -156,23 +171,13 @@ def main(in_args: list[str] = None):
# just get passed args
args = main_parser.parse_args(in_args)

# set log level depending on verbosity argument
if 0 < args.verbosity < 4:
args.verbosity = 40 - (10 * args.verbosity)
elif args.verbosity >= 4:
# debug is the limit
args.verbosity = 40 - (10 * 3)
# log configuration
if args.opt_logfile_disabled:
configure_logger(
verbosity=args.verbosity, logfile=f"{__title_clean__}_{__version__}.log"
)
else:
args.verbosity = 0

logging.basicConfig(
level=args.verbosity,
format="%(asctime)s||%(levelname)s||%(module)s||%(lineno)d||%(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)

console = logging.StreamHandler()
console.setLevel(args.verbosity)
configure_logger(verbosity=args.verbosity)

# add the handler to the root logger
logger = logging.getLogger(__title_clean__)
Expand All @@ -181,9 +186,6 @@ def main(in_args: list[str] = None):
# -- RUN LOGIC --
if hasattr(args, "func"):
args.func(args)
else:
# if no args, run deployment
main(["deploy"] + in_args)


# #############################################################################
Expand Down
138 changes: 138 additions & 0 deletions qgis_deployment_toolbelt/utils/journalizer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
#! python3 # noqa: E265

"""Helper to configure logging depending on CLI options."""

# ############################################################################
# ########## IMPORTS #############
# ################################

# standard library
import logging
from getpass import getuser
from logging.handlers import RotatingFileHandler
from os import getenv
from os.path import expanduser, expandvars
from pathlib import Path
from platform import architecture
from platform import platform as opersys
from socket import gethostname

# package
from qgis_deployment_toolbelt.__about__ import __title__, __version__
from qgis_deployment_toolbelt.constants import get_qdt_working_directory
from qgis_deployment_toolbelt.utils.check_path import check_path
from qgis_deployment_toolbelt.utils.proxies import get_proxy_settings

# ############################################################################
# ########## GLOBALS #############
# ################################

# logs
logger = logging.getLogger(__name__)

# ############################################################################
# ########## FUNCTIONS ###########
# ################################


def configure_logger(verbosity: int = 1, logfile: Path = None):
"""Configure logging according to verbosity from CLI.
Args:
verbosity (int): verbosity level
logfile (Path, optional): file where to store log. Defaults to None.
"""
# handle log level overridden by environment variable
verbosity = getenv("QDT_LOGS_LEVEL", verbosity)
try:
verbosity = int(verbosity)
except ValueError as err:
logger.error(f"Bad verbosity value type: {err}. Fallback to 1.")
verbosity = 1

# set log level depending on verbosity argument
if 0 < verbosity < 4:
verbosity = 40 - (10 * verbosity)
elif verbosity >= 4:
# debug is the limit
verbosity = 40 - (10 * 3)
else:
verbosity = 0

# set console handler
log_console_handler = logging.StreamHandler()
log_console_handler.setLevel(verbosity)

# set log file
if not logfile:
logging.basicConfig(
level=verbosity,
format="%(asctime)s||%(levelname)s||%(module)s||%(lineno)d||%(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
handlers=[log_console_handler],
)

else:
if getenv("QDT_LOGS_DIR") and check_path(
input_path=Path(expandvars(expanduser(getenv("QDT_LOGS_DIR")))),
must_be_a_file=False,
must_be_a_folder=True,
must_be_writable=True,
raise_error=False,
):
logs_folder = Path(expandvars(expanduser(getenv("QDT_LOGS_DIR"))))
logger.debug(
f"Logs folder set with QDT_LOGS_DIR environment variable: {logs_folder}"
)
else:
logs_folder = get_qdt_working_directory().parent / "logs"
logger.debug(
"Logs folder specified in QDT_LOGS_DIR environment variable "
f"{getenv('QDT_LOGS_DIR')} can't be used (see logs above). Fallback on "
f"default folder: {logs_folder}"
)

# make sure folder exists
logs_folder.mkdir(exist_ok=True, parents=True)
logs_filepath = Path(logs_folder, logfile)

log_file_handler = RotatingFileHandler(
backupCount=10,
delay=True,
encoding="UTF-8",
filename=logs_filepath,
maxBytes=3000000,
mode="a",
)
# force new file by execution
if logs_filepath.is_file():
log_file_handler.doRollover()

logging.basicConfig(
level=verbosity,
format="%(asctime)s||%(levelname)s||%(module)s||%(lineno)d||%(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
handlers=[log_console_handler, log_file_handler],
)

logger.info(f"Log file: {logs_filepath}")

headers()


def headers():
"""Basic information to log before other message."""
# initialize the log
logger.info(f"{'='*10} {__title__} - {__version__} {'='*10}")
logger.debug(f"Operating System: {opersys()}")
logger.debug(f"Architecture: {architecture()[0]}")
logger.debug(f"Computer: {gethostname()}")
logger.debug(f"Launched by user: {getuser()}")

if getenv("userdomain"):
logger.debug(f"OS Domain: {getenv('userdomain')}")

if get_proxy_settings():
logger.debug(f"Network proxies detected: {get_proxy_settings()}")
else:
logger.debug("No network proxies detected")

0 comments on commit bb1ee6a

Please sign in to comment.