From c0c5c3f2d89a32f1677ab8b8fcf67e226528ae84 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Thu, 22 Mar 2018 21:57:07 -0700 Subject: [PATCH 01/33] ENH: Helper function to find current log files Instead of repeating the code to find a handler of a certain name, a generic function was created to find a handler of any name. This is then used to find the base log file name and search the directory for any other log messages that may have been created. --- hutch_python/log_setup.py | 46 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/hutch_python/log_setup.py b/hutch_python/log_setup.py index 432821a9..49941033 100644 --- a/hutch_python/log_setup.py +++ b/hutch_python/log_setup.py @@ -16,6 +16,7 @@ logger = logging.getLogger(__name__) + def setup_logging(dir_logs=None): """ Sets up the ``logging`` configuration. @@ -63,6 +64,24 @@ def setup_logging(dir_logs=None): logging.getLogger('parso.cache').disabled = True +def get_session_logfiles(): + """ + Get the path to the current debug log file + + Returns + ------- + logs : list + List of absolute paths to log files that were created by this session + """ + # Grab the debug file handler + handler = get_debug_handler() + # Find all the log files that were generated by this session + base = Path(handler.baseFilename) + return [str(base.parent / log) + for log in os.listdir(base.parent) + if log.startswith(base.name)] + + def get_console_handler(): """ Helper function to find the console ``StreamHandler``. @@ -72,11 +91,34 @@ def get_console_handler(): console: ``StreamHandler`` The ``Handler`` that prints to the screen. """ + return get_handler('console') + + +def get_debug_handler(): + """ + Helper function to find the debug ``RotatingFileHandler`` + + Returns + ------- + debug: ``RotatingFileHandler`` + The ``Handler`` that prints to the log files + """ + return get_handler('debug') + + +def get_handler(name): + """ + Helper function to get an arbitrary `Handler` + + Returns + ------- + hander : `Handler` + """ root = logging.getLogger('') for handler in root.handlers: - if handler.name == 'console': + if handler.name == name: return handler - raise RuntimeError('No console handler') + raise RuntimeError('No {} handler'.format(name)) def get_console_level(): From bc9d60b62970fc7f4511c6cd79fa748f74acc1b1 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Thu, 22 Mar 2018 22:01:21 -0700 Subject: [PATCH 02/33] TST: Add test to find log files from test session --- hutch_python/tests/test_log_setup.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/hutch_python/tests/test_log_setup.py b/hutch_python/tests/test_log_setup.py index 38f93e3b..92c69eaa 100644 --- a/hutch_python/tests/test_log_setup.py +++ b/hutch_python/tests/test_log_setup.py @@ -4,9 +4,10 @@ import pytest -from hutch_python.log_setup import (setup_logging, +from hutch_python.log_setup import (setup_logging, get_session_logfiles, get_console_handler, set_console_level, - debug_mode, debug_context, debug_wrapper) + debug_mode, debug_context, debug_wrapper, + get_debug_handler) from conftest import restore_logging @@ -38,6 +39,18 @@ def test_console_handler(log_queue): assert isinstance(handler, QueueHandler) +def test_get_session_logfiles(): + logger.debug('test_get_session_logfiles') + with restore_logging(): + # Create a parent log file + setup_logging(dir_logs=Path(__file__).parent / 'logs') + debug_handler = get_debug_handler() + debug_handler.doRollover() + debug_handler.doRollover() + assert len(get_session_logfiles()) == 3 + assert all([log.startswith(debug_handler.baseFilename) + for log in get_session_logfiles()]) + def setup_queue_console(): root_logger = logging.getLogger('') for handler in root_logger.handlers: From 9ee76d1c26a9c6e05de7cb26f20c36bfea659ae0 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Fri, 23 Mar 2018 15:27:25 -0700 Subject: [PATCH 03/33] TST: Add a test for generating bug report --- hutch_python/bug.py | 213 +++++++++++++++++++++++++++++++++ hutch_python/tests/test_bug.py | 28 +++++ 2 files changed, 241 insertions(+) create mode 100644 hutch_python/bug.py create mode 100644 hutch_python/tests/test_bug.py diff --git a/hutch_python/bug.py b/hutch_python/bug.py new file mode 100644 index 00000000..89e14060 --- /dev/null +++ b/hutch_python/bug.py @@ -0,0 +1,213 @@ +""" +Automated Bug Reporting +""" +import os +import getpass +import logging +import textwrap +import tempfile +import warnings +import subprocess + +from IPython.core.history import HistoryAccessor + +from .log_setup import get_session_logfiles + + +logger = logging.getLogger(__name__) + + +# This function is aliased here so it can be easily be patched by the test +# suite. This allows us to eloquently simulate user inputted data +request_input = input + + +def get_current_environment(): + """ + Get the current CONDA environment + + The environment name is gathered by first checking environment variable + ``$CONDA_ENVNAME`` that is set by the hutch-python startup scripts. If for + whatever reason that is not available we check $CONDA_DEFAULT_ENV which is + set by Conda itself. The reason this is not relied on primarily is it has a + strange name and is entirely undocumented but seems to work effectively. + + In addition the hutch-python startup script sets the PYTHONPATH to pick up + packages in "development" mode. The list of package names installed this + way is found to help inform how the current Python environment differs from + the enforced CONDA environment + + Returns + ------- + env: str + Name of environment + + dev_pkgs :list + List of packages installed in the development folder + """ + # Search for the environment variable set by the hutch python setup + env = os.getenv('CONDA_ENVNAME') + # Otherwise look for built-in Conda environment variables + if not env: + env = os.getenv('CONDA_DEFAULT_ENV') + # Check the top level PYTHONPATH to see if we have packages installed in + # development mode + dev = os.getenv('PYTHONPATH') + if dev: + try: + dev_pkgs = os.listdir(dev) + except FileNotFoundError: + logger.debug("No dev folder found") + else: + dev_pkgs = list() + return env, dev_pkgs + + +def get_last_n_commands(n): + """ + Find the last n commands entered in the IPython session + + Parameters + ---------- + n : int + Number of commands to retrieve + + Returns + ------- + commands : str + Block of text representing user input + """ + # Ignore warnings generated by the HistoryAccessor. This can be removed + # when https://github.com/ipython/ipython/pull/11054 reaches our release + # environment + with warnings.catch_warnings(): + warnings.simplefilter('ignore', UserWarning) + ha = HistoryAccessor() + return '\n'.join([cmd[2] for cmd in ha.get_tail(n, include_latest=True)]) + + +def get_text_from_editor(): + """ + Request a description written in a text editor + + Opens a vim session with a prompt to request a detailed response from the + operator. + + Returns + ------- + text : str + Block of text of the user input + """ + with tempfile.NamedTemporaryFile(suffix='.tmp', mode='w+t') as f: + # Create a temporary file with instructions on describing bug + f.write(message + '\n\n') + f.flush() + # Open the editor and allow the user to type + subprocess.call(['vim', f.name]) + # Read and clean the file + f.seek(0) + text = ''.join([line for line in f.readlines() + if line and not line.lstrip().startswith('#')]) + return textwrap.wrap(text, width=90) + + +def report_bug(title=None, description=None, author=None, + prior_commands=None, + captured_output=None): + """ + Report a bug from the IPython session + + The purpose of this command is to collect the necessary information to + later help diagnose and troubleshoot the issue. This is written as an + interactive tool, but it can be used in a non-interactive way by entering + the information on the call. + + By the end we should have gathered: + + * A brief description of the problem + * The relevant commands to the bug report + * Name of the current CONDA environment + * Any packages installed in "development" mode + * Relevant logfiles + * Name of bug report author + + Parameters + ---------- + title :str, optional + One sentence description of the issue + + description : str, optional + Written description of problem. If this is not provided, a text editor + is launched that request the information from the user + + author : str, optional + Name of bug report author. If not provided, this is requested from the + user via command line + + prior_commands : int, optional + Number of prior commands to capture. If this is not provided, this is + requested from the user via command line. + + captured_output : str, optional + Captured output from the command + + Returns + ------- + report : dict + A dictionary with the keys: + + * title + * description + * author + * commands + * env + * logfiles + * output + * dev_pkgs + """ + logger.debug("Reporting a bug from the IPython terminal ...") + if not title: + title = request_input('Please provide a one sentence description of ' + 'the problem you are encountering: ') + # Grab relevant commands + if not prior_commands: + try: + n = request_input("How many of the previous commands are relevant " + "to the issue you would like investigated?: ") + prior_commands = int(n) + except ValueError: + logger.error("Invalid input %s", n) + # Only select the last command by default + prior_commands = 1 + # Grab specified number of commands + commands = get_last_n_commands(prior_commands) + # Get a more specific description + description = description or get_text_from_editor() + # Find the author + if not author: + author = request_input('Please enter a name so we can follow-up with ' + 'additional questions: ') + author = author or getpass.getuser() + # Gather environment information + conda_env, dev_pkgs = get_current_environment() + # Gather logfiles + try: + logfiles = get_session_logfiles() + except RuntimeError: + logger.warning("No debug RotatingFileHandler configured for session") + logfiles = list() + return {'title': title, 'author': author, 'commands': commands, + 'description': description, 'env': conda_env, + 'logfiles': logfiles, 'output': captured_output, + 'dev_pkgs': dev_pkgs} + + +message = """\ +# Please describe the issue you are wishing to report. What did you expect to +# happen? What actually happened? Does this issue occur every time you use this +# function? + +# Lines that start with a '#' will be ignored. Save the session and exit to +# store the description. Press "i" to be begin typing, then "Esc" followed by +# ":wq" to exit. +""" diff --git a/hutch_python/tests/test_bug.py b/hutch_python/tests/test_bug.py new file mode 100644 index 00000000..f8d3184a --- /dev/null +++ b/hutch_python/tests/test_bug.py @@ -0,0 +1,28 @@ +import os +import logging +from unittest.mock import patch + +import hutch_python.bug +from hutch_python.bug import report_bug + +logger = logging.getLogger(__name__) + + +def user_input(prompt): + return prompt + +def test_bug_report(monkeypatch): + logger.debug('test_bug_report') + # Patch in our fake user input function + monkeypatch.setattr(hutch_python.bug, 'request_input', user_input) + # Set a fake environment name + os.environ['CONDA_ENVNAME'] = 'test-environment' + # Gather report + bug = report_bug(captured_output='A printed message', + description='A description of the bug') + # See that we catch the fake environment + assert bug['env'] == 'test-environment' + # See that bug reporting captures all keys + bug_keys = ['author', 'commands', 'description', 'env', 'logfiles', + 'title', 'output', 'dev_pkgs'] + assert all([key in bug for key in bug_keys]) From 9ff2c5361c21a2f715b02d555dff41b727b0bd50 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Fri, 23 Mar 2018 15:28:02 -0700 Subject: [PATCH 04/33] DEV: Github issue Jinja2 template --- hutch_python/templates/issue.template | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 hutch_python/templates/issue.template diff --git a/hutch_python/templates/issue.template b/hutch_python/templates/issue.template new file mode 100644 index 00000000..9ec48add --- /dev/null +++ b/hutch_python/templates/issue.template @@ -0,0 +1,34 @@ +## Description +Authored by: `{{author}}` + +{{description}} + +## Commands +```python +{{commands}} +``` +{% if output %} +
+ Captured Output +
+{{output}}
+
+
+ +{% endif %} +{% if logfiles %} +## Relevant Log Files +``` +{% for log in logfiles %} +{{ log }} +{% endfor %}``` +{% endif %} +## Environment +Conda Environment : `{{env}}` {% if dev_pkgs %}with development packages: +``` +{% for pkg in dev_pkgs %} +{{pkg}} +{% endfor %}``` +{% endif %} + +*This report was automatically generated by [hutch-python](https://github.com/pcdshub/hutch-python)* From ad2aa12f6fea1e08608da34b685ca1b2298f64f5 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Sat, 24 Mar 2018 11:38:13 -0700 Subject: [PATCH 05/33] MAINT: Keep internal variable for report directory --- hutch_python/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hutch_python/constants.py b/hutch_python/constants.py index 1a8a8589..221b87e1 100644 --- a/hutch_python/constants.py +++ b/hutch_python/constants.py @@ -4,6 +4,8 @@ CLASS_SEARCH_PATH = ['pcdsdevices.device_types'] +BUG_REPORT_PATH = '/reg/g/pcds/pyps/apps/hutch-python/Bug-Reports/reports' + DAQ_MAP = dict(amo=0, sxr=0, xpp=1, From b98dedc59ffca0341868600bde09bcb4f2e01679 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Sat, 24 Mar 2018 11:39:20 -0700 Subject: [PATCH 06/33] BLD: Explicit dependencies on requests, simplejson and jinja2 --- conda-recipe/meta.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 100331ca..30fa322e 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -28,6 +28,9 @@ requirements: - lightpath >=0.3.0 - cookiecutter >=1.6.0 - matplotlib + - simplejson + - requests + - jinja2 test: imports: From 335d145a88086d959eb3f1f991a8db16476f48ef Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Sat, 24 Mar 2018 11:42:44 -0700 Subject: [PATCH 07/33] ENH: JSON report saving and GitHub posting --- hutch_python/bug.py | 57 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index 89e14060..4f46771b 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -2,6 +2,7 @@ Automated Bug Reporting """ import os +import uuid import getpass import logging import textwrap @@ -9,9 +10,13 @@ import warnings import subprocess +import requests +import simplejson from IPython.core.history import HistoryAccessor +from jinja2 import Environment, PackageLoader from .log_setup import get_session_logfiles +from .constants import BUG_REPORT_PATH logger = logging.getLogger(__name__) @@ -196,10 +201,54 @@ def report_bug(title=None, description=None, author=None, except RuntimeError: logger.warning("No debug RotatingFileHandler configured for session") logfiles = list() - return {'title': title, 'author': author, 'commands': commands, - 'description': description, 'env': conda_env, - 'logfiles': logfiles, 'output': captured_output, - 'dev_pkgs': dev_pkgs} + # Save the report to JSON + return save_report({'title': title, 'author': author, 'commands': commands, + 'description': description, 'env': conda_env, + 'logfiles': logfiles, 'output': captured_output, + 'dev_pkgs': dev_pkgs}) + + +def save_report(report): + """ + Ship the report to the bug report directory + """ + path = os.path.join(BUG_REPORT_PATH, '{}.json'.format(str(uuid.uuid4()))) + logger.info("Saving JSON representation of bug report %s", path) + simplejson.dump(report, open(path, 'w+')) + return path + + +def post_to_github(path, user, pw=None, delete=True): + """ + Load a saved report and post an issue to GitHub + """ + report = simplejson.load(open(path, 'r')) + if not pw: + pw = getpass.getpass() + # Our url to create issues via POST + url = 'https://api.github.com/repos/teddyrendahl/Bug-Reports/issues' + # Create the body of the template + env = Environment(loader=PackageLoader('hutch_python'), + trim_blocks=True, lstrip_blocks=True) + template = env.get_template('issue.template') + body = template.render(report) + # Requests session + session = requests.Session() + session.auth = (user, pw) + issue = {'title': report['title'], + 'body': body, + 'assignee': None, + 'milestone': None, + 'labels': []} # TODO: Determine hutch to create issue for + # Post to GitHub + r = session.post(url, simplejson.dumps(issue)) + if r.status_code == 201: + logger.info("Succesfully created GitHub issue") + if delete: + os.remove(path) + else: + logger.exception("Could not create GitHub issue. HTTP Status Code: %s", + r.status_code) message = """\ From 257afd0ab691a1149db50cd3977c817f1f94cd59 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Sat, 24 Mar 2018 11:44:06 -0700 Subject: [PATCH 08/33] TST: Test GitHub posting with fake session --- hutch_python/tests/test_bug.py | 43 ++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/hutch_python/tests/test_bug.py b/hutch_python/tests/test_bug.py index f8d3184a..a2085060 100644 --- a/hutch_python/tests/test_bug.py +++ b/hutch_python/tests/test_bug.py @@ -1,28 +1,61 @@ import os import logging -from unittest.mock import patch +import tempfile + +import simplejson +from requests import Response import hutch_python.bug -from hutch_python.bug import report_bug +from hutch_python.bug import post_to_github, report_bug + logger = logging.getLogger(__name__) +# Mock user input function that just returns the prompt def user_input(prompt): return prompt + +# Mock requests.Session to avoid actual GitHub posts +class FakeSession: + + def __init__(self): + self.auth = None + def post(self, url, json_dump): + r = Response() + r.status_code = 201 + return r + + def test_bug_report(monkeypatch): logger.debug('test_bug_report') # Patch in our fake user input function monkeypatch.setattr(hutch_python.bug, 'request_input', user_input) # Set a fake environment name os.environ['CONDA_ENVNAME'] = 'test-environment' - # Gather report - bug = report_bug(captured_output='A printed message', - description='A description of the bug') + with tempfile.TemporaryDirectory() as tmp: + # Make sure we write to this directory + hutch_python.bug.BUG_REPORT_PATH = tmp + # Gather report + bug_path = report_bug(captured_output='A printed message', + description='A description of the bug') + bug = simplejson.load(open(bug_path, 'r')) # See that we catch the fake environment assert bug['env'] == 'test-environment' # See that bug reporting captures all keys bug_keys = ['author', 'commands', 'description', 'env', 'logfiles', 'title', 'output', 'dev_pkgs'] assert all([key in bug for key in bug_keys]) + + +def test_post_to_github(monkeypatch): + logger.debug('test_post_to_github') + # Create a fake issue and requests.Session + monkeypatch.setattr(hutch_python.bug.requests, 'Session', FakeSession) + fake_post = {'title': 'This is a Test Issue'} + with tempfile.NamedTemporaryFile('w+', delete=False) as tmp: + simplejson.dump(fake_post, open(tmp.name, 'w+')) + post_to_github(tmp.name, 'user', pw='pw') + fname = tmp.name + assert not os.path.exists(fname) From 6513714f53c078b6e2134bcc9d1445f72c5d3ba7 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Sat, 24 Mar 2018 11:47:50 -0700 Subject: [PATCH 09/33] FIX: Rejoin textwrap from editor --- hutch_python/bug.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index 4f46771b..310309eb 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -111,9 +111,9 @@ def get_text_from_editor(): subprocess.call(['vim', f.name]) # Read and clean the file f.seek(0) - text = ''.join([line for line in f.readlines() + text = ''.join([line.lstrip() for line in f.readlines() if line and not line.lstrip().startswith('#')]) - return textwrap.wrap(text, width=90) + return '\n'.join(textwrap.wrap(text, width=100)) def report_bug(title=None, description=None, author=None, From 881035334e6dfc49c8e97d80c7199ef03d73fa80 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Sat, 24 Mar 2018 11:49:57 -0700 Subject: [PATCH 10/33] STY: Various PEP8 violations --- hutch_python/log_setup.py | 1 - hutch_python/tests/test_bug.py | 1 + hutch_python/tests/test_log_setup.py | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hutch_python/log_setup.py b/hutch_python/log_setup.py index 49941033..bffa85b0 100644 --- a/hutch_python/log_setup.py +++ b/hutch_python/log_setup.py @@ -16,7 +16,6 @@ logger = logging.getLogger(__name__) - def setup_logging(dir_logs=None): """ Sets up the ``logging`` configuration. diff --git a/hutch_python/tests/test_bug.py b/hutch_python/tests/test_bug.py index a2085060..db4bf614 100644 --- a/hutch_python/tests/test_bug.py +++ b/hutch_python/tests/test_bug.py @@ -22,6 +22,7 @@ class FakeSession: def __init__(self): self.auth = None + def post(self, url, json_dump): r = Response() r.status_code = 201 diff --git a/hutch_python/tests/test_log_setup.py b/hutch_python/tests/test_log_setup.py index 92c69eaa..caafe171 100644 --- a/hutch_python/tests/test_log_setup.py +++ b/hutch_python/tests/test_log_setup.py @@ -51,6 +51,7 @@ def test_get_session_logfiles(): assert all([log.startswith(debug_handler.baseFilename) for log in get_session_logfiles()]) + def setup_queue_console(): root_logger = logging.getLogger('') for handler in root_logger.handlers: From f487ed2764e523f15d3c7bd6546dcbabf79b3f9d Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Mon, 26 Mar 2018 09:58:35 -0700 Subject: [PATCH 11/33] BLD: Include jinja2 template in package --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 60396f4c..4f67c2a7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include versioneer.py include hutch_python/_version.py +include hutch_python/templates/* include hutch_python/logging.yml include hutch_python/cookiecutter/* include hutch_python/cookiecutter/*/* From 29ffd9c6228aaa110d8ae851c5cf135df43f9f19 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Mon, 26 Mar 2018 09:59:23 -0700 Subject: [PATCH 12/33] MAINT: Try / except block for no IPython history --- hutch_python/bug.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index 310309eb..e0b63885 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -185,7 +185,13 @@ def report_bug(title=None, description=None, author=None, # Only select the last command by default prior_commands = 1 # Grab specified number of commands - commands = get_last_n_commands(prior_commands) + try: + commands = get_last_n_commands(prior_commands) + # If the user somehow has no IPython history this is raised. A very rare + # occurence except for in Continuous Integration tests + except OSError: + logger.exception + commands = '' # Get a more specific description description = description or get_text_from_editor() # Find the author From d3d555fca1dec485f0411313274eecdb825db0b7 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Mon, 26 Mar 2018 14:32:17 -0700 Subject: [PATCH 13/33] ENH: Create report_bug magics --- hutch_python/bug.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index e0b63885..552d1126 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -13,6 +13,8 @@ import requests import simplejson from IPython.core.history import HistoryAccessor +from IPython.core.magic import Magics, magics_class, line_magic +from IPython.utils.io import capture_output from jinja2 import Environment, PackageLoader from .log_setup import get_session_logfiles @@ -257,6 +259,20 @@ def post_to_github(path, user, pw=None, delete=True): r.status_code) +@magics_class +class BugMagics(Magics): + + @line_magic + def report_bug(self, line): + # Enter both the DEBUG context and store the output of our command + with capture_output() as shell_output: + self.shell.run_cell(line) + # Show the capture output to the user + shell_output.show() + # Create the report + report_bug(prior_commands=1, captured_output=shell_output.stdout) + + message = """\ # Please describe the issue you are wishing to report. What did you expect to # happen? What actually happened? Does this issue occur every time you use this From 47fd7cefeffc13a37d24f4661b879b530f04240a Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Mon, 26 Mar 2018 14:32:57 -0700 Subject: [PATCH 14/33] MAINT: Register report_bug magics in IPython shell --- hutch_python/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hutch_python/cli.py b/hutch_python/cli.py index d3ded349..21d30e5f 100644 --- a/hutch_python/cli.py +++ b/hutch_python/cli.py @@ -16,6 +16,7 @@ from .constants import DIR_MODULE from .ipython_log import init_ipython_logger from .load_conf import load +from .bug import BugMagics from .log_setup import (setup_logging, set_console_level, debug_mode, debug_context, debug_wrapper) @@ -115,6 +116,9 @@ def hutch_ipython_embed(stack_offset=0): init_ipython_logger(shell) shell.enable_matplotlib() plt.ion() + # Add our Bug Reporting Magic + logger.debug('Registering bug_report magics') + shell.register_magics(BugMagics) shell(stack_depth=stack_depth) From 9e7f59de7a40ec8cae25aff17ba2229ee54cda8b Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Mon, 26 Mar 2018 16:13:15 -0700 Subject: [PATCH 15/33] ENH: post-issues command line function --- bin/post-issues | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100755 bin/post-issues diff --git a/bin/post-issues b/bin/post-issues new file mode 100755 index 00000000..75b4b17e --- /dev/null +++ b/bin/post-issues @@ -0,0 +1,56 @@ +#!/usr/bin/env python +import os +import getpass +import logging +import argparse + +from hutch_python.constants import BUG_REPORT_PATH +from hutch_python.bug import post_to_github + +logger = logging.getLogger(__name__) + + +description = """\ +Post all of the stored issue to https://github.com/pcdshub/Bug-Reports + +This requires a valid GitHub account that is authorized to post issues in +the pcdshub organization. By default this script will be run automatically at a +frequent interval, but if that is failing or you feel the need to immediately +push an issue upstream this script can be utilized. + + +""" + + +def main(): + """Post all issues in BUG_REPORT_PATH""" + # Create argument parser + fclass = argparse.RawDescriptionHelpFormatter + parser = argparse.ArgumentParser(epilog=description, + formatter_class=fclass) + parser.add_argument('-u', '--user', dest='username', + default=None, + help="Username for GitHub account") + parser.add_argument('-p', '--pw', dest='password', + default=None, + help='Password for GitHub account. This will be ' + 'queried for if not entered at call time.') + args = parser.parse_args() + # Ask for the password once so if it isn't asked to repetitively + pw = args.password or getpass.getpass() + # Run post_to_github on every stored issue + for issue in os.listdir(BUG_REPORT_PATH): + if issue.endswith('.json'): + try: + logger.info('Posting %s ...', issue) + post_to_github(os.path.join(BUG_REPORT_PATH, issue), + user=args.username, pw=pw) + except Exception: + logger.exception("Error posting %s", issue) + + +if __name__ == '__main__': + # Configure logger + logging.basicConfig(level=logging.INFO) + # Post issues + main() diff --git a/setup.py b/setup.py index f7e727c7..27555efa 100644 --- a/setup.py +++ b/setup.py @@ -10,5 +10,5 @@ include_package_data=True, description=('Launcher and Config Reader for ' 'LCLS Interactive IPython Sessions'), - scripts=['bin/hutch-python'] + scripts=['bin/hutch-python', 'bin/post-issues'] ) From b83111a0f230dd43f78de58a7d59a60babe9d542 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Mon, 26 Mar 2018 17:08:12 -0700 Subject: [PATCH 16/33] DOC: Full API documentation and module explanation --- hutch_python/bug.py | 80 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index 552d1126..f5cf9665 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -1,5 +1,33 @@ """ -Automated Bug Reporting +One advantage of standardizing the logging system and startup scripts with +hutch-python is programatically being able to gather information about the +currrent Python environment. One common use case for much of this information +is being able to diagnose software bugs after the fact if we are diligent about +recording the pertinent information. The `hutch_python.bug.report_bug` command +wraps much of this functionality up by asking a few simple questions of the +operator and noting the current environment and log file. By the end of the +function we should have: + + * A one line description of the problem + * A more verbose explanation of the issue and how it affets operations + * A name to follow up with additional questions / closeout + * The commands entered by the operator + * The current CONDA environment + * Relevant logfiles written to by this Python session + * Capture output printed to the terminal + * Any packages installed in "development" mode + +This command is available as both an actual Python function and an IPython +magic. The ladder allows you to pinpoint the exact function that is causing +issues. + +.. code-block:: python + + %report_bug my_buggy_function() + +If you call `report_bug` as a regular function you can specify a number of past +commands you want to include in the overall report. This allows you to +posthumously report issues without running the actual function again. """ import os import uuid @@ -160,17 +188,8 @@ def report_bug(title=None, description=None, author=None, Returns ------- - report : dict - A dictionary with the keys: - - * title - * description - * author - * commands - * env - * logfiles - * output - * dev_pkgs + path : str + A path to the created JSON file """ logger.debug("Reporting a bug from the IPython terminal ...") if not title: @@ -219,6 +238,25 @@ def report_bug(title=None, description=None, author=None, def save_report(report): """ Ship the report to the bug report directory + + Parameters + ---------- + report : dict + A dictionary with the keys: + + * title + * description + * author + * commands + * env + * logfiles + * output + * dev_pkgs + + Returns + ------- + path : str + A path to the created JSON file """ path = os.path.join(BUG_REPORT_PATH, '{}.json'.format(str(uuid.uuid4()))) logger.info("Saving JSON representation of bug report %s", path) @@ -229,6 +267,22 @@ def save_report(report): def post_to_github(path, user, pw=None, delete=True): """ Load a saved report and post an issue to GitHub + + Parameters + ---------- + path: str + Path to issue JSON file + + user: str + Username of GitHub profile + + pw : str, optional + Password for GitHub profile. This will be queried for if not provided + in the function call. + + delete : bool, optional + Delete the JSON file after the GitHub issue has been created + succesfully """ report = simplejson.load(open(path, 'r')) if not pw: @@ -261,9 +315,11 @@ def post_to_github(path, user, pw=None, delete=True): @magics_class class BugMagics(Magics): + """Magics function for report_bug function""" @line_magic def report_bug(self, line): + """Creates a bug_report while running the given line""" # Enter both the DEBUG context and store the output of our command with capture_output() as shell_output: self.shell.run_cell(line) From 6175dbf82b199be0e05e42fcb5b44e19c07cf38f Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Mon, 26 Mar 2018 17:08:31 -0700 Subject: [PATCH 17/33] DOC: Add get_session_logfiles to API --- docs/source/log_setup.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/log_setup.rst b/docs/source/log_setup.rst index c93ffe18..8a372048 100644 --- a/docs/source/log_setup.rst +++ b/docs/source/log_setup.rst @@ -8,6 +8,7 @@ log_setup.py :nosignatures: setup_logging + get_session_logfiles get_console_handler get_console_level set_console_level From bd8da30a4746ee6b8e578874a8b8a256992180f5 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Mon, 26 Mar 2018 17:08:57 -0700 Subject: [PATCH 18/33] DOC: Break up utils into sections, include bug reporting --- docs/source/user_utils.rst | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/source/user_utils.rst b/docs/source/user_utils.rst index 1057515a..d06313fc 100644 --- a/docs/source/user_utils.rst +++ b/docs/source/user_utils.rst @@ -1,9 +1,26 @@ Useful Utilities ================ -The ``hutch_python.utils`` and ``hutch_python.namespace`` modules have -functions that may be broadly useful. - +Reporting Issues +---------------- +.. automodule:: hutch_python.bug + + .. autofunction:: report_bug + +Issue Lifecyle +^^^^^^^^^^^^^^ +We can not expect that every operator on the beamline has a valid Github +account. To get around this, when you call `report_bug` we dump a JSON +description of the issue into a standard NFS directory. Then by a regular CRON +job we will post this issue to https://github.com/pcdshub/Bug-Reports. This +leads to a slight delay but also allows to have issues posted by persons +without valid GitHub accounts. Once issues are received on GitHub the +appropriate action will be made by the PCDS staff. This may mean a deeper look +at the linked log files and/or creating a distilled issue or action item in a +different repository. + +Safe Loading +------------ `hutch_python.utils.safe_load` can be used as a shortcut for wrapping code that may or may not succeed to prevent a bad submodule from interrupting the ``hutch-python`` load sequence. This means that if the ``Attenuator`` class is @@ -23,6 +40,8 @@ For example, this will complete successfully but show a warning: The reason for the failure with the full traceback will be saved to the log file and will be visible in the terminal if you are in `debug mode `. +User Namespaces +--------------- `hutch_python.namespace.class_namespace` can be used to create your own object groupings by type. This will find all objects loaded by hutch python plus all objects in your global environment, and accumulate them if they match a given From 01ecf5f92e069a07dc1067c9b5941942a742539f Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Mon, 26 Mar 2018 17:09:22 -0700 Subject: [PATCH 19/33] DOC: Full API of bug.py available in autosummary --- docs/source/bug.rst | 14 ++++++++++++++ docs/source/index.rst | 1 + 2 files changed, 15 insertions(+) create mode 100644 docs/source/bug.rst diff --git a/docs/source/bug.rst b/docs/source/bug.rst new file mode 100644 index 00000000..52de7a0b --- /dev/null +++ b/docs/source/bug.rst @@ -0,0 +1,14 @@ +bug.py +====== +Functionality associatted with bug reports + +.. autosummary:: + :toctree: generated + :nosignatures: + + hutch_python.bug.get_current_environment + hutch_python.bug.get_last_n_commands + hutch_python.bug.get_text_from_editor + hutch_python.bug.report_bug + hutch_python.bug.post_to_github + diff --git a/docs/source/index.rst b/docs/source/index.rst index fdb616f3..03bda2d4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -40,6 +40,7 @@ :caption: Internal API Reference :hidden: + bug cache cli ipython_log From b9266800d836464e4964c30ed28cd9360d44fd94 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 09:24:41 -0700 Subject: [PATCH 20/33] MAINT: Switch to official Bug-Reports repository --- hutch_python/bug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index f5cf9665..9834f144 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -288,7 +288,7 @@ def post_to_github(path, user, pw=None, delete=True): if not pw: pw = getpass.getpass() # Our url to create issues via POST - url = 'https://api.github.com/repos/teddyrendahl/Bug-Reports/issues' + url = 'https://api.github.com/repos/pcdshub/Bug-Reports/issues' # Create the body of the template env = Environment(loader=PackageLoader('hutch_python'), trim_blocks=True, lstrip_blocks=True) From 0adc78c8a2ee6fd7f67f2d70c93ad370aa024a6d Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 11:30:35 -0700 Subject: [PATCH 21/33] REV: Remove CRON job post-issues script --- bin/post-issues | 56 ------------------------------------------------- setup.py | 2 +- 2 files changed, 1 insertion(+), 57 deletions(-) delete mode 100755 bin/post-issues diff --git a/bin/post-issues b/bin/post-issues deleted file mode 100755 index 75b4b17e..00000000 --- a/bin/post-issues +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -import os -import getpass -import logging -import argparse - -from hutch_python.constants import BUG_REPORT_PATH -from hutch_python.bug import post_to_github - -logger = logging.getLogger(__name__) - - -description = """\ -Post all of the stored issue to https://github.com/pcdshub/Bug-Reports - -This requires a valid GitHub account that is authorized to post issues in -the pcdshub organization. By default this script will be run automatically at a -frequent interval, but if that is failing or you feel the need to immediately -push an issue upstream this script can be utilized. - - -""" - - -def main(): - """Post all issues in BUG_REPORT_PATH""" - # Create argument parser - fclass = argparse.RawDescriptionHelpFormatter - parser = argparse.ArgumentParser(epilog=description, - formatter_class=fclass) - parser.add_argument('-u', '--user', dest='username', - default=None, - help="Username for GitHub account") - parser.add_argument('-p', '--pw', dest='password', - default=None, - help='Password for GitHub account. This will be ' - 'queried for if not entered at call time.') - args = parser.parse_args() - # Ask for the password once so if it isn't asked to repetitively - pw = args.password or getpass.getpass() - # Run post_to_github on every stored issue - for issue in os.listdir(BUG_REPORT_PATH): - if issue.endswith('.json'): - try: - logger.info('Posting %s ...', issue) - post_to_github(os.path.join(BUG_REPORT_PATH, issue), - user=args.username, pw=pw) - except Exception: - logger.exception("Error posting %s", issue) - - -if __name__ == '__main__': - # Configure logger - logging.basicConfig(level=logging.INFO) - # Post issues - main() diff --git a/setup.py b/setup.py index 27555efa..f7e727c7 100644 --- a/setup.py +++ b/setup.py @@ -10,5 +10,5 @@ include_package_data=True, description=('Launcher and Config Reader for ' 'LCLS Interactive IPython Sessions'), - scripts=['bin/hutch-python', 'bin/post-issues'] + scripts=['bin/hutch-python'] ) From 7c9ff1435dcec9a046f22f5634ed611cc48cac40 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 11:38:13 -0700 Subject: [PATCH 22/33] DOC: Various typos and formatting issues --- docs/source/bug.rst | 2 +- hutch_python/bug.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/source/bug.rst b/docs/source/bug.rst index 52de7a0b..1997827e 100644 --- a/docs/source/bug.rst +++ b/docs/source/bug.rst @@ -1,6 +1,6 @@ bug.py ====== -Functionality associatted with bug reports +Functionality associated with bug reports .. autosummary:: :toctree: generated diff --git a/hutch_python/bug.py b/hutch_python/bug.py index 9834f144..dc52093e 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -1,7 +1,7 @@ """ One advantage of standardizing the logging system and startup scripts with -hutch-python is programatically being able to gather information about the -currrent Python environment. One common use case for much of this information +hutch-python is programmatically being able to gather information about the +current Python environment. One common use case for much of this information is being able to diagnose software bugs after the fact if we are diligent about recording the pertinent information. The `hutch_python.bug.report_bug` command wraps much of this functionality up by asking a few simple questions of the @@ -9,7 +9,7 @@ function we should have: * A one line description of the problem - * A more verbose explanation of the issue and how it affets operations + * A more verbose explanation of the issue and how it affects operations * A name to follow up with additional questions / closeout * The commands entered by the operator * The current CONDA environment @@ -63,12 +63,13 @@ def get_current_environment(): The environment name is gathered by first checking environment variable ``$CONDA_ENVNAME`` that is set by the hutch-python startup scripts. If for - whatever reason that is not available we check $CONDA_DEFAULT_ENV which is - set by Conda itself. The reason this is not relied on primarily is it has a - strange name and is entirely undocumented but seems to work effectively. + whatever reason that is not available we check ``$CONDA_DEFAULT_ENV`` which + is set by Conda itself. The reason this is not relied on primarily is it + has a strange name and is entirely undocumented but seems to work + effectively. - In addition the hutch-python startup script sets the PYTHONPATH to pick up - packages in "development" mode. The list of package names installed this + In addition the hutch-python startup script sets the ``PYTHONPATH`` to pick + up packages in "development" mode. The list of package names installed this way is found to help inform how the current Python environment differs from the enforced CONDA environment @@ -168,7 +169,7 @@ def report_bug(title=None, description=None, author=None, Parameters ---------- - title :str, optional + title : str, optional One sentence description of the issue description : str, optional From 5b380289075ecbae66fe6c5aad63a084c4d4f6e7 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 11:42:24 -0700 Subject: [PATCH 23/33] REV: Remove bug report information for Useful Utilities --- docs/source/user_utils.rst | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/docs/source/user_utils.rst b/docs/source/user_utils.rst index d06313fc..1057515a 100644 --- a/docs/source/user_utils.rst +++ b/docs/source/user_utils.rst @@ -1,26 +1,9 @@ Useful Utilities ================ -Reporting Issues ----------------- -.. automodule:: hutch_python.bug - - .. autofunction:: report_bug - -Issue Lifecyle -^^^^^^^^^^^^^^ -We can not expect that every operator on the beamline has a valid Github -account. To get around this, when you call `report_bug` we dump a JSON -description of the issue into a standard NFS directory. Then by a regular CRON -job we will post this issue to https://github.com/pcdshub/Bug-Reports. This -leads to a slight delay but also allows to have issues posted by persons -without valid GitHub accounts. Once issues are received on GitHub the -appropriate action will be made by the PCDS staff. This may mean a deeper look -at the linked log files and/or creating a distilled issue or action item in a -different repository. - -Safe Loading ------------- +The ``hutch_python.utils`` and ``hutch_python.namespace`` modules have +functions that may be broadly useful. + `hutch_python.utils.safe_load` can be used as a shortcut for wrapping code that may or may not succeed to prevent a bad submodule from interrupting the ``hutch-python`` load sequence. This means that if the ``Attenuator`` class is @@ -40,8 +23,6 @@ For example, this will complete successfully but show a warning: The reason for the failure with the full traceback will be saved to the log file and will be visible in the terminal if you are in `debug mode `. -User Namespaces ---------------- `hutch_python.namespace.class_namespace` can be used to create your own object groupings by type. This will find all objects loaded by hutch python plus all objects in your global environment, and accumulate them if they match a given From 6d805e0206086cbef82fd6340033ea8bb2022a64 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 11:50:39 -0700 Subject: [PATCH 24/33] DOC: Reorganize placement of various docstrings --- docs/source/bug.rst | 18 ++++++++--------- docs/source/index.rst | 1 + docs/source/report.rst | 45 ++++++++++++++++++++++++++++++++++++++++++ hutch_python/bug.py | 31 ++--------------------------- 4 files changed, 57 insertions(+), 38 deletions(-) create mode 100644 docs/source/report.rst diff --git a/docs/source/bug.rst b/docs/source/bug.rst index 1997827e..b1569d12 100644 --- a/docs/source/bug.rst +++ b/docs/source/bug.rst @@ -1,14 +1,14 @@ bug.py ====== -Functionality associated with bug reports +.. automodule:: hutch_python.bug -.. autosummary:: - :toctree: generated - :nosignatures: + .. autosummary:: + :toctree: generated + :nosignatures: - hutch_python.bug.get_current_environment - hutch_python.bug.get_last_n_commands - hutch_python.bug.get_text_from_editor - hutch_python.bug.report_bug - hutch_python.bug.post_to_github + hutch_python.bug.get_current_environment + hutch_python.bug.get_last_n_commands + hutch_python.bug.get_text_from_editor + hutch_python.bug.report_bug + hutch_python.bug.post_to_github diff --git a/docs/source/index.rst b/docs/source/index.rst index 03bda2d4..9f97077b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -22,6 +22,7 @@ database beamline experiment + report user_utils other_modules diff --git a/docs/source/report.rst b/docs/source/report.rst new file mode 100644 index 00000000..a577ed0a --- /dev/null +++ b/docs/source/report.rst @@ -0,0 +1,45 @@ +Reporting Issues +================ +One advantage of standardizing the logging system and startup scripts with +hutch-python is programmatically being able to gather information about the +current Python environment. One common use case for much of this information +is being able to diagnose software bugs after the fact if we are diligent about +recording the pertinent information. The `hutch_python.bug.report_bug` command +wraps much of this functionality up by asking a few simple questions of the +operator and noting the current environment and log file. By the end of the +function we should have: + + * A one line description of the problem + * A more verbose explanation of the issue and how it affects operations + * A name to follow up with additional questions / closeout + * The commands entered by the operator + * The current CONDA environment + * Relevant logfiles written to by this Python session + * Capture output printed to the terminal + * Any packages installed in "development" mode + +This command is available as both an actual Python function and an IPython +magic. The ladder allows you to pinpoint the exact function that is causing +issues. + +.. code-block:: python + + %report_bug my_buggy_function() + +If you call `report_bug` as a regular function you can specify a number of past +commands you want to include in the overall report. This allows you to +posthumously report issues without running the actual function again. + + .. autofunction:: report_bug + +Issue Lifecyle +^^^^^^^^^^^^^^ +We can not expect that every operator on the beamline has a valid Github +account. To get around this, when you call `report_bug` we dump a JSON +description of the issue into a standard NFS directory. Then by a regular CRON +job we will post this issue to https://github.com/pcdshub/Bug-Reports. This +leads to a slight delay but also allows to have issues posted by persons +without valid GitHub accounts. Once issues are received on GitHub the +appropriate action will be made by the PCDS staff. This may mean a deeper look +at the linked log files and/or creating a distilled issue or action item in a +different repository. diff --git a/hutch_python/bug.py b/hutch_python/bug.py index dc52093e..2d15aaf2 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -1,33 +1,6 @@ """ -One advantage of standardizing the logging system and startup scripts with -hutch-python is programmatically being able to gather information about the -current Python environment. One common use case for much of this information -is being able to diagnose software bugs after the fact if we are diligent about -recording the pertinent information. The `hutch_python.bug.report_bug` command -wraps much of this functionality up by asking a few simple questions of the -operator and noting the current environment and log file. By the end of the -function we should have: - - * A one line description of the problem - * A more verbose explanation of the issue and how it affects operations - * A name to follow up with additional questions / closeout - * The commands entered by the operator - * The current CONDA environment - * Relevant logfiles written to by this Python session - * Capture output printed to the terminal - * Any packages installed in "development" mode - -This command is available as both an actual Python function and an IPython -magic. The ladder allows you to pinpoint the exact function that is causing -issues. - -.. code-block:: python - - %report_bug my_buggy_function() - -If you call `report_bug` as a regular function you can specify a number of past -commands you want to include in the overall report. This allows you to -posthumously report issues without running the actual function again. +This module is used to both gather and report information for the purpose of +identifying bugs """ import os import uuid From 5f72c03c977a093a3b6bbd6ce1292cc5f5721e91 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 11:54:10 -0700 Subject: [PATCH 25/33] DOC: Remove references to CRON job --- docs/source/report.rst | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/source/report.rst b/docs/source/report.rst index a577ed0a..4ec2e569 100644 --- a/docs/source/report.rst +++ b/docs/source/report.rst @@ -34,12 +34,8 @@ posthumously report issues without running the actual function again. Issue Lifecyle ^^^^^^^^^^^^^^ -We can not expect that every operator on the beamline has a valid Github -account. To get around this, when you call `report_bug` we dump a JSON -description of the issue into a standard NFS directory. Then by a regular CRON -job we will post this issue to https://github.com/pcdshub/Bug-Reports. This -leads to a slight delay but also allows to have issues posted by persons -without valid GitHub accounts. Once issues are received on GitHub the -appropriate action will be made by the PCDS staff. This may mean a deeper look -at the linked log files and/or creating a distilled issue or action item in a -different repository. +After reporting the issue on the command line, an issue will be created at +https://github.com/pcdshub/Bug-Reports. This will alert those subscribed to +this repository via email about the current issue and appropriate action will +be made by the PCDS staff. This may mean a deeper look at the linked log files +and/or creating a distilled issue or action item in a different repository. From 6113df5749617af6d2da42b7b033bc56898fc1cb Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 11:55:28 -0700 Subject: [PATCH 26/33] FIX: Forgotten description in exception logging --- hutch_python/bug.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index 2d15aaf2..675fec7d 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -185,7 +185,8 @@ def report_bug(title=None, description=None, author=None, # If the user somehow has no IPython history this is raised. A very rare # occurence except for in Continuous Integration tests except OSError: - logger.exception + logger.exception('Unable to retrieve commmands from the ' + 'IPython session') commands = '' # Get a more specific description description = description or get_text_from_editor() From 1309b0d7be11fd11e904b40a07c2fd8c3328418d Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 11:58:41 -0700 Subject: [PATCH 27/33] MAINT: Clarify various comments --- hutch_python/bug.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index 675fec7d..b57df548 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -92,6 +92,8 @@ def get_last_n_commands(n): with warnings.catch_warnings(): warnings.simplefilter('ignore', UserWarning) ha = HistoryAccessor() + # Parse the last n commands of the IPython history, joining printed + # messages return '\n'.join([cmd[2] for cmd in ha.get_tail(n, include_latest=True)]) @@ -295,7 +297,7 @@ class BugMagics(Magics): @line_magic def report_bug(self, line): """Creates a bug_report while running the given line""" - # Enter both the DEBUG context and store the output of our command + # Store the output of our command with capture_output() as shell_output: self.shell.run_cell(line) # Show the capture output to the user From 666bc8adb4a1fe2f9bafb9f91f0ac39b6fd2c64c Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 12:00:09 -0700 Subject: [PATCH 28/33] MAINT: Check EDITOR environment variable instead of assuming vim --- hutch_python/bug.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index b57df548..7b70e2d4 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -114,7 +114,8 @@ def get_text_from_editor(): f.write(message + '\n\n') f.flush() # Open the editor and allow the user to type - subprocess.call(['vim', f.name]) + editor = os.environ.get('EDITOR', 'vim') + subprocess.call([editor, f.name]) # Read and clean the file f.seek(0) text = ''.join([line.lstrip() for line in f.readlines() From 4f59dabc9d869ffcdacb1f02e625f3f1b57f3b62 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 12:05:54 -0700 Subject: [PATCH 29/33] MAINT: Place try/except block in lower level get_session_logfiles --- hutch_python/bug.py | 6 +----- hutch_python/log_setup.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index 7b70e2d4..be554776 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -201,11 +201,7 @@ def report_bug(title=None, description=None, author=None, # Gather environment information conda_env, dev_pkgs = get_current_environment() # Gather logfiles - try: - logfiles = get_session_logfiles() - except RuntimeError: - logger.warning("No debug RotatingFileHandler configured for session") - logfiles = list() + logfiles = get_session_logfiles() # Save the report to JSON return save_report({'title': title, 'author': author, 'commands': commands, 'description': description, 'env': conda_env, diff --git a/hutch_python/log_setup.py b/hutch_python/log_setup.py index bffa85b0..8baa8362 100644 --- a/hutch_python/log_setup.py +++ b/hutch_python/log_setup.py @@ -70,10 +70,16 @@ def get_session_logfiles(): Returns ------- logs : list - List of absolute paths to log files that were created by this session + List of absolute paths to log files that were created by this session. + Returns an empty list if there is no ``RotatingFileHandler`` with the + name ``debug`` """ # Grab the debug file handler - handler = get_debug_handler() + try: + handler = get_debug_handler() + except RuntimeError: + logger.warning("No debug RotatingFileHandler configured for session") + return list() # Find all the log files that were generated by this session base = Path(handler.baseFilename) return [str(base.parent / log) From f5c01132acedec6d2074a4ccb4b07b0241a32506 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Tue, 27 Mar 2018 12:08:54 -0700 Subject: [PATCH 30/33] STY: One blank line instead of two between imports and logger init --- hutch_python/bug.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index be554776..f8ccd17a 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -21,7 +21,6 @@ from .log_setup import get_session_logfiles from .constants import BUG_REPORT_PATH - logger = logging.getLogger(__name__) From ac0715e4c0c868e04ee62ddeb5e081fe78ab4868 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Wed, 28 Mar 2018 16:01:06 -0700 Subject: [PATCH 31/33] TST: Add explicit test for dev_pkgs and CONDA env variables --- hutch_python/tests/test_bug.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/hutch_python/tests/test_bug.py b/hutch_python/tests/test_bug.py index db4bf614..7af33549 100644 --- a/hutch_python/tests/test_bug.py +++ b/hutch_python/tests/test_bug.py @@ -1,13 +1,13 @@ import os import logging +import pathlib import tempfile - import simplejson from requests import Response import hutch_python.bug -from hutch_python.bug import post_to_github, report_bug - +from hutch_python.bug import (get_current_environment, post_to_github, + report_bug) logger = logging.getLogger(__name__) @@ -29,6 +29,23 @@ def post(self, url, json_dump): return r +def test_get_current_environment(): + logger.debug('test_get_current_environment') + # Development packages + fk_pkgs = ['one', 'two', 'three'] + os.environ['CONDA_ENVNAME'] = 'test-environment' + with tempfile.TemporaryDirectory() as tmp: + logger.debug('Creating temp directory {}', tmp) + # Set a PythonPath + os.environ['PYTHONPATH'] = tmp + # Create fake packages + for pkg in fk_pkgs: + pathlib.Path(os.path.join(tmp, pkg)).touch() + env, pkgs = get_current_environment() + assert all([pkg in pkgs for pkg in fk_pkgs]) + assert env == 'test-environment' + + def test_bug_report(monkeypatch): logger.debug('test_bug_report') # Patch in our fake user input function From 83d0ace79ab4756ccfb5a6ffc1903d7748efae32 Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Wed, 28 Mar 2018 16:01:36 -0700 Subject: [PATCH 32/33] FIX: Return empty list even if PYTHONPATH is not found --- hutch_python/bug.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hutch_python/bug.py b/hutch_python/bug.py index f8ccd17a..81b37dbd 100644 --- a/hutch_python/bug.py +++ b/hutch_python/bug.py @@ -66,6 +66,7 @@ def get_current_environment(): dev_pkgs = os.listdir(dev) except FileNotFoundError: logger.debug("No dev folder found") + dev_pkgs = list() else: dev_pkgs = list() return env, dev_pkgs From 56d7601fe4fd42e506157e2391fd86e72b45c4ce Mon Sep 17 00:00:00 2001 From: teddyrendahl Date: Wed, 28 Mar 2018 16:17:03 -0700 Subject: [PATCH 33/33] TST: Add mock test for get_text_from_editor --- hutch_python/tests/test_bug.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/hutch_python/tests/test_bug.py b/hutch_python/tests/test_bug.py index 7af33549..72496a2b 100644 --- a/hutch_python/tests/test_bug.py +++ b/hutch_python/tests/test_bug.py @@ -7,7 +7,7 @@ import hutch_python.bug from hutch_python.bug import (get_current_environment, post_to_github, - report_bug) + report_bug, get_text_from_editor) logger = logging.getLogger(__name__) @@ -77,3 +77,16 @@ def test_post_to_github(monkeypatch): post_to_github(tmp.name, 'user', pw='pw') fname = tmp.name assert not os.path.exists(fname) + + +def test_get_text_from_editor(monkeypatch): + logger.debug("test_get_text_from_editor") + + # Mock write function + def write_text(info): + open(info[1], 'w+').write(info[0]) + + # Patch in our fake write_text + monkeypatch.setattr(hutch_python.bug.subprocess, 'call', write_text) + os.environ['EDITOR'] = 'vim' + assert get_text_from_editor() == 'vim'