Merge branch 'master' into doc_jupy_usage

.gitignore

@@ -118,3 +118,6 @@ hutch_python/tests/tst/db.txt
 
 # vscode
 .vscode
+
+# unit test artifacts
+*.sqlite

docs/source/releases.rst

@@ -2,6 +2,45 @@ Release History
 ###############
 
 
+v1.20.0 (2024-04-19)
+====================
+
+Features
+--------
+- Allow per-session separate ipython histories via ``--hist-file``
+  command-line argument. If omitted, default ipython behavior is used.
+  When ``--hist-file`` is included with no argument, the history file
+  will be written a local operator console hard drive, if available.
+  An argument can be provided to use any file as the history.sqlite file.
+
+Contributors
+------------
+- zllentz
+
+
+
+v1.19.0 (2024-04-15)
+====================
+
+Features
+--------
+- Updates EpicsArch script, adding several debug/testin flags and ensuring
+  information is up-to-date.  Includes:
+
+  - ``update_file``: feature that updates EpicsArch based on current questionnaire
+  - ``--softlink``: point arch file to new experiment via symbolic link
+  - ``--level``: specify a debug level
+  - ``--cds-items``: displays questionnaire data for a given run and experiment
+  - ``--link-path``: allows user to provide custom filepath for softlinks
+- Adds load_level conf.yaml key for choosing the method used to gather happi devices
+
+Contributors
+------------
+- c-tsoi
+- tangkong
+
+
+
 v1.18.5 (2023-09-14)
 ====================
 

docs/source/yaml_files.rst

@@ -3,8 +3,8 @@ Yaml Files
 
 ``hutch-python`` uses a ``conf.yml`` file for basic configuration. This is a
 standard yaml file with the following valid keys:
-``hutch``, ``db``, ``load``, ``experiment``, ``obj_config``, ``daq_type``,
-``daq_host``, and ``daq_platform``.
+``hutch``, ``db``, ``load``, ``load_level``, ``experiment``, ``obj_config``,
+``daq_type``, ``daq_host``, and ``daq_platform``.
 
 
 hutch
@@ -69,6 +69,22 @@ This key is used to include hutch-specific code.
 ``from module import *`` for each of these modules.
 
 
+load_level
+----------
+The ``load_level`` key expects one of the following strings, corresponding to the
+amount of ophyd devices to load:
+
+- ``UPSTREAM``: The hutch's devices, and devices upstream from the requested hutch.
+If there are multiple paths to the requested hutch, all paths' devices are loaded.
+- ``STANDARD``: Devices gathered via ``UPSTREAM``, plus devices that share the
+"beamline" field in happi with the ``UPSTREAM`` devices.  (The current standard)
+- ``ALL``: All devices in the happi database.  Use this option at your own risk.
+
+.. code-block:: YAML
+
+   load_level: UPSTREAM
+
+
 experiment
 ----------
 

hutch_python/cli.py

@@ -3,11 +3,15 @@
 ``hutch-python`` script. It also provides utilities that are only used at
 startup.
 """
+from __future__ import annotations
+
 import argparse
+import dataclasses
 import logging
 import os
 import sys
 from pathlib import Path
+from string import Template
 
 import IPython
 import matplotlib
@@ -22,25 +26,59 @@
 
 logger = logging.getLogger(__name__)
 opts_cache = {}
+DEFAULT_HISTFILE = "/u1/${USER}/hutch-python/history.sqlite"
+
 
 # Define the parser
-parser = argparse.ArgumentParser(prog='hutch-python',
-                                 description='Launch LCLS Hutch Python')
-parser.add_argument('--cfg', required=False, default=None,
-                    help='Configuration yaml file')
-parser.add_argument('--exp', required=False, default=None,
-                    help='Experiment number override')
-parser.add_argument('--debug', action='store_true', default=False,
-                    help='Start in debug mode')
-parser.add_argument('--sim', action='store_true', default=False,
-                    help='Run with simulated DAQ (lcls1 only)')
-parser.add_argument('--create', action='store', default=False,
-                    help='Create a new hutch deployment')
-parser.add_argument('script', nargs='?',
-                    help='Run a script instead of running interactively')
+def get_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="hutch-python",
+                                     description="Launch LCLS Hutch Python")
+    parser.add_argument("--cfg", required=False, default=None,
+                        help="Configuration yaml file")
+    parser.add_argument("--exp", required=False, default=None,
+                        help="Experiment number override")
+    parser.add_argument("--debug", action="store_true", default=False,
+                        help="Start in debug mode")
+    parser.add_argument("--sim", action="store_true", default=False,
+                        help="Run with simulated DAQ (lcls1 only)")
+    parser.add_argument("--create", action="store", default=False,
+                        help="Create a new hutch deployment")
+    parser.add_argument("--hist-file", nargs="?", action="store",
+                        default=None, const=DEFAULT_HISTFILE,
+                        help=(
+                            "File to store the sqlite session history in. "
+                            "${VARIABLES} will be substituted for "
+                            "via shell environment variables, "
+                            "Though in some cases these may be expanded by the shell "
+                            "prior to reaching the python layer. "
+                            "If omitted, defaults to the ipython default location. "
+                            "If included but left blank, defaults to "
+                            f"{DEFAULT_HISTFILE} "
+                            "if the folder exists, "
+                            "otherwise uses the ipython default location. "
+                            "This folder is a local hard-drive location for lcls "
+                            "operator consoles."
+                        ))
+    parser.add_argument("script", nargs="?",
+                        help="Run a script instead of running interactively")
+    return parser
+
+
+parser = get_parser()
 
 # Append to module docs
-__doc__ += '\n::\n\n    ' + parser.format_help().replace('\n', '\n    ')
+__doc__ += "\n::\n\n    " + parser.format_help().replace("\n", "\n    ")
+
+
+@dataclasses.dataclass
+class HutchPythonArgs:
+    cfg: str | None = None
+    exp: str | None = None
+    debug: bool = False
+    sim: bool = False
+    create: bool = False
+    hist_file: str | None = None
+    script: str | None = None
 
 
 def configure_tab_completion(ipy_config):
@@ -64,13 +102,13 @@ def configure_tab_completion(ipy_config):
         # First, access it to see that the internals have not changed:
         IPython.core.completer.dir2
     except AttributeError:
-        logger.debug('Looks like the IPython API changed!')
+        logger.debug("Looks like the IPython API changed!")
     else:
         # Then monkeypatch it in:
         IPython.core.completer.dir2 = dir
 
 
-def configure_ipython_session():
+def configure_ipython_session(args: HutchPythonArgs):
     """
     Configure a new IPython session.
 
@@ -82,17 +120,17 @@ def configure_ipython_session():
     ipy_config = Config()
     # Important Utilities
     ipy_config.InteractiveShellApp.extensions = [
-        'hutch_python.ipython_log',
-        'hutch_python.bug',
-        'hutch_python.pt_app_config'
+        "hutch_python.ipython_log",
+        "hutch_python.bug",
+        "hutch_python.pt_app_config"
     ]
     # Matplotlib setup for ipython (automatically do %matplotlib)
-    backend = matplotlib.get_backend().replace('Agg', '').lower()
+    backend = matplotlib.get_backend().replace("Agg", "").lower()
     ipy_config.InteractiveShellApp.matplotlib = backend
-    if backend == 'agg':
-        logger.warning('No matplotlib rendering available. '
-                       'Methods that create plots will not '
-                       'function properly.')
+    if backend == "agg":
+        logger.warning("No matplotlib rendering available. "
+                       "Methods that create plots will not "
+                       "function properly.")
 
     # Disable reformatting input with black
     ipy_config.TerminalInteractiveShell.autoformatter = None
@@ -109,6 +147,20 @@ def configure_ipython_session():
     ]
     ipy_config.InteractiveShellApp.exec_files = files
 
+    # Custom history file with sensible non-NFS default for opr accounts
+    if args.hist_file is not None:
+        hist_file = Template(args.hist_file).safe_substitute(os.environ)
+        if hist_file == ":memory:" or Path(hist_file).parent.exists():
+            ipy_config.HistoryManager.hist_file = hist_file
+        else:
+            msg = f"No such directory for history file {hist_file}, using ipython default instead."
+            if args.hist_file == DEFAULT_HISTFILE:
+                # We expect this to be missing for non-opr users
+                logger.debug(msg)
+            else:
+                # You specified a file, so we need to warn about this
+                logger.warning(msg)
+
     return ipy_config
 
 
@@ -120,13 +172,13 @@ def main():
     setup functions.
     """
     # Parse the user's arguments
-    args = parser.parse_args()
+    args = parser.parse_args(namespace=HutchPythonArgs())
 
     # Set up logging first
     if args.cfg is None:
         log_dir = None
     else:
-        log_dir = os.path.join(os.path.dirname(args.cfg), 'logs')
+        log_dir = os.path.join(os.path.dirname(args.cfg), "logs")
 
     configure_log_directory(log_dir)
     setup_logging()
@@ -136,49 +188,49 @@ def main():
         debug_mode(True)
 
     # Do the first log message, now that logging is ready
-    logger.debug('cli starting with args %s', args)
+    logger.debug("cli starting with args %s", args)
 
     # Check and display the environment info as appropriate (very early)
     log_env()
 
     # Options that mean skipping the python environment
     if args.create:
         hutch = args.create
-        envs_dir = CONDA_BASE / 'envs'
+        envs_dir = CONDA_BASE / "envs"
         if envs_dir.exists():
             # Pick most recent pcds release in our common env
             base = str(CONDA_BASE)
-            path_obj = sorted(envs_dir.glob('pcds-*'))[-1]
+            path_obj = sorted(envs_dir.glob("pcds-*"))[-1]
             env = path_obj.name
         else:
             # Fallback: pick current env
             try:
-                base = str(Path(os.environ['CONDA_EXE']).parent.parent)
-                env = os.environ['CONDA_DEFAULT_ENV']
+                base = str(Path(os.environ["CONDA_EXE"]).parent.parent)
+                env = os.environ["CONDA_DEFAULT_ENV"]
             except KeyError:
                 # Take a stab at some non-conda defaults; ideally these would
                 # be configurable with argparse.
                 base = str(Path(sys.executable).parent)
                 env = hutch
-        logger.info(('Creating hutch-python dir for hutch %s using'
-                     ' base=%s env=%s'), hutch, base, env)
-        cookiecutter(str(DIR_MODULE / 'cookiecutter'), no_input=True,
+        logger.info(("Creating hutch-python dir for hutch %s using"
+                     " base=%s env=%s"), hutch, base, env)
+        cookiecutter(str(DIR_MODULE / "cookiecutter"), no_input=True,
                      extra_context=dict(base=base, env=env, hutch=hutch))
         return
 
     # Save whether we are an interactive session or a script session
-    opts_cache['script'] = args.script
+    opts_cache["script"] = args.script
 
     # Load objects based on the configuration file
     objs = load(cfg=args.cfg, args=args)
 
-    script = opts_cache.get('script')
+    script = opts_cache.get("script")
     if script is None:
         # Finally start the interactive session
-        start_ipython(argv=['--quick'], user_ns=objs,
-                      config=configure_ipython_session())
+        start_ipython(argv=["--quick"], user_ns=objs,
+                      config=configure_ipython_session(args))
     else:
         # Instead of setting up ipython, run the script with objs
         with open(script) as fn:
-            code = compile(fn.read(), script, 'exec')
+            code = compile(fn.read(), script, "exec")
             exec(code, objs, objs)

hutch_python/constants.py

@@ -32,6 +32,7 @@
     'hutch',
     'db',
     'load',
+    'load_level',
     'experiment',
     'daq_platform',
     'daq_type',