Merge branch 'master' into WorkflowUpdate

.github/workflows/build.yml

@@ -12,7 +12,7 @@ jobs:
         python-version: ['3.7', '3.8', '3.9', '3.10']
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:

docs/source/_templates/crown-copyright.html

@@ -0,0 +1,11 @@
+{# Crown copyright is displayed differently to normal. #}
+{# Configured from conf.py as per usual. #}
+{% if show_copyright and copyright %}
+<p class="copyright">
+    {% if hasdoc('copyright') %}
+    {% trans path=pathto('copyright'), copyright=copyright|e %}© <a href="{{ path }}">Copyright</a> {{ copyright }}.{% endtrans %}
+    {% else %}
+    {% trans copyright=copyright|e %}© Crown Copyright {{ copyright }}.{% endtrans %}
+    {% endif %}
+</p>
+{% endif %}

docs/source/advanced_config.rst

@@ -315,7 +315,7 @@ which most Fab steps accept. (See :ref:`Overriding default collections`)
 
     @step
     def custom_step(state):
-            state._artefact_store['custom_artefacts'] = do_something(state._artefact_store['step 1 artefacts'])
+        state.artefact_store['custom_artefacts'] = do_something(state.artefact_store['step 1 artefacts'])
 
 
     with BuildConfig(project_label='<project label>') as state:
@@ -332,7 +332,7 @@ Steps have access to multiprocessing methods through the
 
     @step
     def custom_step(state):
-        input_files = artefact_store['custom_artefacts']
+        input_files = state.artefact_store['custom_artefacts']
         results = run_mp(state, items=input_files, func=do_something)
 
 

docs/source/api.rst

@@ -0,0 +1,10 @@
+API Documentation
+=================
+
+This API documentation is generated from comments within the source code.
+
+.. autosummary::
+   :toctree: api
+   :recursive:
+
+   fab

docs/source/conf.py

@@ -17,8 +17,8 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'fab'
-copyright = '2023'
+project = 'Fab'
+copyright = '2024 Met Office. All rights reserved.'
 author = 'Fab Team'
 
 # The full version, including alpha/beta/rc tags
@@ -52,24 +52,35 @@
 exclude_patterns = []
 
 
-# -- Options for HTML output -------------------------------------------------
+# -- Autodoc -----------------------------------------------------------------
 
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
+autodoc_default_options = {
+    'members': True,
+    'show-inheritane': True
+}
 
-# ugly
-# html_theme = "classic"
+autoclass_content = 'both'
 
-# poor contrast between title, h1 & h2
-# html_theme = "sphinxdoc"
-# html_theme = "sphinx_rtd_theme"
-# html_theme = 'python_docs_theme'
 
-# good contrast between title, h1 & h2
-# html_theme = 'alabaster'
-html_theme = 'sphinx_material'
+# -- Options for HTML output -------------------------------------------------
 
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'pydata_sphinx_theme'
+
+html_theme_options = {
+    "icon_links": [
+        {
+            "name": "GitHub",
+            "url": "https://github.com/metomi/fab",
+            "icon": "fa-brands fa-github"
+        }
+    ],
+    "footer_start": ["crown-copyright"],
+    "footer_center": ["sphinx-version"],
+    "footer_end": ["theme-version"],
+}
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -93,12 +104,3 @@
 
 # include default values in argument descriptions
 typehints_defaults = 'braces-after'
-
-# needed when using material theme
-html_sidebars = {
-    "**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]
-}
-# needed when not using material theme
-# html_sidebars = {
-#     "**": ["globaltoc.html", "searchbox.html"]
-# }

docs/source/index.rst

@@ -81,7 +81,7 @@ See also
    writing_config
    advanced_config
    features
-   Api Reference <apidoc/modules>
+   Api Reference <api>
    development
    glossary
    genindex

pyproject.toml

@@ -22,8 +22,11 @@ classifiers = [
 c-language = ['python-clang']
 plots = ['matplotlib']
 tests = ['pytest', 'pytest-cov', 'pytest-mock']
-checks = ['flake8', 'mypy']
-docs = ['sphinx', 'sphinx-material', 'sphinx-autodoc-typehints', 'sphinx-copybutton']
+checks = ['flake8>=5.0.4', 'mypy']
+docs = ['sphinx',
+        'pydata-sphinx-theme>=0.13.3',
+        'sphinx-autodoc-typehints',
+        'sphinx-copybutton']
 dev = ['sci-fab[plots, tests, checks, docs]']
 
 [project.scripts]

source/fab/artefacts.py

@@ -15,7 +15,7 @@
 from pathlib import Path
 from typing import Iterable, Union, Dict, List
 
-from fab.constants import BUILD_TREES
+from fab.constants import BUILD_TREES, CURRENT_PREBUILDS
 from fab.dep_tree import filter_source_tree, AnalysedDependent
 from fab.util import suffix_filter
 
@@ -32,7 +32,8 @@ def __call__(self, artefact_store):
             The artefact store from which to retrieve.
 
         """
-        pass
+        raise NotImplementedError(f"__call__ must be implemented for "
+                                  f"'{type(self).__name__}'.")
 
 
 class CollectionGetter(ArtefactsGetter):
@@ -53,7 +54,6 @@ def __init__(self, collection_name):
         self.collection_name = collection_name
 
     def __call__(self, artefact_store):
-        super().__call__(artefact_store)
         return artefact_store.get(self.collection_name, [])
 
 
@@ -84,7 +84,6 @@ def __init__(self, collections: Iterable[Union[str, ArtefactsGetter]]):
 
     # todo: ensure the labelled values are iterables
     def __call__(self, artefact_store: Dict):
-        super().__call__(artefact_store)
         # todo: this should be a set, in case a file appears in multiple collections
         result = []
         for collection in self.collections:
@@ -118,7 +117,6 @@ def __init__(self, collection_name: str, suffix: Union[str, List[str]]):
         self.suffixes = [suffix] if isinstance(suffix, str) else suffix
 
     def __call__(self, artefact_store):
-        super().__call__(artefact_store)
         # todo: returning an empty list is probably "dishonest" if the collection doesn't exist - return None instead?
         fpaths: Iterable[Path] = artefact_store.get(self.collection_name, [])
         return suffix_filter(fpaths, self.suffixes)
@@ -149,7 +147,6 @@ def __init__(self, suffix: Union[str, List[str]], collection_name: str = BUILD_T
         self.suffixes = [suffix] if isinstance(suffix, str) else suffix
 
     def __call__(self, artefact_store):
-        super().__call__(artefact_store)
 
         build_trees = artefact_store[self.collection_name]
 
@@ -158,3 +155,18 @@ def __call__(self, artefact_store):
             build_lists[root] = filter_source_tree(source_tree=tree, suffixes=self.suffixes)
 
         return build_lists
+
+
+class ArtefactStore(dict):
+    '''This object stores artefacts (which can be of any type). Each artefact
+    is indexed by a string.
+    '''
+    def __init__(self):
+        super().__init__()
+        self.reset()
+
+    def reset(self):
+        '''Clears the artefact store (but does not delete any files).
+        '''
+        self.clear()
+        self[CURRENT_PREBUILDS] = set()

source/fab/build_config.py

@@ -18,16 +18,18 @@
 from multiprocessing import cpu_count
 from pathlib import Path
 from string import Template
-from typing import List, Optional, Dict, Any, Iterable
+from typing import List, Optional, Iterable
 
+from fab.artefacts import ArtefactStore
 from fab.constants import BUILD_OUTPUT, SOURCE_ROOT, PREBUILD, CURRENT_PREBUILDS
 from fab.metrics import send_metric, init_metrics, stop_metrics, metrics_summary
+from fab.steps.cleanup_prebuilds import CLEANUP_COUNT, cleanup_prebuilds
 from fab.util import TimerLogger, by_type, get_fab_workspace
 
 logger = logging.getLogger(__name__)
 
 
-class BuildConfig(object):
+class BuildConfig():
     """
     Contains and runs a list of build steps.
 
@@ -105,9 +107,10 @@ def __init__(self, project_label: str, multiprocessing: bool = True, n_procs: Op
 
         # todo: should probably pull the artefact store out of the config
         # runtime
-        # todo: either make this public, add get/setters, or extract into a class.
-        self._artefact_store: Dict[str, Any] = {}
-        self.init_artefact_store()  # note: the artefact store is reset with every call to run()
+        self._artefact_store = ArtefactStore()
+
+        self._build_timer = None
+        self._start_time = None
 
     def __enter__(self):
 
@@ -130,8 +133,7 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
 
         if not exc_type:  # None if there's no error.
-            from fab.steps.cleanup_prebuilds import CLEANUP_COUNT, cleanup_prebuilds
-            if CLEANUP_COUNT not in self._artefact_store:
+            if CLEANUP_COUNT not in self.artefact_store:
                 logger.info("no housekeeping step was run, using a default hard cleanup")
                 cleanup_prebuilds(config=self, all_unused=True)
 
@@ -142,19 +144,23 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self._finalise_logging()
 
     @property
-    def build_output(self):
-        return self.project_workspace / BUILD_OUTPUT
+    def artefact_store(self) -> ArtefactStore:
+        ''':returns: the Artefact instance for this configuration.
+        '''
+        return self._artefact_store
 
-    def init_artefact_store(self):
-        # there's no point writing to this from a child process of Step.run_mp() because you'll be modifying a copy.
-        self._artefact_store = {CURRENT_PREBUILDS: set()}
+    @property
+    def build_output(self) -> Path:
+        ''':returns: the build output path.
+        '''
+        return self.project_workspace / BUILD_OUTPUT
 
     def add_current_prebuilds(self, artefacts: Iterable[Path]):
         """
         Mark the given file paths as being current prebuilds, not to be cleaned during housekeeping.
 
         """
-        self._artefact_store[CURRENT_PREBUILDS].update(artefacts)
+        self.artefact_store[CURRENT_PREBUILDS].update(artefacts)
 
     def _run_prep(self):
         self._init_logging()
@@ -168,7 +174,7 @@ def _run_prep(self):
         init_metrics(metrics_folder=self.metrics_folder)
 
         # note: initialising here gives a new set of artefacts each run
-        self.init_artefact_store()
+        self.artefact_store.reset()
 
     def _prep_folders(self):
         self.source_root.mkdir(parents=True, exist_ok=True)
@@ -210,7 +216,7 @@ def _finalise_metrics(self, start_time, steps_timer):
 
 
 # todo: better name? perhaps PathFlags?
-class AddFlags(object):
+class AddFlags():
     """
     Add command-line flags when our path filter matches.
     Generally used inside a :class:`~fab.build_config.FlagsConfig`.
@@ -265,14 +271,13 @@ def run(self, fpath: Path, input_flags: List[str], config):
             input_flags += add_flags
 
 
-class FlagsConfig(object):
+class FlagsConfig():
     """
     Return command-line flags for a given path.
 
     Simply allows appending flags but may evolve to also replace and remove flags.
 
     """
-
     def __init__(self, common_flags: Optional[List[str]] = None, path_flags: Optional[List[AddFlags]] = None):
         """
         :param common_flags:

source/fab/parse/fortran_common.py

@@ -134,8 +134,6 @@ def run(self, fpath: Path) \
 
         # find things in the node tree
         analysed_file = self.walk_nodes(fpath=fpath, file_hash=file_hash, node_tree=node_tree)
-
-        analysis_fpath = self._get_analysis_fpath(fpath, file_hash)
         analysed_file.save(analysis_fpath)
 
         return analysed_file, analysis_fpath

source/fab/steps/analyse.py

@@ -167,7 +167,7 @@ def analyse(
     c_analyser._config = config
 
     # parse
-    files: List[Path] = source_getter(config._artefact_store)
+    files: List[Path] = source_getter(config.artefact_store)
     analysed_files = _parse_files(config, files=files, fortran_analyser=fortran_analyser, c_analyser=c_analyser)
     _add_manual_results(special_measure_analysis_results, analysed_files)
 
@@ -206,7 +206,7 @@ def analyse(
         _add_unreferenced_deps(unreferenced_deps, symbol_table, project_source_tree, build_tree)
         validate_dependencies(build_tree)
 
-    config._artefact_store[BUILD_TREES] = build_trees
+    config.artefact_store[BUILD_TREES] = build_trees
 
 
 def _analyse_dependencies(analysed_files: Iterable[AnalysedDependent]):

source/fab/steps/archive_objects.py

@@ -95,14 +95,14 @@ def archive_objects(config: BuildConfig, source: Optional[ArtefactsGetter] = Non
     output_fpath = str(output_fpath) if output_fpath else None
     output_collection = output_collection
 
-    target_objects = source_getter(config._artefact_store)
+    target_objects = source_getter(config.artefact_store)
     assert target_objects.keys()
     if output_fpath and list(target_objects.keys()) != [None]:
         raise ValueError("You must not specify an output path (library) when there are root symbols (exes)")
     if not output_fpath and list(target_objects.keys()) == [None]:
         raise ValueError("You must specify an output path when building a library.")
 
-    output_archives = config._artefact_store.setdefault(output_collection, {})
+    output_archives = config.artefact_store.setdefault(output_collection, {})
     for root, objects in target_objects.items():
 
         if root:

source/fab/steps/c_pragma_injector.py

@@ -43,9 +43,9 @@ def c_pragma_injector(config, source: Optional[ArtefactsGetter] = None, output_n
     source_getter = source or DEFAULT_SOURCE_GETTER
     output_name = output_name or PRAGMAD_C
 
-    files = source_getter(config._artefact_store)
+    files = source_getter(config.artefact_store)
     results = run_mp(config, items=files, func=_process_artefact)
-    config._artefact_store[output_name] = list(results)
+    config.artefact_store[output_name] = list(results)
 
 
 def _process_artefact(fpath: Path):