Issue22 pip installer

.github/workflows/pip_install.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04, ubuntu-22.04, macos-latest, windows-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     runs-on: ${{ matrix.os }}
 
@@ -29,6 +29,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
+          pip install -r requirements.txt
           pip install .
         continue-on-error: false  # Allow the job to continue even if installation fails
 

README.md

@@ -82,18 +82,44 @@ and run the the example script:
 python workflow_example.py
 ```
 This will produce a `json` file listing requested variables for each CMIP7 experiment.
-The same functionality is available from a command-line interface. 
-For example:
+The same functionality is available from a command-line interface. To access this interface 
+we recommend installing the python package using pip (see below) and then using the
+`export_dreq_lists_json` command.
+
+## Pip installation
+
+If you have a conda or virtual (venv, virtualenv) environment which already has the dependencies of
+this package you can install the code using
+```bash
+python -m pip install git+https://github.com/CMIP-Data-Request/CMIP7_DReq_Software.git@<tag>
 ```
-./export_dreq_lists_json.py v1.0 dreq_list.json --all_opportunities
+where `<tag>` needs to be replaced with the version you wish to
+install.
+
+If installation is successful you should be able to run the command 
+```bash
+export_dreq_lists_json --all_opportunities v1.0 amip.json --experiments amip
 ```
-will produce a file a listing the variables requested from all CMIP7 AR7 Fast Track experiments assuming that all Data Request Opportunities are supported.
-The list can be customized by specifying which Opportunities are supported or filtering based on one or more experiment names.
-The `-h` option provides further usage guidance:
+To confirm that the variable list for the amip experiment can be
+produced.
+
+To install from a local copy for development purposes cd to the root
+of the repository and run
+```bash
+python -m pip install -e .
 ```
-./export_dreq_lists_json.py -h
+
+The package can be uninstalled using
+```bash
+python -m pip uninstall CMIP7_data_request_api
 ```
 
+### Development: addition of command line tools
+
+Command line utilities should be hosted under the
+`data_request_api.command_line` package and pointed at by adding
+references to the appropriate `main()` routine into the 
+`[project:scripts]` section of the pyproject.toml file.
 
 ## Further details
 
@@ -125,3 +151,9 @@ Or to remove all locally cached versions:
 ```
 dc.delete()
 ```
+
+## Contributors
+
+[![Contributors](https://contrib.rocks/image?repo=CMIP-Data-Request/CMIP7_DReq_Software)](https://github.com/CMIP-Data-Request/CMIP7_DReq_Software/graphs/contributors/)
+
+Thanks to our contributors!

scripts/export_dreq_lists_json.py → data_request_api/command_line/export_dreq_lists_json.py

@@ -8,21 +8,9 @@
 import os
 import argparse
 
-# The following should be removed once python packaging is completed.
-sys.path.append(os.path.join(os.path.dirname('__file__'), '..'))
-
 # import data_request_api.stable.content.dreq_api as dreq_api
-# import data_request_api.stable.content.dreq_api.dreq_content as dc
-# import data_request_api.stable.query.dreq_query as dq
-add_paths = []
-add_paths.append('../data_request_api/stable/content/dreq_api')
-add_paths.append('../data_request_api/stable/query')
-add_paths.append('../data_request_api/stable/transform')
-for path in add_paths:
-   if path not in sys.path:
-       sys.path.append(path)
-import dreq_content as dc
-import dreq_query as dq
+import data_request_api.stable.content.dreq_api.dreq_content as dc
+import data_request_api.stable.query.dreq_query as dq
 
 
 def parse_args():
@@ -99,5 +87,6 @@ def main():
     else:
         print(f'\nFor data request version {use_dreq_version}, no requested variables were found')
 
+
 if __name__ == '__main__':
     main()

data_request_api/dev/JA/check_plev_requests.py

@@ -8,15 +8,12 @@
 '''
 
 import os
-import sys
 import json
-add_paths = ['../../sandbox/MS/dreq_api/']
-for path in add_paths:
-    if path not in sys.path:
-        sys.path.append(path)
-import dreq_content as dc
-import dreq_query as dq
-import dreq_classes
+
+
+import data_request_api.stable.content.dreq_api.dreq_content as dc
+import data_request_api.stable.query.dreq_query as dq
+from data_request_api.stable.query import dreq_classes
 
 from collections import OrderedDict, defaultdict
 from copy import deepcopy
@@ -32,8 +29,6 @@
 # use_dreq_version = 'v1.0alpha'
 use_dreq_version = 'v1.0beta'
 
-
-
 # # Download specified version of data request content (if not locally cached)
 # dc.retrieve(use_dreq_version, export='raw')
 # # Load content into python dict

data_request_api/dev/JA/workflow_example_GRtest.py

@@ -6,24 +6,16 @@
 """
 from __future__ import division, print_function, unicode_literals, absolute_import
 
-import sys
 import pprint
 from collections import defaultdict
 
 import six
 
 import json
 
-# add_paths = ['../sandbox/MS/dreq_api/', '../sandbox/JA', '../sandbox/GR']
-add_paths = ['../MS/dreq_api/', '../GR']
-for path in add_paths:
-    if path not in sys.path:
-        sys.path.append(path)
-
-
-import dreq_content as dc
-from data_request import DataRequest
-from logger import change_log_file, change_log_level
+from data_request_api.stable.content.dreq_api import dreq_content as dc
+from data_request_api.stable.query.data_request import DataRequest
+from data_request_api.stable.utilities.logger import change_log_file, change_log_level
 
 # Set up log file (default to stdout) and log level
 change_log_file(default=True)

data_request_api/dev/JA/workflow_example_test.py

@@ -49,14 +49,9 @@
 
     stop
 
-import sys
 import json
-add_paths = ['../sandbox/MS/dreq_api/', '../sandbox/JA', '../sandbox/GR']
-for path in add_paths:
-    if path not in sys.path:
-        sys.path.append(path)
-import dreq_content as dc
-import dreq_query as dq
+from data_request_api.stable.content.dreq_api import dreq_content as dc
+from data_request_api.stable.query import dreq_query as dq
 
 from importlib import reload
 reload(dq)

data_request_api/stable/content/README.MD

@@ -62,7 +62,7 @@ These are:
 -----------------
 
 ```python
-import dreq_content as dc
+import data_request_api.stable.content.dreq_api.dreq_content as dc
 
 # Get the list of available versions
 versions = dc.get_versions()

data_request_api/stable/content/dreq_api/consolidate_export.py

@@ -1,19 +1,10 @@
 import json
 import os
 import re
-import sys
 import warnings
 
-# TODO: remove after initial "sandbox" dev period
-add_paths = [os.path.abspath(os.sep.join([os.path.abspath(os.path.dirname(__file__)), "../.."])), ]
-for path in add_paths:
-   if path not in sys.path:
-       sys.path.append(path)
-from utilities.logger import get_logger  # noqa
-# from ...transform.logger import get_logger  # noqa
-
-# from .mapping_table import version_consistency
-from mapping_table import version_consistency
+from data_request_api.stable.utilities.logger import get_logger  # noqa
+from data_request_api.stable.content.dreq_api.mapping_table import version_consistency
 
 # UID generation
 default_count = 0

data_request_api/stable/content/dreq_api/dreq_content.py

@@ -1,27 +1,18 @@
 import json
 import os
 import re
-import sys
 import time
 import warnings
 from filecmp import cmp
 from shutil import move
 
-# from . import consolidate_export as ce
-import consolidate_export as ce
 import pooch
 import requests
 from bs4 import BeautifulSoup
-# from .mapping_table import mapping_table
-from mapping_table import mapping_table
 
-# TODO: remove after initial "sandbox" dev period
-add_paths = [os.path.abspath(os.sep.join([os.path.abspath(os.path.dirname(__file__)), "../.."])), ]
-for path in add_paths:
-   if path not in sys.path:
-       sys.path.append(path)
-from utilities.logger import get_logger  # noqa
-# from ...transform.logger import get_logger  # noqa
+from . import consolidate_export as ce
+from data_request_api.stable.content.dreq_api.mapping_table import mapping_table
+from data_request_api.stable.utilities.logger import get_logger  # noqa
 
 
 # Suppress pooch info output
@@ -67,6 +58,7 @@
 
 _dreq_content_loaded = {}
 
+
 def _parse_version(version):
     """Parse a version tag and return a tuple for sorting.
 

data_request_api/stable/content/dreq_api/test_dreq_content.py

@@ -1,6 +1,6 @@
 import os
 
-import dreq_content as dc
+from data_request_api.stable.content.dreq_api import dreq_content as dc
 import pytest
 
 

data_request_api/stable/content/dump_transformation.py

@@ -16,8 +16,8 @@
 
 import six
 
-from utilities.logger import get_logger, change_log_level, change_log_file
-from utilities.tools import read_json_input_file_content, write_json_output_file_content
+from data_request_api.stable.utilities.logger import get_logger, change_log_level, change_log_file
+from data_request_api.stable.utilities.tools import read_json_input_file_content, write_json_output_file_content
 
 
 def correct_key_string(input_string, *to_remove_strings):

data_request_api/stable/query/data_request.py

@@ -14,10 +14,10 @@
 
 import six
 
-from utilities.logger import get_logger, change_log_file, change_log_level
-from content.dump_transformation import transform_content
-from utilities.tools import read_json_file
-from query.vocabulary_server import VocabularyServer, is_link_id_or_value, build_link_from_id
+from data_request_api.stable.utilities.logger import get_logger, change_log_file, change_log_level
+from data_request_api.stable.content.dump_transformation import transform_content
+from data_request_api.stable.utilities.tools import read_json_file
+from data_request_api.stable.query.vocabulary_server import VocabularyServer, is_link_id_or_value, build_link_from_id
 
 version = "0.1"
 

data_request_api/stable/query/dreq_query.py

@@ -13,12 +13,9 @@
 import json
 from collections import OrderedDict
 
-# from importlib import reload
-# import dreq_classes
-# reload(dreq_classes)
 
-# from .dreq_classes import dreq_table, expt_request, UNIQUE_VAR_NAME, PRIORITY_LEVELS
-from dreq_classes import dreq_table, expt_request, UNIQUE_VAR_NAME, PRIORITY_LEVELS
+from data_request_api.stable.query.dreq_classes import (
+    dreq_table, expt_request, UNIQUE_VAR_NAME, PRIORITY_LEVELS)
 
 DREQ_VERSION = ''  # if a tagged version is being used, set this in calling script
 

data_request_api/stable/query/get_variables_metadata.py

@@ -2,22 +2,16 @@
 '''
 Extract metadata of CMOR variables and write to json
 '''
-import sys
 import json
 import os
 import hashlib
-add_paths = []
-add_paths.append('../content/dreq_api')
-add_paths.append('../transform')
-for path in add_paths:
-    if path not in sys.path:
-        sys.path.append(path)
-import dreq_content as dc
-import dreq_query as dq
-import dreq_classes
-
 from collections import OrderedDict
 
+import data_request_api.stable.content.dreq_api.dreq_content as dc
+import data_request_api.stable.query.dreq_query as dq
+from data_request_api.stable.query import dreq_classes
+
+
 # from importlib import reload
 # reload(dq)
 # reload(dc)

data_request_api/stable/query/vocabulary_server.py

@@ -12,8 +12,8 @@
 
 import six
 
-from utilities.logger import get_logger
-from utilities.tools import read_json_file
+from data_request_api.stable.utilities.logger import get_logger
+from data_request_api.stable.utilities.tools import read_json_file
 
 
 def is_link_id_or_value(elt):

data_request_api/stable/utilities/tools.py

@@ -9,7 +9,7 @@
 import json
 import os
 
-from utilities.logger import get_logger
+from data_request_api.stable.utilities.logger import get_logger
 
 
 def read_json_file(filename):