diff --git a/.requirements_dev.txt b/.requirements_dev.txt index 82f7066..5e78c7a 100644 --- a/.requirements_dev.txt +++ b/.requirements_dev.txt @@ -5,3 +5,4 @@ pytest==4.5.0 pytest-cov==2.7.1 pytest-runner==4.4 codecov==2.0.15 +xrootd>=4.10.0 diff --git a/.travis.yml b/.travis.yml index b598242..cebfb0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,26 @@ python: - "3.7" install: + - sudo apt-get update + # We do this conditionally because it saves us some downloading if the + # version is the same. + - if [[ "$TRAVIS_PYTHON_VERSION" == 3.[678]* ]]; then + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + bash miniconda.sh -b -p $HOME/miniconda; + source "$HOME/miniconda/etc/profile.d/conda.sh"; + hash -r; + conda config --set always_yes yes --set changeps1 no; + conda update -q conda; + conda info -a; + + conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION; + conda activate test-environment; + conda install -c conda-forge xrootd; + fi - pip install -r .requirements_dev.txt - pip install . + script: pytest -v --doctest-modules --cov-report=xml --cov=fast_curator tests/ after_success: diff --git a/CHANGELOG.md b/CHANGELOG.md index ff28e35..d3ad280 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed ### Removed +## [0.6.0] - 2020-04-17 +### Changed +- Use Xrootd's implementation of xrootd_glob not our own version PR #43 [@benkrikler](https://github.com/benkrikler) + ## [0.5.0] - 2020-01-27 ### Added - Support for handling files which can't be opened PR #36 [@seriksen](https://github.com/seriksen) diff --git a/fast_curator/catalogues/__init__.py b/fast_curator/catalogues/__init__.py index 03d3bac..561bde3 100644 --- a/fast_curator/catalogues/__init__.py +++ b/fast_curator/catalogues/__init__.py @@ -1,18 +1,23 @@ import os +import sys import uproot from collections import defaultdict, Counter +from functools import partial +if sys.version[0] > '2': + from urllib.parse import urlparse +else: + from urlparse import urlparse class XrootdExpander(): """ Expand wild-carded file paths, including with xrootd-served files """ - from .. import xrootd_glob - @staticmethod def expand_file_list(files, prefix=None): - glob = XrootdExpander.xrootd_glob.glob - return expand_file_list_generic(files, prefix, glob=glob) + from XRootD.client.glob_funcs import glob + return expand_file_list_generic(files, prefix, + glob=partial(glob, raise_error=True)) @staticmethod def check_files(*args, **kwargs): @@ -38,7 +43,8 @@ def check_files(*args, **kwargs): def expand_file_list_generic(files, prefix, glob): full_list = [] for name in files: - if not os.path.isabs(name): + scheme = urlparse(name).scheme + if not scheme and not os.path.isabs(name): if prefix: name = os.path.join(prefix, name) else: diff --git a/fast_curator/version.py b/fast_curator/version.py index d1947dd..06ce1a9 100644 --- a/fast_curator/version.py +++ b/fast_curator/version.py @@ -12,5 +12,5 @@ def split_version(version): return tuple(result) -__version__ = '0.5.0' +__version__ = '0.6.0' version_info = split_version(__version__) # noqa diff --git a/fast_curator/xrootd_glob.py b/fast_curator/xrootd_glob.py deleted file mode 100644 index 959cf5e..0000000 --- a/fast_curator/xrootd_glob.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -Reproduce the standard glob package behaviour -""" -import glob as gl -import os -import fnmatch -import sys -if sys.version_info[0] > 2: - from urllib.parse import urlparse -else: - from urlparse import urlparse - - -__all__ = ["glob", "iglob"] - - -def split_url(url): - parsed_uri = urlparse(url) - domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri) - path = parsed_uri.path - return domain, path - - -def glob(pathname): - # Let normal python glob try first - try_glob = gl.glob(pathname) - if try_glob: - return try_glob - - # If pathname does not contain a wildcard: - if not gl.has_magic(pathname): - return [pathname] - - # Else try xrootd instead - return xrootd_glob(pathname) - - -def xrootd_glob(pathname): - from pyxrootd.client import FileSystem - # Split the pathname into a directory and basename - dirs, basename = os.path.split(pathname) - - if gl.has_magic(dirs): - dirs = xrootd_glob(dirs) - else: - dirs = [dirs] - - files = [] - for dirname in dirs: - host, path = split_url(dirname) - query = FileSystem(host) - - if not query: - raise RuntimeError("Cannot prepare xrootd query") - - _, dirlist = query.dirlist(path) - for entry in dirlist["dirlist"]: - filename = entry["name"] - if filename in [".", ".."]: - continue - if not fnmatch.fnmatchcase(filename, basename): - continue - files.append(os.path.join(dirname, filename)) - - return files - - -def iglob(pathname): - for name in glob(pathname): - yield name diff --git a/setup.cfg b/setup.cfg index f9ff585..2bee756 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.5.0 +current_version = 0.6.0 commit = True tag = False diff --git a/setup.py b/setup.py index 2466dbf..66eb7e2 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def get_version(): setup_requirements = ['pytest-runner', ] -test_requirements = ['pytest', 'flake8', 'pytest-cov'] +test_requirements = ['pytest', 'flake8', 'pytest-cov', 'xrootd'] setup( author="F.A.S.T",