From cb70df3c6d11b41e9702b487b041a3c1df65deb7 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Wed, 8 Jan 2025 02:28:56 +1100 Subject: [PATCH 1/3] Removed requirements.txt files from all datainterfaces (#1171) --- src/neuroconv/datainterfaces/behavior/audio/requirements.txt | 1 - .../datainterfaces/behavior/deeplabcut/requirements.txt | 3 --- src/neuroconv/datainterfaces/behavior/fictrac/requirements.txt | 0 .../datainterfaces/behavior/lightningpose/requirements.txt | 2 -- src/neuroconv/datainterfaces/behavior/medpc/requirements.txt | 1 - .../datainterfaces/behavior/miniscope/requirements.txt | 2 -- src/neuroconv/datainterfaces/behavior/sleap/requirements.txt | 3 --- src/neuroconv/datainterfaces/behavior/video/requirements.txt | 1 - .../datainterfaces/ecephys/cellexplorer/requirements.txt | 2 -- src/neuroconv/datainterfaces/ecephys/edf/requirements.txt | 1 - src/neuroconv/datainterfaces/ecephys/intan/requirements.txt | 0 src/neuroconv/datainterfaces/ecephys/mearec/requirements.txt | 1 - .../datainterfaces/ecephys/neuralynx/requirements.txt | 1 - .../datainterfaces/ecephys/neuroscope/requirements.txt | 1 - .../datainterfaces/ecephys/openephys/requirements.txt | 1 - src/neuroconv/datainterfaces/ecephys/plexon/requirements.txt | 1 - src/neuroconv/datainterfaces/ecephys/requirements.txt | 2 -- src/neuroconv/datainterfaces/ecephys/spike2/requirements.txt | 1 - src/neuroconv/datainterfaces/icephys/abf/requirements.txt | 1 - src/neuroconv/datainterfaces/icephys/requirements.txt | 1 - src/neuroconv/datainterfaces/ophys/brukertiff/requirements.txt | 1 - .../datainterfaces/ophys/micromanagertiff/requirements.txt | 1 - src/neuroconv/datainterfaces/ophys/miniscope/requirements.txt | 2 -- src/neuroconv/datainterfaces/ophys/requirements.txt | 1 - src/neuroconv/datainterfaces/ophys/scanimage/requirements.txt | 1 - src/neuroconv/datainterfaces/ophys/tdt_fp/requirements.txt | 2 -- src/neuroconv/datainterfaces/ophys/tiff/requirements.txt | 1 - src/neuroconv/datainterfaces/text/excel/requirements.txt | 2 -- 28 files changed, 37 deletions(-) delete mode 100644 src/neuroconv/datainterfaces/behavior/audio/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/fictrac/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/lightningpose/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/medpc/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/miniscope/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/sleap/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/behavior/video/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/cellexplorer/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/edf/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/intan/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/mearec/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/neuralynx/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/neuroscope/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/openephys/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/plexon/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ecephys/spike2/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/icephys/abf/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/icephys/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/brukertiff/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/micromanagertiff/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/miniscope/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/scanimage/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/tdt_fp/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/ophys/tiff/requirements.txt delete mode 100644 src/neuroconv/datainterfaces/text/excel/requirements.txt diff --git a/src/neuroconv/datainterfaces/behavior/audio/requirements.txt b/src/neuroconv/datainterfaces/behavior/audio/requirements.txt deleted file mode 100644 index 5f5d607ea..000000000 --- a/src/neuroconv/datainterfaces/behavior/audio/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ndx-sound>=0.2.0 diff --git a/src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt b/src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt deleted file mode 100644 index 3a5f2fd19..000000000 --- a/src/neuroconv/datainterfaces/behavior/deeplabcut/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -tables>=3.10.1; platform_system == 'Darwin' and python_version >= '3.10' -tables; platform_system != 'Darwin' -ndx-pose==0.1.1 diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/requirements.txt b/src/neuroconv/datainterfaces/behavior/fictrac/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/neuroconv/datainterfaces/behavior/lightningpose/requirements.txt b/src/neuroconv/datainterfaces/behavior/lightningpose/requirements.txt deleted file mode 100644 index 73aa32089..000000000 --- a/src/neuroconv/datainterfaces/behavior/lightningpose/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ndx-pose==0.1.1 -neuroconv[video] diff --git a/src/neuroconv/datainterfaces/behavior/medpc/requirements.txt b/src/neuroconv/datainterfaces/behavior/medpc/requirements.txt deleted file mode 100644 index f0f292cab..000000000 --- a/src/neuroconv/datainterfaces/behavior/medpc/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ndx-events==0.2.0 diff --git a/src/neuroconv/datainterfaces/behavior/miniscope/requirements.txt b/src/neuroconv/datainterfaces/behavior/miniscope/requirements.txt deleted file mode 100644 index 90ab7d140..000000000 --- a/src/neuroconv/datainterfaces/behavior/miniscope/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ndx-miniscope>=0.5.1 -natsort>=8.3.1 diff --git a/src/neuroconv/datainterfaces/behavior/sleap/requirements.txt b/src/neuroconv/datainterfaces/behavior/sleap/requirements.txt deleted file mode 100644 index a62a14965..000000000 --- a/src/neuroconv/datainterfaces/behavior/sleap/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sleap-io>=0.0.2; python_version>='3.9' -sleap-io>=0.0.2,<0.0.12; python_version<'3.9' -av>=10.0.0 diff --git a/src/neuroconv/datainterfaces/behavior/video/requirements.txt b/src/neuroconv/datainterfaces/behavior/video/requirements.txt deleted file mode 100644 index 650f0426f..000000000 --- a/src/neuroconv/datainterfaces/behavior/video/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -opencv-python-headless>=4.8.1.78 diff --git a/src/neuroconv/datainterfaces/ecephys/cellexplorer/requirements.txt b/src/neuroconv/datainterfaces/ecephys/cellexplorer/requirements.txt deleted file mode 100644 index c7417a085..000000000 --- a/src/neuroconv/datainterfaces/ecephys/cellexplorer/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -hdf5storage>=0.1.18 -pymatreader>=0.0.32 diff --git a/src/neuroconv/datainterfaces/ecephys/edf/requirements.txt b/src/neuroconv/datainterfaces/ecephys/edf/requirements.txt deleted file mode 100644 index 2ca43feaa..000000000 --- a/src/neuroconv/datainterfaces/ecephys/edf/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pyedflib>=0.1.36 diff --git a/src/neuroconv/datainterfaces/ecephys/intan/requirements.txt b/src/neuroconv/datainterfaces/ecephys/intan/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/neuroconv/datainterfaces/ecephys/mearec/requirements.txt b/src/neuroconv/datainterfaces/ecephys/mearec/requirements.txt deleted file mode 100644 index 6cc8b3266..000000000 --- a/src/neuroconv/datainterfaces/ecephys/mearec/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -MEArec>=1.8.0 diff --git a/src/neuroconv/datainterfaces/ecephys/neuralynx/requirements.txt b/src/neuroconv/datainterfaces/ecephys/neuralynx/requirements.txt deleted file mode 100644 index 8e45f67d0..000000000 --- a/src/neuroconv/datainterfaces/ecephys/neuralynx/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -natsort>=7.1.1 diff --git a/src/neuroconv/datainterfaces/ecephys/neuroscope/requirements.txt b/src/neuroconv/datainterfaces/ecephys/neuroscope/requirements.txt deleted file mode 100644 index ae68fd6f2..000000000 --- a/src/neuroconv/datainterfaces/ecephys/neuroscope/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -lxml>=4.6.5 diff --git a/src/neuroconv/datainterfaces/ecephys/openephys/requirements.txt b/src/neuroconv/datainterfaces/ecephys/openephys/requirements.txt deleted file mode 100644 index 2378cff71..000000000 --- a/src/neuroconv/datainterfaces/ecephys/openephys/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -lxml>=4.9.4 diff --git a/src/neuroconv/datainterfaces/ecephys/plexon/requirements.txt b/src/neuroconv/datainterfaces/ecephys/plexon/requirements.txt deleted file mode 100644 index 1f3b3018c..000000000 --- a/src/neuroconv/datainterfaces/ecephys/plexon/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -zugbruecke >= 0.2.1; platform_system != "Windows" diff --git a/src/neuroconv/datainterfaces/ecephys/requirements.txt b/src/neuroconv/datainterfaces/ecephys/requirements.txt deleted file mode 100644 index bcd36de3f..000000000 --- a/src/neuroconv/datainterfaces/ecephys/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -spikeinterface>=0.101.0 -neo>=0.13.3 diff --git a/src/neuroconv/datainterfaces/ecephys/spike2/requirements.txt b/src/neuroconv/datainterfaces/ecephys/spike2/requirements.txt deleted file mode 100644 index 7c00e91a8..000000000 --- a/src/neuroconv/datainterfaces/ecephys/spike2/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -sonpy>=1.7.1;python_version=='3.9' diff --git a/src/neuroconv/datainterfaces/icephys/abf/requirements.txt b/src/neuroconv/datainterfaces/icephys/abf/requirements.txt deleted file mode 100644 index b459ee3b4..000000000 --- a/src/neuroconv/datainterfaces/icephys/abf/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ndx-dandi-icephys>=0.4.0 diff --git a/src/neuroconv/datainterfaces/icephys/requirements.txt b/src/neuroconv/datainterfaces/icephys/requirements.txt deleted file mode 100644 index 7c4979c99..000000000 --- a/src/neuroconv/datainterfaces/icephys/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -neo>=0.13.2 diff --git a/src/neuroconv/datainterfaces/ophys/brukertiff/requirements.txt b/src/neuroconv/datainterfaces/ophys/brukertiff/requirements.txt deleted file mode 100644 index 1022c0578..000000000 --- a/src/neuroconv/datainterfaces/ophys/brukertiff/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tifffile>=2023.3.21 diff --git a/src/neuroconv/datainterfaces/ophys/micromanagertiff/requirements.txt b/src/neuroconv/datainterfaces/ophys/micromanagertiff/requirements.txt deleted file mode 100644 index 1022c0578..000000000 --- a/src/neuroconv/datainterfaces/ophys/micromanagertiff/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tifffile>=2023.3.21 diff --git a/src/neuroconv/datainterfaces/ophys/miniscope/requirements.txt b/src/neuroconv/datainterfaces/ophys/miniscope/requirements.txt deleted file mode 100644 index 90ab7d140..000000000 --- a/src/neuroconv/datainterfaces/ophys/miniscope/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ndx-miniscope>=0.5.1 -natsort>=8.3.1 diff --git a/src/neuroconv/datainterfaces/ophys/requirements.txt b/src/neuroconv/datainterfaces/ophys/requirements.txt deleted file mode 100644 index 1aac92108..000000000 --- a/src/neuroconv/datainterfaces/ophys/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -roiextractors>=0.5.7 diff --git a/src/neuroconv/datainterfaces/ophys/scanimage/requirements.txt b/src/neuroconv/datainterfaces/ophys/scanimage/requirements.txt deleted file mode 100644 index 0c734c8f4..000000000 --- a/src/neuroconv/datainterfaces/ophys/scanimage/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -scanimage-tiff-reader>=1.4.1 diff --git a/src/neuroconv/datainterfaces/ophys/tdt_fp/requirements.txt b/src/neuroconv/datainterfaces/ophys/tdt_fp/requirements.txt deleted file mode 100644 index 268db6599..000000000 --- a/src/neuroconv/datainterfaces/ophys/tdt_fp/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -tdt -ndx-fiber-photometry diff --git a/src/neuroconv/datainterfaces/ophys/tiff/requirements.txt b/src/neuroconv/datainterfaces/ophys/tiff/requirements.txt deleted file mode 100644 index 530ee6f2d..000000000 --- a/src/neuroconv/datainterfaces/ophys/tiff/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tiffile>=2018.10.18 diff --git a/src/neuroconv/datainterfaces/text/excel/requirements.txt b/src/neuroconv/datainterfaces/text/excel/requirements.txt deleted file mode 100644 index 95079672f..000000000 --- a/src/neuroconv/datainterfaces/text/excel/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -openpyxl -xlrd From cd7764e699ea77bb279ba7f04c974d5e27f12393 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 14 Jan 2025 09:00:41 -0600 Subject: [PATCH 2/3] Set ceiling for HDMF to avoid failures with the latest release (#1175) --- .gitignore | 7 +++++++ CHANGELOG.md | 1 + pyproject.toml | 2 +- tests/test_on_data/setup_paths.py | 9 ++++----- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 04504e9e0..c1e9ea2b8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,13 @@ __pycache__/ # C extensions *.so + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +uv.lock + # Distribution / packaging .Python build/ diff --git a/CHANGELOG.md b/CHANGELOG.md index ba0e4068f..f20217c6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Small fixes should be here. ## Deprecations ## Bug Fixes +* Temporary set a ceiling for hdmf to avoid a chunking bug [PR #1175](https://github.com/catalystneuro/neuroconv/pull/1175) ## Features diff --git a/pyproject.toml b/pyproject.toml index cd92852ec..9debeb5e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "PyYAML>=5.4", "scipy>=1.4.1", "h5py>=3.9.0", - "hdmf>=3.13.0", + "hdmf>=3.13.0,<=3.14.5", # Chunking bug "hdmf_zarr>=0.7.0", "pynwb>=2.7.0", "pydantic>=2.0.0", diff --git a/tests/test_on_data/setup_paths.py b/tests/test_on_data/setup_paths.py index 3f7bf4123..939a9d74b 100644 --- a/tests/test_on_data/setup_paths.py +++ b/tests/test_on_data/setup_paths.py @@ -10,7 +10,7 @@ # Load the configuration for the data tests - +project_root_path = Path(__file__).parent.parent.parent if os.getenv("CI"): LOCAL_PATH = Path(".") # Must be set to "." for CI @@ -18,12 +18,11 @@ else: # Override LOCAL_PATH in the `gin_test_config.json` file to a point on your system that contains the dataset folder # Use DANDIHub at hub.dandiarchive.org for open, free use of data found in the /shared/catalystneuro/ directory - test_config_path = Path(__file__).parent / "gin_test_config.json" + test_config_path = project_root_path / "tests" / "test_on_data" / "gin_test_config.json" config_file_exists = test_config_path.exists() if not config_file_exists: - root = test_config_path.parent.parent - base_test_config_path = root / "base_gin_test_config.json" + base_test_config_path = project_root_path / "base_gin_test_config.json" test_config_path.parent.mkdir(parents=True, exist_ok=True) copy(src=base_test_config_path, dst=test_config_path) @@ -40,4 +39,4 @@ ECEPHY_DATA_PATH = LOCAL_PATH / "ephy_testing_data" OPHYS_DATA_PATH = LOCAL_PATH / "ophys_testing_data" -TEXT_DATA_PATH = Path(__file__).parent.parent.parent / "tests" / "test_text" +TEXT_DATA_PATH = project_root_path / "tests" / "test_text" From fe5bfb4a7922a9f7f3ae11fd822dd5d618e2b0ab Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 14 Jan 2025 12:37:20 -0600 Subject: [PATCH 3/3] Remove source validation from converters and interfaces (#1168) --- CHANGELOG.md | 1 + docs/conversion_examples_gallery/behavior/lightningpose.rst | 1 - docs/conversion_examples_gallery/recording/spikeglx.rst | 1 - src/neuroconv/basedatainterface.py | 2 -- src/neuroconv/nwbconverter.py | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f20217c6c..1e7dfc4df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Bug Fixes ## Features +* Source validation is no longer performed when initializing interfaces or converters [PR #1168](https://github.com/catalystneuro/neuroconv/pull/1168) ## Improvements diff --git a/docs/conversion_examples_gallery/behavior/lightningpose.rst b/docs/conversion_examples_gallery/behavior/lightningpose.rst index a024a6799..f0b91e804 100644 --- a/docs/conversion_examples_gallery/behavior/lightningpose.rst +++ b/docs/conversion_examples_gallery/behavior/lightningpose.rst @@ -23,7 +23,6 @@ Convert LightningPose pose estimation data to NWB using :py:class:`~neuroconv.da >>> labeled_video_file_path = str(folder_path / "labeled_videos/test_vid_labeled.mp4") >>> converter = LightningPoseConverter(file_path=file_path, original_video_file_path=original_video_file_path, labeled_video_file_path=labeled_video_file_path, verbose=False) - Source data is valid! >>> metadata = converter.get_metadata() >>> # For data provenance we add the time zone information to the conversion >>> session_start_time = metadata["NWBFile"]["session_start_time"] diff --git a/docs/conversion_examples_gallery/recording/spikeglx.rst b/docs/conversion_examples_gallery/recording/spikeglx.rst index 0bc67fc1d..e6ff6bf03 100644 --- a/docs/conversion_examples_gallery/recording/spikeglx.rst +++ b/docs/conversion_examples_gallery/recording/spikeglx.rst @@ -24,7 +24,6 @@ We can easily convert all data stored in the native SpikeGLX folder structure to >>> >>> folder_path = f"{ECEPHY_DATA_PATH}/spikeglx/Noise4Sam_g0" >>> converter = SpikeGLXConverterPipe(folder_path=folder_path) - Source data is valid! >>> # Extract what metadata we can from the source files >>> metadata = converter.get_metadata() >>> # For data provenance we add the time zone information to the conversion diff --git a/src/neuroconv/basedatainterface.py b/src/neuroconv/basedatainterface.py index d9e9dc11e..64af908e3 100644 --- a/src/neuroconv/basedatainterface.py +++ b/src/neuroconv/basedatainterface.py @@ -68,8 +68,6 @@ def __init__(self, verbose: bool = False, **source_data): self.verbose = verbose self.source_data = source_data - self._validate_source_data(source_data=source_data, verbose=verbose) - def get_metadata_schema(self) -> dict: """Retrieve JSON schema for metadata.""" metadata_schema = load_dict_from_file(Path(__file__).parent / "schemas" / "base_metadata_schema.json") diff --git a/src/neuroconv/nwbconverter.py b/src/neuroconv/nwbconverter.py index ff066ad6b..5b024f126 100644 --- a/src/neuroconv/nwbconverter.py +++ b/src/neuroconv/nwbconverter.py @@ -80,7 +80,6 @@ def _validate_source_data(self, source_data: dict[str, dict], verbose: bool = Tr def __init__(self, source_data: dict[str, dict], verbose: bool = True): """Validate source_data against source_schema and initialize all data interfaces.""" self.verbose = verbose - self._validate_source_data(source_data=source_data, verbose=self.verbose) self.data_interface_objects = { name: data_interface(**source_data[name]) for name, data_interface in self.data_interface_classes.items()