Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Verify pandas read with compression kwarg. #380

Merged
merged 2 commits into from
Aug 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/hipscat_import/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
"""All modules for hipscat-import package"""

from ._version import __version__
from .runtime_arguments import RuntimeArguments
17 changes: 17 additions & 0 deletions tests/hipscat_import/catalog/test_file_readers.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,23 @@ def test_csv_reader(small_sky_single_file):
assert total_chunks == 1


def test_csv_gzip_reader(formats_dir):
"""Verify we can read the csv file into a single data frame."""
total_chunks = 0
for frame in CsvReader(compression="zip").read(formats_dir / "catalog.zip"):
total_chunks += 1
assert len(frame) == 131

assert total_chunks == 1

total_chunks = 0
for frame in CsvReader(compression="gzip").read(formats_dir / "catalog.csv.gz"):
total_chunks += 1
assert len(frame) == 131

assert total_chunks == 1


def test_csv_reader_chunks(small_sky_single_file):
"""Verify we can read the csv file into multiple data frames."""
total_chunks = 0
Expand Down
Binary file not shown.
Binary file added tests/hipscat_import/data/test_formats/catalog.zip
Binary file not shown.