Skip to content

Commit

Permalink
Pass compression kwargs through file system open. (#334)
Browse files Browse the repository at this point in the history
* Pass compression kwargs through file system open.

* Undo weird isort result.

* More pylint warnings.
  • Loading branch information
delucchi-cmu authored Aug 13, 2024
1 parent 0459650 commit 8139cbc
Show file tree
Hide file tree
Showing 4 changed files with 6 additions and 21 deletions.
2 changes: 1 addition & 1 deletion src/hipscat/io/file_io/file_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ def load_csv_to_pandas_generator(
pandas dataframe loaded from CSV
"""
file_system, file_pointer = get_fs(file_pointer, storage_options=storage_options)
with file_system.open(file_pointer, "r") as csv_file:
with file_system.open(file_pointer, "r", **kwargs) as csv_file:
with pd.read_csv(csv_file, chunksize=chunksize, **kwargs) as reader:
yield from reader

Expand Down
2 changes: 1 addition & 1 deletion tests/hipscat/io/file_io/test_file_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ def test_load_csv_to_pandas(small_sky_source_dir):
def test_load_csv_to_pandas_generator(small_sky_source_dir):
partition_info_path = small_sky_source_dir / "partition_info.csv"
num_reads = 0
for frame in load_csv_to_pandas_generator(partition_info_path, chunksize=7):
for frame in load_csv_to_pandas_generator(partition_info_path, chunksize=7, compression=None):
assert len(frame) == 7
num_reads += 1
assert num_reads == 2
Expand Down
19 changes: 2 additions & 17 deletions tests/hipscat/pixel_tree/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,23 +15,8 @@ def pixel_tree_1():


@pytest.fixture
def pixel_tree_2_pixels():
return [
HealpixPixel(2, 128),
HealpixPixel(2, 130),
HealpixPixel(2, 131),
HealpixPixel(1, 33),
HealpixPixel(1, 35),
HealpixPixel(0, 10),
HealpixPixel(1, 44),
HealpixPixel(1, 45),
HealpixPixel(1, 46),
]


@pytest.fixture
def pixel_tree_2(pixel_tree_2_pixels):
return PixelTree.from_healpix(pixel_tree_2_pixels)
def pixel_tree_2(pixel_list_breadth_first):
return PixelTree.from_healpix(pixel_list_breadth_first)


@pytest.fixture
Expand Down
4 changes: 2 additions & 2 deletions tests/hipscat/pixel_tree/test_pixel_tree.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ def test_pixel_tree_length():
assert len(tree) == length


def test_get_healpix_pixels(pixel_tree_2, pixel_tree_2_pixels):
assert pixel_tree_2.get_healpix_pixels() == pixel_tree_2_pixels
def test_get_healpix_pixels(pixel_tree_2, pixel_list_breadth_first):
assert pixel_tree_2.get_healpix_pixels() == pixel_list_breadth_first


def test_pixel_tree_max_depth(pixel_tree_1, pixel_tree_2, pixel_tree_3):
Expand Down

0 comments on commit 8139cbc

Please sign in to comment.