From f6b0ac6d733e67e0f6b7a278e94ef1de89948199 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Mon, 12 Aug 2024 14:47:28 -0400 Subject: [PATCH 1/3] Pass compression kwargs through file system open. --- src/hipscat/io/file_io/file_io.py | 2 +- tests/hipscat/io/file_io/test_file_io.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hipscat/io/file_io/file_io.py b/src/hipscat/io/file_io/file_io.py index 11c8211d..4cb0dabd 100644 --- a/src/hipscat/io/file_io/file_io.py +++ b/src/hipscat/io/file_io/file_io.py @@ -166,7 +166,7 @@ def load_csv_to_pandas_generator( pandas dataframe loaded from CSV """ file_system, file_pointer = get_fs(file_pointer, storage_options=storage_options) - with file_system.open(file_pointer, "r") as csv_file: + with file_system.open(file_pointer, "r", **kwargs) as csv_file: with pd.read_csv(csv_file, chunksize=chunksize, **kwargs) as reader: yield from reader diff --git a/tests/hipscat/io/file_io/test_file_io.py b/tests/hipscat/io/file_io/test_file_io.py index 038b8e34..440efca3 100644 --- a/tests/hipscat/io/file_io/test_file_io.py +++ b/tests/hipscat/io/file_io/test_file_io.py @@ -1,9 +1,9 @@ import json import numpy as np -import pandas as pd import pytest +import pandas as pd from hipscat.io.file_io import ( delete_file, get_file_pointer_from_path, @@ -126,7 +126,7 @@ def test_load_csv_to_pandas(small_sky_source_dir): def test_load_csv_to_pandas_generator(small_sky_source_dir): partition_info_path = small_sky_source_dir / "partition_info.csv" num_reads = 0 - for frame in load_csv_to_pandas_generator(partition_info_path, chunksize=7): + for frame in load_csv_to_pandas_generator(partition_info_path, chunksize=7, compression=None): assert len(frame) == 7 num_reads += 1 assert num_reads == 2 From a7399b898b1d637c73f27d81047804679bc442b7 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Tue, 13 Aug 2024 11:37:34 -0400 Subject: [PATCH 2/3] Undo weird isort result. --- tests/hipscat/io/file_io/test_file_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/hipscat/io/file_io/test_file_io.py b/tests/hipscat/io/file_io/test_file_io.py index 440efca3..6ac8fd89 100644 --- a/tests/hipscat/io/file_io/test_file_io.py +++ b/tests/hipscat/io/file_io/test_file_io.py @@ -1,9 +1,9 @@ import json import numpy as np +import pandas as pd import pytest -import pandas as pd from hipscat.io.file_io import ( delete_file, get_file_pointer_from_path, From 414e6243661a1e675e58921d37a9c231d0dfaa0b Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi Date: Tue, 13 Aug 2024 11:43:12 -0400 Subject: [PATCH 3/3] More pylint warnings. --- tests/hipscat/pixel_tree/conftest.py | 19 ++----------------- tests/hipscat/pixel_tree/test_pixel_tree.py | 4 ++-- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/tests/hipscat/pixel_tree/conftest.py b/tests/hipscat/pixel_tree/conftest.py index cf520084..0c77be80 100644 --- a/tests/hipscat/pixel_tree/conftest.py +++ b/tests/hipscat/pixel_tree/conftest.py @@ -15,23 +15,8 @@ def pixel_tree_1(): @pytest.fixture -def pixel_tree_2_pixels(): - return [ - HealpixPixel(2, 128), - HealpixPixel(2, 130), - HealpixPixel(2, 131), - HealpixPixel(1, 33), - HealpixPixel(1, 35), - HealpixPixel(0, 10), - HealpixPixel(1, 44), - HealpixPixel(1, 45), - HealpixPixel(1, 46), - ] - - -@pytest.fixture -def pixel_tree_2(pixel_tree_2_pixels): - return PixelTree.from_healpix(pixel_tree_2_pixels) +def pixel_tree_2(pixel_list_breadth_first): + return PixelTree.from_healpix(pixel_list_breadth_first) @pytest.fixture diff --git a/tests/hipscat/pixel_tree/test_pixel_tree.py b/tests/hipscat/pixel_tree/test_pixel_tree.py index 5614cee2..6eb5c1f2 100644 --- a/tests/hipscat/pixel_tree/test_pixel_tree.py +++ b/tests/hipscat/pixel_tree/test_pixel_tree.py @@ -15,8 +15,8 @@ def test_pixel_tree_length(): assert len(tree) == length -def test_get_healpix_pixels(pixel_tree_2, pixel_tree_2_pixels): - assert pixel_tree_2.get_healpix_pixels() == pixel_tree_2_pixels +def test_get_healpix_pixels(pixel_tree_2, pixel_list_breadth_first): + assert pixel_tree_2.get_healpix_pixels() == pixel_list_breadth_first def test_pixel_tree_max_depth(pixel_tree_1, pixel_tree_2, pixel_tree_3):