From 8d0a64414fab2e262c49ef3eb4c5a1852adbd6d3 Mon Sep 17 00:00:00 2001 From: ucapbba Date: Fri, 29 Mar 2024 13:31:00 +0000 Subject: [PATCH 01/16] ENH: async FITS creator (#156) --- .github/test-constraints.txt | 2 +- .github/workflows/test.yml | 2 +- glass/core/fitsIO.py | 60 ++++++++++++++++++++++++++++++++++++ glass/core/test/test_fits.py | 37 ++++++++++++++++++++++ 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 glass/core/fitsIO.py create mode 100644 glass/core/test/test_fits.py diff --git a/.github/test-constraints.txt b/.github/test-constraints.txt index 70a2563a..42c712d5 100644 --- a/.github/test-constraints.txt +++ b/.github/test-constraints.txt @@ -1,2 +1,2 @@ ---prefer-binary +--prefer-binary fitsio --only-binary numpy,scipy,healpy,healpix diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8b5636e..1324c61e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/glass/core/fitsIO.py b/glass/core/fitsIO.py new file mode 100644 index 00000000..e3132088 --- /dev/null +++ b/glass/core/fitsIO.py @@ -0,0 +1,60 @@ +"""Module for writing catalogue output.""" + +from contextlib import contextmanager +from threading import Thread + + +class AsyncHduWriter: + """Writer that asynchronously appends rows to a HDU.""" + + def __init__(self, fits, ext=None): + """Create a new, uninitialised writer.""" + self.fits = fits + self.ext = ext + self.thread = None + + def _append(self, data, names=None): + """Write routine for FITS data.""" + + if self.ext is None or self.ext not in self.fits: + self.fits.write_table(data, names=names, extname=self.ext) + if self.ext is None: + self.ext = self.fits[-1].get_extnum() + else: + hdu = self.fits[self.ext] + # not using hdu.append here because of incompatibilities + hdu.write(data, names=names, firstrow=hdu.get_nrows()) + + def write(self, data=None, /, **columns): + """Asynchronously append to FITS.""" + + # if data is given, write it as it is + if data is not None: + if self.thread: + self.thread.join() + self.thread = Thread(target=self._append, args=(data,)) + self.thread.start() + + # if keyword arguments are given, treat them as names and columns + if columns: + names, values = list(columns.keys()), list(columns.values()) + if self.thread: + self.thread.join() + self.thread = Thread(target=self._append, args=(values, names)) + self.thread.start() + + +@contextmanager +def awrite(filename, *, ext=None): + """Context manager for an asynchronous FITS catalogue writer.""" + + import fitsio + + with fitsio.FITS(filename, "rw", clobber=True) as fits: + fits.write(None) + writer = AsyncHduWriter(fits, ext) + try: + yield writer + finally: + if writer.thread: + writer.thread.join() diff --git a/glass/core/test/test_fits.py b/glass/core/test/test_fits.py new file mode 100644 index 00000000..128d9531 --- /dev/null +++ b/glass/core/test/test_fits.py @@ -0,0 +1,37 @@ +import pytest + +# check if fitsio is available for testing +try: + import fitsio +except ImportError: + HAVE_FITSIO = False +else: + del fitsio + HAVE_FITSIO = True + +filename = "newfiles.fits" + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_out_filename(): + import glass.core.fitsIO as GfitsIO + from fitsio import FITS + fileToWrite = "myFile.FITS" + fits = FITS(fileToWrite, "rw", clobber=True) + writer = GfitsIO.AsyncHduWriter(fits) + assert writer.fits._filename == fileToWrite + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_write_none(): + import glass.core.fitsIO as GfitsIO + with GfitsIO.awrite(filename, ext="CATALOG") as out: + out.write() + assert 1 == 1 + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_awrite_yield(): + import glass.core.fitsIO as fitsIO + with fitsIO.awrite(filename, ext="CATALOG") as out: + assert type(out) is fitsIO.AsyncHduWriter From 8aa7fb01c040b3ed661b2af30c63c1f7c3759e81 Mon Sep 17 00:00:00 2001 From: ucapbba Date: Sun, 31 Mar 2024 09:11:17 +0100 Subject: [PATCH 02/16] small rework on fits testing --- glass/core/test/test_fits.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/glass/core/test/test_fits.py b/glass/core/test/test_fits.py index 128d9531..1284facc 100644 --- a/glass/core/test/test_fits.py +++ b/glass/core/test/test_fits.py @@ -9,17 +9,16 @@ del fitsio HAVE_FITSIO = True -filename = "newfiles.fits" +filename = "myFile.FITS" @pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") def test_out_filename(): import glass.core.fitsIO as GfitsIO from fitsio import FITS - fileToWrite = "myFile.FITS" - fits = FITS(fileToWrite, "rw", clobber=True) + fits = FITS(filename, "rw", clobber=True) writer = GfitsIO.AsyncHduWriter(fits) - assert writer.fits._filename == fileToWrite + assert writer.fits._filename == filename @pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") From 0ea270ae5d7a8780ef074302387df1309392b70f Mon Sep 17 00:00:00 2001 From: ucapbba Date: Thu, 4 Apr 2024 10:15:02 +0100 Subject: [PATCH 03/16] test documentation update push --- glass/shells.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glass/shells.py b/glass/shells.py index b1667d6d..0bef3129 100644 --- a/glass/shells.py +++ b/glass/shells.py @@ -6,7 +6,7 @@ .. currentmodule:: glass.shells -The :mod:`glass.shells` module provides functions for the definition of +The :mod:`glass.shells` super awesome module provides functions for the definition of matter shells, i.e. the radial discretisation of the light cone. From aa90d311940a79f0fedb756fa67d2b59aa54c335 Mon Sep 17 00:00:00 2001 From: ucapbba Date: Wed, 10 Apr 2024 08:48:15 +0100 Subject: [PATCH 04/16] Move fits writer to user.py from core/fitsIO.py and tests from core/test to test/ --- glass/core/fitsIO.py | 60 -------------------- glass/core/test/test_fits.py | 36 ------------ glass/test/test_fits.py | 107 +++++++++++++++++++++++++++++++++++ glass/user.py | 68 +++++++++++++++++++++- 4 files changed, 174 insertions(+), 97 deletions(-) delete mode 100644 glass/core/fitsIO.py delete mode 100644 glass/core/test/test_fits.py create mode 100644 glass/test/test_fits.py diff --git a/glass/core/fitsIO.py b/glass/core/fitsIO.py deleted file mode 100644 index e3132088..00000000 --- a/glass/core/fitsIO.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Module for writing catalogue output.""" - -from contextlib import contextmanager -from threading import Thread - - -class AsyncHduWriter: - """Writer that asynchronously appends rows to a HDU.""" - - def __init__(self, fits, ext=None): - """Create a new, uninitialised writer.""" - self.fits = fits - self.ext = ext - self.thread = None - - def _append(self, data, names=None): - """Write routine for FITS data.""" - - if self.ext is None or self.ext not in self.fits: - self.fits.write_table(data, names=names, extname=self.ext) - if self.ext is None: - self.ext = self.fits[-1].get_extnum() - else: - hdu = self.fits[self.ext] - # not using hdu.append here because of incompatibilities - hdu.write(data, names=names, firstrow=hdu.get_nrows()) - - def write(self, data=None, /, **columns): - """Asynchronously append to FITS.""" - - # if data is given, write it as it is - if data is not None: - if self.thread: - self.thread.join() - self.thread = Thread(target=self._append, args=(data,)) - self.thread.start() - - # if keyword arguments are given, treat them as names and columns - if columns: - names, values = list(columns.keys()), list(columns.values()) - if self.thread: - self.thread.join() - self.thread = Thread(target=self._append, args=(values, names)) - self.thread.start() - - -@contextmanager -def awrite(filename, *, ext=None): - """Context manager for an asynchronous FITS catalogue writer.""" - - import fitsio - - with fitsio.FITS(filename, "rw", clobber=True) as fits: - fits.write(None) - writer = AsyncHduWriter(fits, ext) - try: - yield writer - finally: - if writer.thread: - writer.thread.join() diff --git a/glass/core/test/test_fits.py b/glass/core/test/test_fits.py deleted file mode 100644 index 1284facc..00000000 --- a/glass/core/test/test_fits.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest - -# check if fitsio is available for testing -try: - import fitsio -except ImportError: - HAVE_FITSIO = False -else: - del fitsio - HAVE_FITSIO = True - -filename = "myFile.FITS" - - -@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") -def test_out_filename(): - import glass.core.fitsIO as GfitsIO - from fitsio import FITS - fits = FITS(filename, "rw", clobber=True) - writer = GfitsIO.AsyncHduWriter(fits) - assert writer.fits._filename == filename - - -@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") -def test_write_none(): - import glass.core.fitsIO as GfitsIO - with GfitsIO.awrite(filename, ext="CATALOG") as out: - out.write() - assert 1 == 1 - - -@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") -def test_awrite_yield(): - import glass.core.fitsIO as fitsIO - with fitsIO.awrite(filename, ext="CATALOG") as out: - assert type(out) is fitsIO.AsyncHduWriter diff --git a/glass/test/test_fits.py b/glass/test/test_fits.py new file mode 100644 index 00000000..c145bf92 --- /dev/null +++ b/glass/test/test_fits.py @@ -0,0 +1,107 @@ +import pytest + +# check if fitsio is available for testing +try: + import fitsio +except ImportError: + HAVE_FITSIO = False +else: + del fitsio + HAVE_FITSIO = True + + + +'''def _append(filename, data, names): + import time + import fitsio + """Write routine for FITS data.""" + with fitsio.FITS(filename, "rw", clobber=True) as fits: + fits.write(data, names=names) + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_basic_write(tmp_path): + import glass.user as Gfits + import numpy as np + d = tmp_path / "sub" + d.mkdir() + filename_gfits = "gfits.fits" + filename_fits = "fits.fits" + delta = 0.1 #determines number of points in arrays + myMax = 100 #target number of threads without exception + + with Gfits.awrite(d / filename_gfits, ext="CATALOG") as out: + for i in range(0,myMax): + array = np.arange(i, i+1, delta) #array of size 1/delta + array2 = np.arange(i+1, i+2, delta) #array of size 1/delta + out.write(RA=array,RB=array2) + arrays = [array, array2] + names = ['RA','RB'] + _append(d / filename_fits, arrays, names) + + from astropy.io import fits + with fits.open(d / filename_gfits) as g_fits, fits.open(d / filename_fits) as my_fits: + g_data = g_fits[1].data + my_data = my_fits[1].data + assert g_data['RA'].size == my_data['RA'].size + assert g_data['RB'].size == my_data['RA'].size''' + +filename = "myFile.FITS" +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_awrite_exception(tmp_path): + import glass.user as Gfits + import numpy as np + d = tmp_path / "sub" + d.mkdir() + filename = "testAwriteException.fits" + + delta = 0.001 #determines number of points in arrays + myMax = 10000 #target number of threads without exception + exceptInt = 7500 #where we raise exception in loop + + try: + with Gfits.awrite(d / filename, ext="CATALOG") as out: + for i in range(0,myMax): + if i == exceptInt : + raise Exception("Unhandled exception") + array = np.arange(i, i+1, delta) #array of size 1/delta + array2 = np.arange(i+1, i+2, delta) #array of size 1/delta + out.write(RA=array,RB=array2) + + except: + from astropy.io import fits + with fits.open(d / filename) as hdul: + data = hdul[1].data + assert data['RA'].size == exceptInt/delta + assert data['RB'].size == exceptInt/delta + + fitsMat = data['RA'].reshape(exceptInt,int(1/delta)) + fitsMat2 = data['RB'].reshape(exceptInt,int(1/delta)) + for i in range(0,exceptInt): + array = np.arange(i, i+1, delta) #re-create array to compare to read data + array2 = np.arange(i+1, i+2, delta) + assert array.tolist() == fitsMat[i].tolist() + assert array2.tolist() == fitsMat2[i].tolist() + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_out_filename(): + import glass.user as Gfits + from fitsio import FITS + fits = FITS(filename, "rw", clobber=True) + writer = Gfits.AsyncHduWriter(fits) + assert writer.fits._filename == filename + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_write_none(): + import glass.user as Gfits + with Gfits.awrite(filename, ext="CATALOG") as out: + out.write() + assert 1 == 1 + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_awrite_yield(): + import glass.user as Gfits + with Gfits.awrite(filename, ext="CATALOG") as out: + assert type(out) is Gfits.AsyncHduWriter diff --git a/glass/user.py b/glass/user.py index af2f13bd..6a556355 100644 --- a/glass/user.py +++ b/glass/user.py @@ -10,12 +10,17 @@ library. -Input and output +Basic IO ---------------- .. autofunction:: save_cls .. autofunction:: load_cls +FITS creation +---------------- +.. autofunction:: awrite +.. autoclass:: AsyncHduWriter + ''' import numpy as np @@ -45,3 +50,64 @@ def load_cls(filename): values = npz['values'] split = npz['split'] return np.split(values, split) + + +"""Module for writing catalogue output.""" + +from contextlib import contextmanager +from threading import Thread + +class AsyncHduWriter: + """Writer that asynchronously appends rows to a HDU.""" + + def __init__(self, fits, ext=None): + """Create a new, uninitialised writer.""" + self.fits = fits + self.ext = ext + self.thread = None + + def _append(self, data, names=None): + """Write routine for FITS data.""" + + if self.ext is None or self.ext not in self.fits: + self.fits.write_table(data, names=names, extname=self.ext) + if self.ext is None: + self.ext = self.fits[-1].get_extnum() + else: + hdu = self.fits[self.ext] + # not using hdu.append here because of incompatibilities + hdu.write(data, names=names, firstrow=hdu.get_nrows()) + + def write(self, data=None, /, **columns): + """Asynchronously append to FITS.""" + + # if data is given, write it as it is + if data is not None: + if self.thread: + self.thread.join() + self.thread = Thread(target=self._append, args=(data,)) + self.thread.start() + + # if keyword arguments are given, treat them as names and columns + if columns: + names, values = list(columns.keys()), list(columns.values()) + if self.thread: + self.thread.join() + self.thread = Thread(target=self._append, args=(values, names)) + self.thread.start() + + +@contextmanager +def awrite(filename, *, ext=None): + """Context manager for an asynchronous FITS catalogue writer.""" + + import fitsio + + with fitsio.FITS(filename, "rw", clobber=True) as fits: + fits.write(None) + writer = AsyncHduWriter(fits, ext) + try: + yield writer + finally: + if writer.thread: + writer.thread.join() From e36221ca9bbb27b3ca2c8e8600f8259a37654703 Mon Sep 17 00:00:00 2001 From: ucapbba Date: Sun, 14 Apr 2024 16:49:12 +0100 Subject: [PATCH 05/16] FITS Writer class and test cases --- .github/test-constraints.txt | 2 +- .github/workflows/test.yml | 2 +- glass/test/test_fits.py | 109 +++++++++++++++++++++++++++++++++++ glass/user.py | 50 +++++++++++++++- 4 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 glass/test/test_fits.py diff --git a/.github/test-constraints.txt b/.github/test-constraints.txt index 70a2563a..42c712d5 100644 --- a/.github/test-constraints.txt +++ b/.github/test-constraints.txt @@ -1,2 +1,2 @@ ---prefer-binary +--prefer-binary fitsio --only-binary numpy,scipy,healpy,healpix diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8b5636e..1324c61e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/glass/test/test_fits.py b/glass/test/test_fits.py new file mode 100644 index 00000000..4c160139 --- /dev/null +++ b/glass/test/test_fits.py @@ -0,0 +1,109 @@ +import pytest + +# check if fitsio is available for testing +try: + import fitsio +except ImportError: + HAVE_FITSIO = False +else: + del fitsio + HAVE_FITSIO = True + +import glass.user as user +import numpy as np +import fitsio + + +def _test_append(fits, data, names): + """Write routine for FITS data.""" + cat_name = 'CATALOG' + if cat_name not in fits: + fits.write_table(data, names=names, extname=cat_name) + else: + hdu = fits[cat_name] + hdu.write(data, names=names, firstrow=hdu.get_nrows()) + + +delta = 0.001 # Number of points in arrays +myMax = 1000 # Typically number of galaxies in loop +exceptInt = 750 # Where test exception occurs in loop +filename = "MyFile.Fits" + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_basic_write(tmp_path): + d = tmp_path / "sub" + d.mkdir() + filename_gfits = "gfits.fits" # what GLASS creates + filename_tfits = "tfits.fits" # file create on the fly to test against + + with user.swrite(d / filename_gfits, ext="CATALOG") as out, fitsio.FITS(d / filename_tfits, "rw", clobber=True) as myFits: + for i in range(0, myMax): + array = np.arange(i, i+1, delta) # array of size 1/delta + array2 = np.arange(i+1, i+2, delta) # array of size 1/delta + out.write(RA=array, RB=array2) + arrays = [array, array2] + names = ['RA', 'RB'] + _test_append(myFits, arrays, names) + + from astropy.io import fits + with fits.open(d / filename_gfits) as g_fits, fits.open(d / filename_tfits) as t_fits: + glass_data = g_fits[1].data + test_data = t_fits[1].data + assert glass_data['RA'].size == test_data['RA'].size + assert glass_data['RB'].size == test_data['RA'].size + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_write_exception(tmp_path): + d = tmp_path / "sub" + d.mkdir() + + try: + with user.swrite(d / filename, ext="CATALOG") as out: + for i in range(0, myMax): + if i == exceptInt: + raise Exception("Unhandled exception") + array = np.arange(i, i+1, delta) # array of size 1/delta + array2 = np.arange(i+1, i+2, delta) # array of size 1/delta + out.write(RA=array, RB=array2) + + except Exception: + from astropy.io import fits + with fits.open(d / filename) as hdul: + data = hdul[1].data + assert data['RA'].size == exceptInt/delta + assert data['RB'].size == exceptInt/delta + + fitsMat = data['RA'].reshape(exceptInt, int(1/delta)) + fitsMat2 = data['RB'].reshape(exceptInt, int(1/delta)) + for i in range(0, exceptInt): + array = np.arange(i, i+1, delta) # re-create array to compare to read data + array2 = np.arange(i+1, i+2, delta) + assert array.tolist() == fitsMat[i].tolist() + assert array2.tolist() == fitsMat2[i].tolist() + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_out_filename(tmp_path): + + fits = fitsio.FITS(filename, "rw", clobber=True) + writer = user.FitsWriter(fits) + assert writer.fits._filename == filename + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_write_none(tmp_path): + d = tmp_path / "sub" + d.mkdir() + with user.swrite(d / filename, ext="CATALOG") as out: + out.write() + assert 1 == 1 + + +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") +def test_awrite_yield(tmp_path): + d = tmp_path / "sub" + d.mkdir() + with user.swrite(d / filename, ext="CATALOG") as out: + assert type(out) is user.FitsWriter diff --git a/glass/user.py b/glass/user.py index af2f13bd..0fb95c84 100644 --- a/glass/user.py +++ b/glass/user.py @@ -10,15 +10,21 @@ library. -Input and output +Basic IO ---------------- .. autofunction:: save_cls .. autofunction:: load_cls +FITS creation +---------------- +.. autofunction:: swrite +.. autoclass:: SyncHduWriter + ''' import numpy as np +from contextlib import contextmanager def save_cls(filename, cls): @@ -45,3 +51,45 @@ def load_cls(filename): values = npz['values'] split = npz['split'] return np.split(values, split) + + +class FitsWriter: + """Writer that appends rows to a HDU.""" + + def __init__(self, fits, ext=None): + """Create a new, uninitialised writer.""" + self.fits = fits + self.ext = ext + + def _append(self, data, names=None): + """Write routine for FITS data.""" + + if self.ext is None or self.ext not in self.fits: + self.fits.write_table(data, names=names, extname=self.ext) + if self.ext is None: + self.ext = self.fits[-1].get_extnum() + else: + hdu = self.fits[self.ext] + # not using hdu.append here because of incompatibilities + hdu.write(data, names=names, firstrow=hdu.get_nrows()) + + def write(self, data=None, /, **columns): + """Append to FITS.""" + # if data is given, write it as it is + if data is not None: + self._append(data) + + # if keyword arguments are given, treat them as names and columns + if columns: + names, values = list(columns.keys()), list(columns.values()) + self._append(values, names) + + +@contextmanager +def swrite(filename, *, ext=None): + """Context manager for a FITS catalogue writer.""" + import fitsio + with fitsio.FITS(filename, "rw", clobber=True) as fits: + fits.write(None) + writer = FitsWriter(fits, ext) + yield writer \ No newline at end of file From 05f22a4aea5ab1c81f630d6965854711a0313931 Mon Sep 17 00:00:00 2001 From: ucapbba Date: Tue, 30 Apr 2024 09:22:31 +0100 Subject: [PATCH 06/16] update function comments in FitsWriter --- glass/test/test_fits.py | 11 ++++++----- glass/user.py | 30 +++++++++++++++++++----------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/glass/test/test_fits.py b/glass/test/test_fits.py index 4c160139..800e5f6b 100644 --- a/glass/test/test_fits.py +++ b/glass/test/test_fits.py @@ -14,6 +14,7 @@ import fitsio +@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") def _test_append(fits, data, names): """Write routine for FITS data.""" cat_name = 'CATALOG' @@ -37,7 +38,7 @@ def test_basic_write(tmp_path): filename_gfits = "gfits.fits" # what GLASS creates filename_tfits = "tfits.fits" # file create on the fly to test against - with user.swrite(d / filename_gfits, ext="CATALOG") as out, fitsio.FITS(d / filename_tfits, "rw", clobber=True) as myFits: + with user.write_context(d / filename_gfits, ext="CATALOG") as out, fitsio.FITS(d / filename_tfits, "rw", clobber=True) as myFits: for i in range(0, myMax): array = np.arange(i, i+1, delta) # array of size 1/delta array2 = np.arange(i+1, i+2, delta) # array of size 1/delta @@ -60,7 +61,7 @@ def test_write_exception(tmp_path): d.mkdir() try: - with user.swrite(d / filename, ext="CATALOG") as out: + with user.write_context(d / filename, ext="CATALOG") as out: for i in range(0, myMax): if i == exceptInt: raise Exception("Unhandled exception") @@ -96,14 +97,14 @@ def test_out_filename(tmp_path): def test_write_none(tmp_path): d = tmp_path / "sub" d.mkdir() - with user.swrite(d / filename, ext="CATALOG") as out: + with user.write_context(d / filename, ext="CATALOG") as out: out.write() assert 1 == 1 @pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") -def test_awrite_yield(tmp_path): +def test_write_yield(tmp_path): d = tmp_path / "sub" d.mkdir() - with user.swrite(d / filename, ext="CATALOG") as out: + with user.write_context(d / filename, ext="CATALOG") as out: assert type(out) is user.FitsWriter diff --git a/glass/user.py b/glass/user.py index 0fb95c84..fdc4c5bd 100644 --- a/glass/user.py +++ b/glass/user.py @@ -18,9 +18,10 @@ FITS creation ---------------- -.. autofunction:: swrite -.. autoclass:: SyncHduWriter - +.. autoclass:: FitsWriter +.. automethod:: write +.. automethod:: append +.. autofunction:: write_context ''' import numpy as np @@ -54,15 +55,15 @@ def load_cls(filename): class FitsWriter: - """Writer that appends rows to a HDU.""" + '''Writer that creates a FITS file. Initialised with the fits object and extention name.''' def __init__(self, fits, ext=None): - """Create a new, uninitialised writer.""" + '''Create a new, uninitialised writer.''' self.fits = fits self.ext = ext def _append(self, data, names=None): - """Write routine for FITS data.""" + '''Internal method where the FITS writing is done''' if self.ext is None or self.ext not in self.fits: self.fits.write_table(data, names=names, extname=self.ext) @@ -74,11 +75,14 @@ def _append(self, data, names=None): hdu.write(data, names=names, firstrow=hdu.get_nrows()) def write(self, data=None, /, **columns): - """Append to FITS.""" + '''Writes to FITS by calling the internal _append method. + Pass either a positional variable (data) + or multiple named arguments (**columns)''' + # if data is given, write it as it is if data is not None: self._append(data) - + # if keyword arguments are given, treat them as names and columns if columns: names, values = list(columns.keys()), list(columns.values()) @@ -86,10 +90,14 @@ def write(self, data=None, /, **columns): @contextmanager -def swrite(filename, *, ext=None): - """Context manager for a FITS catalogue writer.""" +def write_context(filename, *, ext=None): + '''Context manager for a FITS catalogue writer. Calls class FitsWriter. + + ext is the name of the HDU extension + + ''' import fitsio with fitsio.FITS(filename, "rw", clobber=True) as fits: fits.write(None) writer = FitsWriter(fits, ext) - yield writer \ No newline at end of file + yield writer From 51b6b8b7fe63674667b409d2abf467fa793f0461 Mon Sep 17 00:00:00 2001 From: ucapbba Date: Tue, 30 Apr 2024 10:51:32 +0100 Subject: [PATCH 07/16] remove double fitsio import --- glass/test/test_fits.py | 2 -- glass/user.py | 1 - 2 files changed, 3 deletions(-) diff --git a/glass/test/test_fits.py b/glass/test/test_fits.py index 800e5f6b..589220a2 100644 --- a/glass/test/test_fits.py +++ b/glass/test/test_fits.py @@ -11,8 +11,6 @@ import glass.user as user import numpy as np -import fitsio - @pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") def _test_append(fits, data, names): diff --git a/glass/user.py b/glass/user.py index fdc4c5bd..5e178e10 100644 --- a/glass/user.py +++ b/glass/user.py @@ -20,7 +20,6 @@ ---------------- .. autoclass:: FitsWriter .. automethod:: write -.. automethod:: append .. autofunction:: write_context ''' From 677d34b6e67451923d1ebe10cef8b5dbf6848c89 Mon Sep 17 00:00:00 2001 From: ucapbba Date: Tue, 30 Apr 2024 11:09:49 +0100 Subject: [PATCH 08/16] small flake8 issue in test_fits.py --- glass/test/test_fits.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/glass/test/test_fits.py b/glass/test/test_fits.py index 589220a2..968e10b8 100644 --- a/glass/test/test_fits.py +++ b/glass/test/test_fits.py @@ -12,6 +12,7 @@ import glass.user as user import numpy as np + @pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") def _test_append(fits, data, names): """Write routine for FITS data.""" @@ -31,6 +32,7 @@ def _test_append(fits, data, names): @pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") def test_basic_write(tmp_path): + import fitsio d = tmp_path / "sub" d.mkdir() filename_gfits = "gfits.fits" # what GLASS creates @@ -85,7 +87,7 @@ def test_write_exception(tmp_path): @pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") def test_out_filename(tmp_path): - + import fitsio fits = fitsio.FITS(filename, "rw", clobber=True) writer = user.FitsWriter(fits) assert writer.fits._filename == filename From 386a10b73bef1c7c73a4338afaed7f8d0ccd0fca Mon Sep 17 00:00:00 2001 From: ucapbba Date: Tue, 30 Apr 2024 11:44:12 +0100 Subject: [PATCH 09/16] remove faulty doc for FitsWriter --- glass/user.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glass/user.py b/glass/user.py index 5e178e10..b33e401c 100644 --- a/glass/user.py +++ b/glass/user.py @@ -18,9 +18,9 @@ FITS creation ---------------- -.. autoclass:: FitsWriter -.. automethod:: write + .. autofunction:: write_context + ''' import numpy as np From 8d69e852a457939571f1c0175354e675d3f66c56 Mon Sep 17 00:00:00 2001 From: Bradley Augstein Date: Sun, 16 Jun 2024 06:44:11 +0100 Subject: [PATCH 10/16] #156 - small updates based on review --- .github/test-constraints.txt | 2 +- glass/test/test_fits.py | 30 ++++++++++++++---------------- pyproject.toml | 1 + 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/test-constraints.txt b/.github/test-constraints.txt index 42c712d5..70a2563a 100644 --- a/.github/test-constraints.txt +++ b/.github/test-constraints.txt @@ -1,2 +1,2 @@ ---prefer-binary fitsio +--prefer-binary --only-binary numpy,scipy,healpy,healpix diff --git a/glass/test/test_fits.py b/glass/test/test_fits.py index 968e10b8..8ea40955 100644 --- a/glass/test/test_fits.py +++ b/glass/test/test_fits.py @@ -1,13 +1,11 @@ import pytest # check if fitsio is available for testing -try: - import fitsio -except ImportError: - HAVE_FITSIO = False -else: - del fitsio +import importlib.util +if importlib.util.find_spec("fitsio") is not None: HAVE_FITSIO = True +else: + HAVE_FITSIO = False import glass.user as user import numpy as np @@ -25,8 +23,8 @@ def _test_append(fits, data, names): delta = 0.001 # Number of points in arrays -myMax = 1000 # Typically number of galaxies in loop -exceptInt = 750 # Where test exception occurs in loop +my_max = 1000 # Typically number of galaxies in loop +except_int = 750 # Where test exception occurs in loop filename = "MyFile.Fits" @@ -39,7 +37,7 @@ def test_basic_write(tmp_path): filename_tfits = "tfits.fits" # file create on the fly to test against with user.write_context(d / filename_gfits, ext="CATALOG") as out, fitsio.FITS(d / filename_tfits, "rw", clobber=True) as myFits: - for i in range(0, myMax): + for i in range(0, my_max): array = np.arange(i, i+1, delta) # array of size 1/delta array2 = np.arange(i+1, i+2, delta) # array of size 1/delta out.write(RA=array, RB=array2) @@ -62,8 +60,8 @@ def test_write_exception(tmp_path): try: with user.write_context(d / filename, ext="CATALOG") as out: - for i in range(0, myMax): - if i == exceptInt: + for i in range(0, my_max): + if i == except_int: raise Exception("Unhandled exception") array = np.arange(i, i+1, delta) # array of size 1/delta array2 = np.arange(i+1, i+2, delta) # array of size 1/delta @@ -73,12 +71,12 @@ def test_write_exception(tmp_path): from astropy.io import fits with fits.open(d / filename) as hdul: data = hdul[1].data - assert data['RA'].size == exceptInt/delta - assert data['RB'].size == exceptInt/delta + assert data['RA'].size == except_int/delta + assert data['RB'].size == except_int/delta - fitsMat = data['RA'].reshape(exceptInt, int(1/delta)) - fitsMat2 = data['RB'].reshape(exceptInt, int(1/delta)) - for i in range(0, exceptInt): + fitsMat = data['RA'].reshape(except_int, int(1/delta)) + fitsMat2 = data['RB'].reshape(except_int, int(1/delta)) + for i in range(0, except_int): array = np.arange(i, i+1, delta) # re-create array to compare to read data array2 = np.arange(i+1, i+2, delta) assert array.tolist() == fitsMat[i].tolist() diff --git a/pyproject.toml b/pyproject.toml index af1e1db7..366dce25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dynamic = ["version"] test = [ "pytest", "scipy", + "fitsio", ] docs = [ "sphinx", From 8dcd47f4d21a623bc80ce6425905961c0a169999 Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Sun, 16 Jun 2024 08:02:25 +0100 Subject: [PATCH 11/16] revert shells.py --- glass/shells.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glass/shells.py b/glass/shells.py index 0bef3129..b1667d6d 100644 --- a/glass/shells.py +++ b/glass/shells.py @@ -6,7 +6,7 @@ .. currentmodule:: glass.shells -The :mod:`glass.shells` super awesome module provides functions for the definition of +The :mod:`glass.shells` module provides functions for the definition of matter shells, i.e. the radial discretisation of the light cone. From bff6a04b1272c4117571d9b206f795f2a55c08fe Mon Sep 17 00:00:00 2001 From: baugstein <87702063+ucapbba@users.noreply.github.com> Date: Sun, 16 Jun 2024 08:05:01 +0100 Subject: [PATCH 12/16] Update user.py based on review, just one Basic IO heading for save_cls, load_cls and write_context --- glass/user.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/glass/user.py b/glass/user.py index b33e401c..e7876258 100644 --- a/glass/user.py +++ b/glass/user.py @@ -15,10 +15,6 @@ .. autofunction:: save_cls .. autofunction:: load_cls - -FITS creation ----------------- - .. autofunction:: write_context ''' From 17c4ff7a4981073fdd27500f3a571163723fe727 Mon Sep 17 00:00:00 2001 From: Bradley Augstein Date: Sun, 16 Jun 2024 09:21:44 +0100 Subject: [PATCH 13/16] #156 code review - updating names, removing unised test cases --- glass/test/test_fits.py | 26 ++++---------------------- glass/user.py | 6 +++--- pyproject.toml | 2 +- 3 files changed, 8 insertions(+), 26 deletions(-) diff --git a/glass/test/test_fits.py b/glass/test/test_fits.py index 8ea40955..cfbbc660 100644 --- a/glass/test/test_fits.py +++ b/glass/test/test_fits.py @@ -11,9 +11,8 @@ import numpy as np -@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") def _test_append(fits, data, names): - """Write routine for FITS data.""" + '''Write routine for FITS test cases''' cat_name = 'CATALOG' if cat_name not in fits: fits.write_table(data, names=names, extname=cat_name) @@ -36,7 +35,7 @@ def test_basic_write(tmp_path): filename_gfits = "gfits.fits" # what GLASS creates filename_tfits = "tfits.fits" # file create on the fly to test against - with user.write_context(d / filename_gfits, ext="CATALOG") as out, fitsio.FITS(d / filename_tfits, "rw", clobber=True) as myFits: + with user.write_catalog(d / filename_gfits, ext="CATALOG") as out, fitsio.FITS(d / filename_tfits, "rw", clobber=True) as myFits: for i in range(0, my_max): array = np.arange(i, i+1, delta) # array of size 1/delta array2 = np.arange(i+1, i+2, delta) # array of size 1/delta @@ -59,7 +58,7 @@ def test_write_exception(tmp_path): d.mkdir() try: - with user.write_context(d / filename, ext="CATALOG") as out: + with user.write_catalog(d / filename, ext="CATALOG") as out: for i in range(0, my_max): if i == except_int: raise Exception("Unhandled exception") @@ -88,21 +87,4 @@ def test_out_filename(tmp_path): import fitsio fits = fitsio.FITS(filename, "rw", clobber=True) writer = user.FitsWriter(fits) - assert writer.fits._filename == filename - - -@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") -def test_write_none(tmp_path): - d = tmp_path / "sub" - d.mkdir() - with user.write_context(d / filename, ext="CATALOG") as out: - out.write() - assert 1 == 1 - - -@pytest.mark.skipif(not HAVE_FITSIO, reason="test requires fitsio") -def test_write_yield(tmp_path): - d = tmp_path / "sub" - d.mkdir() - with user.write_context(d / filename, ext="CATALOG") as out: - assert type(out) is user.FitsWriter + assert writer.fits._filename == filename \ No newline at end of file diff --git a/glass/user.py b/glass/user.py index e7876258..97005d1d 100644 --- a/glass/user.py +++ b/glass/user.py @@ -49,7 +49,7 @@ def load_cls(filename): return np.split(values, split) -class FitsWriter: +class _FitsWriter: '''Writer that creates a FITS file. Initialised with the fits object and extention name.''' def __init__(self, fits, ext=None): @@ -85,7 +85,7 @@ def write(self, data=None, /, **columns): @contextmanager -def write_context(filename, *, ext=None): +def write_catalog(filename, *, ext=None): '''Context manager for a FITS catalogue writer. Calls class FitsWriter. ext is the name of the HDU extension @@ -94,5 +94,5 @@ def write_context(filename, *, ext=None): import fitsio with fitsio.FITS(filename, "rw", clobber=True) as fits: fits.write(None) - writer = FitsWriter(fits, ext) + writer = _FitsWriter(fits, ext) yield writer diff --git a/pyproject.toml b/pyproject.toml index 366dce25..4cdc575b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "glass" description = "Generator for Large Scale Structure" readme = "README.md" -requires-python = ">=3.6" +requires-python = ">=3.8" license = {file = "LICENSE"} maintainers = [ {name = "Nicolas Tessore", email = "n.tessore@ucl.ac.uk"}, From bd9d5e89dba73f6558b4bea2a5c868e0d73b90e8 Mon Sep 17 00:00:00 2001 From: Bradley Augstein Date: Sun, 16 Jun 2024 09:23:48 +0100 Subject: [PATCH 14/16] #156 - formatting fail fix --- glass/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glass/user.py b/glass/user.py index 97005d1d..28009166 100644 --- a/glass/user.py +++ b/glass/user.py @@ -10,7 +10,7 @@ library. -Basic IO +Input and Output ---------------- .. autofunction:: save_cls From 7d852c562aa2b2f583c4ffcb569332f991985ea6 Mon Sep 17 00:00:00 2001 From: Bradley Augstein Date: Tue, 23 Jul 2024 10:43:41 +0100 Subject: [PATCH 15/16] Remove astropy from test_fits --- glass/test/test_fits.py | 19 +++++++++---------- glass/user.py | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/glass/test/test_fits.py b/glass/test/test_fits.py index cfbbc660..25549388 100644 --- a/glass/test/test_fits.py +++ b/glass/test/test_fits.py @@ -33,7 +33,7 @@ def test_basic_write(tmp_path): d = tmp_path / "sub" d.mkdir() filename_gfits = "gfits.fits" # what GLASS creates - filename_tfits = "tfits.fits" # file create on the fly to test against + filename_tfits = "tfits.fits" # file created on the fly to test against with user.write_catalog(d / filename_gfits, ext="CATALOG") as out, fitsio.FITS(d / filename_tfits, "rw", clobber=True) as myFits: for i in range(0, my_max): @@ -44,10 +44,9 @@ def test_basic_write(tmp_path): names = ['RA', 'RB'] _test_append(myFits, arrays, names) - from astropy.io import fits - with fits.open(d / filename_gfits) as g_fits, fits.open(d / filename_tfits) as t_fits: - glass_data = g_fits[1].data - test_data = t_fits[1].data + with fitsio.FITS(d / filename_gfits) as g_fits, fitsio.FITS(d / filename_tfits) as t_fits: + glass_data = g_fits[1].read() + test_data = t_fits[1].read() assert glass_data['RA'].size == test_data['RA'].size assert glass_data['RB'].size == test_data['RA'].size @@ -67,9 +66,9 @@ def test_write_exception(tmp_path): out.write(RA=array, RB=array2) except Exception: - from astropy.io import fits - with fits.open(d / filename) as hdul: - data = hdul[1].data + import fitsio + with fitsio.FITS(d / filename) as hdul: + data = hdul[1].read() assert data['RA'].size == except_int/delta assert data['RB'].size == except_int/delta @@ -86,5 +85,5 @@ def test_write_exception(tmp_path): def test_out_filename(tmp_path): import fitsio fits = fitsio.FITS(filename, "rw", clobber=True) - writer = user.FitsWriter(fits) - assert writer.fits._filename == filename \ No newline at end of file + writer = user._FitsWriter(fits) + assert writer.fits._filename == filename diff --git a/glass/user.py b/glass/user.py index 28009166..dc3474ea 100644 --- a/glass/user.py +++ b/glass/user.py @@ -15,7 +15,7 @@ .. autofunction:: save_cls .. autofunction:: load_cls -.. autofunction:: write_context +.. autofunction:: write_catalog ''' From 499257cc4dc600fb042ec634fc751f1f800790d9 Mon Sep 17 00:00:00 2001 From: Nicolas Tessore Date: Fri, 23 Aug 2024 17:06:40 +0100 Subject: [PATCH 16/16] Update glass/user.py --- glass/user.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/glass/user.py b/glass/user.py index dc3474ea..7ff63b74 100644 --- a/glass/user.py +++ b/glass/user.py @@ -86,11 +86,20 @@ def write(self, data=None, /, **columns): @contextmanager def write_catalog(filename, *, ext=None): - '''Context manager for a FITS catalogue writer. Calls class FitsWriter. + """ + Write a catalogue into a FITS file, where *ext* is the optional + name of the extension. To be used as a context manager:: - ext is the name of the HDU extension + # create the catalogue writer + with write_catalog("catalog.fits") as out: + ... + # write catalogue columns RA, DEC, E1, E2, WHT with given arrays + out.write(RA=lon, DEC=lat, E1=eps1, E2=e2, WHT=w) - ''' + .. note:: + Requires the ``fitsio`` package. + + """ import fitsio with fitsio.FITS(filename, "rw", clobber=True) as fits: fits.write(None)