From 59eb0e2451d3516a3b09bf9628d0c6bf16b55b5e Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Wed, 26 Apr 2023 07:29:48 -0600 Subject: [PATCH 01/16] test for SSL error on linux (issue #1246) --- test/tst_dap.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/tst_dap.py b/test/tst_dap.py index 7924e289b..db43926ec 100644 --- a/test/tst_dap.py +++ b/test/tst_dap.py @@ -35,6 +35,9 @@ def runTest(self): ncfile = netCDF4.Dataset(URL_https) assert(ncfile['sst'].long_name=='Sea Surface Temperature') ncfile.close() + # check for SSL error on linux (issue #1246) + ncfile=Dataset('https://icdc.cen.uni-hamburg.de/thredds/dodsC/ftpthredds/hamtide//m2.hamtide11a.nc') + if __name__ == '__main__': unittest.main() From f31facfaf5b4154fd156ab7c451f97f7b68d0a47 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Wed, 26 Apr 2023 07:38:52 -0600 Subject: [PATCH 02/16] update --- test/tst_dap.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/tst_dap.py b/test/tst_dap.py index db43926ec..46a26ae43 100644 --- a/test/tst_dap.py +++ b/test/tst_dap.py @@ -8,7 +8,7 @@ yesterday = datetime.utcnow() - timedelta(days=1) URL = "http://nomads.ncep.noaa.gov/dods/gfs_1p00/gfs%s/gfs_1p00_00z" % yesterday.strftime('%Y%m%d') -URL_https = 'https://podaac-opendap.jpl.nasa.gov/opendap/allData/modis/L3/aqua/11um/v2019.0/4km/daily/2017/365/AQUA_MODIS.20171231.L3m.DAY.NSST.sst.4km.nc' +URL_https='https://icdc.cen.uni-hamburg.de/thredds/dodsC/ftpthredds/hamtide//m2.hamtide11a.nc' varname = 'hgtsfc' data_min = -40; data_max = 5900 varshape = (181, 360) @@ -33,10 +33,8 @@ def runTest(self): ncfile.close() # test https support (linked curl lib must built with openssl support) ncfile = netCDF4.Dataset(URL_https) - assert(ncfile['sst'].long_name=='Sea Surface Temperature') + assert(ncfile['PHAS'].long_name=='Phase') ncfile.close() - # check for SSL error on linux (issue #1246) - ncfile=Dataset('https://icdc.cen.uni-hamburg.de/thredds/dodsC/ftpthredds/hamtide//m2.hamtide11a.nc') if __name__ == '__main__': From 0b161e4c058489382559448df4eb93d4a6d69fcf Mon Sep 17 00:00:00 2001 From: jswhit Date: Sun, 30 Apr 2023 10:06:38 -0600 Subject: [PATCH 03/16] add nc_rc_set --- include/netCDF4.pxi | 1 + 1 file changed, 1 insertion(+) diff --git a/include/netCDF4.pxi b/include/netCDF4.pxi index 9233c8165..7eae57750 100644 --- a/include/netCDF4.pxi +++ b/include/netCDF4.pxi @@ -366,6 +366,7 @@ cdef extern from "netcdf.h": int nc_inq_enum_member(int ncid, nc_type xtype, int idx, char *name, void *value) nogil int nc_inq_enum_ident(int ncid, nc_type xtype, long long value, char *identifier) nogil + int nc_rc_set(char key, char* value) nogil IF HAS_QUANTIZATION_SUPPORT: cdef extern from "netcdf.h": From 5d0130f65ec05b787d08cd3b1b5d6cadb2ce9621 Mon Sep 17 00:00:00 2001 From: jswhit Date: Sun, 30 Apr 2023 10:07:42 -0600 Subject: [PATCH 04/16] fix typo in previous commit --- include/netCDF4.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/netCDF4.pxi b/include/netCDF4.pxi index 7eae57750..72b876585 100644 --- a/include/netCDF4.pxi +++ b/include/netCDF4.pxi @@ -366,7 +366,7 @@ cdef extern from "netcdf.h": int nc_inq_enum_member(int ncid, nc_type xtype, int idx, char *name, void *value) nogil int nc_inq_enum_ident(int ncid, nc_type xtype, long long value, char *identifier) nogil - int nc_rc_set(char key, char* value) nogil + int nc_rc_set(char* key, char* value) nogil IF HAS_QUANTIZATION_SUPPORT: cdef extern from "netcdf.h": From 8f5334dc612d365a2ea0869724793b9c69faea86 Mon Sep 17 00:00:00 2001 From: jswhit Date: Sun, 30 Apr 2023 13:53:38 -0600 Subject: [PATCH 05/16] use certifi and nc_rc_set to set path to ssl certificates --- src/netCDF4/_netCDF4.pyx | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/netCDF4/_netCDF4.pyx b/src/netCDF4/_netCDF4.pyx index a0ed5e5d0..13403ed69 100644 --- a/src/netCDF4/_netCDF4.pyx +++ b/src/netCDF4/_netCDF4.pyx @@ -1258,6 +1258,20 @@ ELSE: ctypedef object Comm ctypedef object Info +import certifi + +# set path to SSL certificates (issue #1246) +cdef _set_curl_certpath(certpath): + cdef char *cert_path + cdef char *key + cdef int ierr + bytestr = _strencode(certpath) + cert_path = bytestr + ierr = nc_rc_set("HTTP.SSL.CAINFO",cert_path) + if ierr != 0: + raise RuntimeError('error setting path to SSL certificates') +_set_curl_certpath(certifi.where()) + # check for required version of netcdf-4 and hdf5. def _gethdf5libversion(): @@ -1265,7 +1279,7 @@ def _gethdf5libversion(): cdef herr_t ierr with nogil: ierr = H5get_libversion( &majorvers, &minorvers, &releasevers) - if ierr < 0: + if ierr != 0: raise RuntimeError('error getting HDF5 library version info') return '%d.%d.%d' % (majorvers,minorvers,releasevers) From 8b55eeef7a4686ca13ce24a57361bfc98c5664d1 Mon Sep 17 00:00:00 2001 From: jswhit Date: Sun, 30 Apr 2023 15:54:59 -0600 Subject: [PATCH 06/16] update --- Changelog | 6 ++++++ setup.py | 15 +++++++++++++-- src/netCDF4/_netCDF4.pyx | 28 ++++++++++++++-------------- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/Changelog b/Changelog index f89ab4e91..fae77aa5f 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,9 @@ + version 1.6.4 (not yet released) +================================= + * set path to SSL certificates internally, so https DAP URLs work with wheels + (issue #1246, requires nc_rc_set function available starting with netcdf-c 4.9.2). + Added certifi as a dependency. + version 1.6.3 (tag v1.6.3rel) ============================== * Use ``nc_put_vars`` for strided writes for netcdf-c >= 4.6.2 (issue #1222). diff --git a/setup.py b/setup.py index 016e7b5b9..b72c9b2cf 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ def check_api(inc_dirs,netcdf_lib_version): has_blosc = False has_ncfilter = False has_set_alignment = False + has_nc_rc_set = False for d in inc_dirs: try: @@ -83,6 +84,8 @@ def check_api(inc_dirs,netcdf_lib_version): has_quantize = True if line.startswith('nc_set_alignment'): has_set_alignment = True + if line.startswith('nc_rc_set'): + has_nc_rc_set = True if has_nc_open_mem: try: @@ -150,7 +153,8 @@ def check_api(inc_dirs,netcdf_lib_version): return has_rename_grp, has_nc_inq_path, has_nc_inq_format_extended, \ has_cdf5_format, has_nc_open_mem, has_nc_create_mem, \ has_parallel4_support, has_pnetcdf_support, has_szip_support, has_quantize, \ - has_zstandard, has_bzip2, has_blosc, has_set_alignment, has_ncfilter + has_zstandard, has_bzip2, has_blosc, has_set_alignment, has_ncfilter, \ + has_nc_rc_set def getnetcdfvers(libdirs): @@ -564,7 +568,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): has_rename_grp, has_nc_inq_path, has_nc_inq_format_extended, \ has_cdf5_format, has_nc_open_mem, has_nc_create_mem, \ has_parallel4_support, has_pnetcdf_support, has_szip_support, has_quantize, \ - has_zstandard, has_bzip2, has_blosc, has_set_alignment, has_ncfilter = \ + has_zstandard, has_bzip2, has_blosc, has_set_alignment, has_ncfilter, has_nc_rc_set = \ check_api(inc_dirs,netcdf_lib_version) # for netcdf 4.4.x CDF5 format is always enabled. if netcdf_lib_version is not None and\ @@ -686,6 +690,13 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): sys.stdout.write('netcdf lib does not have nc_inq_filter_avail function\n') f.write('DEF HAS_NCFILTER = 0\n') + if has_nc_rc_set: + sys.stdout.write('netcdf lib has nc_rc_set function\n') + f.write('DEF HAS_NCRCSET = 1\n') + else: + sys.stdout.write('netcdf lib does not have nc_rc_set function\n') + f.write('DEF HAS_NCRCSET = 0\n') + f.close() if has_parallel4_support or has_pnetcdf_support: diff --git a/src/netCDF4/_netCDF4.pyx b/src/netCDF4/_netCDF4.pyx index 13403ed69..ed6af81c1 100644 --- a/src/netCDF4/_netCDF4.pyx +++ b/src/netCDF4/_netCDF4.pyx @@ -1,5 +1,5 @@ """ -Version 1.6.3 +Version 1.6.4 ------------- # Introduction @@ -1227,7 +1227,7 @@ from .utils import (_StartCountStride, _quantize, _find_dim, _walk_grps, _out_array_shape, _sortbylist, _tostr, _safecast, _is_int) import sys -__version__ = "1.6.3" +__version__ = "1.6.4" # Initialize numpy import posixpath @@ -1258,19 +1258,19 @@ ELSE: ctypedef object Comm ctypedef object Info -import certifi - # set path to SSL certificates (issue #1246) -cdef _set_curl_certpath(certpath): - cdef char *cert_path - cdef char *key - cdef int ierr - bytestr = _strencode(certpath) - cert_path = bytestr - ierr = nc_rc_set("HTTP.SSL.CAINFO",cert_path) - if ierr != 0: - raise RuntimeError('error setting path to SSL certificates') -_set_curl_certpath(certifi.where()) +IF HAS_NCRCSET: # available starting in version 4.9.2 + import certifi + cdef _set_curl_certpath(certpath): + cdef char *cert_path + cdef char *key + cdef int ierr + bytestr = _strencode(certpath) + cert_path = bytestr + ierr = nc_rc_set("HTTP.SSL.CAINFO",cert_path) + if ierr != 0: + raise RuntimeError('error setting path to SSL certificates') + _set_curl_certpath(certifi.where()) # check for required version of netcdf-4 and hdf5. From 1128c6d841764e8e2cfb516b819a43d66805dc92 Mon Sep 17 00:00:00 2001 From: jswhit Date: Sun, 30 Apr 2023 15:59:18 -0600 Subject: [PATCH 07/16] update --- README.md | 4 ++++ pyproject.toml | 1 + 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index ec55f5a5c..8ad22ee4b 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ ## News For details on the latest updates, see the [Changelog](https://github.com/Unidata/netcdf4-python/blob/master/Changelog). +?/?/2023: Version [1.6.4](https://pypi.python.org/pypi/netCDF4/1.6.4) released. Now requires +[certifi](https://github.com/certifi/python-certifi) to locate SSL certificates - this allows +OpenDAP https URLs to work with wheels (issue [#1246)(https://github.com/Unidata/netcdf4-python/issues/1246). + 3/3/2023: Version [1.6.3](https://pypi.python.org/pypi/netCDF4/1.6.3) released. 11/15/2022: Version [1.6.2](https://pypi.python.org/pypi/netCDF4/1.6.2) released. Fix for diff --git a/pyproject.toml b/pyproject.toml index 0060eac30..f755e1679 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ classifiers = [ ] dependencies = [ "cftime", + "certifi", "numpy", ] dynamic = ["version"] From 829a665c6a587572f06cc9b23e47065cb01ad9d7 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sun, 30 Apr 2023 16:42:12 -0600 Subject: [PATCH 08/16] build version 4.9.2 --- .github/workflows/build_latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index f7798b255..320190ddb 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest env: PNETCDF_VERSION: 1.12.1 - NETCDF_VERSION: 4.9.1 + NETCDF_VERSION: 4.9.2 NETCDF_DIR: ${{ github.workspace }}/.. NETCDF_EXTRA_CONFIG: --enable-pnetcdf CC: mpicc.mpich From a859dd1271e536cbdb8e082ad643b3fc72f2b614 Mon Sep 17 00:00:00 2001 From: jswhit Date: Sun, 30 Apr 2023 17:36:40 -0600 Subject: [PATCH 09/16] fix nc_rc_set test --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b72c9b2cf..5bf61e16a 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def check_api(inc_dirs,netcdf_lib_version): has_quantize = True if line.startswith('nc_set_alignment'): has_set_alignment = True - if line.startswith('nc_rc_set'): + if line.startswith('EXTERNL int nc_rc_set'): has_nc_rc_set = True if has_nc_open_mem: From 602554d63c2e1ed5c1ab17ce8d26d2b32b14242c Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sun, 30 Apr 2023 20:53:18 -0600 Subject: [PATCH 10/16] update --- Changelog | 3 ++- README.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Changelog b/Changelog index fae77aa5f..2e37e9845 100644 --- a/Changelog +++ b/Changelog @@ -1,7 +1,8 @@ version 1.6.4 (not yet released) ================================= * set path to SSL certificates internally, so https DAP URLs work with wheels - (issue #1246, requires nc_rc_set function available starting with netcdf-c 4.9.2). + (issue #1246, requires nc_rc_set function available starting with netcdf-c + 4.9.2, plus bugfix in netcdf-c PR #2690). Added certifi as a dependency. version 1.6.3 (tag v1.6.3rel) diff --git a/README.md b/README.md index 8ad22ee4b..c11ca205d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ For details on the latest updates, see the [Changelog](https://github.com/Unidat ?/?/2023: Version [1.6.4](https://pypi.python.org/pypi/netCDF4/1.6.4) released. Now requires [certifi](https://github.com/certifi/python-certifi) to locate SSL certificates - this allows -OpenDAP https URLs to work with wheels (issue [#1246)(https://github.com/Unidata/netcdf4-python/issues/1246). +OpenDAP https URLs to work with linux wheels (issue [#1246](https://github.com/Unidata/netcdf4-python/issues/1246). 3/3/2023: Version [1.6.3](https://pypi.python.org/pypi/netCDF4/1.6.3) released. From 2b4f447b35eb5890986979b5f6f2c45a5e52bf42 Mon Sep 17 00:00:00 2001 From: jswhit Date: Mon, 1 May 2023 11:06:16 -0600 Subject: [PATCH 11/16] update docstrings, README --- Changelog | 2 +- src/netCDF4/_netCDF4.pyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog b/Changelog index 2e37e9845..4d3f1025a 100644 --- a/Changelog +++ b/Changelog @@ -2,7 +2,7 @@ ================================= * set path to SSL certificates internally, so https DAP URLs work with wheels (issue #1246, requires nc_rc_set function available starting with netcdf-c - 4.9.2, plus bugfix in netcdf-c PR #2690). + 4.9.1, plus bugfix in netcdf-c PR #2690). Added certifi as a dependency. version 1.6.3 (tag v1.6.3rel) diff --git a/src/netCDF4/_netCDF4.pyx b/src/netCDF4/_netCDF4.pyx index ed6af81c1..687dafb3c 100644 --- a/src/netCDF4/_netCDF4.pyx +++ b/src/netCDF4/_netCDF4.pyx @@ -1259,7 +1259,7 @@ ELSE: ctypedef object Info # set path to SSL certificates (issue #1246) -IF HAS_NCRCSET: # available starting in version 4.9.2 +IF HAS_NCRCSET: # available starting in version 4.9.1 import certifi cdef _set_curl_certpath(certpath): cdef char *cert_path From 30278275e65a2bcf3c03c9b06a03c01e586a0827 Mon Sep 17 00:00:00 2001 From: jswhit Date: Mon, 1 May 2023 11:42:44 -0600 Subject: [PATCH 12/16] include certifi dependency --- .github/workflows/miniconda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index a4a463bb3..009f4266b 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -31,7 +31,7 @@ jobs: - name: Python ${{ matrix.python-version }} shell: bash -l {0} run: | - micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest hdf5 libnetcdf cftime zlib --channel conda-forge + micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest hdf5 libnetcdf cftime certifi zlib --channel conda-forge micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config pip install -v -e . --no-deps --force-reinstall @@ -67,7 +67,7 @@ jobs: - name: Python ${{ matrix.python-version }} shell: bash -l {0} run: | - micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib --channel conda-forge + micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime certifi zlib --channel conda-forge micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all From c474baa6d2a125a0afa5b00b6182dd6b49064307 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Tue, 23 May 2023 15:45:09 -0600 Subject: [PATCH 13/16] Update tst_dap.py --- test/tst_dap.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/tst_dap.py b/test/tst_dap.py index 9aeec4750..2b033e93a 100644 --- a/test/tst_dap.py +++ b/test/tst_dap.py @@ -7,10 +7,8 @@ # test accessing data over http with opendap. yesterday = datetime.utcnow() - timedelta(days=1) - URL = f'http://nomads.ncep.noaa.gov/dods/gfs_1p00/gfs{yesterday:%Y%m%d}/gfs_1p00_00z' URL_https = 'https://www.neracoos.org/erddap/griddap/WW3_EastCoast_latest' - varname = 'hgtsfc' data_min = -40; data_max = 5900 varshape = (181, 360) @@ -38,6 +36,5 @@ def runTest(self): assert(ncfile['hs'].long_name=='Significant Wave Height') ncfile.close() - if __name__ == '__main__': unittest.main() From 5fb16dd50d8609dd269b7e9a938fbc9fb883e104 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Tue, 30 May 2023 09:17:30 -0600 Subject: [PATCH 14/16] add certifi dep --- .github/workflows/miniconda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 30b336804..8db0307f1 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -30,7 +30,7 @@ jobs: init-shell: bash create-args: >- python=${{ matrix.python-version }} - numpy cython pip pytest hdf5 libnetcdf cftime zlib + numpy cython pip pytest hdf5 libnetcdf cftime zlib certifi --channel conda-forge - name: Install netcdf4-python @@ -61,7 +61,7 @@ jobs: init-shell: bash create-args: >- python=${{ matrix.python-version }} - numpy cython pip pytest mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib + numpy cython pip pytest mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib certifi --channel conda-forge - name: Install netcdf4-python with mpi From ddf810709687134534cae353cd92db6cea90e7c1 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Tue, 30 May 2023 09:19:42 -0600 Subject: [PATCH 15/16] update --- Changelog | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Changelog b/Changelog index 44a74a395..1efad0da0 100644 --- a/Changelog +++ b/Changelog @@ -1,6 +1,5 @@ - - version 1.6.4 (not yet released) -================================= + version 1.6.4 (tag v1.6.4rel) +=============================== * set path to SSL certificates internally, so https DAP URLs work with wheels (issue #1246, requires nc_rc_set function available starting with netcdf-c 4.9.1, plus bugfix in netcdf-c PR #2690). From a79802db579acf6a69b35e902fe9eada9f30e1b6 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Wed, 31 May 2023 20:26:24 -0600 Subject: [PATCH 16/16] update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c11ca205d..2545bc101 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ ## News For details on the latest updates, see the [Changelog](https://github.com/Unidata/netcdf4-python/blob/master/Changelog). -?/?/2023: Version [1.6.4](https://pypi.python.org/pypi/netCDF4/1.6.4) released. Now requires +6/4/2023: Version [1.6.4](https://pypi.python.org/pypi/netCDF4/1.6.4) released. Now requires [certifi](https://github.com/certifi/python-certifi) to locate SSL certificates - this allows OpenDAP https URLs to work with linux wheels (issue [#1246](https://github.com/Unidata/netcdf4-python/issues/1246).