diff --git a/.github/stubtest-allowlist b/.github/stubtest-allowlist index a3bd3a1a4..0352fdcf7 100644 --- a/.github/stubtest-allowlist +++ b/.github/stubtest-allowlist @@ -1,33 +1,42 @@ -netCDF4.AccessModeOptions -netCDF4.CompressionLevelOptions -netCDF4.CompressionOptions -netCDF4.DatatypeOptions -netCDF4.DimensionsOptions -netCDF4.DiskFormatOptions -netCDF4.EndianOptions -netCDF4.FormatOptions -netCDF4.QuantizeOptions -netCDF4.CalendarOptions -netCDF4.ellipsis +netCDF4.RealTypeLiteral +netCDF4.ComplexTypeLiteral +netCDF4.NumericTypeLiteral +netCDF4.CharTypeLiteral +netCDF4.TypeLiteral +netCDF4.NumPyRealType +netCDF4.NumPyComplexType +netCDF4.NumPyNumericType +netCDF4.NetCDFUDTClass +netCDF4.AccessMode +netCDF4.CompressionLevel +netCDF4.CompressionType +netCDF4.DatatypeType +netCDF4.DimensionsType +netCDF4.DiskFormat +netCDF4.EndianType +netCDF4.Format +netCDF4.QuantizeMode +netCDF4.CalendarType netCDF4.DateTimeArray netCDF4.FiltersDict netCDF4.SzipInfo netCDF4.BloscInfo netCDF4.BoolInt -netCDF4.GetSetItemKey -netCDF4.T_Datatype -netCDF4.T_DatatypeNC -netCDF4.Dataset.__dealloc +netCDF4.VarT +netCDF4.RealVarT +netCDF4.ComplexVarT +netCDF4.NumericVarT netCDF4.Dimension.__reduce_cython__ netCDF4.Dimension.__setstate_cython__ netCDF4.Variable.auto_complex -netCDF4._netCDF4.Dataset.__dealloc +netCDF4.Variable.__iter__ netCDF4._netCDF4.Dimension.__reduce_cython__ netCDF4._netCDF4.Dimension.__setstate_cython__ netCDF4._netCDF4.NC_DISKLESS netCDF4._netCDF4.NC_PERSIST netCDF4._netCDF4.Variable.auto_complex +netCDF4._netCDF4.Variable.__iter__ netCDF4._netCDF4.__reduce_cython__ netCDF4._netCDF4.__setstate_cython__ netCDF4._netCDF4.__test__ -netCDF4.utils.bytes \ No newline at end of file +netCDF4.utils \ No newline at end of file diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index bd797bd1e..cca163d45 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -58,7 +58,7 @@ jobs: - name: Install python dependencies via pip run: | python -m pip install --upgrade pip - pip install numpy cython cftime pytest twine wheel check-manifest mpi4py + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py typing-extensions - name: Install netcdf4-python run: | diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index 48f901648..4e0e34fe5 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -47,7 +47,7 @@ jobs: - name: Install python dependencies via pip run: | python -m pip install --upgrade pip - pip install numpy cython cftime pytest twine wheel check-manifest mpi4py mypy types-setuptools + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py mypy types-setuptools typing-extensions - name: Install netcdf4-python run: | diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index 484a627e2..047c892de 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -59,7 +59,7 @@ jobs: - name: Install python dependencies via pip run: | python -m pip install --upgrade pip - pip install numpy cython cftime pytest twine wheel check-manifest mpi4py + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py typing-extensions - name: Install netcdf4-python run: | diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index 15a87d6ef..831c120b2 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -103,7 +103,7 @@ jobs: CIBW_TEST_SKIP: "cp38-*_aarch64 cp39-*_aarch64 cp310-*_aarch64 cp311-*_aarch64" CIBW_ENVIRONMENT: ${{ matrix.CIBW_ENVIRONMENT }} CIBW_BEFORE_BUILD_MACOS: brew install hdf5 netcdf - CIBW_TEST_REQUIRES: pytest cython packaging + CIBW_TEST_REQUIRES: pytest cython packaging typing-extensions CIBW_TEST_COMMAND: > python -c "import netCDF4; print(f'netCDF4 v{netCDF4.__version__}')" && pytest -s -rxs -v {project}/test @@ -155,7 +155,7 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: > delvewheel show {wheel} && delvewheel repair -w {dest_dir} {wheel} - CIBW_TEST_REQUIRES: pytest cython packaging + CIBW_TEST_REQUIRES: pytest cython packaging typing-extensions CIBW_TEST_COMMAND: > python -c "import netCDF4; print(f'netCDF4 v{netCDF4.__version__}')" && pytest -s -rxs -v {project}\\test diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 893e60cf5..6619db072 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -16,7 +16,7 @@ jobs: os: [windows-latest, ubuntu-latest, macos-latest] platform: [x64, x32] exclude: - - os: macos-latest + - os: macos-latest platform: x32 fail-fast: false defaults: @@ -35,7 +35,7 @@ jobs: init-shell: bash create-args: >- python=${{ matrix.python-version }} - numpy cython pip pytest hdf5 libnetcdf cftime zlib certifi + numpy cython pip pytest hdf5 libnetcdf cftime zlib certifi typing-extensions --channel conda-forge - name: Install netcdf4-python @@ -69,7 +69,7 @@ jobs: init-shell: bash create-args: >- python=${{ matrix.python-version }} - numpy cython pip pytest openmpi mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib certifi + numpy cython pip pytest openmpi mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib certifi typing-extensions --channel conda-forge - name: Install netcdf4-python with mpi @@ -82,7 +82,7 @@ jobs: run: | cd test && python run_all.py cd ../examples - export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" + export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" which mpirun mpirun --version mpirun -np 4 --oversubscribe python mpi_example.py # for openmpi diff --git a/examples/bench.py b/examples/bench.py index f3ad75246..08f95d48f 100644 --- a/examples/bench.py +++ b/examples/bench.py @@ -1,9 +1,14 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform import netCDF4 from timeit import Timer import os, sys +if TYPE_CHECKING: + from netCDF4 import Format as NCFormat +else: + NCFormat = Any # create an n1dim by n2dim by n3dim random array. n1dim = 30 @@ -14,7 +19,7 @@ sys.stdout.write('reading and writing a %s by %s by %s by %s random array ..\n'%(n1dim,n2dim,n3dim,n4dim)) array = uniform(size=(n1dim,n2dim,n3dim,n4dim)) -def write_netcdf(filename,zlib=False,least_significant_digit=None,format='NETCDF4'): +def write_netcdf(filename,zlib=False,least_significant_digit=None,format: NCFormat='NETCDF4'): file = netCDF4.Dataset(filename,'w',format=format) file.createDimension('n1', n1dim) file.createDimension('n2', n2dim) diff --git a/examples/bench_compress.py b/examples/bench_compress.py index 39bffe906..f094a6ffa 100644 --- a/examples/bench_compress.py +++ b/examples/bench_compress.py @@ -1,9 +1,15 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform import netCDF4 +import netCDF4.utils from timeit import Timer import os, sys +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any # create an n1dim by n2dim by n3dim random array. n1dim = 30 @@ -13,10 +19,9 @@ ntrials = 10 sys.stdout.write('reading and writing a %s by %s by %s by %s random array ..\n'%(n1dim,n2dim,n3dim,n4dim)) sys.stdout.write('(average of %s trials)\n' % ntrials) -array = netCDF4.utils._quantize(uniform(size=(n1dim,n2dim,n3dim,n4dim)),4) # type: ignore +array = netCDF4.utils._quantize(uniform(size=(n1dim,n2dim,n3dim,n4dim)),4) - -def write_netcdf(filename,zlib=False,shuffle=False,complevel=6): +def write_netcdf(filename,zlib=False,shuffle=False,complevel: CompressionLevel = 6): file = netCDF4.Dataset(filename,'w',format='NETCDF4') file.createDimension('n1', n1dim) file.createDimension('n2', n2dim) diff --git a/examples/bench_compress4.py b/examples/bench_compress4.py index a7d0e1033..d8f643935 100644 --- a/examples/bench_compress4.py +++ b/examples/bench_compress4.py @@ -1,5 +1,6 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. +from typing import Literal from numpy.random.mtrand import uniform import netCDF4 from timeit import Timer @@ -19,7 +20,11 @@ array = nc.variables['hgt'][0:n1dim,5,:,:] -def write_netcdf(filename,nsd,quantize_mode='BitGroom'): +def write_netcdf( + filename, + nsd, + quantize_mode: Literal["BitGroom", "BitRound", "GranularBitRound"] = "BitGroom" + ): file = netCDF4.Dataset(filename,'w',format='NETCDF4') file.createDimension('n1', None) file.createDimension('n3', n3dim) diff --git a/examples/bench_diskless.py b/examples/bench_diskless.py index dd7a78315..076f446b4 100644 --- a/examples/bench_diskless.py +++ b/examples/bench_diskless.py @@ -1,9 +1,14 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. +from typing import TYPE_CHECKING, Any, Literal from numpy.random.mtrand import uniform import netCDF4 from timeit import Timer import os, sys +if TYPE_CHECKING: + from netCDF4 import Format as NCFormat +else: + NCFormat = Any # create an n1dim by n2dim by n3dim random array. n1dim = 30 @@ -14,7 +19,7 @@ sys.stdout.write('reading and writing a %s by %s by %s by %s random array ..\n'%(n1dim,n2dim,n3dim,n4dim)) array = uniform(size=(n1dim,n2dim,n3dim,n4dim)) -def write_netcdf(filename,zlib=False,least_significant_digit=None,format='NETCDF4',closeit=False): +def write_netcdf(filename, zlib=False, least_significant_digit=None, format: NCFormat='NETCDF4',closeit=False): file = netCDF4.Dataset(filename,'w',format=format,diskless=True,persist=True) file.createDimension('n1', n1dim) file.createDimension('n2', n2dim) @@ -42,13 +47,13 @@ def read_netcdf(ncfile): sys.stdout.write('writing took %s seconds\n' %\ repr(sum(t.repeat(ntrials,1))/ntrials)) # test reading. - ncfile = write_netcdf('test1.nc',format=format) + ncfile = write_netcdf('test1.nc',format=format) # type: ignore t = Timer("read_netcdf(ncfile)","from __main__ import read_netcdf,ncfile") sys.stdout.write('reading took %s seconds\n' % repr(sum(t.repeat(ntrials,1))/ntrials)) # test diskless=True in nc_open -format='NETCDF3_CLASSIC' +format: Literal["NETCDF3_CLASSIC"] = 'NETCDF3_CLASSIC' # mypy should know this but it needs help... trials=50 sys.stdout.write('test caching of file in memory on open for %s\n' % format) sys.stdout.write('testing file format %s ...\n' % format) diff --git a/examples/mpi_example.py b/examples/mpi_example.py index 7966bdafe..93ca57bc7 100644 --- a/examples/mpi_example.py +++ b/examples/mpi_example.py @@ -1,27 +1,23 @@ # to run: mpirun -np 4 python mpi_example.py import sys -from typing import Literal from mpi4py import MPI import numpy as np from netCDF4 import Dataset -format: Literal[ - 'NETCDF4', - 'NETCDF4_CLASSIC', - 'NETCDF3_CLASSIC', - 'NETCDF3_64BIT_OFFSET', - 'NETCDF3_64BIT_DATA' -] -if len(sys.argv) == 2: - format = sys.argv[1] # type: ignore -else: - format = 'NETCDF4_CLASSIC' + +nc_format = 'NETCDF4_CLASSIC' if len(sys.argv) < 2 else sys.argv[1] rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run) if rank == 0: - print('Creating file with format {}'.format(format)) -nc = Dataset('parallel_test.nc', 'w', parallel=True, comm=MPI.COMM_WORLD, - info=MPI.Info(), format=format) + print('Creating file with format {}'.format(nc_format)) +nc = Dataset( + "parallel_test.nc", + "w", + parallel=True, + comm=MPI.COMM_WORLD, + info=MPI.Info(), + format=nc_format, # type: ignore # we'll assume it's OK +) # below should work also - MPI_COMM_WORLD and MPI_INFO_NULL will be used. #nc = Dataset('parallel_test.nc', 'w', parallel=True) d = nc.createDimension('dim',4) diff --git a/pyproject.toml b/pyproject.toml index 750e5f59f..c577e761a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,12 @@ filterwarnings = [ [tool.mypy] files = ["src/netCDF4"] +exclude = "utils.py" +check_untyped_defs = true +allow_redefinition = true +# next 2 lines workarounds for mypy dealing with type_guards.py +mypy_path = "test" +explicit_package_bases = true [[tool.mypy.overrides]] ignore_missing_imports = true diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 68e18dcae..6ed415f7a 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -1,72 +1,165 @@ +"""__init__.pyi - Type stubs for the netCDF4 Python package""" +# Notes: +# +# - The stubs in this file are manually-generated and must be updated if and when the API is changed. +# - The following **ruff** commands may be used to format this file according to +# https://typing.readthedocs.io/en/latest/source/stubs.html +# +# ruff format --line-length 130 src/netCDF4/__init__.pyi # format code +# ruff check --line-length 130 --select I --fix src/netCDF4/__init__.pyi # sort imports +# +# - The Variable class is a generic and may thus be statically typed, but this has limited utility for the following reasons: +# - The return type of `Variable.__getitem__()` (and `Variable.getValue()`) depends on a number of factors (e.g. variable +# shape, key shape, whether masking is enabled) that cannot be easily determined statically. +# - Similarly, the types and shapes of data that `Variable.__setitem__()` may accept varies widely depending on many factors +# and is intractable to determine statically. +# - Some facility for automatically typing a Variable on creation has been provided, however it is not exhaustive as a variable +# may created with a string literal indicating its type and it would require an excessive number of overloads to enumerate +# each of these cases. +# - It is not possible to statically type a Variable of any user-defined type (CompoundType, EnumType, VLType) as these types +# are created dynamically. +# Thus it is most often best for the user to implement TypeGuards and/or perform other mixed static/runtime type-checking to +# ensure the type and shape of data retrieved from this library. +# - The return type of some functions or properties (such as `Dataset.__getitem__()`) may one of a number of types. Where it is +# not possible to narrow the type with overloads, the authors of these stubs have generally chosen to use `Any` as the return +# type rather than a union of the possible types. +# - `MFDataset.dimensions` returns `dict[str, Dimension]` and `MFDataset.variables` returns `dict[str, Variable]` even though the +# dict value types may actually be `_Dimension` and `_Variable`, respectively. The original authors of this stubfile have +# elected to do this for simplicity's sake, but it may make sense to change this in the future, or just return `dict[str, Any]`. +import datetime as dt import os import sys -import datetime as dt -from typing import (Any, Callable, Final, Generic, Iterable, Literal, Mapping, - NoReturn, Self, Sequence, TypeAlias, TypedDict, TypeVar, Union, overload) -from typing_extensions import Buffer +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Final, + Generic, + Iterable, + Iterator, + Literal, + Mapping, + NoReturn, + Sequence, + TypedDict, + TypeVar, + Union, + overload, +) import cftime import numpy as np import numpy.typing as npt +from typing_extensions import Buffer, Self, TypeAlias __all__ = [ - 'Dataset', 'Variable', 'Dimension', 'Group', 'MFDataset', 'MFTime', 'CompoundType', - 'VLType', 'date2num', 'num2date', 'date2index', 'stringtochar', 'chartostring', - 'stringtoarr', 'getlibversion', 'EnumType', 'get_chunk_cache', 'set_chunk_cache', - 'set_alignment', 'get_alignment', 'rc_get', 'rc_set', + "Dataset", + "Variable", + "Dimension", + "Group", + "MFDataset", + "MFTime", + "CompoundType", + "VLType", + "date2num", + "num2date", + "rc_get", + "rc_set", + "date2index", + "stringtochar", + "chartostring", + "stringtoarr", + "getlibversion", + "EnumType", + "get_chunk_cache", + "set_chunk_cache", + "set_alignment", + "get_alignment", ] -__pdoc__ = {'utils': False} - - -if sys.version_info >= (3, 10): - from types import EllipsisType - - ellipsis = EllipsisType -elif not TYPE_CHECKING: - ellipsis = type(Ellipsis) # keeps ruff happy until ruff uses typeshed +__pdoc__ = {"utils": False} + +# string type specifiers +# fmt: off +RealTypeLiteral: TypeAlias = Literal[ + "i1", "b", "B", "int8", # NC_BYTE + "u1", "uint8", # NC_UBYTE + "i2", "h", "s", "int16", # NC_SHORT + "u2", "uint16", # NC_USHORT + "i4", "i", "l", "int32", # NC_INT + "u4", "uint32", # NC_UINT + "i8", "int64", "int", # NC_INT64 + "u8", "uint64", # NC_UINT64 + "f4", "f", "float32", # NC_FLOAT + "f8", "d", "float64", "float" # NC_DOUBLE +] +# fmt: on +ComplexTypeLiteral: TypeAlias = Literal["c8", "c16", "complex64", "complex128"] +NumericTypeLiteral: TypeAlias = RealTypeLiteral | ComplexTypeLiteral +CharTypeLiteral: TypeAlias = Literal["S1", "c"] # NC_CHAR +TypeLiteral: TypeAlias = NumericTypeLiteral | CharTypeLiteral + +# Numpy types +NumPyRealType: TypeAlias = ( + np.int8 | np.uint8 | np.int16 | np.uint16 | np.int32 | np.uint32 | np.int64 | np.uint64 | np.float16 | np.float32 | np.float64 +) +NumPyComplexType: TypeAlias = np.complex64 | np.complex128 +NumPyNumericType: TypeAlias = NumPyRealType | NumPyComplexType +# Classes that can create instances of NetCDF user-defined types +NetCDFUDTClass: TypeAlias = CompoundType | VLType | EnumType +# Possible argument types for the datatype argument used in Variable creation. At this time, it is not possible to allow unknown +# strings arguments in the datatype field but exclude and string literals that are not one of `TypeLiteral`, so really +# `TypeLiteral` is made irrelevant, except for anyone who looks at this file. +DatatypeType: TypeAlias = ( + TypeLiteral + | str + | np.dtype[NumPyNumericType | np.str_] + | type[int | float | NumPyNumericType | str | np.str_] + | NetCDFUDTClass +) +VarT = TypeVar("VarT") +RealVarT = TypeVar("RealVarT", bound=NumPyRealType) +ComplexVarT = TypeVar("ComplexVarT", bound=NumPyComplexType) +NumericVarT = TypeVar("NumericVarT", bound=NumPyNumericType) -_DatatypeStrOptions: TypeAlias = Literal[ - 'S1', 'c', 'i1', 'b', 'B', 'u1', 'i2', 'h', 's', 'u2', 'i4', - 'i', 'l', 'u4', 'i8', 'u8', 'f4', 'f', 'f8', 'd', 'c8', 'c16' -] -_DatatypeNCOptions: TypeAlias = Union[CompoundType, VLType, EnumType] -DatatypeOptions: TypeAlias = Union[_DatatypeStrOptions, _DatatypeNCOptions, npt.DTypeLike] -T_Datatype = TypeVar("T_Datatype", bound=DatatypeOptions) -T_DatatypeNC = TypeVar("T_DatatypeNC", CompoundType, VLType, EnumType) - -DimensionsOptions: TypeAlias = Union[str, bytes, Dimension, Iterable[Union[str, bytes, Dimension]]] -CompressionOptions: TypeAlias = Literal[ - 'zlib', 'szip', 'zstd', 'blosc_lz','blosc_lz4', - 'blosc_lz4hc', 'blosc_zlib', 'blosc_zstd' +DimensionsType: TypeAlias = Union[str, bytes, Dimension, Iterable[Union[str, bytes, Dimension]]] +CompressionType: TypeAlias = Literal[ + "zlib", "szip", "zstd", "bzip2", "blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd" ] -CompressionLevelOptions: TypeAlias = Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] -AccessModeOptions: TypeAlias = Literal['r', 'w', 'r+', 'a', 'x', 'rs', 'ws', 'r+s', 'as'] -FormatOptions: TypeAlias = Literal[ - 'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_CLASSIC', - 'NETCDF3_64BIT_OFFSET', 'NETCDF3_64BIT_DATA' -] -DiskFormatOptions: TypeAlias = Literal['NETCDF3', 'HDF5', 'HDF4', 'PNETCDF', 'DAP2', 'DAP4', 'UNDEFINED'] -QuantizeOptions: TypeAlias = Literal['BitGroom', 'BitRound', 'GranularBitRound'] -EndianOptions: TypeAlias = Literal['native', 'little', 'big'] -CalendarOptions: TypeAlias = Literal[ - 'standard', 'gregorian', 'proleptic_gregorian' 'noleap', - '365_day', '360_day', 'julian', 'all_leap', '366_day' +CompressionLevel: TypeAlias = Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +AccessMode: TypeAlias = Literal["r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"] +Format: TypeAlias = Literal["NETCDF4", "NETCDF4_CLASSIC", "NETCDF3_CLASSIC", "NETCDF3_64BIT_OFFSET", "NETCDF3_64BIT_DATA"] +DiskFormat: TypeAlias = Literal["NETCDF3", "HDF5", "HDF4", "PNETCDF", "DAP2", "DAP4", "UNDEFINED"] +QuantizeMode: TypeAlias = Literal["BitGroom", "BitRound", "GranularBitRound"] +EndianType: TypeAlias = Literal["native", "little", "big"] +CalendarType: TypeAlias = Literal[ + "standard", "gregorian", "proleptic_gregorian", "noleap", "365_day", "360_day", "julian", "all_leap", "366_day" ] BoolInt: TypeAlias = Literal[0, 1] DateTimeArray: TypeAlias = npt.NDArray[np.object_] """numpy array of datetime.datetime or cftime.datetime""" -GetSetItemKey: TypeAlias = ( - int - | slice - | ellipsis - | list[int | bool] - | npt.NDArray[np.integer | np.bool_] - | tuple[int | slice | ellipsis | Sequence[int | bool] | npt.NDArray[np.integer | np.bool_], ...] -) +class BloscInfo(TypedDict): + compressor: Literal["blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd"] + shuffle: Literal[0, 1, 2] + +class SzipInfo(TypedDict): + coding: Literal["nn", "ec"] + pixels_per_block: Literal[4, 8, 16, 32] + +class FiltersDict(TypedDict): + """Dict returned from netCDF4.Variable.filters()""" + + zlib: bool + szip: Literal[False] | SzipInfo + zstd: bool + bzip2: bool + blosc: Literal[False] | BloscInfo + shuffle: bool + complevel: int + fletcher32: bool __version__: str __netcdf4libversion__: str @@ -90,70 +183,47 @@ __has_ncfilter__: BoolInt __has_nc_rc_set__: BoolInt is_native_little: bool is_native_big: bool -default_encoding: Final = 'utf-8' -unicode_error: Final = 'replace' +default_encoding: Final = "utf-8" +unicode_error: Final = "replace" default_fillvals: dict[str, int | float | str] - # date2index, date2num, and num2date are actually provided by cftime and if stubs for # cftime are completed these should be removed. def date2index( dates: dt.datetime | cftime.datetime | Sequence[dt.datetime | cftime.datetime] | DateTimeArray, nctime: Variable, - calendar: CalendarOptions | None = None, + calendar: CalendarType | str | None = None, select: Literal["exact", "before", "after", "nearest"] = "exact", has_year_zero: bool | None = None, ) -> int | npt.NDArray[np.int_]: ... def date2num( dates: dt.datetime | cftime.datetime | Sequence[dt.datetime | cftime.datetime] | DateTimeArray, units: str, - calendar: CalendarOptions | None = None, + calendar: CalendarType | str | None = None, has_year_zero: bool | None = None, longdouble: bool = False, ) -> np.number | npt.NDArray[np.number]: ... def num2date( times: Sequence[int | float | np.number] | npt.NDArray[np.number], units: str, - calendar: CalendarOptions = "standard", + calendar: CalendarType | str = "standard", only_use_cftime_datetimes: bool = True, only_use_python_datetimes: bool = False, has_year_zero: bool | None = None, ) -> dt.datetime | DateTimeArray: ... - -class BloscInfo(TypedDict): - compressor: Literal["blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd"] - shuffle: Literal[0, 1, 2] - - -class SzipInfo(TypedDict): - coding: Literal["nn", "ec"] - pixels_per_block: Literal[4, 8, 16, 32] - - -class FiltersDict(TypedDict): - """Dict returned from netCDF4.Variable.filters()""" - zlib: bool - szip: Literal[False] | SzipInfo - zstd: bool - bzip2: bool - blosc: Literal[False] | BloscInfo - shuffle: bool - complevel: int - fletcher32: bool - - class NetCDF4MissingFeatureException(Exception): def __init__(self, feature: str, version: str): ... +def dtype_is_complex(dtype: str) -> bool: ... class Dataset: def __init__( self, filename: str | os.PathLike, - mode: AccessModeOptions = 'r', + mode: AccessMode = "r", clobber: bool = True, - format: FormatOptions = 'NETCDF4', + format: Format = "NETCDF4", diskless: bool = False, persist: bool = False, keepweakref: bool = False, @@ -163,9 +233,8 @@ class Dataset: comm: Any = None, info: Any = None, auto_complex: bool = False, - **kwargs: Any + **kwargs: Any, ): ... - @property def name(self) -> str: ... @property @@ -181,11 +250,11 @@ class Dataset: @property def enumtypes(self) -> dict[str, EnumType]: ... @property - def data_model(self) -> FormatOptions: ... + def data_model(self) -> Format: ... @property - def file_format(self) -> FormatOptions: ... + def file_format(self) -> Format: ... @property - def disk_format(self) -> DiskFormatOptions: ... + def disk_format(self) -> DiskFormat: ... @property def parent(self) -> Dataset | None: ... @property @@ -198,65 +267,85 @@ class Dataset: def _ncstring_attrs__(self) -> bool: ... @property def __orthogonal_indexing__(self) -> bool: ... - def filepath(self, encoding: str | None = None) -> str: ... def isopen(self) -> bool: ... def close(self) -> memoryview: ... # only if writing and memory != None, but otherwise people ignore the return None anyway def sync(self) -> None: ... def set_fill_on(self) -> None: ... def set_fill_off(self) -> None: ... - def createDimension(self, dimname: str, size: int | None = None) -> Dimension: ... - def renameDimension( self, oldname: str, newname: str) -> None: ... + def renameDimension(self, oldname: str, newname: str) -> None: ... + @overload + def createVariable( + self, + varname: str, + datatype: np.dtype[NumericVarT] | type[NumericVarT], + dimensions: DimensionsType = (), + compression: CompressionType | None = None, + zlib: bool = False, + complevel: CompressionLevel | None = 4, + shuffle: bool = True, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, + fletcher32: bool = False, + contiguous: bool = False, + chunksizes: Sequence[int] | None = None, + endian: EndianType = "native", + least_significant_digit: int | None = None, + significant_digits: int | None = None, + quantize_mode: QuantizeMode = "BitGroom", + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, + chunk_cache: int | None = None, + ) -> Variable[NumericVarT]: ... @overload - def createVariable( # type: ignore + def createVariable( self, varname: str, - datatype: T_DatatypeNC, - dimensions: DimensionsOptions = (), - compression: CompressionOptions | None = None, + datatype: np.dtype[np.str_] | type[str | np.str_], + dimensions: DimensionsType = (), + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevelOptions | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal['nn', 'ec'] = 'nn', + szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: int | None = None, - endian: EndianOptions = 'native', + chunksizes: Sequence[int] | None = None, + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = 'BitGroom', - fill_value: int | float | str | bytes | Literal[False] | None = None, - chunk_cache: int | None = None - ) -> Variable[T_DatatypeNC]: ... + quantize_mode: QuantizeMode = "BitGroom", + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, + chunk_cache: int | None = None, + ) -> Variable[str]: ... @overload def createVariable( self, varname: str, - datatype: _DatatypeStrOptions | npt.DTypeLike, - dimensions: DimensionsOptions = (), - compression: CompressionOptions | None = None, + datatype: DatatypeType, + dimensions: DimensionsType = (), + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevelOptions | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal['nn', 'ec'] = 'nn', + szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: int | None = None, - endian: EndianOptions = 'native', + chunksizes: Sequence[int] | None = None, + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = 'BitGroom', - fill_value: int | float | str | bytes | Literal[False] | None = None, - chunk_cache: int | None = None - ) -> Variable[np.dtype]: ... + quantize_mode: QuantizeMode = "BitGroom", + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, + chunk_cache: int | None = None, + ) -> Variable: ... def renameVariable(self, oldname: str, newname: str) -> None: ... def createGroup(self, groupname: str) -> Group: ... - def renameGroup(self, oldname: str, newname: str) -> None: ... def renameAttribute(self, oldname: str, newname: str) -> None: ... def createCompoundType( @@ -264,14 +353,16 @@ class Dataset: ) -> CompoundType: ... def createVLType(self, datatype: npt.DTypeLike, datatype_name: str) -> VLType: ... def createEnumType( - self, datatype: np.dtype[np.integer] | type[np.integer] | type[int], datatype_name: str, enum_dict: dict[str, int] + self, + datatype: np.dtype[np.integer] | type[np.integer] | type[int], + datatype_name: str, + enum_dict: Mapping[str, int | np.integer], ) -> EnumType: ... - def ncattrs(self) -> list[str]: ... def setncattr_string(self, name: str, value: Any) -> None: ... def setncattr(self, name: str, value: Any) -> None: ... def setncatts(self, attdict: Mapping[str, Any]) -> None: ... - def getncattr(self, name: str, encoding: str = 'utf-8') -> Any: ... + def getncattr(self, name: str, encoding: str = "utf-8") -> Any: ... def delncattr(self, name: str) -> None: ... def set_auto_chartostring(self, value: bool) -> None: ... def set_auto_maskandscale(self, value: bool) -> None: ... @@ -280,152 +371,168 @@ class Dataset: def set_always_mask(self, value: bool) -> None: ... def set_ncstring_attrs(self, value: bool) -> None: ... def get_variables_by_attributes(self, **kwargs: Callable[[Any], bool] | Any) -> list[Variable]: ... - @staticmethod def fromcdl( - cdlfilename: str, - ncfilename: str | None = None, - mode: AccessModeOptions = 'a', - format: FormatOptions = 'NETCDF4' + cdlfilename: str, ncfilename: str | None = None, mode: AccessMode = "a", format: Format = "NETCDF4" ) -> Dataset: ... @overload - def tocdl( - self, - coordvars: bool = False, - data: bool = False, - outfile: None = None - ) -> str: ... + def tocdl(self, coordvars: bool = False, data: bool = False, outfile: None = None) -> str: ... @overload - def tocdl( - self, - coordvars: bool = False, - data: bool = False, - *, - outfile: str | os.PathLike - ) -> None: ... - + def tocdl(self, coordvars: bool = False, data: bool = False, *, outfile: str | os.PathLike) -> None: ... def has_blosc_filter(self) -> bool: ... def has_zstd_filter(self) -> bool: ... def has_bzip2_filter(self) -> bool: ... def has_szip_filter(self) -> bool: ... - def __getitem__(self, elem: str) -> Any: ... # should be Group | Variable, but this causes too many problems def __setattr__(self, name: str, value: Any) -> None: ... def __getattr__(self, name: str) -> Any: ... def __delattr__(self, name: str): ... - def __dealloc(self) -> None: ... def __reduce__(self) -> NoReturn: ... def __enter__(self) -> Self: ... def __exit__(self, atype, value, traceback) -> None: ... - class Group(Dataset): def __init__(self, parent: Dataset, name: str, **kwargs: Any) -> None: ... - def close(self) -> NoReturn: ... - class Dimension: def __init__(self, grp: Dataset, name: str, size: int | None = None, **kwargs: Any) -> None: ... - @property def name(self) -> str: ... @property def size(self) -> int: ... - def group(self) -> Dataset: ... def isunlimited(self) -> bool: ... - def __len__(self) -> int: ... +class _VarDatatypeProperty: + # A faux descriptor definition of the property to allow overloads + @overload + def __get__(self, instance: Variable[RealVarT], owner: Any) -> np.dtype[RealVarT]: ... + @overload + def __get__(self, instance: Variable[ComplexVarT], owner: Any) -> CompoundType: ... + @overload + def __get__(self, instance: Variable[str], owner: Any) -> VLType: ... + @overload + def __get__( + self, instance: Variable, owner: Any + ) -> Any: ... # actual return type np.dtype | CompoundType | VLType | EnumType -class Variable(Generic[T_Datatype]): +class _VarDtypeProperty: + # A faux descriptor definition of the property to allow overloads + @overload + def __get__(self, instance: Variable[NumericVarT], owner: Any) -> np.dtype[NumericVarT]: ... + @overload + def __get__(self, instance: Variable[str], owner: Any) -> type[str]: ... + @overload + def __get__(self, instance: Variable, owner: Any) -> Any: ... # actual return type np.dtype | Type[str] +class Variable(Generic[VarT]): + # Overloads of __new__ are provided for some cases where the Variable's type may be statically inferred from the datatype arg @overload - def __new__( # type: ignore - self, + def __new__( + cls, grp: Dataset, name: str, - datatype: T_DatatypeNC, - dimensions: DimensionsOptions = (), - compression: CompressionOptions | None = None, + datatype: np.dtype[NumericVarT] | type[NumericVarT], + dimensions: DimensionsType = (), + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevelOptions | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal['nn', 'ec'] = 'nn', + szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: Sequence[int] | None = None, - endian: EndianOptions = 'native', + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = 'BitGroom', - fill_value: int | float | str | bytes | Literal[False] | None = None, + quantize_mode: QuantizeMode = "BitGroom", + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, chunk_cache: int | None = None, - **kwargs: Any - ) -> Variable[T_DatatypeNC]: ... - + **kwargs: Any, + ) -> Variable[NumericVarT]: ... @overload def __new__( - self, + cls, grp: Dataset, name: str, - datatype: _DatatypeStrOptions | npt.DTypeLike, - dimensions: DimensionsOptions = (), - compression: CompressionOptions | None = None, + datatype: np.dtype[np.str_] | type[str | np.str_], + dimensions: DimensionsType = (), + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevelOptions | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal['nn', 'ec'] = 'nn', + szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: Sequence[int] | None = None, - endian: EndianOptions = 'native', + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = 'BitGroom', - fill_value: int | float | str | bytes | Literal[False] | None = None, + quantize_mode: QuantizeMode = "BitGroom", + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, chunk_cache: int | None = None, - **kwargs: Any - ) -> Variable[np.dtype]: ... - - + **kwargs: Any, + ) -> Variable[str]: ... + @overload + def __new__( + cls, + grp: Dataset, + name: str, + datatype: DatatypeType, + dimensions: DimensionsType = (), + compression: CompressionType | None = None, + zlib: bool = False, + complevel: CompressionLevel | None = 4, + shuffle: bool = True, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, + fletcher32: bool = False, + contiguous: bool = False, + chunksizes: Sequence[int] | None = None, + endian: EndianType = "native", + least_significant_digit: int | None = None, + significant_digits: int | None = None, + quantize_mode: QuantizeMode = "BitGroom", + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, + chunk_cache: int | None = None, + **kwargs: Any, + ) -> Variable: ... def __init__( self, grp: Dataset, name: str, - datatype: T_Datatype, - dimensions: DimensionsOptions = (), - compression: CompressionOptions | None = None, + datatype: DatatypeType, + dimensions: DimensionsType = (), + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevelOptions | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal['nn', 'ec'] = 'nn', + szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: Sequence[int] | None = None, - endian: EndianOptions = 'native', + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = 'BitGroom', - fill_value: int | float | str | bytes | Literal[False] | None = None, + quantize_mode: QuantizeMode = "BitGroom", + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, chunk_cache: int | None = None, - **kwargs: Any + **kwargs: Any, ) -> None: ... - + datatype: _VarDatatypeProperty + dtype: _VarDtypeProperty @property def name(self) -> str: ... @property - def dtype(self) -> np.dtype | type[str]: ... - @property - def datatype(self) -> T_Datatype: ... - @property def shape(self) -> tuple[int, ...]: ... @property def size(self) -> int: ... @@ -443,24 +550,20 @@ class Variable(Generic[T_Datatype]): def always_mask(self) -> bool: ... @property def __orthogonal_indexing__(self) -> bool: ... - def group(self) -> Dataset: ... def ncattrs(self) -> list[str]: ... def setncattr(self, name: str, value: Any) -> None: ... def setncattr_string(self, name: str, value: Any) -> None: ... def setncatts(self, attdict: Mapping[str, Any]) -> None: ... - def getncattr(self, name: str, encoding='utf-8'): ... + def getncattr(self, name: str, encoding="utf-8"): ... def delncattr(self, name: str) -> None: ... def filters(self) -> FiltersDict: ... - def quantization(self) -> tuple[int, QuantizeOptions] | None: ... - def endian(self) -> EndianOptions: ... - def chunking(self) -> Literal['contiguous'] | list[int]: ... + def quantization(self) -> tuple[int, QuantizeMode] | None: ... + def endian(self) -> EndianType: ... + def chunking(self) -> Literal["contiguous"] | list[int]: ... def get_var_chunk_cache(self) -> tuple[int, int, float]: ... def set_var_chunk_cache( - self, - size: int | None = None, - nelems: int | None = None, - preemption: float | None = None + self, size: int | None = None, nelems: int | None = None, preemption: float | None = None ) -> None: ... def renameAttribute(self, oldname: str, newname: str) -> None: ... def assignValue(self, val: Any) -> None: ... @@ -475,15 +578,14 @@ class Variable(Generic[T_Datatype]): def set_ncstring_attrs(self, ncstring_attrs: bool) -> None: ... def set_collective(self, value: bool) -> None: ... def get_dims(self) -> tuple[Dimension, ...]: ... - def __delattr__(self, name: str) -> None: ... def __setattr__(self, name: str, value: Any) -> None: ... def __getattr__(self, name: str) -> Any: ... - def __getitem__(self, elem: GetSetItemKey) -> np.ndarray: ... - def __setitem__(self, elem: GetSetItemKey, data: npt.ArrayLike) -> None: ... + def __getitem__(self, elem: Any) -> Any: ... + def __setitem__(self, elem: Any, data: npt.ArrayLike) -> None: ... def __array__(self) -> np.ndarray: ... def __len__(self) -> int: ... - + def __iter__(self) -> Iterator[Any]: ... # faux method so mypy believes Variable is iterable class CompoundType: dtype: np.dtype @@ -493,19 +595,15 @@ class CompoundType: def __init__( self, grp: Dataset, dt: npt.DTypeLike | Sequence[tuple[str, npt.DTypeLike]], dtype_name: str, **kwargs: Any ) -> None: ... - def __reduce__(self) -> NoReturn: ... - class VLType: dtype: np.dtype name: str | None def __init__(self, grp: Dataset, dt: npt.DTypeLike, dtype_name: str, **kwargs: Any) -> None: ... - def __reduce__(self) -> NoReturn: ... - class EnumType: dtype: np.dtype[np.integer] name: str @@ -516,13 +614,11 @@ class EnumType: grp: Dataset, dt: np.dtype[np.integer] | type[np.integer] | type[int] | str, dtype_name: str, - enum_dict: Mapping[str, int], - **kwargs: Any + enum_dict: Mapping[str, int | np.integer], + **kwargs: Any, ) -> None: ... - def __reduce__(self) -> NoReturn: ... - class MFDataset(Dataset): def __init__( self, @@ -530,27 +626,21 @@ class MFDataset(Dataset): check: bool = False, aggdim: str | None = None, exclude: Sequence[str] = [], - master_file: str | os.PathLike | None = None + master_file: str | os.PathLike | None = None, ) -> None: ... - @property def dimensions(self) -> dict[str, Dimension]: ... # this should be: dict[str, Dimension | _Dimension] @property def variables(self) -> dict[str, Variable[Any]]: ... # this should be: dict[str, _Variable[Any] | _Variable] - class _Dimension: dimlens: list[int] dimtolen: int - def __init__( - self, dimname: str, dim: Dimension, dimlens: list[int], dimtotlen: int - ) -> None: ... - + def __init__(self, dimname: str, dim: Dimension, dimlens: list[int], dimtotlen: int) -> None: ... def __len__(self) -> int: ... def isunlimited(self) -> Literal[True]: ... - class _Variable: dimensions: tuple[str, ...] dtype: np.dtype | type[str] @@ -564,7 +654,6 @@ class _Variable: def ndim(self) -> int: ... @property def name(self) -> str: ... - def typecode(self) -> np.dtype | type[str]: ... def ncattrs(self) -> list[str]: ... def _shape(self) -> tuple[int, ...]: ... @@ -573,30 +662,22 @@ class _Variable: def set_auto_mask(self, val: bool) -> None: ... def set_auto_scale(self, val: bool) -> None: ... def set_always_mask(self, val: bool) -> None: ... - def __getattr__(self, name: str) -> Any: ... - def __getitem__(self, elem: GetSetItemKey) -> Any: ... + def __getitem__(self, elem: Any) -> Any: ... def __len__(self) -> int: ... - class MFTime(_Variable): - calendar: CalendarOptions | None + calendar: CalendarType | None units: str | None - def __init__( - self, - time: Variable, - units: str | None = None, - calendar: CalendarOptions | None = None - ): ... - def __getitem__(self, elem: GetSetItemKey) -> np.ndarray: ... - + def __init__(self, time: Variable, units: str | None = None, calendar: CalendarType | str | None = None): ... + def __getitem__(self, elem: Any) -> np.ndarray: ... @overload def stringtoarr( string: str, NUMCHARS: int, - dtype: Literal["S"] | np.dtype[np.bytes_]= "S", + dtype: Literal["S"] | np.dtype[np.bytes_] = "S", ) -> npt.NDArray[np.bytes_]: ... @overload def stringtoarr( @@ -624,17 +705,10 @@ def chartostring( b: npt.NDArray[np.character], encoding: str = ..., ) -> npt.NDArray[np.str_] | npt.NDArray[np.bytes_]: ... - def getlibversion() -> str: ... def rc_get(key: str) -> str | None: ... -def rc_set(key: str, value: str)-> None: ... - +def rc_set(key: str, value: str) -> None: ... def set_alignment(threshold: int, alignment: int): ... def get_alignment() -> tuple[int, int]: ... - -def set_chunk_cache( - size: int | None = None, - nelems: int | None = None, - preemption: float | None = None -) -> None: ... +def set_chunk_cache(size: int | None = None, nelems: int | None = None, preemption: float | None = None) -> None: ... def get_chunk_cache() -> tuple[int, int, float]: ... diff --git a/src/netCDF4/_netCDF4.pyi b/src/netCDF4/_netCDF4.pyi index 304e50e69..d0bdb389f 100644 --- a/src/netCDF4/_netCDF4.pyi +++ b/src/netCDF4/_netCDF4.pyi @@ -1,20 +1,53 @@ -# The definitions are intendionally done in the __init__. +# The definitions are intentionally done in the __init__. # This file only exists in case someone imports from netCDF4._netCDF4 from . import ( - Dataset, Variable, Dimension, Group, MFDataset, MFTime, CompoundType, - VLType, date2num, num2date, date2index, stringtochar, chartostring, - stringtoarr, getlibversion, EnumType, get_chunk_cache, set_chunk_cache, - set_alignment, get_alignment, default_fillvals, default_encoding, - NetCDF4MissingFeatureException, is_native_big, is_native_little, unicode_error, - rc_get, rc_set, - __version__, __netcdf4libversion__, __hdf5libversion__, __has_rename_grp__, - __has_nc_inq_path__, __has_nc_inq_format_extended__, __has_nc_open_mem__, - __has_nc_create_mem__, __has_cdf5_format__, __has_parallel4_support__, - __has_pnetcdf_support__, __has_parallel_support__, - __has_quantization_support__, __has_zstandard_support__, - __has_bzip2_support__, __has_blosc_support__, __has_szip_support__, - __has_set_alignment__, __has_ncfilter__, __has_nc_rc_set__, + CompoundType, + Dataset, + Dimension, + EnumType, + Group, + MFDataset, + MFTime, + NetCDF4MissingFeatureException, + Variable, + VLType, + __has_blosc_support__, + __has_bzip2_support__, + __has_cdf5_format__, + __has_nc_create_mem__, + __has_nc_inq_format_extended__, + __has_nc_inq_path__, + __has_nc_open_mem__, + __has_nc_rc_set__, + __has_ncfilter__, + __has_parallel4_support__, + __has_parallel_support__, + __has_pnetcdf_support__, + __has_quantization_support__, + __has_rename_grp__, + __has_set_alignment__, + __has_szip_support__, + __has_zstandard_support__, + __hdf5libversion__, + __netcdf4libversion__, + __version__, + chartostring, + date2index, + date2num, + default_encoding, + default_fillvals, + dtype_is_complex, + get_alignment, + get_chunk_cache, + getlibversion, + is_native_big, + is_native_little, + num2date, + rc_get, + rc_set, + set_alignment, + set_chunk_cache, + stringtoarr, + stringtochar, + unicode_error, ) - - -def dtype_is_complex(dtype: str) -> bool: ... diff --git a/test/run_all.py b/test/run_all.py index deff8c6af..b99565963 100755 --- a/test/run_all.py +++ b/test/run_all.py @@ -24,7 +24,7 @@ sys.stdout.write('netcdf lib version: %s\n' % __netcdf4libversion__) sys.stdout.write('numpy version %s\n' % numpy.__version__) sys.stdout.write('cython version %s\n' % cython.__version__) - runner = unittest.TextTestRunner(verbosity=1) + runner = unittest.TextTestRunner(verbosity=(2 if "-v" in sys.argv else 1)) result = runner.run(testsuite) if not result.wasSuccessful(): sys.exit(1) diff --git a/test/test_compound_alignment.py b/test/test_compound_alignment.py index 0e1c36919..7b8c553b6 100644 --- a/test/test_compound_alignment.py +++ b/test/test_compound_alignment.py @@ -99,7 +99,7 @@ def runTest(self): f = netCDF4.Dataset(self.file, 'r') new_cells = f.variables["cells"][:] assert new_cells.shape == cells.shape - assert sorted(new_cells.dtype.names) == sorted(cells.dtype.names) + assert cells.dtype.names and sorted(new_cells.dtype.names) == sorted(cells.dtype.names) for name in cells.dtype.names: assert cells[name].dtype.name == new_cells[name].dtype.name assert cells[name].shape == new_cells[name].shape diff --git a/test/test_compression.py b/test/test_compression.py index 78827ddff..6c9351af6 100644 --- a/test/test_compression.py +++ b/test/test_compression.py @@ -1,8 +1,13 @@ +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform from netCDF4 import Dataset from netCDF4.utils import _quantize from numpy.testing import assert_almost_equal import os, tempfile, unittest +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any ndim = 100000 ndim2 = 100 @@ -14,7 +19,7 @@ lsd = 3 def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,contiguous=False,\ - chunksizes=None,complevel=6,fletcher32=False): + chunksizes=None, complevel: CompressionLevel = 6, fletcher32=False): file = Dataset(filename,'w') file.createDimension('n', ndim) foo = file.createVariable('data',\ @@ -30,8 +35,8 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=F #compression='' #compression=0 #compression='gzip' # should fail - foo2 = file.createVariable('data2',\ - dtype,('n'),compression=compression,least_significant_digit=least_significant_digit,\ + foo2 = file.createVariable('data2', + dtype,('n'),compression=compression,least_significant_digit=least_significant_digit, # type: ignore # mypy doesn't like compression shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) foo[:] = data foo2[:] = data @@ -42,7 +47,7 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=F file.close() def write_netcdf2(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,contiguous=False,\ - chunksizes=None,complevel=6,fletcher32=False): + chunksizes=None, complevel: CompressionLevel = 6, fletcher32=False): file = Dataset(filename,'w') file.createDimension('n', ndim) file.createDimension('n2', ndim2) @@ -103,7 +108,7 @@ def runTest(self): {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':6,'fletcher32':False} assert f.variables['data2'].filters() ==\ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':6,'fletcher32':False} - assert size < 0.95*uncompressed_size + assert size < 0.95*uncompressed_size f.close() # check compression with shuffle f = Dataset(self.files[2]) @@ -114,7 +119,7 @@ def runTest(self): {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':False} assert f.variables['data2'].filters() ==\ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':False} - assert size < 0.85*uncompressed_size + assert size < 0.85*uncompressed_size f.close() # check lossy compression without shuffle f = Dataset(self.files[3]) @@ -122,14 +127,14 @@ def runTest(self): checkarray = _quantize(array,lsd) assert_almost_equal(checkarray,f.variables['data'][:]) assert_almost_equal(checkarray,f.variables['data2'][:]) - assert size < 0.27*uncompressed_size + assert size < 0.27*uncompressed_size f.close() # check lossy compression with shuffle f = Dataset(self.files[4]) size = os.stat(self.files[4]).st_size assert_almost_equal(checkarray,f.variables['data'][:]) assert_almost_equal(checkarray,f.variables['data2'][:]) - assert size < 0.20*uncompressed_size + assert size < 0.20*uncompressed_size size_save = size f.close() # check lossy compression with shuffle and fletcher32 checksum. @@ -141,9 +146,9 @@ def runTest(self): {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':True} assert f.variables['data2'].filters() ==\ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':True} - assert size < 0.20*uncompressed_size + assert size < 0.20*uncompressed_size # should be slightly larger than without fletcher32 - assert size > size_save + assert size > size_save # check chunksizes f.close() f = Dataset(self.files[6]) diff --git a/test/test_compression_blosc.py b/test/test_compression_blosc.py index 57d8e0f56..fda01e216 100644 --- a/test/test_compression_blosc.py +++ b/test/test_compression_blosc.py @@ -1,8 +1,13 @@ +from typing import TYPE_CHECKING, Any, Literal from numpy.random.mtrand import uniform from netCDF4 import Dataset from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_blosc_filter +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any ndim = 100000 @@ -11,7 +16,7 @@ filename = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name datarr = uniform(size=(ndim,)) -def write_netcdf(filename,dtype='f8',blosc_shuffle=1,complevel=6): +def write_netcdf(filename, dtype='f8', blosc_shuffle: Literal[0, 1, 2] = 1, complevel: CompressionLevel = 6): nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ @@ -38,7 +43,7 @@ def write_netcdf(filename,dtype='f8',blosc_shuffle=1,complevel=6): class CompressionTestCase(unittest.TestCase): def setUp(self): self.filename = filename - write_netcdf(self.filename,complevel=iblosc_complevel,blosc_shuffle=iblosc_shuffle) + write_netcdf(self.filename,complevel=iblosc_complevel,blosc_shuffle=iblosc_shuffle) # type: ignore def tearDown(self): # Remove the temporary files diff --git a/test/test_compression_bzip2.py b/test/test_compression_bzip2.py index 75c0fced1..15ef8eaec 100644 --- a/test/test_compression_bzip2.py +++ b/test/test_compression_bzip2.py @@ -1,15 +1,20 @@ +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform from netCDF4 import Dataset from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_bzip2_filter +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name filename2 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name array = uniform(size=(ndim,)) -def write_netcdf(filename,dtype='f8',complevel=6): +def write_netcdf(filename,dtype='f8',complevel: CompressionLevel = 6): nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ @@ -47,7 +52,7 @@ def runTest(self): assert_almost_equal(array,f.variables['data'][:]) assert f.variables['data'].filters() ==\ {'zlib':False,'szip':False,'zstd':False,'bzip2':True,'blosc':False,'shuffle':False,'complevel':4,'fletcher32':False} - assert size < 0.96*uncompressed_size + assert size < 0.96*uncompressed_size f.close() diff --git a/test/test_compression_quant.py b/test/test_compression_quant.py index 3654bf9d5..b56739624 100644 --- a/test/test_compression_quant.py +++ b/test/test_compression_quant.py @@ -1,8 +1,14 @@ +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform from netCDF4 import Dataset, __has_quantization_support__ from numpy.testing import assert_almost_equal import numpy as np import os, tempfile, unittest +if TYPE_CHECKING: + from netCDF4 import CompressionLevel, QuantizeMode +else: + CompressionLevel = Any + QuantizeMode = Any ndim = 100000 nfiles = 7 @@ -13,7 +19,7 @@ complevel = 6 def write_netcdf(filename,zlib,significant_digits,data,dtype='f8',shuffle=False,\ - complevel=6,quantize_mode="BitGroom"): + complevel: CompressionLevel = 6, quantize_mode: QuantizeMode = "BitGroom"): file = Dataset(filename,'w') file.createDimension('n', ndim) foo = file.createVariable('data',\ @@ -61,7 +67,7 @@ def runTest(self): assert_almost_equal(data_array,f.variables['data'][:]) assert f.variables['data'].filters() ==\ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':complevel,'fletcher32':False} - assert size < 0.95*uncompressed_size + assert size < 0.95*uncompressed_size f.close() # check compression with shuffle f = Dataset(self.files[2]) @@ -70,43 +76,43 @@ def runTest(self): assert_almost_equal(data_array,f.variables['data'][:]) assert f.variables['data'].filters() ==\ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':complevel,'fletcher32':False} - assert size < 0.85*uncompressed_size + assert size < 0.85*uncompressed_size f.close() # check lossy compression without shuffle f = Dataset(self.files[3]) size = os.stat(self.files[3]).st_size errmax = (np.abs(data_array-f.variables['data'][:])).max() #print('compressed lossy no shuffle = ',size,' max err = ',errmax) - assert f.variables['data'].quantization() == (nsd,'BitGroom') - assert errmax < 1.e-3 - assert size < 0.35*uncompressed_size + assert f.variables['data'].quantization() == (nsd,'BitGroom') + assert errmax < 1.e-3 + assert size < 0.35*uncompressed_size f.close() # check lossy compression with shuffle f = Dataset(self.files[4]) size = os.stat(self.files[4]).st_size errmax = (np.abs(data_array-f.variables['data'][:])).max() print('compressed lossy with shuffle and standard quantization = ',size,' max err = ',errmax) - assert f.variables['data'].quantization() == (nsd,'BitGroom') - assert errmax < 1.e-3 - assert size < 0.24*uncompressed_size + assert f.variables['data'].quantization() == (nsd,'BitGroom') + assert errmax < 1.e-3 + assert size < 0.24*uncompressed_size f.close() # check lossy compression with shuffle and alternate quantization f = Dataset(self.files[5]) size = os.stat(self.files[5]).st_size errmax = (np.abs(data_array-f.variables['data'][:])).max() print('compressed lossy with shuffle and alternate quantization = ',size,' max err = ',errmax) - assert f.variables['data'].quantization() == (nsd,'GranularBitRound') - assert errmax < 1.e-3 - assert size < 0.24*uncompressed_size + assert f.variables['data'].quantization() == (nsd,'GranularBitRound') + assert errmax < 1.e-3 + assert size < 0.24*uncompressed_size f.close() # check lossy compression with shuffle and alternate quantization f = Dataset(self.files[6]) size = os.stat(self.files[6]).st_size errmax = (np.abs(data_array-f.variables['data'][:])).max() print('compressed lossy with shuffle and alternate quantization = ',size,' max err = ',errmax) - assert f.variables['data'].quantization() == (nsb,'BitRound') - assert errmax < 1.e-3 - assert size < 0.24*uncompressed_size + assert f.variables['data'].quantization() == (nsb,'BitRound') + assert errmax < 1.e-3 + assert size < 0.24*uncompressed_size f.close() if __name__ == '__main__': diff --git a/test/test_compression_zstd.py b/test/test_compression_zstd.py index 9f4259fd0..3557d5d0c 100644 --- a/test/test_compression_zstd.py +++ b/test/test_compression_zstd.py @@ -1,15 +1,20 @@ +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform from netCDF4 import Dataset from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_zstd_filter +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name filename2 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name array = uniform(size=(ndim,)) -def write_netcdf(filename,dtype='f8',complevel=6): +def write_netcdf(filename,dtype='f8',complevel: CompressionLevel = 6): nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ @@ -47,7 +52,7 @@ def runTest(self): assert_almost_equal(array,f.variables['data'][:]) assert f.variables['data'].filters() ==\ {'zlib':False,'szip':False,'zstd':True,'bzip2':False,'blosc':False,'shuffle':False,'complevel':4,'fletcher32':False} - assert size < 0.96*uncompressed_size + assert size < 0.96*uncompressed_size f.close() if __name__ == '__main__': diff --git a/test/test_endian.py b/test/test_endian.py index 99d6b44f8..716d1a958 100644 --- a/test/test_endian.py +++ b/test/test_endian.py @@ -73,13 +73,13 @@ def issue310(file): endian='little' else: raise ValueError('cannot determine native endianness') - var_big_endian = nc.createVariable(\ - 'obs_big_endian', '>f8', ('obs', ),\ - endian=endian,fill_value=fval) + var_big_endian = nc.createVariable( + 'obs_big_endian', '>f8', ('obs', ), endian=endian, fill_value=fval, # type: ignore # mypy is bad at narrowing endian + ) # use default _FillValue - var_big_endian2 = nc.createVariable(\ - 'obs_big_endian2', '>f8', ('obs', ),\ - endian=endian) + var_big_endian2 = nc.createVariable( + 'obs_big_endian2', '>f8', ('obs', ), endian=endian, # type: ignore # mypy is bad at narrowing endian + ) # NOTE: missing_value be written in same byte order # as variable, or masked array won't be masked correctly # when data is read in. diff --git a/test/test_enum.py b/test/test_enum.py index 0ab42ec6e..945905058 100644 --- a/test/test_enum.py +++ b/test/test_enum.py @@ -1,9 +1,10 @@ -import sys -import unittest import os import tempfile -from netCDF4 import Dataset +import unittest + +import netCDF4 import numpy as np +from netCDF4 import Dataset, EnumType from numpy.testing import assert_array_equal FILE_NAME = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -26,7 +27,7 @@ def setUp(self): cloud_type = f.createEnumType(ENUM_BASETYPE,ENUM_NAME,ENUM_DICT) # make sure KeyError raised if non-integer basetype used. try: - cloud_typ2 = f.createEnumType(np.float32,ENUM_NAME,ENUM_DICT) + cloud_typ2 = f.createEnumType(np.float32,ENUM_NAME,ENUM_DICT) # type: ignore[arg-type] # mypy correctly doesn't like float32 except KeyError: pass f.createDimension('time',None) @@ -49,6 +50,7 @@ def runTest(self): """testing enum data type""" f = Dataset(self.file, 'r') v = f.variables[VAR_NAME] + assert isinstance(v.datatype, EnumType) assert v.datatype.enum_dict == ENUM_DICT assert list(f.enumtypes.keys()) == [ENUM_NAME] assert f.enumtypes[ENUM_NAME].name == ENUM_NAME # issue 775 @@ -64,26 +66,30 @@ def runTest(self): f.close() class EnumDictTestCase(unittest.TestCase): + # issue 1128 def setUp(self): DT = np.int16; BITS = 8 self.STORED_VAL = DT(2**BITS) self.VAL_MAP = {f'bits_{n}': DT(2**n) for n in range(1,BITS+1)} - self.VAL_MAP['invalid'] = 0 + self.VAL_MAP['invalid'] = DT(0) self.file = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name - with netCDF4.Dataset(file, 'w') as nc: + with netCDF4.Dataset(self.file, 'w') as nc: # The enum is created with dtype=int16, so it will allow BITS values up to 15 et = nc.createEnumType(DT, 'etype', self.VAL_MAP) ev = nc.createVariable('evar', et) # Succeeds because the created EnumType does keep the correct dict ev[...] = self.STORED_VAL - def tearDown(self): - os.remove(self.file) - def runTest(self): - with netCDF4.Dataset(file, 'r') as nc: - read_var = nc['evar'] - assert read_var[...] == self.STORED_VAL - assert read_et.enum_dict == self.VAL_MAP + + def tearDown(self): + os.remove(self.file) + + def runTest(self): + with netCDF4.Dataset(self.file, 'r') as nc: + read_var = nc['evar'] + read_et = nc.enumtypes["etype"] + assert read_var[...] == self.STORED_VAL + assert read_et.enum_dict == self.VAL_MAP if __name__ == '__main__': unittest.main() diff --git a/test/test_masked.py b/test/test_masked.py index 0775f29ad..3cb2bf786 100644 --- a/test/test_masked.py +++ b/test/test_masked.py @@ -93,7 +93,8 @@ def setUp(self): # of raising an exception when auto-converted slice to a # masked array with netCDF4.Dataset(pathlib.Path(__file__).parent / "issue1152.nc") as dataset: - data = dataset['v'][:] + with self.assertWarns(UserWarning): + data = dataset['v'][:] # issue #1271 (mask is ignored when assigning bool array to uint8 var) ds = netCDF4.Dataset(self.file3, "w") diff --git a/test/test_masked3.py b/test/test_masked3.py index ed88109f2..c7cd4f2f5 100755 --- a/test/test_masked3.py +++ b/test/test_masked3.py @@ -59,7 +59,7 @@ def test_unscaled(self): self.assertEqual(v.dtype, "i2") self.assertTrue(isinstance(v, np.ndarray)) - self.assertTrue(not isinstance(v, ma.core.MaskedArray)) + self.assertTrue(not isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v) f.close() @@ -85,7 +85,7 @@ def test_scaled(self): self.assertEqual(v.dtype, "f8") self.assertTrue(isinstance(v, np.ndarray)) - self.assertTrue(not isinstance(v, ma.core.MaskedArray)) + self.assertTrue(not isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v_scaled) f.close() @@ -104,7 +104,7 @@ def test_unscaled(self): self.assertEqual(v_ma.dtype, "i2") self.assertTrue(isinstance(v_ma, np.ndarray)) - self.assertTrue(isinstance(v_ma, ma.core.MaskedArray)) + self.assertTrue(isinstance(v_ma, ma.masked_array)) assert_array_almost_equal(v_ma, self.v_ma) f.close() @@ -128,7 +128,7 @@ def test_scaled(self): self.assertEqual(v_ma.dtype, "f8") self.assertTrue(isinstance(v_ma, np.ndarray)) - self.assertTrue(isinstance(v_ma, ma.core.MaskedArray)) + self.assertTrue(isinstance(v_ma, ma.masked_array)) assert_array_almost_equal(v_ma, self.v_ma_scaled) f.close() diff --git a/test/test_masked4.py b/test/test_masked4.py index 14dd14fc4..61d9a1690 100755 --- a/test/test_masked4.py +++ b/test/test_masked4.py @@ -88,11 +88,11 @@ def test_scaled(self): v3 = f.variables["v3"][:] self.assertEqual(v.dtype, "f8") self.assertTrue(isinstance(v, np.ndarray)) - self.assertTrue(isinstance(v, ma.core.MaskedArray)) + self.assertTrue(isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v_scaled) self.assertEqual(v2.dtype, "f8") self.assertTrue(isinstance(v2, np.ndarray)) - self.assertTrue(isinstance(v2, ma.core.MaskedArray)) + self.assertTrue(isinstance(v2, ma.masked_array)) assert_array_almost_equal(v2, self.v_scaled) self.assertTrue(np.all(self.v_ma.mask == v.mask)) self.assertTrue(np.all(self.v_ma.mask == v2.mask)) diff --git a/test/test_masked5.py b/test/test_masked5.py index 3d8dba4db..87734024a 100755 --- a/test/test_masked5.py +++ b/test/test_masked5.py @@ -45,7 +45,7 @@ def test_scaled(self): f = Dataset(self.testfile) v = f.variables["v"] v2 = f.variables["v2"] - self.assertTrue(isinstance(v[:], ma.core.MaskedArray)) + self.assertTrue(isinstance(v[:], ma.masked_array)) assert_array_equal(v[:], self.v_ma) assert_array_equal(v[2],self.v[2]) # issue #624. v.set_auto_mask(False) diff --git a/test/test_masked6.py b/test/test_masked6.py index 65db53dde..dc77da99e 100644 --- a/test/test_masked6.py +++ b/test/test_masked6.py @@ -47,13 +47,13 @@ def test_always_mask(self): v = f.variables['v'][:] self.assertTrue(isinstance(v, np.ndarray)) - self.assertTrue(isinstance(v, ma.core.MaskedArray)) + self.assertTrue(isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v) w = f.variables['w'][:] self.assertTrue(isinstance(w, np.ndarray)) - self.assertTrue(isinstance(w, ma.core.MaskedArray)) + self.assertTrue(isinstance(w, ma.masked_array)) assert_array_almost_equal(w, self.w) f.close() @@ -69,13 +69,13 @@ def test_always_mask(self): v = f.variables['v'][:] self.assertTrue(isinstance(v, np.ndarray)) - self.assertFalse(isinstance(v, ma.core.MaskedArray)) + self.assertFalse(isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v) w = f.variables['w'][:] self.assertTrue(isinstance(w, np.ndarray)) - self.assertTrue(isinstance(w, ma.core.MaskedArray)) + self.assertTrue(isinstance(w, ma.masked_array)) assert_array_almost_equal(w, self.w) f.close() diff --git a/test/test_multifile.py b/test/test_multifile.py index 93d72bded..ddfc05ada 100644 --- a/test/test_multifile.py +++ b/test/test_multifile.py @@ -52,6 +52,7 @@ def runTest(self): assert_array_equal(np.arange(0,nx),f.variables['x'][:]) varin = f.variables['data'] datin = varin[:] + assert isinstance(data, np.ma.masked_array) assert_array_equal(datin.mask,data.mask) varin.set_auto_maskandscale(False) data2 = data.filled() @@ -73,7 +74,7 @@ def runTest(self): # testing multi-file get_variables_by_attributes f = MFDataset(self.files,check=True) assert f.get_variables_by_attributes(axis='T') == [] - f.get_variables_by_attributes(units='zlotys')[0] == f['x'] + assert f.get_variables_by_attributes(units='zlotys')[0] == f['x'] assert f.isopen() f.close() assert not f.isopen() @@ -123,13 +124,13 @@ def runTest(self): # Get the real dates dates = [] for file in self.files: - f = Dataset(file) - t = f.variables['time'] + ds = Dataset(file) + t = ds.variables['time'] dates.extend(cftime.num2date(t[:], t.units, calendar)) - f.close() + ds.close() # Compare with the MF dates - f = MFDataset(self.files,check=True) - t = f.variables['time'] + ds = MFDataset(self.files,check=True) + t = ds.variables['time'] T = MFTime(t, calendar=calendar) assert_equal(T.calendar, calendar) assert_equal(len(T), len(t)) @@ -141,7 +142,7 @@ def runTest(self): if Version(cftime.__version__) >= Version('1.0.1'): assert_array_equal(cftime.num2date(T[:], T.units, T.calendar), dates) assert_equal(cftime.date2index(datetime.datetime(1980, 1, 2), T), 366) - f.close() + ds.close() # Test exception is raised when no calendar attribute is available on the # time variable. @@ -153,8 +154,8 @@ def runTest(self): # variables. First, add calendar attributes to file. Note this will modify # the files inplace. calendars = ['standard', 'gregorian'] - for idx, f in enumerate(self.files): - with Dataset(f, 'a') as ds: + for idx, file in enumerate(self.files): + with Dataset(file, 'a') as ds: ds.variables['time'].calendar = calendars[idx] with MFDataset(self.files, check=True) as ds: with self.assertRaises(ValueError): diff --git a/test/test_multifile2.py b/test/test_multifile2.py index f8e8552a6..2c37a6f83 100644 --- a/test/test_multifile2.py +++ b/test/test_multifile2.py @@ -23,7 +23,7 @@ def setUp(self): for nfile,file in enumerate(self.files): f = Dataset(file,'w',format='NETCDF4_CLASSIC') #f.createDimension('x',None) - f.createDimension('x',ninc) + f.createDimension('x',int(ninc)) f.createDimension('y',ydim) f.createDimension('z',zdim) f.history = 'created today' @@ -52,6 +52,7 @@ def runTest(self): assert_array_equal(np.arange(0,nx),f.variables['x'][:]) varin = f.variables['data'] datin = varin[:] + assert isinstance(data, np.ma.masked_array) assert_array_equal(datin.mask,data.mask) varin.set_auto_maskandscale(False) data2 = data.filled() @@ -106,8 +107,8 @@ def runTest(self): # Get the real dates # skip this until cftime pull request #55 is in a released # version (1.0.1?). Otherwise, fix for issue #808 breaks this + dates = [] if Version(cftime.__version__) >= Version('1.0.1'): - dates = [] for file in self.files: f = Dataset(file) t = f.variables['time'] diff --git a/test/test_scaled.py b/test/test_scaled.py index 4a73ba3f7..5c1ce9542 100755 --- a/test/test_scaled.py +++ b/test/test_scaled.py @@ -68,7 +68,7 @@ def test_unmasked(self): self.assertEqual(v.dtype, "i2") self.assertTrue(isinstance(v, np.ndarray)) # issue 785: always return masked array by default - self.assertTrue(isinstance(v, ma.core.MaskedArray)) + self.assertTrue(isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v) f.close() @@ -93,7 +93,7 @@ def test_masked(self): self.assertEqual(v_ma.dtype, "i2") self.assertTrue(isinstance(v_ma, np.ndarray)) - self.assertTrue(isinstance(v_ma, ma.core.MaskedArray)) + self.assertTrue(isinstance(v_ma, ma.masked_array)) assert_array_almost_equal(v_ma, self.v_ma) f.close() @@ -118,7 +118,7 @@ def test_unmasked(self): self.assertEqual(v_scaled.dtype, "f8") self.assertTrue(isinstance(v_scaled, np.ndarray)) # issue 785: always return masked array by default - self.assertTrue(isinstance(v_scaled, ma.core.MaskedArray)) + self.assertTrue(isinstance(v_scaled, ma.masked_array)) assert_array_almost_equal(v_scaled, self.v_scaled) f.close() @@ -141,7 +141,7 @@ def test_masked(self): self.assertEqual(v_ma_scaled.dtype, "f8") self.assertTrue(isinstance(v_ma_scaled, np.ndarray)) - self.assertTrue(isinstance(v_ma_scaled, ma.core.MaskedArray)) + self.assertTrue(isinstance(v_ma_scaled, ma.masked_array)) assert_array_almost_equal(v_ma_scaled, self.v_ma_scaled) f.close() diff --git a/test/test_slicing.py b/test/test_slicing.py index 8d3f88b7d..16d384ded 100644 --- a/test/test_slicing.py +++ b/test/test_slicing.py @@ -103,7 +103,7 @@ def test_0d(self): assert_equal(v.shape, v[...].shape) # issue #785: always return masked array #assert type(v[...]) == np.ndarray - assert type(v[...]) == np.ma.core.MaskedArray + assert type(v[...]) == np.ma.masked_array f.set_auto_mask(False) assert type(v[...]) == np.ndarray f.close() diff --git a/test/test_stringarr.py b/test/test_stringarr.py index 24c890fb2..9d4fcd909 100644 --- a/test/test_stringarr.py +++ b/test/test_stringarr.py @@ -24,7 +24,7 @@ class StringArrayTestCase(unittest.TestCase): def setUp(self): self.file = FILE_NAME - nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT) + nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT) # type: ignore # FILE_FORMAT nc.createDimension('n1',None) nc.createDimension('n2',n2) nc.createDimension('nchar',nchar) @@ -70,19 +70,19 @@ def runTest(self): assert_array_equal(data3,datau) # these slices should return a char array, not a string array data4 = v2[:,:,0] - assert data4.dtype.itemsize == 1 + assert data4.dtype.itemsize == 1 assert_array_equal(data4, datac[:,:,0]) data5 = v2[0,0:nchar,0] - assert data5.dtype.itemsize == 1 + assert data5.dtype.itemsize == 1 assert_array_equal(data5, datac[0,0:nchar,0]) # test turning auto-conversion off. v2.set_auto_chartostring(False) data6 = v2[:] - assert data6.dtype.itemsize == 1 + assert data6.dtype.itemsize == 1 assert_array_equal(data6, datac) nc.set_auto_chartostring(False) data7 = v3[:] - assert data7.dtype.itemsize == 1 + assert data7.dtype.itemsize == 1 assert_array_equal(data7, datac) nc.close() diff --git a/test/test_types.py b/test/test_types.py index 048843a98..3c5054bbd 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -1,4 +1,5 @@ import sys +from typing import TYPE_CHECKING, Any import unittest import os import tempfile @@ -6,6 +7,10 @@ from numpy.testing import assert_array_equal, assert_array_almost_equal from numpy.random.mtrand import uniform import netCDF4 +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any # test primitive data types. @@ -14,7 +19,7 @@ n1dim = 5 n2dim = 10 ranarr = 100.*uniform(size=(n1dim,n2dim)) -zlib=False;complevel=0;shuffle=0;least_significant_digit=None +zlib=False; complevel=0; shuffle=False; least_significant_digit=None datatypes = ['f8','f4','i1','i2','i4','i8','u1','u2','u4','u8','S1'] FillValue = 1.0 issue273_data = np.ma.array(['z']*10,dtype='S1',\ @@ -28,7 +33,16 @@ def setUp(self): f.createDimension('n1', None) f.createDimension('n2', n2dim) for typ in datatypes: - foo = f.createVariable('data_'+typ, typ, ('n1','n2',),zlib=zlib,complevel=complevel,shuffle=shuffle,least_significant_digit=least_significant_digit,fill_value=FillValue) + foo = f.createVariable( + f"data_{typ}", + typ, + ('n1','n2',), + zlib=zlib, + complevel=complevel, # type: ignore # type checkers bad at narrowing + shuffle=shuffle, + least_significant_digit=least_significant_digit, + fill_value=FillValue, + ) #foo._FillValue = FillValue # test writing of _FillValue attribute for diff types # (should be cast to type of variable silently) @@ -52,10 +66,11 @@ def runTest(self): for typ in datatypes: data = f.variables['data_'+typ] data.set_auto_maskandscale(False) - datarr = data[1:n1dim] + datarr: np.ndarray = data[1:n1dim] # fill missing data with _FillValue # ('S1' array will have some missing values) if hasattr(datarr, 'mask'): + assert isinstance(datarr, np.ma.masked_array) datarr = datarr.filled() datfilled = data[0] # check to see that data type is correct @@ -81,7 +96,7 @@ def runTest(self): v2 = f.variables['issue273'] assert type(v2._FillValue) == bytes assert v2._FillValue == b'\x00' - assert str(issue273_data) == str(v2[:]) + assert str(issue273_data) == str(v2[:]) # issue 707 (don't apply missing_value if cast to variable type is # unsafe) v3 = f.variables['issue707'] diff --git a/test/test_utils.py b/test/test_utils.py index af205be4c..88ee12c62 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -53,14 +53,14 @@ def test_fancy(self): try: - elem = ( np.arange(6).reshape((3,2)), slice(None), slice(None) ) - start, count, stride, put_ind = _StartCountStride(elem, (3,4,5)) + elem2 = ( np.arange(6).reshape((3,2)), slice(None), slice(None) ) + start, count, stride, put_ind = _StartCountStride(elem2, (3,4,5)) except IndexError: pass # this one should be converted to a slice - elem = [slice(None), [1,3,5], 8] - start, count, stride, put_ind = _StartCountStride(elem, (50, 6, 10)) + elem3 = [slice(None), [1,3,5], 8] + start, count, stride, put_ind = _StartCountStride(elem3, (50, 6, 10)) # pull request #683 now does not convert integer sequences to strided # slices. PR #1224 reverts this behavior. assert_equal(put_ind[...,1].squeeze(), slice(None,None,None)) @@ -81,8 +81,8 @@ def test_multiple_sequences(self): assert_equal(count[...,2], 10) i = [1,3,4] - elem = (i, i, i) - start, count, stride, put_ind = _StartCountStride(elem, (50, 5, 10)) + elem2 = (i, i, i) + start, count, stride, put_ind = _StartCountStride(elem2, (50, 5, 10)) assert_equal(_out_array_shape(count), (3,3,3)) def test_put_indices(self): @@ -198,15 +198,15 @@ def test_ellipsis(self): assert_equal(put_ind[0,0,0,0,0], (slice(None), slice(None), slice(None), slice(None), slice(None))) try: - elem=(Ellipsis, [15,16,17,18,19], slice(None)) - start, count, stride, put_ind = _StartCountStride(elem, (2,10,20,10,10)) + elem2=(Ellipsis, [15,16,17,18,19], slice(None)) + start, count, stride, put_ind = _StartCountStride(elem2, (2,10,20,10,10)) assert_equal(None, 'Should throw an exception') except IndexError as e: assert_equal(str(e), "integer index exceeds dimension size") try: - elem=(Ellipsis, [15,16,17,18,19], Ellipsis) - start, count, stride, put_ind = _StartCountStride(elem, (2,10, 20,10,10)) + elem3=(Ellipsis, [15,16,17,18,19], Ellipsis) + start, count, stride, put_ind = _StartCountStride(elem3, (2,10, 20,10,10)) assert_equal(None, 'Should throw an exception') except IndexError as e: assert_equal(str(e), "At most one ellipsis allowed in a slicing expression") @@ -303,8 +303,8 @@ def test_ellipsis(self): assert_equal(take_ind[0,0,0,0,0], (slice(None), slice(None), slice(None), slice(None), slice(None))) try: - elem=(Ellipsis, [15,16,17,18,19], slice(None)) - start, count, stride, take_ind = _StartCountStride(elem, (2,10,20,10,10),\ + elem2=(Ellipsis, [15,16,17,18,19], slice(None)) + start, count, stride, take_ind = _StartCountStride(elem2, (2,10,20,10,10),\ ['time', 'z', 'y', 'x'], grp, (2,10,5,10,10), put=True) assert_equal(None, 'Should throw an exception') except IndexError as e: @@ -312,8 +312,8 @@ def test_ellipsis(self): assert_equal(str(e), "list index out of range") try: - elem=(Ellipsis, [15,16,17,18,19], Ellipsis) - start, count, stride, take_ind = _StartCountStride(elem, (2,10, 20,10,10),\ + elem3=(Ellipsis, [15,16,17,18,19], Ellipsis) + start, count, stride, take_ind = _StartCountStride(elem3, (2,10, 20,10,10),\ ['time', 'z', 'y', 'x'], grp, (2,10,5,10,10), put=True) assert_equal(None, 'Should throw an exception') except IndexError as e: