From 851d45327d476f59650b66afe585a361d96ddec7 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 11 Jul 2024 09:08:50 -0700 Subject: [PATCH 01/81] Reformat type stubs with ruff to 130 chars --- src/netCDF4/__init__.pyi | 227 +++++++++++++++------------------------ src/netCDF4/_netCDF4.pyi | 58 +++++++--- 2 files changed, 133 insertions(+), 152 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 413336f0e..698185a5f 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -1,23 +1,52 @@ - +import datetime as dt import os import sys -import datetime as dt -from typing import (Any, Callable, Final, Generic, Iterable, Literal, Mapping, - NoReturn, Self, Sequence, TypeAlias, TypedDict, TypeVar, Union, overload) -from typing_extensions import Buffer +from typing import ( + Any, + Callable, + Final, + Generic, + Iterable, + Literal, + Mapping, + NoReturn, + Self, + Sequence, + TypeAlias, + TypedDict, + TypeVar, + Union, + overload, +) import cftime import numpy as np import numpy.typing as npt +from typing_extensions import Buffer __all__ = [ - 'Dataset', 'Variable', 'Dimension', 'Group', 'MFDataset', 'MFTime', 'CompoundType', - 'VLType', 'date2num', 'num2date', 'date2index', 'stringtochar', 'chartostring', - 'stringtoarr', 'getlibversion', 'EnumType', 'get_chunk_cache', 'set_chunk_cache', - 'set_alignment', 'get_alignment' + "Dataset", + "Variable", + "Dimension", + "Group", + "MFDataset", + "MFTime", + "CompoundType", + "VLType", + "date2num", + "num2date", + "date2index", + "stringtochar", + "chartostring", + "stringtoarr", + "getlibversion", + "EnumType", + "get_chunk_cache", + "set_chunk_cache", + "set_alignment", + "get_alignment", ] -__pdoc__ = {'utils': False} - +__pdoc__ = {"utils": False} if sys.version_info >= (3, 10): from types import EllipsisType @@ -26,10 +55,8 @@ if sys.version_info >= (3, 10): elif not TYPE_CHECKING: ellipsis = type(Ellipsis) # keeps ruff happy until ruff uses typeshed - _DatatypeStrOptions: TypeAlias = Literal[ - 'S1', 'c', 'i1', 'b', 'B', 'u1', 'i2', 'h', 's', 'u2', 'i4', - 'i', 'l', 'u4', 'i8', 'u8', 'f4', 'f', 'f8', 'd', 'c8', 'c16' + "S1", "c", "i1", "b", "B", "u1", "i2", "h", "s", "u2", "i4", "i", "l", "u4", "i8", "u8", "f4", "f", "f8", "d", "c8", "c16" ] _DatatypeNCOptions: TypeAlias = Union[CompoundType, VLType, EnumType] DatatypeOptions: TypeAlias = Union[_DatatypeStrOptions, _DatatypeNCOptions, npt.DTypeLike] @@ -38,21 +65,16 @@ T_DatatypeNC = TypeVar("T_DatatypeNC", CompoundType, VLType, EnumType) DimensionsOptions: TypeAlias = Union[str, bytes, Dimension, Iterable[Union[str, bytes, Dimension]]] CompressionOptions: TypeAlias = Literal[ - 'zlib', 'szip', 'zstd', 'blosc_lz','blosc_lz4', - 'blosc_lz4hc', 'blosc_zlib', 'blosc_zstd' + "zlib", "szip", "zstd", "blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd" ] CompressionLevelOptions: TypeAlias = Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] -AccessModeOptions: TypeAlias = Literal['r', 'w', 'r+', 'a', 'x', 'rs', 'ws', 'r+s', 'as'] -FormatOptions: TypeAlias = Literal[ - 'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_CLASSIC', - 'NETCDF3_64BIT_OFFSET', 'NETCDF3_64BIT_DATA' -] -DiskFormatOptions: TypeAlias = Literal['NETCDF3', 'HDF5', 'HDF4', 'PNETCDF', 'DAP2', 'DAP4', 'UNDEFINED'] -QuantizeOptions: TypeAlias = Literal['BitGroom', 'BitRound', 'GranularBitRound'] -EndianOptions: TypeAlias = Literal['native', 'little', 'big'] +AccessModeOptions: TypeAlias = Literal["r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"] +FormatOptions: TypeAlias = Literal["NETCDF4", "NETCDF4_CLASSIC", "NETCDF3_CLASSIC", "NETCDF3_64BIT_OFFSET", "NETCDF3_64BIT_DATA"] +DiskFormatOptions: TypeAlias = Literal["NETCDF3", "HDF5", "HDF4", "PNETCDF", "DAP2", "DAP4", "UNDEFINED"] +QuantizeOptions: TypeAlias = Literal["BitGroom", "BitRound", "GranularBitRound"] +EndianOptions: TypeAlias = Literal["native", "little", "big"] CalendarOptions: TypeAlias = Literal[ - 'standard', 'gregorian', 'proleptic_gregorian' 'noleap', - '365_day', '360_day', 'julian', 'all_leap', '366_day' + "standard", "gregorian", "proleptic_gregorian" "noleap", "365_day", "360_day", "julian", "all_leap", "366_day" ] BoolInt: TypeAlias = Literal[0, 1] @@ -89,11 +111,10 @@ __has_set_alignment__: BoolInt __has_ncfilter__: BoolInt is_native_little: bool is_native_big: bool -default_encoding: Final = 'utf-8' -unicode_error: Final = 'replace' +default_encoding: Final = "utf-8" +unicode_error: Final = "replace" default_fillvals: dict[str, int | float | str] - # date2index, date2num, and num2date are actually provided by cftime and if stubs for # cftime are completed these should be removed. def date2index( @@ -119,19 +140,17 @@ def num2date( has_year_zero: bool | None = None, ) -> dt.datetime | DateTimeArray: ... - class BloscInfo(TypedDict): compressor: Literal["blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd"] shuffle: Literal[0, 1, 2] - class SzipInfo(TypedDict): coding: Literal["nn", "ec"] pixels_per_block: Literal[4, 8, 16, 32] - class FiltersDict(TypedDict): """Dict returned from netCDF4.Variable.filters()""" + zlib: bool szip: Literal[False] | SzipInfo zstd: bool @@ -141,18 +160,16 @@ class FiltersDict(TypedDict): complevel: int fletcher32: bool - class NetCDF4MissingFeatureException(Exception): def __init__(self, feature: str, version: str): ... - class Dataset: def __init__( self, filename: str | os.PathLike, - mode: AccessModeOptions = 'r', + mode: AccessModeOptions = "r", clobber: bool = True, - format: FormatOptions = 'NETCDF4', + format: FormatOptions = "NETCDF4", diskless: bool = False, persist: bool = False, keepweakref: bool = False, @@ -162,9 +179,8 @@ class Dataset: comm: Any = None, info: Any = None, auto_complex: bool = False, - **kwargs: Any + **kwargs: Any, ): ... - @property def name(self) -> str: ... @property @@ -197,16 +213,14 @@ class Dataset: def _ncstring_attrs__(self) -> bool: ... @property def __orthogonal_indexing__(self) -> bool: ... - def filepath(self, encoding: str | None = None) -> str: ... def isopen(self) -> bool: ... def close(self) -> memoryview: ... # only if writing and memory != None, but otherwise people ignore the return None anyway def sync(self) -> None: ... def set_fill_on(self) -> None: ... def set_fill_off(self) -> None: ... - def createDimension(self, dimname: str, size: int | None = None) -> Dimension: ... - def renameDimension( self, oldname: str, newname: str) -> None: ... + def renameDimension(self, oldname: str, newname: str) -> None: ... @overload def createVariable( # type: ignore self, @@ -217,18 +231,18 @@ class Dataset: zlib: bool = False, complevel: CompressionLevelOptions | None = 4, shuffle: bool = True, - szip_coding: Literal['nn', 'ec'] = 'nn', + szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: int | None = None, - endian: EndianOptions = 'native', + endian: EndianOptions = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = 'BitGroom', + quantize_mode: QuantizeOptions = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, - chunk_cache: int | None = None + chunk_cache: int | None = None, ) -> Variable[T_DatatypeNC]: ... @overload def createVariable( @@ -240,22 +254,21 @@ class Dataset: zlib: bool = False, complevel: CompressionLevelOptions | None = 4, shuffle: bool = True, - szip_coding: Literal['nn', 'ec'] = 'nn', + szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: int | None = None, - endian: EndianOptions = 'native', + endian: EndianOptions = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = 'BitGroom', + quantize_mode: QuantizeOptions = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, - chunk_cache: int | None = None + chunk_cache: int | None = None, ) -> Variable[np.dtype]: ... def renameVariable(self, oldname: str, newname: str) -> None: ... def createGroup(self, groupname: str) -> Group: ... - def renameGroup(self, oldname: str, newname: str) -> None: ... def renameAttribute(self, oldname: str, newname: str) -> None: ... def createCompoundType( @@ -265,12 +278,11 @@ class Dataset: def createEnumType( self, datatype: np.dtype[np.integer] | type[np.integer] | type[int], datatype_name: str, enum_dict: dict[str, int] ) -> EnumType: ... - def ncattrs(self) -> list[str]: ... def setncattr_string(self, name: str, value: Any) -> None: ... def setncattr(self, name: str, value: Any) -> None: ... def setncatts(self, attdict: Mapping[str, Any]) -> None: ... - def getncattr(self, name: str, encoding: str = 'utf-8') -> Any: ... + def getncattr(self, name: str, encoding: str = "utf-8") -> Any: ... def delncattr(self, name: str) -> None: ... def set_auto_chartostring(self, value: bool) -> None: ... def set_auto_maskandscale(self, value: bool) -> None: ... @@ -279,35 +291,18 @@ class Dataset: def set_always_mask(self, value: bool) -> None: ... def set_ncstring_attrs(self, value: bool) -> None: ... def get_variables_by_attributes(self, **kwargs: Callable[[Any], bool] | Any) -> list[Variable]: ... - @staticmethod def fromcdl( - cdlfilename: str, - ncfilename: str | None = None, - mode: AccessModeOptions = 'a', - format: FormatOptions = 'NETCDF4' + cdlfilename: str, ncfilename: str | None = None, mode: AccessModeOptions = "a", format: FormatOptions = "NETCDF4" ) -> Dataset: ... @overload - def tocdl( - self, - coordvars: bool = False, - data: bool = False, - outfile: None = None - ) -> str: ... + def tocdl(self, coordvars: bool = False, data: bool = False, outfile: None = None) -> str: ... @overload - def tocdl( - self, - coordvars: bool = False, - data: bool = False, - *, - outfile: str | os.PathLike - ) -> None: ... - + def tocdl(self, coordvars: bool = False, data: bool = False, *, outfile: str | os.PathLike) -> None: ... def has_blosc_filter(self) -> bool: ... def has_zstd_filter(self) -> bool: ... def has_bzip2_filter(self) -> bool: ... def has_szip_filter(self) -> bool: ... - def __getitem__(self, elem: str) -> Any: ... # should be Group | Variable, but this causes too many problems def __setattr__(self, name: str, value: Any) -> None: ... def __getattr__(self, name: str) -> Any: ... @@ -317,29 +312,21 @@ class Dataset: def __enter__(self) -> Self: ... def __exit__(self, atype, value, traceback) -> None: ... - class Group(Dataset): def __init__(self, parent: Dataset, name: str, **kwargs: Any) -> None: ... - def close(self) -> NoReturn: ... - class Dimension: def __init__(self, grp: Dataset, name: str, size: int | None = None, **kwargs: Any) -> None: ... - @property def name(self) -> str: ... @property def size(self) -> int: ... - def group(self) -> Dataset: ... def isunlimited(self) -> bool: ... - def __len__(self) -> int: ... - class Variable(Generic[T_Datatype]): - @overload def __new__( # type: ignore self, @@ -351,21 +338,20 @@ class Variable(Generic[T_Datatype]): zlib: bool = False, complevel: CompressionLevelOptions | None = 4, shuffle: bool = True, - szip_coding: Literal['nn', 'ec'] = 'nn', + szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: Sequence[int] | None = None, - endian: EndianOptions = 'native', + endian: EndianOptions = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = 'BitGroom', + quantize_mode: QuantizeOptions = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, - **kwargs: Any + **kwargs: Any, ) -> Variable[T_DatatypeNC]: ... - @overload def __new__( self, @@ -377,22 +363,20 @@ class Variable(Generic[T_Datatype]): zlib: bool = False, complevel: CompressionLevelOptions | None = 4, shuffle: bool = True, - szip_coding: Literal['nn', 'ec'] = 'nn', + szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: Sequence[int] | None = None, - endian: EndianOptions = 'native', + endian: EndianOptions = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = 'BitGroom', + quantize_mode: QuantizeOptions = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, - **kwargs: Any + **kwargs: Any, ) -> Variable[np.dtype]: ... - - def __init__( self, grp: Dataset, @@ -403,21 +387,20 @@ class Variable(Generic[T_Datatype]): zlib: bool = False, complevel: CompressionLevelOptions | None = 4, shuffle: bool = True, - szip_coding: Literal['nn', 'ec'] = 'nn', + szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: Sequence[int] | None = None, - endian: EndianOptions = 'native', + endian: EndianOptions = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = 'BitGroom', + quantize_mode: QuantizeOptions = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, - **kwargs: Any + **kwargs: Any, ) -> None: ... - @property def name(self) -> str: ... @property @@ -442,24 +425,20 @@ class Variable(Generic[T_Datatype]): def always_mask(self) -> bool: ... @property def __orthogonal_indexing__(self) -> bool: ... - def group(self) -> Dataset: ... def ncattrs(self) -> list[str]: ... def setncattr(self, name: str, value: Any) -> None: ... def setncattr_string(self, name: str, value: Any) -> None: ... def setncatts(self, attdict: Mapping[str, Any]) -> None: ... - def getncattr(self, name: str, encoding='utf-8'): ... + def getncattr(self, name: str, encoding="utf-8"): ... def delncattr(self, name: str) -> None: ... def filters(self) -> FiltersDict: ... def quantization(self) -> tuple[int, QuantizeOptions] | None: ... def endian(self) -> EndianOptions: ... - def chunking(self) -> Literal['contiguous'] | list[int]: ... + def chunking(self) -> Literal["contiguous"] | list[int]: ... def get_var_chunk_cache(self) -> tuple[int, int, float]: ... def set_var_chunk_cache( - self, - size: int | None = None, - nelems: int | None = None, - preemption: float | None = None + self, size: int | None = None, nelems: int | None = None, preemption: float | None = None ) -> None: ... def renameAttribute(self, oldname: str, newname: str) -> None: ... def assignValue(self, val: Any) -> None: ... @@ -473,7 +452,6 @@ class Variable(Generic[T_Datatype]): def set_ncstring_attrs(self, ncstring_attrs: bool) -> None: ... def set_collective(self, value: bool) -> None: ... def get_dims(self) -> tuple[Dimension, ...]: ... - def __delattr__(self, name: str) -> None: ... def __setattr__(self, name: str, value: Any) -> None: ... def __getattr__(self, name: str) -> Any: ... @@ -482,7 +460,6 @@ class Variable(Generic[T_Datatype]): def __array__(self) -> np.ndarray: ... def __len__(self) -> int: ... - class CompoundType: dtype: np.dtype dtype_view: np.dtype @@ -491,19 +468,15 @@ class CompoundType: def __init__( self, grp: Dataset, dt: npt.DTypeLike | Sequence[tuple[str, npt.DTypeLike]], dtype_name: str, **kwargs: Any ) -> None: ... - def __reduce__(self) -> NoReturn: ... - class VLType: dtype: np.dtype name: str | None def __init__(self, grp: Dataset, dt: npt.DTypeLike, dtype_name: str, **kwargs: Any) -> None: ... - def __reduce__(self) -> NoReturn: ... - class EnumType: dtype: np.dtype[np.integer] name: str @@ -515,12 +488,10 @@ class EnumType: dt: np.dtype[np.integer] | type[np.integer] | type[int] | str, dtype_name: str, enum_dict: Mapping[str, int], - **kwargs: Any + **kwargs: Any, ) -> None: ... - def __reduce__(self) -> NoReturn: ... - class MFDataset(Dataset): def __init__( self, @@ -528,27 +499,21 @@ class MFDataset(Dataset): check: bool = False, aggdim: str | None = None, exclude: Sequence[str] = [], - master_file: str | os.PathLike | None = None + master_file: str | os.PathLike | None = None, ) -> None: ... - @property def dimensions(self) -> dict[str, Dimension]: ... # this should be: dict[str, Dimension | _Dimension] @property def variables(self) -> dict[str, Variable[Any]]: ... # this should be: dict[str, _Variable[Any] | _Variable] - class _Dimension: dimlens: list[int] dimtolen: int - def __init__( - self, dimname: str, dim: Dimension, dimlens: list[int], dimtotlen: int - ) -> None: ... - + def __init__(self, dimname: str, dim: Dimension, dimlens: list[int], dimtotlen: int) -> None: ... def __len__(self) -> int: ... def isunlimited(self) -> Literal[True]: ... - class _Variable: dimensions: tuple[str, ...] dtype: np.dtype | type[str] @@ -562,7 +527,6 @@ class _Variable: def ndim(self) -> int: ... @property def name(self) -> str: ... - def typecode(self) -> np.dtype | type[str]: ... def ncattrs(self) -> list[str]: ... def _shape(self) -> tuple[int, ...]: ... @@ -571,30 +535,22 @@ class _Variable: def set_auto_mask(self, val: bool) -> None: ... def set_auto_scale(self, val: bool) -> None: ... def set_always_mask(self, val: bool) -> None: ... - def __getattr__(self, name: str) -> Any: ... def __getitem__(self, elem: GetSetItemKey) -> Any: ... def __len__(self) -> int: ... - class MFTime(_Variable): calendar: CalendarOptions | None units: str | None - def __init__( - self, - time: Variable, - units: str | None = None, - calendar: CalendarOptions | None = None - ): ... + def __init__(self, time: Variable, units: str | None = None, calendar: CalendarOptions | None = None): ... def __getitem__(self, elem: GetSetItemKey) -> np.ndarray: ... - @overload def stringtoarr( string: str, NUMCHARS: int, - dtype: Literal["S"] | np.dtype[np.bytes_]= "S", + dtype: Literal["S"] | np.dtype[np.bytes_] = "S", ) -> npt.NDArray[np.bytes_]: ... @overload def stringtoarr( @@ -622,15 +578,8 @@ def chartostring( b: npt.NDArray[np.character], encoding: str = ..., ) -> npt.NDArray[np.str_] | npt.NDArray[np.bytes_]: ... - def getlibversion() -> str: ... - def set_alignment(threshold: int, alignment: int): ... def get_alignment() -> tuple[int, int]: ... - -def set_chunk_cache( - size: int | None = None, - nelems: int | None = None, - preemption: float | None = None -) -> None: ... +def set_chunk_cache(size: int | None = None, nelems: int | None = None, preemption: float | None = None) -> None: ... def get_chunk_cache() -> tuple[int, int, float]: ... diff --git a/src/netCDF4/_netCDF4.pyi b/src/netCDF4/_netCDF4.pyi index 536d4f63b..5c407a9dc 100644 --- a/src/netCDF4/_netCDF4.pyi +++ b/src/netCDF4/_netCDF4.pyi @@ -1,19 +1,51 @@ # The definitions are intendionally done in the __init__. # This file only exists in case someone imports from netCDF4._netCDF4 from . import ( - Dataset, Variable, Dimension, Group, MFDataset, MFTime, CompoundType, - VLType, date2num, num2date, date2index, stringtochar, chartostring, - stringtoarr, getlibversion, EnumType, get_chunk_cache, set_chunk_cache, - set_alignment, get_alignment, default_fillvals, default_encoding, - NetCDF4MissingFeatureException, is_native_big, is_native_little, unicode_error, - __version__, __netcdf4libversion__, __hdf5libversion__, __has_rename_grp__, - __has_nc_inq_path__, __has_nc_inq_format_extended__, __has_nc_open_mem__, - __has_nc_create_mem__, __has_cdf5_format__, __has_parallel4_support__, - __has_pnetcdf_support__, __has_parallel_support__, - __has_quantization_support__, __has_zstandard_support__, - __has_bzip2_support__, __has_blosc_support__, __has_szip_support__, - __has_set_alignment__, __has_ncfilter__ + CompoundType, + Dataset, + Dimension, + EnumType, + Group, + MFDataset, + MFTime, + NetCDF4MissingFeatureException, + Variable, + VLType, + __has_blosc_support__, + __has_bzip2_support__, + __has_cdf5_format__, + __has_nc_create_mem__, + __has_nc_inq_format_extended__, + __has_nc_inq_path__, + __has_nc_open_mem__, + __has_ncfilter__, + __has_parallel4_support__, + __has_parallel_support__, + __has_pnetcdf_support__, + __has_quantization_support__, + __has_rename_grp__, + __has_set_alignment__, + __has_szip_support__, + __has_zstandard_support__, + __hdf5libversion__, + __netcdf4libversion__, + __version__, + chartostring, + date2index, + date2num, + default_encoding, + default_fillvals, + get_alignment, + get_chunk_cache, + getlibversion, + is_native_big, + is_native_little, + num2date, + set_alignment, + set_chunk_cache, + stringtoarr, + stringtochar, + unicode_error, ) - def dtype_is_complex(dtype: str) -> bool: ... From 5098b5e99249de4a30062daaf775f77b9ab0a3b1 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 11 Jul 2024 09:18:50 -0700 Subject: [PATCH 02/81] Import typing.TYPE_CHECKING, import Self and TypeAlias from typing_extensions for 3.8 compat --- src/netCDF4/__init__.pyi | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 698185a5f..f484b5629 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -2,6 +2,7 @@ import datetime as dt import os import sys from typing import ( + TYPE_CHECKING, Any, Callable, Final, @@ -10,9 +11,7 @@ from typing import ( Literal, Mapping, NoReturn, - Self, Sequence, - TypeAlias, TypedDict, TypeVar, Union, @@ -22,7 +21,7 @@ from typing import ( import cftime import numpy as np import numpy.typing as npt -from typing_extensions import Buffer +from typing_extensions import Buffer, Self, TypeAlias __all__ = [ "Dataset", From c45529d41f143bd833691ba130cc9b7e082b9a55 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 11 Jul 2024 09:20:28 -0700 Subject: [PATCH 03/81] Add missing comma to strings in CalendarOptions --- src/netCDF4/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index f484b5629..ed8b516bf 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -73,7 +73,7 @@ DiskFormatOptions: TypeAlias = Literal["NETCDF3", "HDF5", "HDF4", "PNETCDF", "DA QuantizeOptions: TypeAlias = Literal["BitGroom", "BitRound", "GranularBitRound"] EndianOptions: TypeAlias = Literal["native", "little", "big"] CalendarOptions: TypeAlias = Literal[ - "standard", "gregorian", "proleptic_gregorian" "noleap", "365_day", "360_day", "julian", "all_leap", "366_day" + "standard", "gregorian", "proleptic_gregorian", "noleap", "365_day", "360_day", "julian", "all_leap", "366_day" ] BoolInt: TypeAlias = Literal[0, 1] From 361ead079d503f1b74d93929cb86bdb8eaf83225 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 11 Jul 2024 09:24:18 -0700 Subject: [PATCH 04/81] dtype_is_complex is publicly available under the netCDF4 module due to 'from ... import *' without '__all__' in _netCDF4.pyx --- src/netCDF4/__init__.pyi | 4 ++++ src/netCDF4/_netCDF4.pyi | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index ed8b516bf..cf544575a 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -162,6 +162,10 @@ class FiltersDict(TypedDict): class NetCDF4MissingFeatureException(Exception): def __init__(self, feature: str, version: str): ... + +def dtype_is_complex(dtype: str) -> bool: ... + + class Dataset: def __init__( self, diff --git a/src/netCDF4/_netCDF4.pyi b/src/netCDF4/_netCDF4.pyi index 5c407a9dc..f649647fe 100644 --- a/src/netCDF4/_netCDF4.pyi +++ b/src/netCDF4/_netCDF4.pyi @@ -35,6 +35,7 @@ from . import ( date2num, default_encoding, default_fillvals, + dtype_is_complex, get_alignment, get_chunk_cache, getlibversion, @@ -47,5 +48,3 @@ from . import ( stringtochar, unicode_error, ) - -def dtype_is_complex(dtype: str) -> bool: ... From 147337d55c804daa22b22e9c78801af31c1b6be0 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 11 Jul 2024 09:29:38 -0700 Subject: [PATCH 05/81] minor whitespace --- src/netCDF4/__init__.pyi | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index cf544575a..6be7c9f3a 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -162,10 +162,8 @@ class FiltersDict(TypedDict): class NetCDF4MissingFeatureException(Exception): def __init__(self, feature: str, version: str): ... - def dtype_is_complex(dtype: str) -> bool: ... - class Dataset: def __init__( self, From 03e57b1f713dd6d6fcf3930f2cdb750396ba783c Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 11 Jul 2024 09:31:14 -0700 Subject: [PATCH 06/81] Use cls instead of self in __new__ --- src/netCDF4/__init__.pyi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 6be7c9f3a..efb1c7780 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -329,8 +329,8 @@ class Dimension: class Variable(Generic[T_Datatype]): @overload - def __new__( # type: ignore - self, + def __new__( + cls, grp: Dataset, name: str, datatype: T_DatatypeNC, @@ -355,7 +355,7 @@ class Variable(Generic[T_Datatype]): ) -> Variable[T_DatatypeNC]: ... @overload def __new__( - self, + cls, grp: Dataset, name: str, datatype: _DatatypeStrOptions | npt.DTypeLike, From 4badee2f8bdb1f9b288f3f208afca7410e4230d0 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 11 Jul 2024 09:32:51 -0700 Subject: [PATCH 07/81] Fix '__dealloc' >> '__dealloc__' --- src/netCDF4/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index efb1c7780..5e62f2f1c 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -308,7 +308,7 @@ class Dataset: def __setattr__(self, name: str, value: Any) -> None: ... def __getattr__(self, name: str) -> Any: ... def __delattr__(self, name: str): ... - def __dealloc(self) -> None: ... + def __dealloc__(self) -> None: ... def __reduce__(self) -> NoReturn: ... def __enter__(self) -> Self: ... def __exit__(self, atype, value, traceback) -> None: ... From d91bc54d99dacada5d59e8774569e2df369ef986 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 11 Jul 2024 09:57:04 -0700 Subject: [PATCH 08/81] Rename type aliases to singular names --- src/netCDF4/__init__.pyi | 108 +++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 5e62f2f1c..a210a3d06 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -54,25 +54,23 @@ if sys.version_info >= (3, 10): elif not TYPE_CHECKING: ellipsis = type(Ellipsis) # keeps ruff happy until ruff uses typeshed -_DatatypeStrOptions: TypeAlias = Literal[ +DatatypeCode: TypeAlias = Literal[ "S1", "c", "i1", "b", "B", "u1", "i2", "h", "s", "u2", "i4", "i", "l", "u4", "i8", "u8", "f4", "f", "f8", "d", "c8", "c16" ] -_DatatypeNCOptions: TypeAlias = Union[CompoundType, VLType, EnumType] -DatatypeOptions: TypeAlias = Union[_DatatypeStrOptions, _DatatypeNCOptions, npt.DTypeLike] -T_Datatype = TypeVar("T_Datatype", bound=DatatypeOptions) +NCComplexDatatype: TypeAlias = Union[CompoundType, VLType, EnumType] +Datatype: TypeAlias = Union[DatatypeCode, NCComplexDatatype, npt.DTypeLike] +T_Datatype = TypeVar("T_Datatype", bound=Datatype) T_DatatypeNC = TypeVar("T_DatatypeNC", CompoundType, VLType, EnumType) -DimensionsOptions: TypeAlias = Union[str, bytes, Dimension, Iterable[Union[str, bytes, Dimension]]] -CompressionOptions: TypeAlias = Literal[ - "zlib", "szip", "zstd", "blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd" -] -CompressionLevelOptions: TypeAlias = Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] -AccessModeOptions: TypeAlias = Literal["r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"] -FormatOptions: TypeAlias = Literal["NETCDF4", "NETCDF4_CLASSIC", "NETCDF3_CLASSIC", "NETCDF3_64BIT_OFFSET", "NETCDF3_64BIT_DATA"] -DiskFormatOptions: TypeAlias = Literal["NETCDF3", "HDF5", "HDF4", "PNETCDF", "DAP2", "DAP4", "UNDEFINED"] -QuantizeOptions: TypeAlias = Literal["BitGroom", "BitRound", "GranularBitRound"] -EndianOptions: TypeAlias = Literal["native", "little", "big"] -CalendarOptions: TypeAlias = Literal[ +DimensionsSpecifier: TypeAlias = Union[str, bytes, Dimension, Iterable[Union[str, bytes, Dimension]]] +CompressionType: TypeAlias = Literal["zlib", "szip", "zstd", "blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd"] +CompressionLevel: TypeAlias = Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +AccessMode: TypeAlias = Literal["r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"] +Format: TypeAlias = Literal["NETCDF4", "NETCDF4_CLASSIC", "NETCDF3_CLASSIC", "NETCDF3_64BIT_OFFSET", "NETCDF3_64BIT_DATA"] +DiskFormat: TypeAlias = Literal["NETCDF3", "HDF5", "HDF4", "PNETCDF", "DAP2", "DAP4", "UNDEFINED"] +QuantizeMode: TypeAlias = Literal["BitGroom", "BitRound", "GranularBitRound"] +EndianType: TypeAlias = Literal["native", "little", "big"] +CalendarType: TypeAlias = Literal[ "standard", "gregorian", "proleptic_gregorian", "noleap", "365_day", "360_day", "julian", "all_leap", "366_day" ] BoolInt: TypeAlias = Literal[0, 1] @@ -119,21 +117,21 @@ default_fillvals: dict[str, int | float | str] def date2index( dates: dt.datetime | cftime.datetime | Sequence[dt.datetime | cftime.datetime] | DateTimeArray, nctime: Variable, - calendar: CalendarOptions | None = None, + calendar: CalendarType | None = None, select: Literal["exact", "before", "after", "nearest"] = "exact", has_year_zero: bool | None = None, ) -> int | npt.NDArray[np.int_]: ... def date2num( dates: dt.datetime | cftime.datetime | Sequence[dt.datetime | cftime.datetime] | DateTimeArray, units: str, - calendar: CalendarOptions | None = None, + calendar: CalendarType | None = None, has_year_zero: bool | None = None, longdouble: bool = False, ) -> np.number | npt.NDArray[np.number]: ... def num2date( times: Sequence[int | float | np.number] | npt.NDArray[np.number], units: str, - calendar: CalendarOptions = "standard", + calendar: CalendarType = "standard", only_use_cftime_datetimes: bool = True, only_use_python_datetimes: bool = False, has_year_zero: bool | None = None, @@ -168,9 +166,9 @@ class Dataset: def __init__( self, filename: str | os.PathLike, - mode: AccessModeOptions = "r", + mode: AccessMode = "r", clobber: bool = True, - format: FormatOptions = "NETCDF4", + format: Format = "NETCDF4", diskless: bool = False, persist: bool = False, keepweakref: bool = False, @@ -197,11 +195,11 @@ class Dataset: @property def enumtypes(self) -> dict[str, EnumType]: ... @property - def data_model(self) -> FormatOptions: ... + def data_model(self) -> Format: ... @property - def file_format(self) -> FormatOptions: ... + def file_format(self) -> Format: ... @property - def disk_format(self) -> DiskFormatOptions: ... + def disk_format(self) -> DiskFormat: ... @property def parent(self) -> Dataset | None: ... @property @@ -227,10 +225,10 @@ class Dataset: self, varname: str, datatype: T_DatatypeNC, - dimensions: DimensionsOptions = (), - compression: CompressionOptions | None = None, + dimensions: DimensionsSpecifier = (), + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevelOptions | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, @@ -238,10 +236,10 @@ class Dataset: fletcher32: bool = False, contiguous: bool = False, chunksizes: int | None = None, - endian: EndianOptions = "native", + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable[T_DatatypeNC]: ... @@ -249,11 +247,11 @@ class Dataset: def createVariable( self, varname: str, - datatype: _DatatypeStrOptions | npt.DTypeLike, - dimensions: DimensionsOptions = (), - compression: CompressionOptions | None = None, + datatype: DatatypeCode | npt.DTypeLike, + dimensions: DimensionsSpecifier = (), + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevelOptions | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, @@ -261,10 +259,10 @@ class Dataset: fletcher32: bool = False, contiguous: bool = False, chunksizes: int | None = None, - endian: EndianOptions = "native", + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable[np.dtype]: ... @@ -294,7 +292,7 @@ class Dataset: def get_variables_by_attributes(self, **kwargs: Callable[[Any], bool] | Any) -> list[Variable]: ... @staticmethod def fromcdl( - cdlfilename: str, ncfilename: str | None = None, mode: AccessModeOptions = "a", format: FormatOptions = "NETCDF4" + cdlfilename: str, ncfilename: str | None = None, mode: AccessMode = "a", format: Format = "NETCDF4" ) -> Dataset: ... @overload def tocdl(self, coordvars: bool = False, data: bool = False, outfile: None = None) -> str: ... @@ -334,10 +332,10 @@ class Variable(Generic[T_Datatype]): grp: Dataset, name: str, datatype: T_DatatypeNC, - dimensions: DimensionsOptions = (), - compression: CompressionOptions | None = None, + dimensions: DimensionsSpecifier = (), + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevelOptions | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, @@ -345,10 +343,10 @@ class Variable(Generic[T_Datatype]): fletcher32: bool = False, contiguous: bool = False, chunksizes: Sequence[int] | None = None, - endian: EndianOptions = "native", + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, @@ -358,11 +356,11 @@ class Variable(Generic[T_Datatype]): cls, grp: Dataset, name: str, - datatype: _DatatypeStrOptions | npt.DTypeLike, - dimensions: DimensionsOptions = (), - compression: CompressionOptions | None = None, + datatype: DatatypeCode | npt.DTypeLike, + dimensions: DimensionsSpecifier = (), + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevelOptions | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, @@ -370,10 +368,10 @@ class Variable(Generic[T_Datatype]): fletcher32: bool = False, contiguous: bool = False, chunksizes: Sequence[int] | None = None, - endian: EndianOptions = "native", + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, @@ -383,10 +381,10 @@ class Variable(Generic[T_Datatype]): grp: Dataset, name: str, datatype: T_Datatype, - dimensions: DimensionsOptions = (), - compression: CompressionOptions | None = None, + dimensions: DimensionsSpecifier = (), + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevelOptions | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, @@ -394,10 +392,10 @@ class Variable(Generic[T_Datatype]): fletcher32: bool = False, contiguous: bool = False, chunksizes: Sequence[int] | None = None, - endian: EndianOptions = "native", + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeOptions = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, @@ -434,8 +432,8 @@ class Variable(Generic[T_Datatype]): def getncattr(self, name: str, encoding="utf-8"): ... def delncattr(self, name: str) -> None: ... def filters(self) -> FiltersDict: ... - def quantization(self) -> tuple[int, QuantizeOptions] | None: ... - def endian(self) -> EndianOptions: ... + def quantization(self) -> tuple[int, QuantizeMode] | None: ... + def endian(self) -> EndianType: ... def chunking(self) -> Literal["contiguous"] | list[int]: ... def get_var_chunk_cache(self) -> tuple[int, int, float]: ... def set_var_chunk_cache( @@ -541,10 +539,10 @@ class _Variable: def __len__(self) -> int: ... class MFTime(_Variable): - calendar: CalendarOptions | None + calendar: CalendarType | None units: str | None - def __init__(self, time: Variable, units: str | None = None, calendar: CalendarOptions | None = None): ... + def __init__(self, time: Variable, units: str | None = None, calendar: CalendarType | None = None): ... def __getitem__(self, elem: GetSetItemKey) -> np.ndarray: ... @overload From 94fc229bd4fc43c862d190ea8926e8e86c74dbf2 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 11 Jul 2024 09:59:15 -0700 Subject: [PATCH 09/81] Move TypedDicts to be adjacent to TypeAliases as they serve a simlar purpose --- src/netCDF4/__init__.pyi | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index a210a3d06..4692f17d4 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -87,6 +87,26 @@ GetSetItemKey: TypeAlias = ( | tuple[int | slice | ellipsis | Sequence[int | bool] | npt.NDArray[np.integer | np.bool_], ...] ) +class BloscInfo(TypedDict): + compressor: Literal["blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd"] + shuffle: Literal[0, 1, 2] + +class SzipInfo(TypedDict): + coding: Literal["nn", "ec"] + pixels_per_block: Literal[4, 8, 16, 32] + +class FiltersDict(TypedDict): + """Dict returned from netCDF4.Variable.filters()""" + + zlib: bool + szip: Literal[False] | SzipInfo + zstd: bool + bzip2: bool + blosc: Literal[False] | BloscInfo + shuffle: bool + complevel: int + fletcher32: bool + __version__: str __netcdf4libversion__: str __hdf5libversion__: str @@ -137,26 +157,6 @@ def num2date( has_year_zero: bool | None = None, ) -> dt.datetime | DateTimeArray: ... -class BloscInfo(TypedDict): - compressor: Literal["blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd"] - shuffle: Literal[0, 1, 2] - -class SzipInfo(TypedDict): - coding: Literal["nn", "ec"] - pixels_per_block: Literal[4, 8, 16, 32] - -class FiltersDict(TypedDict): - """Dict returned from netCDF4.Variable.filters()""" - - zlib: bool - szip: Literal[False] | SzipInfo - zstd: bool - bzip2: bool - blosc: Literal[False] | BloscInfo - shuffle: bool - complevel: int - fletcher32: bool - class NetCDF4MissingFeatureException(Exception): def __init__(self, feature: str, version: str): ... From 4ec155897caabd748edb387b3aa70732b8476242 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 11 Jul 2024 10:04:03 -0700 Subject: [PATCH 10/81] minor - spelling in comment --- src/netCDF4/_netCDF4.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/netCDF4/_netCDF4.pyi b/src/netCDF4/_netCDF4.pyi index f649647fe..facdb5b18 100644 --- a/src/netCDF4/_netCDF4.pyi +++ b/src/netCDF4/_netCDF4.pyi @@ -1,4 +1,4 @@ -# The definitions are intendionally done in the __init__. +# The definitions are intentionally done in the __init__. # This file only exists in case someone imports from netCDF4._netCDF4 from . import ( CompoundType, From 559429b1793317753396cdb8b220a8cd4eb4b005 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Fri, 12 Jul 2024 15:12:43 -0700 Subject: [PATCH 11/81] Rework all possible type arguments and overloads for createVariable. TypeVar for variable type is now unbound. --- src/netCDF4/__init__.pyi | 113 ++++++++++++++++++++++++++++++++------- 1 file changed, 95 insertions(+), 18 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 4692f17d4..a87078caf 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -54,13 +54,41 @@ if sys.version_info >= (3, 10): elif not TYPE_CHECKING: ellipsis = type(Ellipsis) # keeps ruff happy until ruff uses typeshed -DatatypeCode: TypeAlias = Literal[ - "S1", "c", "i1", "b", "B", "u1", "i2", "h", "s", "u2", "i4", "i", "l", "u4", "i8", "u8", "f4", "f", "f8", "d", "c8", "c16" +# string type specifiers +# fmt: off +RealTypeLiteral: TypeAlias = Literal[ + "i1", "b", "B", "int8", # NC_BYTE + "u1", "uint8", # NC_UBYTE + "i2", "h", "s", "int16", # NC_SHORT + "u2", "uint16", # NC_USHORT + "i4", "i", "l", "int32", # NC_INT + "u4", "uint32", # NC_UINT + "i8", "int64", "int", # NC_INT64 + "u8", "uint64", # NC_UINT64 + "f4", "f", "float32", # NC_FLOAT + "f8", "d", "float64", "float" # NC_DOUBLE ] -NCComplexDatatype: TypeAlias = Union[CompoundType, VLType, EnumType] -Datatype: TypeAlias = Union[DatatypeCode, NCComplexDatatype, npt.DTypeLike] -T_Datatype = TypeVar("T_Datatype", bound=Datatype) -T_DatatypeNC = TypeVar("T_DatatypeNC", CompoundType, VLType, EnumType) +# fmt: on +ComplexTypeLiteral: TypeAlias = Literal["c8", "c16", "complex64", "complex128"] +NumericTypeLiteral: TypeAlias = RealTypeLiteral | ComplexTypeLiteral +CharTypeLiteral: TypeAlias = Literal["S1", "c"] # NC_CHAR +TypeLiteral: TypeAlias = NumericTypeLiteral | CharTypeLiteral + +# Numpy types +NumPyRealType: TypeAlias = ( + np.int8 | np.uint8 | np.int16 | np.uint16 | np.int32 | np.uint32 | np.int64 | np.uint64 | np.float16 | np.float32 +) +NumPyComplexType: TypeAlias = np.complex64 | np.complex128 +NumPyNumericType: TypeAlias = NumPyRealType | NumPyComplexType +# Classes that can create instances of NetCDF user-defined types +NetCDFUDTClass: TypeAlias = CompoundType | VLType | EnumType +# Possible argument types for the datatype argument used in Variable creation. +DatatypeSpecifier: TypeAlias = ( + TypeLiteral | np.dtype[NumPyNumericType | np.str_] | type[int | float | NumPyNumericType | str | np.str_] | NetCDFUDTClass +) + +VarT = TypeVar("VarT") +NumericVarT = TypeVar("NumericVarT", bound=NumPyNumericType) DimensionsSpecifier: TypeAlias = Union[str, bytes, Dimension, Iterable[Union[str, bytes, Dimension]]] CompressionType: TypeAlias = Literal["zlib", "szip", "zstd", "blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd"] @@ -221,10 +249,33 @@ class Dataset: def createDimension(self, dimname: str, size: int | None = None) -> Dimension: ... def renameDimension(self, oldname: str, newname: str) -> None: ... @overload - def createVariable( # type: ignore + def createVariable( + self, + varname: str, + datatype: np.dtype[NumericVarT] | type[NumericVarT], + dimensions: DimensionsSpecifier = (), + compression: CompressionType | None = None, + zlib: bool = False, + complevel: CompressionLevel | None = 4, + shuffle: bool = True, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, + fletcher32: bool = False, + contiguous: bool = False, + chunksizes: int | None = None, + endian: EndianType = "native", + least_significant_digit: int | None = None, + significant_digits: int | None = None, + quantize_mode: QuantizeMode = "BitGroom", + fill_value: int | float | str | bytes | Literal[False] | None = None, + chunk_cache: int | None = None, + ) -> Variable[NumericVarT]: ... + @overload + def createVariable( self, varname: str, - datatype: T_DatatypeNC, + datatype: np.dtype[np.str_] | type[str | np.str_], dimensions: DimensionsSpecifier = (), compression: CompressionType | None = None, zlib: bool = False, @@ -242,12 +293,12 @@ class Dataset: quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, - ) -> Variable[T_DatatypeNC]: ... + ) -> Variable[str]: ... @overload def createVariable( self, varname: str, - datatype: DatatypeCode | npt.DTypeLike, + datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), compression: CompressionType | None = None, zlib: bool = False, @@ -265,7 +316,7 @@ class Dataset: quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, - ) -> Variable[np.dtype]: ... + ) -> Variable: ... def renameVariable(self, oldname: str, newname: str) -> None: ... def createGroup(self, groupname: str) -> Group: ... def renameGroup(self, oldname: str, newname: str) -> None: ... @@ -325,13 +376,39 @@ class Dimension: def isunlimited(self) -> bool: ... def __len__(self) -> int: ... -class Variable(Generic[T_Datatype]): +class Variable(Generic[VarT]): + # Overloads of __new__ are provided for some cases where the Variable's type may be statically inferred from the datatype arg + @overload + def __new__( + cls, + grp: Dataset, + name: str, + datatype: np.dtype[NumericVarT] | type[NumericVarT], + dimensions: DimensionsSpecifier = (), + compression: CompressionType | None = None, + zlib: bool = False, + complevel: CompressionLevel | None = 4, + shuffle: bool = True, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, + fletcher32: bool = False, + contiguous: bool = False, + chunksizes: Sequence[int] | None = None, + endian: EndianType = "native", + least_significant_digit: int | None = None, + significant_digits: int | None = None, + quantize_mode: QuantizeMode = "BitGroom", + fill_value: int | float | str | bytes | Literal[False] | None = None, + chunk_cache: int | None = None, + **kwargs: Any, + ) -> Variable[NumericVarT]: ... @overload def __new__( cls, grp: Dataset, name: str, - datatype: T_DatatypeNC, + datatype: np.dtype[np.str_] | type[str | np.str_], dimensions: DimensionsSpecifier = (), compression: CompressionType | None = None, zlib: bool = False, @@ -350,13 +427,13 @@ class Variable(Generic[T_Datatype]): fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, - ) -> Variable[T_DatatypeNC]: ... + ) -> Variable[str]: ... @overload def __new__( cls, grp: Dataset, name: str, - datatype: DatatypeCode | npt.DTypeLike, + datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), compression: CompressionType | None = None, zlib: bool = False, @@ -375,12 +452,12 @@ class Variable(Generic[T_Datatype]): fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, - ) -> Variable[np.dtype]: ... + ) -> Variable: ... def __init__( self, grp: Dataset, name: str, - datatype: T_Datatype, + datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), compression: CompressionType | None = None, zlib: bool = False, @@ -405,7 +482,7 @@ class Variable(Generic[T_Datatype]): @property def dtype(self) -> np.dtype | type[str]: ... @property - def datatype(self) -> T_Datatype: ... + def datatype(self) -> np.dtype | NetCDFUDTClass: ... @property def shape(self) -> tuple[int, ...]: ... @property From 516c644545749e91199557177c56c71972cefa56 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Fri, 12 Jul 2024 15:15:24 -0700 Subject: [PATCH 12/81] Add docstring and notes section to top of file --- src/netCDF4/__init__.pyi | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index a87078caf..1143b3877 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -1,3 +1,31 @@ +"""__init__.pyi - Type stubs for the netCDF4 Python package""" +# Notes: +# +# - The stubs in this file are manually-generated and must be updated if and when the API is changed. +# - The following **ruff** commands may be used to properly format this file according to +# https://typing.readthedocs.io/en/latest/source/stubs.html +# +# ruff format --line-length 130 src/netCDF4/__init__.pyi # format code +# ruff check --line-length 130 --select I --fix src/netCDF4/__init__.pyi # sort imports +# +# - The Variable class is a generic and may thus be statically typed, but this has limited utility for the following reasons: +# - The return type of `Variable.__getitem__()` (and `Variable.getValue()`) depends on a number of factors (e.g. variable shape, +# key shape, whether masking is enabled) that cannot be easily determined statically. +# - Similarly, the types and shapes of data that `Variable.__setitem__()` may accept varies widely depending on many factors and +# is intractable to determine statically. +# - Automatic typing of a Variable on variable creation is tedious due to the large number of ways to specify a variable's type +# (in particular all the type literals). +# - It is not possible to statically type a Variable of any user-defined type (CompoundType, EnumType, VLType) as these types +# are created dynamically. +# It is thus best left to the user to implement TypeGuards and/or perform other mixed static/runtime type-checking to ensure the +# type and shape of data retrieved from this library. +# - `Dataset.__getitem__()` may return either a Variable or a Group, depending on the string passed to it. Rather than return a +# Union of Variable and Group, the authors of these stubs have elected to to return Any, leaving it up to users to determine the +# type of the returned value. +# - `MFDataset.dimensions` returns `dict[str, Dimension]` and `MFDataset.variables` returns `dict[str, Variable]` even though the +# dict value types may actually be `_Dimension` and `_Variable`, respectively. The original authors of this stubfile have +# elected to do this for simplicity's sake, but it may make sense to change this in the future, or just return `dict[str, Any]`. + import datetime as dt import os import sys From 226890936e7d4a4bba9da0ba1d2897b2bb05df79 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Fri, 12 Jul 2024 23:59:49 -0700 Subject: [PATCH 13/81] np.float64 is a possible type --- src/netCDF4/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 1143b3877..f834bac19 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -104,7 +104,7 @@ TypeLiteral: TypeAlias = NumericTypeLiteral | CharTypeLiteral # Numpy types NumPyRealType: TypeAlias = ( - np.int8 | np.uint8 | np.int16 | np.uint16 | np.int32 | np.uint32 | np.int64 | np.uint64 | np.float16 | np.float32 + np.int8 | np.uint8 | np.int16 | np.uint16 | np.int32 | np.uint32 | np.int64 | np.uint64 | np.float16 | np.float32 | np.float64 ) NumPyComplexType: TypeAlias = np.complex64 | np.complex128 NumPyNumericType: TypeAlias = NumPyRealType | NumPyComplexType From d5baab3b4878fc6d0c80d43505acb4d330d0511b Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:01:20 -0700 Subject: [PATCH 14/81] datatype specifier can be an arbitrary string --- src/netCDF4/__init__.pyi | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index f834bac19..62738310d 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -110,9 +110,15 @@ NumPyComplexType: TypeAlias = np.complex64 | np.complex128 NumPyNumericType: TypeAlias = NumPyRealType | NumPyComplexType # Classes that can create instances of NetCDF user-defined types NetCDFUDTClass: TypeAlias = CompoundType | VLType | EnumType -# Possible argument types for the datatype argument used in Variable creation. +# Possible argument types for the datatype argument used in Variable creation. At this time, it is not possible to allow unknown +# strings arguments in the datatype field but exclude and string literals that are not one of `TypeLiteral`, so really +# `TypeLiteral` is made irrelevant, except for anyone who looks at this file. DatatypeSpecifier: TypeAlias = ( - TypeLiteral | np.dtype[NumPyNumericType | np.str_] | type[int | float | NumPyNumericType | str | np.str_] | NetCDFUDTClass + TypeLiteral + | str + | np.dtype[NumPyNumericType | np.str_] + | type[int | float | NumPyNumericType | str | np.str_] + | NetCDFUDTClass ) VarT = TypeVar("VarT") From 229d6aeadd5368fd1efa9aac2512d6ec9bb7dcda Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:03:56 -0700 Subject: [PATCH 15/81] Add float and str for GetSetItemKey --- src/netCDF4/__init__.pyi | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 62738310d..2763b2f21 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -140,13 +140,31 @@ BoolInt: TypeAlias = Literal[0, 1] DateTimeArray: TypeAlias = npt.NDArray[np.object_] """numpy array of datetime.datetime or cftime.datetime""" +# netCDF accepts floats that can be cooerced to an integer value, and sometimes strings that can be coerced to an integer value. +# There's currently no way to specify this statically, so we allow all floats and strings. +# Also, we have to specify all the possible combinations of list[...] because lists are invariant. GetSetItemKey: TypeAlias = ( int + | float + | np.number | slice | ellipsis + | list[int] + | list[float] + | list[bool] + | list[str] # strs that can be cooerced to ints + | list[int | float] | list[int | bool] - | npt.NDArray[np.integer | np.bool_] - | tuple[int | slice | ellipsis | Sequence[int | bool] | npt.NDArray[np.integer | np.bool_], ...] + | list[int | str] + | list[float | bool] + | list[float | str] + | list[bool | str] + | list[int | float | bool | str] + | npt.NDArray[np.number | np.bool_] + | tuple[ + int | float | str | slice | ellipsis | Sequence[int] | Sequence[bool] | npt.NDArray[np.number | np.bool_], + ..., + ] ) class BloscInfo(TypedDict): From 06f7238fc3abe3e6d22fd0f9a4b84cde83411344 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:04:32 -0700 Subject: [PATCH 16/81] integral float can be specified for dimension size --- src/netCDF4/__init__.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 2763b2f21..e5a0c47c6 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -298,7 +298,7 @@ class Dataset: def sync(self) -> None: ... def set_fill_on(self) -> None: ... def set_fill_off(self) -> None: ... - def createDimension(self, dimname: str, size: int | None = None) -> Dimension: ... + def createDimension(self, dimname: str, size: int | float | None = None) -> Dimension: ... def renameDimension(self, oldname: str, newname: str) -> None: ... @overload def createVariable( @@ -419,7 +419,7 @@ class Group(Dataset): def close(self) -> NoReturn: ... class Dimension: - def __init__(self, grp: Dataset, name: str, size: int | None = None, **kwargs: Any) -> None: ... + def __init__(self, grp: Dataset, name: str, size: int | float | None = None, **kwargs: Any) -> None: ... @property def name(self) -> str: ... @property From bc3bd3f4890bf8fd6fce80cbd1ec0f911c8b0ec8 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:07:03 -0700 Subject: [PATCH 17/81] arbitrary strings can be used intead of literals, and truthy values in place of bools --- src/netCDF4/__init__.pyi | 134 +++++++++++++++++++-------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index e5a0c47c6..10cb83060 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -306,20 +306,20 @@ class Dataset: varname: str, datatype: np.dtype[NumericVarT] | type[NumericVarT], dimensions: DimensionsSpecifier = (), - compression: CompressionType | None = None, - zlib: bool = False, - complevel: CompressionLevel | None = 4, - shuffle: bool = True, - szip_coding: Literal["nn", "ec"] = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, - blosc_shuffle: Literal[0, 1, 2] = 1, - fletcher32: bool = False, - contiguous: bool = False, + compression: CompressionType | str | None = None, + zlib: Any = False, + complevel: CompressionLevel | int | None = 4, + shuffle: Any = True, + szip_coding: Literal["nn", "ec"] | str = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, + blosc_shuffle: Literal[0, 1, 2] | int = 1, + fletcher32: Any = False, + contiguous: Any = False, chunksizes: int | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode = "BitGroom", + quantize_mode: QuantizeMode | str = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable[NumericVarT]: ... @@ -329,20 +329,20 @@ class Dataset: varname: str, datatype: np.dtype[np.str_] | type[str | np.str_], dimensions: DimensionsSpecifier = (), - compression: CompressionType | None = None, - zlib: bool = False, - complevel: CompressionLevel | None = 4, - shuffle: bool = True, - szip_coding: Literal["nn", "ec"] = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, - blosc_shuffle: Literal[0, 1, 2] = 1, - fletcher32: bool = False, - contiguous: bool = False, + compression: CompressionType | str | None = None, + zlib: Any = False, + complevel: CompressionLevel | int | None = 4, + shuffle: Any = True, + szip_coding: Literal["nn", "ec"] | str = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, + blosc_shuffle: Literal[0, 1, 2] | int = 1, + fletcher32: Any = False, + contiguous: Any = False, chunksizes: int | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode = "BitGroom", + quantize_mode: QuantizeMode | str = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable[str]: ... @@ -352,20 +352,20 @@ class Dataset: varname: str, datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), - compression: CompressionType | None = None, - zlib: bool = False, - complevel: CompressionLevel | None = 4, - shuffle: bool = True, - szip_coding: Literal["nn", "ec"] = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, - blosc_shuffle: Literal[0, 1, 2] = 1, - fletcher32: bool = False, - contiguous: bool = False, + compression: CompressionType | str | None = None, + zlib: Any = False, + complevel: CompressionLevel | int | None = 4, + shuffle: Any = True, + szip_coding: Literal["nn", "ec"] | str = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, + blosc_shuffle: Literal[0, 1, 2] | int = 1, + fletcher32: Any = False, + contiguous: Any = False, chunksizes: int | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode = "BitGroom", + quantize_mode: QuantizeMode | str = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable: ... @@ -437,20 +437,20 @@ class Variable(Generic[VarT]): name: str, datatype: np.dtype[NumericVarT] | type[NumericVarT], dimensions: DimensionsSpecifier = (), - compression: CompressionType | None = None, - zlib: bool = False, - complevel: CompressionLevel | None = 4, - shuffle: bool = True, - szip_coding: Literal["nn", "ec"] = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, - blosc_shuffle: Literal[0, 1, 2] = 1, - fletcher32: bool = False, - contiguous: bool = False, + compression: CompressionType | str | None = None, + zlib: Any = False, + complevel: CompressionLevel | int | None = 4, + shuffle: Any = True, + szip_coding: Literal["nn", "ec"] | str = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, + blosc_shuffle: Literal[0, 1, 2] | int = 1, + fletcher32: Any = False, + contiguous: Any = False, chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode = "BitGroom", + quantize_mode: QuantizeMode | str = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, @@ -462,20 +462,20 @@ class Variable(Generic[VarT]): name: str, datatype: np.dtype[np.str_] | type[str | np.str_], dimensions: DimensionsSpecifier = (), - compression: CompressionType | None = None, - zlib: bool = False, - complevel: CompressionLevel | None = 4, - shuffle: bool = True, - szip_coding: Literal["nn", "ec"] = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, - blosc_shuffle: Literal[0, 1, 2] = 1, - fletcher32: bool = False, - contiguous: bool = False, + compression: CompressionType | str | None = None, + zlib: Any = False, + complevel: CompressionLevel | int | None = 4, + shuffle: Any = True, + szip_coding: Literal["nn", "ec"] | str = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, + blosc_shuffle: Literal[0, 1, 2] | int = 1, + fletcher32: Any = False, + contiguous: Any = False, chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode = "BitGroom", + quantize_mode: QuantizeMode | str = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, @@ -487,20 +487,20 @@ class Variable(Generic[VarT]): name: str, datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), - compression: CompressionType | None = None, - zlib: bool = False, - complevel: CompressionLevel | None = 4, - shuffle: bool = True, - szip_coding: Literal["nn", "ec"] = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, - blosc_shuffle: Literal[0, 1, 2] = 1, - fletcher32: bool = False, - contiguous: bool = False, + compression: CompressionType | str | None = None, + zlib: Any = False, + complevel: CompressionLevel | int | None = 4, + shuffle: Any = True, + szip_coding: Literal["nn", "ec"] | str = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, + blosc_shuffle: Literal[0, 1, 2] | int = 1, + fletcher32: Any = False, + contiguous: Any = False, chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode = "BitGroom", + quantize_mode: QuantizeMode | str = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, @@ -511,20 +511,20 @@ class Variable(Generic[VarT]): name: str, datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), - compression: CompressionType | None = None, - zlib: bool = False, - complevel: CompressionLevel | None = 4, - shuffle: bool = True, + compression: CompressionType | str | None = None, + zlib: Any = False, + complevel: CompressionLevel | int | None = 4, + shuffle: Any = True, szip_coding: Literal["nn", "ec"] = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, blosc_shuffle: Literal[0, 1, 2] = 1, - fletcher32: bool = False, - contiguous: bool = False, + fletcher32: Any = False, + contiguous: Any = False, chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode = "BitGroom", + quantize_mode: QuantizeMode | str = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, From ea0297f4b5f380c823675b337c40b328345e9d7d Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:07:32 -0700 Subject: [PATCH 18/81] enum_dict for createEnum can have int or np.integer values --- src/netCDF4/__init__.pyi | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 10cb83060..98be1e798 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -378,7 +378,10 @@ class Dataset: ) -> CompoundType: ... def createVLType(self, datatype: npt.DTypeLike, datatype_name: str) -> VLType: ... def createEnumType( - self, datatype: np.dtype[np.integer] | type[np.integer] | type[int], datatype_name: str, enum_dict: dict[str, int] + self, + datatype: np.dtype[np.integer] | type[np.integer] | type[int], + datatype_name: str, + enum_dict: Mapping[str, int | np.integer], ) -> EnumType: ... def ncattrs(self) -> list[str]: ... def setncattr_string(self, name: str, value: Any) -> None: ... @@ -615,7 +618,7 @@ class EnumType: grp: Dataset, dt: np.dtype[np.integer] | type[np.integer] | type[int] | str, dtype_name: str, - enum_dict: Mapping[str, int], + enum_dict: Mapping[str, int | np.integer], **kwargs: Any, ) -> None: ... def __reduce__(self) -> NoReturn: ... From 0ba830d39f40a8e83d7de14c0460e3d7513308b4 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:08:03 -0700 Subject: [PATCH 19/81] Return Any from Variable.__getitem__ --- src/netCDF4/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 98be1e798..14eff09a6 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -586,7 +586,7 @@ class Variable(Generic[VarT]): def __delattr__(self, name: str) -> None: ... def __setattr__(self, name: str, value: Any) -> None: ... def __getattr__(self, name: str) -> Any: ... - def __getitem__(self, elem: GetSetItemKey) -> np.ndarray: ... + def __getitem__(self, elem: GetSetItemKey) -> Any: ... def __setitem__(self, elem: GetSetItemKey, data: npt.ArrayLike) -> None: ... def __array__(self) -> np.ndarray: ... def __len__(self) -> int: ... From 0c5524da34393efb991671163302c35192d9cd8f Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:09:00 -0700 Subject: [PATCH 20/81] Simplify Variable.dtype to be Any --- src/netCDF4/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 14eff09a6..951326548 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -535,7 +535,7 @@ class Variable(Generic[VarT]): @property def name(self) -> str: ... @property - def dtype(self) -> np.dtype | type[str]: ... + def dtype(self) -> Any: ... # actually np.dtype | type[str] @property def datatype(self) -> np.dtype | NetCDFUDTClass: ... @property From 290e67160e96a635548bfd6fdcc9e646a7542ef9 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:13:15 -0700 Subject: [PATCH 21/81] minor fixes to enum type tests --- test/test_enum.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/test/test_enum.py b/test/test_enum.py index 0ab42ec6e..536cb0a27 100644 --- a/test/test_enum.py +++ b/test/test_enum.py @@ -1,9 +1,10 @@ -import sys -import unittest import os import tempfile -from netCDF4 import Dataset +import unittest + +import netCDF4 import numpy as np +from netCDF4 import Dataset, EnumType from numpy.testing import assert_array_equal FILE_NAME = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -26,7 +27,7 @@ def setUp(self): cloud_type = f.createEnumType(ENUM_BASETYPE,ENUM_NAME,ENUM_DICT) # make sure KeyError raised if non-integer basetype used. try: - cloud_typ2 = f.createEnumType(np.float32,ENUM_NAME,ENUM_DICT) + cloud_typ2 = f.createEnumType(np.float32,ENUM_NAME,ENUM_DICT) # type: ignore except KeyError: pass f.createDimension('time',None) @@ -49,6 +50,7 @@ def runTest(self): """testing enum data type""" f = Dataset(self.file, 'r') v = f.variables[VAR_NAME] + assert isinstance(v.datatype, EnumType) assert v.datatype.enum_dict == ENUM_DICT assert list(f.enumtypes.keys()) == [ENUM_NAME] assert f.enumtypes[ENUM_NAME].name == ENUM_NAME # issue 775 @@ -69,9 +71,9 @@ def setUp(self): DT = np.int16; BITS = 8 self.STORED_VAL = DT(2**BITS) self.VAL_MAP = {f'bits_{n}': DT(2**n) for n in range(1,BITS+1)} - self.VAL_MAP['invalid'] = 0 + self.VAL_MAP['invalid'] = DT(0) self.file = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name - with netCDF4.Dataset(file, 'w') as nc: + with netCDF4.Dataset(self.file, 'w') as nc: # The enum is created with dtype=int16, so it will allow BITS values up to 15 et = nc.createEnumType(DT, 'etype', self.VAL_MAP) ev = nc.createVariable('evar', et) @@ -80,10 +82,11 @@ def setUp(self): def tearDown(self): os.remove(self.file) def runTest(self): - with netCDF4.Dataset(file, 'r') as nc: + with netCDF4.Dataset(self.file, 'r') as nc: read_var = nc['evar'] - assert read_var[...] == self.STORED_VAL - assert read_et.enum_dict == self.VAL_MAP + read_et = nc.enumtypes["etype"] + assert read_var[...] == self.STORED_VAL + assert read_et.enum_dict == self.VAL_MAP if __name__ == '__main__': unittest.main() From e5d88d007ec93fc0419bdadb5766df494ac59fc4 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:20:52 -0700 Subject: [PATCH 22/81] use np.ma.masked_array in tests for numpy version compatibility --- test/test_masked3.py | 8 ++++---- test/test_masked4.py | 4 ++-- test/test_masked5.py | 2 +- test/test_masked6.py | 8 ++++---- test/test_scaled.py | 8 ++++---- test/test_slicing.py | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/test/test_masked3.py b/test/test_masked3.py index ed88109f2..c7cd4f2f5 100755 --- a/test/test_masked3.py +++ b/test/test_masked3.py @@ -59,7 +59,7 @@ def test_unscaled(self): self.assertEqual(v.dtype, "i2") self.assertTrue(isinstance(v, np.ndarray)) - self.assertTrue(not isinstance(v, ma.core.MaskedArray)) + self.assertTrue(not isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v) f.close() @@ -85,7 +85,7 @@ def test_scaled(self): self.assertEqual(v.dtype, "f8") self.assertTrue(isinstance(v, np.ndarray)) - self.assertTrue(not isinstance(v, ma.core.MaskedArray)) + self.assertTrue(not isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v_scaled) f.close() @@ -104,7 +104,7 @@ def test_unscaled(self): self.assertEqual(v_ma.dtype, "i2") self.assertTrue(isinstance(v_ma, np.ndarray)) - self.assertTrue(isinstance(v_ma, ma.core.MaskedArray)) + self.assertTrue(isinstance(v_ma, ma.masked_array)) assert_array_almost_equal(v_ma, self.v_ma) f.close() @@ -128,7 +128,7 @@ def test_scaled(self): self.assertEqual(v_ma.dtype, "f8") self.assertTrue(isinstance(v_ma, np.ndarray)) - self.assertTrue(isinstance(v_ma, ma.core.MaskedArray)) + self.assertTrue(isinstance(v_ma, ma.masked_array)) assert_array_almost_equal(v_ma, self.v_ma_scaled) f.close() diff --git a/test/test_masked4.py b/test/test_masked4.py index 14dd14fc4..61d9a1690 100755 --- a/test/test_masked4.py +++ b/test/test_masked4.py @@ -88,11 +88,11 @@ def test_scaled(self): v3 = f.variables["v3"][:] self.assertEqual(v.dtype, "f8") self.assertTrue(isinstance(v, np.ndarray)) - self.assertTrue(isinstance(v, ma.core.MaskedArray)) + self.assertTrue(isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v_scaled) self.assertEqual(v2.dtype, "f8") self.assertTrue(isinstance(v2, np.ndarray)) - self.assertTrue(isinstance(v2, ma.core.MaskedArray)) + self.assertTrue(isinstance(v2, ma.masked_array)) assert_array_almost_equal(v2, self.v_scaled) self.assertTrue(np.all(self.v_ma.mask == v.mask)) self.assertTrue(np.all(self.v_ma.mask == v2.mask)) diff --git a/test/test_masked5.py b/test/test_masked5.py index 3d8dba4db..87734024a 100755 --- a/test/test_masked5.py +++ b/test/test_masked5.py @@ -45,7 +45,7 @@ def test_scaled(self): f = Dataset(self.testfile) v = f.variables["v"] v2 = f.variables["v2"] - self.assertTrue(isinstance(v[:], ma.core.MaskedArray)) + self.assertTrue(isinstance(v[:], ma.masked_array)) assert_array_equal(v[:], self.v_ma) assert_array_equal(v[2],self.v[2]) # issue #624. v.set_auto_mask(False) diff --git a/test/test_masked6.py b/test/test_masked6.py index 65db53dde..dc77da99e 100644 --- a/test/test_masked6.py +++ b/test/test_masked6.py @@ -47,13 +47,13 @@ def test_always_mask(self): v = f.variables['v'][:] self.assertTrue(isinstance(v, np.ndarray)) - self.assertTrue(isinstance(v, ma.core.MaskedArray)) + self.assertTrue(isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v) w = f.variables['w'][:] self.assertTrue(isinstance(w, np.ndarray)) - self.assertTrue(isinstance(w, ma.core.MaskedArray)) + self.assertTrue(isinstance(w, ma.masked_array)) assert_array_almost_equal(w, self.w) f.close() @@ -69,13 +69,13 @@ def test_always_mask(self): v = f.variables['v'][:] self.assertTrue(isinstance(v, np.ndarray)) - self.assertFalse(isinstance(v, ma.core.MaskedArray)) + self.assertFalse(isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v) w = f.variables['w'][:] self.assertTrue(isinstance(w, np.ndarray)) - self.assertTrue(isinstance(w, ma.core.MaskedArray)) + self.assertTrue(isinstance(w, ma.masked_array)) assert_array_almost_equal(w, self.w) f.close() diff --git a/test/test_scaled.py b/test/test_scaled.py index 4a73ba3f7..5c1ce9542 100755 --- a/test/test_scaled.py +++ b/test/test_scaled.py @@ -68,7 +68,7 @@ def test_unmasked(self): self.assertEqual(v.dtype, "i2") self.assertTrue(isinstance(v, np.ndarray)) # issue 785: always return masked array by default - self.assertTrue(isinstance(v, ma.core.MaskedArray)) + self.assertTrue(isinstance(v, ma.masked_array)) assert_array_almost_equal(v, self.v) f.close() @@ -93,7 +93,7 @@ def test_masked(self): self.assertEqual(v_ma.dtype, "i2") self.assertTrue(isinstance(v_ma, np.ndarray)) - self.assertTrue(isinstance(v_ma, ma.core.MaskedArray)) + self.assertTrue(isinstance(v_ma, ma.masked_array)) assert_array_almost_equal(v_ma, self.v_ma) f.close() @@ -118,7 +118,7 @@ def test_unmasked(self): self.assertEqual(v_scaled.dtype, "f8") self.assertTrue(isinstance(v_scaled, np.ndarray)) # issue 785: always return masked array by default - self.assertTrue(isinstance(v_scaled, ma.core.MaskedArray)) + self.assertTrue(isinstance(v_scaled, ma.masked_array)) assert_array_almost_equal(v_scaled, self.v_scaled) f.close() @@ -141,7 +141,7 @@ def test_masked(self): self.assertEqual(v_ma_scaled.dtype, "f8") self.assertTrue(isinstance(v_ma_scaled, np.ndarray)) - self.assertTrue(isinstance(v_ma_scaled, ma.core.MaskedArray)) + self.assertTrue(isinstance(v_ma_scaled, ma.masked_array)) assert_array_almost_equal(v_ma_scaled, self.v_ma_scaled) f.close() diff --git a/test/test_slicing.py b/test/test_slicing.py index 8d3f88b7d..16d384ded 100644 --- a/test/test_slicing.py +++ b/test/test_slicing.py @@ -103,7 +103,7 @@ def test_0d(self): assert_equal(v.shape, v[...].shape) # issue #785: always return masked array #assert type(v[...]) == np.ndarray - assert type(v[...]) == np.ma.core.MaskedArray + assert type(v[...]) == np.ma.masked_array f.set_auto_mask(False) assert type(v[...]) == np.ndarray f.close() From 8806f10bb7ab6e7f0c6b01cee95066328ea3135b Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:22:25 -0700 Subject: [PATCH 23/81] Add some assertions for static typing --- test/test_multifile.py | 1 + test/test_multifile2.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/test_multifile.py b/test/test_multifile.py index 93d72bded..2301ebdf9 100644 --- a/test/test_multifile.py +++ b/test/test_multifile.py @@ -52,6 +52,7 @@ def runTest(self): assert_array_equal(np.arange(0,nx),f.variables['x'][:]) varin = f.variables['data'] datin = varin[:] + assert isinstance(data, np.ma.masked_array) assert_array_equal(datin.mask,data.mask) varin.set_auto_maskandscale(False) data2 = data.filled() diff --git a/test/test_multifile2.py b/test/test_multifile2.py index f8e8552a6..d5b310955 100644 --- a/test/test_multifile2.py +++ b/test/test_multifile2.py @@ -52,6 +52,7 @@ def runTest(self): assert_array_equal(np.arange(0,nx),f.variables['x'][:]) varin = f.variables['data'] datin = varin[:] + assert isinstance(data, np.ma.masked_array) assert_array_equal(datin.mask,data.mask) varin.set_auto_maskandscale(False) data2 = data.filled() From bb330c61f561c0757c565b1e169a0631c6cbbf4e Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:23:50 -0700 Subject: [PATCH 24/81] minor static typing additions --- test/test_types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_types.py b/test/test_types.py index 0bd910a3f..e0ec74c76 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -52,10 +52,11 @@ def runTest(self): for typ in datatypes: data = f.variables['data_'+typ] data.set_auto_maskandscale(False) - datarr = data[1:n1dim] + datarr: np.ndarray = data[1:n1dim] # fill missing data with _FillValue # ('S1' array will have some missing values) if hasattr(datarr, 'mask'): + assert isinstance(datarr, np.ma.masked_array) datarr = datarr.filled() datfilled = data[0] # check to see that data type is correct From 513e4d677884cf8ad075c8a184ddb7dfcf3d9d05 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 00:24:18 -0700 Subject: [PATCH 25/81] use np.ndarray.tobytes instead of deprecated tostring --- test/test_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_types.py b/test/test_types.py index e0ec74c76..7e2a998ee 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -69,7 +69,7 @@ def runTest(self): #assert np.allclose(datarr, ranarr[1:n1dim].astype(data.dtype)) assert_array_almost_equal(datarr,ranarr[1:n1dim].astype(data.dtype)) else: - assert datarr.tostring() == ranarr[1:n1dim].astype(data.dtype).tostring() + assert datarr.tobytes() == ranarr[1:n1dim].astype(data.dtype).tobytes() # check that variable elements not yet written are filled # with the specified _FillValue. assert_array_equal(datfilled,np.asarray(data._FillValue,datfilled.dtype)) From c81f60dc166d6f8e4d746850083cb98c3a1d2572 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 22:30:36 -0700 Subject: [PATCH 26/81] __dealloc__ seems to be a cython-only member of Dataset --- src/netCDF4/__init__.pyi | 1 - 1 file changed, 1 deletion(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 951326548..b881912c9 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -412,7 +412,6 @@ class Dataset: def __setattr__(self, name: str, value: Any) -> None: ... def __getattr__(self, name: str) -> Any: ... def __delattr__(self, name: str): ... - def __dealloc__(self) -> None: ... def __reduce__(self) -> NoReturn: ... def __enter__(self) -> Self: ... def __exit__(self, atype, value, traceback) -> None: ... From 62d97f3736b1046dd276ac35136b1697ce19c9d6 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 22:30:48 -0700 Subject: [PATCH 27/81] Update stubtest-allowlist --- .github/stubtest-allowlist | 40 +++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/stubtest-allowlist b/.github/stubtest-allowlist index a3bd3a1a4..afddeb272 100644 --- a/.github/stubtest-allowlist +++ b/.github/stubtest-allowlist @@ -1,13 +1,22 @@ -netCDF4.AccessModeOptions -netCDF4.CompressionLevelOptions -netCDF4.CompressionOptions -netCDF4.DatatypeOptions -netCDF4.DimensionsOptions -netCDF4.DiskFormatOptions -netCDF4.EndianOptions -netCDF4.FormatOptions -netCDF4.QuantizeOptions -netCDF4.CalendarOptions +netCDF4.RealTypeLiteral +netCDF4.ComplexTypeLiteral +netCDF4.NumericTypeLiteral +netCDF4.CharTypeLiteral +netCDF4.TypeLiteral +netCDF4.NumPyRealType +netCDF4.NumPyComplexType +netCDF4.NumPyNumericType +netCDF4.NetCDFUDTClass +netCDF4.AccessMode +netCDF4.CompressionLevel +netCDF4.CompressionType +netCDF4.DatatypeSpecifier +netCDF4.DimensionsSpecifier +netCDF4.DiskFormat +netCDF4.EndianType +netCDF4.Format +netCDF4.QuantizeMode +netCDF4.CalendarType netCDF4.ellipsis netCDF4.DateTimeArray netCDF4.FiltersDict @@ -15,15 +24,10 @@ netCDF4.SzipInfo netCDF4.BloscInfo netCDF4.BoolInt netCDF4.GetSetItemKey -netCDF4.T_Datatype -netCDF4.T_DatatypeNC -netCDF4.Dataset.__dealloc -netCDF4.Dimension.__reduce_cython__ -netCDF4.Dimension.__setstate_cython__ +netCDF4.VarT +netCDF4.NumericVarT +netCDF4.Dataset netCDF4.Variable.auto_complex -netCDF4._netCDF4.Dataset.__dealloc -netCDF4._netCDF4.Dimension.__reduce_cython__ -netCDF4._netCDF4.Dimension.__setstate_cython__ netCDF4._netCDF4.NC_DISKLESS netCDF4._netCDF4.NC_PERSIST netCDF4._netCDF4.Variable.auto_complex From 77bec6b9d237b80e39984cb4a44ffd5a9e4e83aa Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Sat, 13 Jul 2024 23:40:49 -0700 Subject: [PATCH 28/81] Restore some missing needed entires --- .github/stubtest-allowlist | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/stubtest-allowlist b/.github/stubtest-allowlist index afddeb272..ed56536bf 100644 --- a/.github/stubtest-allowlist +++ b/.github/stubtest-allowlist @@ -26,8 +26,11 @@ netCDF4.BoolInt netCDF4.GetSetItemKey netCDF4.VarT netCDF4.NumericVarT -netCDF4.Dataset +netCDF4.Dimension.__reduce_cython__ +netCDF4.Dimension.__setstate_cython__ netCDF4.Variable.auto_complex +netCDF4._netCDF4.Dimension.__reduce_cython__ +netCDF4._netCDF4.Dimension.__setstate_cython__ netCDF4._netCDF4.NC_DISKLESS netCDF4._netCDF4.NC_PERSIST netCDF4._netCDF4.Variable.auto_complex From 13d625a11d065c36fa8318579c016451c1a58d46 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 16 Jul 2024 10:49:03 -0700 Subject: [PATCH 29/81] Return Any rather than Union from datatype for now --- src/netCDF4/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index b881912c9..a3b4e6beb 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -536,7 +536,7 @@ class Variable(Generic[VarT]): @property def dtype(self) -> Any: ... # actually np.dtype | type[str] @property - def datatype(self) -> np.dtype | NetCDFUDTClass: ... + def datatype(self) -> Any: ... # Actually np.dtype | NetCDFUDTClass @property def shape(self) -> tuple[int, ...]: ... @property From e086579e7484da8daa52991a18b8c524d464a78c Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 16 Jul 2024 16:24:40 -0700 Subject: [PATCH 30/81] All arguments that take literals should allow arbitrary strings also. Literals retained for introspection/documentation. --- src/netCDF4/__init__.pyi | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index a3b4e6beb..f9ad9f615 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -217,21 +217,21 @@ default_fillvals: dict[str, int | float | str] def date2index( dates: dt.datetime | cftime.datetime | Sequence[dt.datetime | cftime.datetime] | DateTimeArray, nctime: Variable, - calendar: CalendarType | None = None, + calendar: CalendarType | str | None = None, select: Literal["exact", "before", "after", "nearest"] = "exact", has_year_zero: bool | None = None, ) -> int | npt.NDArray[np.int_]: ... def date2num( dates: dt.datetime | cftime.datetime | Sequence[dt.datetime | cftime.datetime] | DateTimeArray, units: str, - calendar: CalendarType | None = None, + calendar: CalendarType | str | None = None, has_year_zero: bool | None = None, longdouble: bool = False, ) -> np.number | npt.NDArray[np.number]: ... def num2date( times: Sequence[int | float | np.number] | npt.NDArray[np.number], units: str, - calendar: CalendarType = "standard", + calendar: CalendarType | str = "standard", only_use_cftime_datetimes: bool = True, only_use_python_datetimes: bool = False, has_year_zero: bool | None = None, @@ -246,9 +246,9 @@ class Dataset: def __init__( self, filename: str | os.PathLike, - mode: AccessMode = "r", + mode: AccessMode | str = "r", clobber: bool = True, - format: Format = "NETCDF4", + format: Format | str = "NETCDF4", diskless: bool = False, persist: bool = False, keepweakref: bool = False, @@ -316,7 +316,7 @@ class Dataset: fletcher32: Any = False, contiguous: Any = False, chunksizes: int | None = None, - endian: EndianType = "native", + endian: EndianType | str = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode | str = "BitGroom", @@ -339,7 +339,7 @@ class Dataset: fletcher32: Any = False, contiguous: Any = False, chunksizes: int | None = None, - endian: EndianType = "native", + endian: EndianType | str = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode | str = "BitGroom", @@ -362,7 +362,7 @@ class Dataset: fletcher32: Any = False, contiguous: Any = False, chunksizes: int | None = None, - endian: EndianType = "native", + endian: EndianType | str = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode | str = "BitGroom", @@ -398,7 +398,7 @@ class Dataset: def get_variables_by_attributes(self, **kwargs: Callable[[Any], bool] | Any) -> list[Variable]: ... @staticmethod def fromcdl( - cdlfilename: str, ncfilename: str | None = None, mode: AccessMode = "a", format: Format = "NETCDF4" + cdlfilename: str, ncfilename: str | None = None, mode: AccessMode | str = "a", format: Format | str = "NETCDF4" ) -> Dataset: ... @overload def tocdl(self, coordvars: bool = False, data: bool = False, outfile: None = None) -> str: ... @@ -449,7 +449,7 @@ class Variable(Generic[VarT]): fletcher32: Any = False, contiguous: Any = False, chunksizes: Sequence[int] | None = None, - endian: EndianType = "native", + endian: EndianType | str = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode | str = "BitGroom", @@ -474,7 +474,7 @@ class Variable(Generic[VarT]): fletcher32: Any = False, contiguous: Any = False, chunksizes: Sequence[int] | None = None, - endian: EndianType = "native", + endian: EndianType | str = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode | str = "BitGroom", @@ -499,7 +499,7 @@ class Variable(Generic[VarT]): fletcher32: Any = False, contiguous: Any = False, chunksizes: Sequence[int] | None = None, - endian: EndianType = "native", + endian: EndianType | str = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode | str = "BitGroom", @@ -523,7 +523,7 @@ class Variable(Generic[VarT]): fletcher32: Any = False, contiguous: Any = False, chunksizes: Sequence[int] | None = None, - endian: EndianType = "native", + endian: EndianType | str = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode | str = "BitGroom", @@ -673,7 +673,7 @@ class MFTime(_Variable): calendar: CalendarType | None units: str | None - def __init__(self, time: Variable, units: str | None = None, calendar: CalendarType | None = None): ... + def __init__(self, time: Variable, units: str | None = None, calendar: CalendarType | str | None = None): ... def __getitem__(self, elem: GetSetItemKey) -> np.ndarray: ... @overload From 4bad04f8cf471843997369b33edfc591fc2fbeb3 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 16 Jul 2024 16:25:55 -0700 Subject: [PATCH 31/81] Minor test fixes based on mypy/pyright checking --- test/test_compound_alignment.py | 2 +- test/test_multifile.py | 2 +- test/test_multifile2.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_compound_alignment.py b/test/test_compound_alignment.py index 0e1c36919..7b8c553b6 100644 --- a/test/test_compound_alignment.py +++ b/test/test_compound_alignment.py @@ -99,7 +99,7 @@ def runTest(self): f = netCDF4.Dataset(self.file, 'r') new_cells = f.variables["cells"][:] assert new_cells.shape == cells.shape - assert sorted(new_cells.dtype.names) == sorted(cells.dtype.names) + assert cells.dtype.names and sorted(new_cells.dtype.names) == sorted(cells.dtype.names) for name in cells.dtype.names: assert cells[name].dtype.name == new_cells[name].dtype.name assert cells[name].shape == new_cells[name].shape diff --git a/test/test_multifile.py b/test/test_multifile.py index 2301ebdf9..f9251c87f 100644 --- a/test/test_multifile.py +++ b/test/test_multifile.py @@ -74,7 +74,7 @@ def runTest(self): # testing multi-file get_variables_by_attributes f = MFDataset(self.files,check=True) assert f.get_variables_by_attributes(axis='T') == [] - f.get_variables_by_attributes(units='zlotys')[0] == f['x'] + assert f.get_variables_by_attributes(units='zlotys')[0] == f['x'] assert f.isopen() f.close() assert not f.isopen() diff --git a/test/test_multifile2.py b/test/test_multifile2.py index d5b310955..3ab3d482b 100644 --- a/test/test_multifile2.py +++ b/test/test_multifile2.py @@ -107,8 +107,8 @@ def runTest(self): # Get the real dates # skip this until cftime pull request #55 is in a released # version (1.0.1?). Otherwise, fix for issue #808 breaks this + dates = [] if Version(cftime.__version__) >= Version('1.0.1'): - dates = [] for file in self.files: f = Dataset(file) t = f.variables['time'] From 84d5b41dcbf16bf111f5f417db75d2e2affb4c70 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 16 Jul 2024 16:29:55 -0700 Subject: [PATCH 32/81] Use descriptor stubs to overload datatype and dtype properties --- .github/stubtest-allowlist | 2 ++ src/netCDF4/__init__.pyi | 30 ++++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.github/stubtest-allowlist b/.github/stubtest-allowlist index ed56536bf..44fa8ce8d 100644 --- a/.github/stubtest-allowlist +++ b/.github/stubtest-allowlist @@ -25,6 +25,8 @@ netCDF4.BloscInfo netCDF4.BoolInt netCDF4.GetSetItemKey netCDF4.VarT +netCDF4.RealVarT +netCDF4.ComplexVarT netCDF4.NumericVarT netCDF4.Dimension.__reduce_cython__ netCDF4.Dimension.__setstate_cython__ diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index f9ad9f615..4c007356c 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -122,6 +122,8 @@ DatatypeSpecifier: TypeAlias = ( ) VarT = TypeVar("VarT") +RealVarT = TypeVar("RealVarT", bound=NumPyRealType) +ComplexVarT = TypeVar("ComplexVarT", bound=NumPyComplexType) NumericVarT = TypeVar("NumericVarT", bound=NumPyNumericType) DimensionsSpecifier: TypeAlias = Union[str, bytes, Dimension, Iterable[Union[str, bytes, Dimension]]] @@ -430,6 +432,28 @@ class Dimension: def isunlimited(self) -> bool: ... def __len__(self) -> int: ... +class _VarDatatypeProperty: + # A descriptor definition of the property to allow overloads + @overload + def __get__(self, instance: Variable[RealVarT], owner: Any) -> RealVarT: ... + @overload + def __get__(self, instance: Variable[ComplexVarT], owner: Any) -> CompoundType: ... + @overload + def __get__(self, instance: Variable[str], owner: Any) -> VLType: ... + @overload + def __get__( + self, instance: Variable[Any], owner: Any + ) -> Any: ... # actual return type np.dtype | CompoundType | VLType | EnumType + +class _VarDtypeProperty: + # A descriptor definition of the property to allow overloads + @overload + def __get__(self, instance: Variable[NumericVarT], owner: Any) -> np.dtype[NumericVarT]: ... + @overload + def __get__(self, instance: Variable[str], owner: Any) -> type[str]: ... + @overload + def __get__(self, instance: Variable[Any], owner: Any) -> Any: ... # actual return type np.dtype | Type[str] + class Variable(Generic[VarT]): # Overloads of __new__ are provided for some cases where the Variable's type may be statically inferred from the datatype arg @overload @@ -531,13 +555,11 @@ class Variable(Generic[VarT]): chunk_cache: int | None = None, **kwargs: Any, ) -> None: ... + datatype: _VarDatatypeProperty + dtype: _VarDtypeProperty @property def name(self) -> str: ... @property - def dtype(self) -> Any: ... # actually np.dtype | type[str] - @property - def datatype(self) -> Any: ... # Actually np.dtype | NetCDFUDTClass - @property def shape(self) -> tuple[int, ...]: ... @property def size(self) -> int: ... From 75dab94e9a5a2f419d0c3fa5627a3454e9df4577 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 09:02:19 -0700 Subject: [PATCH 33/81] explain type: ignore --- test/test_enum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_enum.py b/test/test_enum.py index 536cb0a27..ca59cff49 100644 --- a/test/test_enum.py +++ b/test/test_enum.py @@ -27,7 +27,7 @@ def setUp(self): cloud_type = f.createEnumType(ENUM_BASETYPE,ENUM_NAME,ENUM_DICT) # make sure KeyError raised if non-integer basetype used. try: - cloud_typ2 = f.createEnumType(np.float32,ENUM_NAME,ENUM_DICT) # type: ignore + cloud_typ2 = f.createEnumType(np.float32,ENUM_NAME,ENUM_DICT) # type: ignore # mypy correctly doesn't like float32 except KeyError: pass f.createDimension('time',None) From 577a437f2c6dceefe85ec5bec59afc29bdb279cf Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 09:04:10 -0700 Subject: [PATCH 34/81] Fix from mypy - don't use the same name for different meanings in a fn --- test/test_multifile.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_multifile.py b/test/test_multifile.py index f9251c87f..ddfc05ada 100644 --- a/test/test_multifile.py +++ b/test/test_multifile.py @@ -124,13 +124,13 @@ def runTest(self): # Get the real dates dates = [] for file in self.files: - f = Dataset(file) - t = f.variables['time'] + ds = Dataset(file) + t = ds.variables['time'] dates.extend(cftime.num2date(t[:], t.units, calendar)) - f.close() + ds.close() # Compare with the MF dates - f = MFDataset(self.files,check=True) - t = f.variables['time'] + ds = MFDataset(self.files,check=True) + t = ds.variables['time'] T = MFTime(t, calendar=calendar) assert_equal(T.calendar, calendar) assert_equal(len(T), len(t)) @@ -142,7 +142,7 @@ def runTest(self): if Version(cftime.__version__) >= Version('1.0.1'): assert_array_equal(cftime.num2date(T[:], T.units, T.calendar), dates) assert_equal(cftime.date2index(datetime.datetime(1980, 1, 2), T), 366) - f.close() + ds.close() # Test exception is raised when no calendar attribute is available on the # time variable. @@ -154,8 +154,8 @@ def runTest(self): # variables. First, add calendar attributes to file. Note this will modify # the files inplace. calendars = ['standard', 'gregorian'] - for idx, f in enumerate(self.files): - with Dataset(f, 'a') as ds: + for idx, file in enumerate(self.files): + with Dataset(file, 'a') as ds: ds.variables['time'].calendar = calendars[idx] with MFDataset(self.files, check=True) as ds: with self.assertRaises(ValueError): From 2fcf116330aa5845eb3e3a0bdf92c1d21f3ad3fd Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 09:15:32 -0700 Subject: [PATCH 35/81] Add some type: ignore for redefitions mypy doesn't like --- test/test_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index af205be4c..65360c42a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -53,7 +53,7 @@ def test_fancy(self): try: - elem = ( np.arange(6).reshape((3,2)), slice(None), slice(None) ) + elem = ( np.arange(6).reshape((3,2)), slice(None), slice(None) ) # type: ignore # mypy doesn't like redefinition of elem here start, count, stride, put_ind = _StartCountStride(elem, (3,4,5)) except IndexError: pass @@ -90,7 +90,7 @@ def test_put_indices(self): start, count, stride, put_ind = _StartCountStride(elem, (3,4,5)) orig = np.arange(60).reshape((3,4,5)) dest = np.empty(_out_array_shape(count)) - dest[tuple(put_ind[0,0,0])] = orig[tuple(elem)] + dest[tuple(put_ind[0,0,0])] = orig[tuple(elem)] # type: ignore # mypy doesn't like indexing of orig, but it's OK def test_boolean(self): elem = (1, slice(None), np.array([True, True, False, False, True])) @@ -198,14 +198,14 @@ def test_ellipsis(self): assert_equal(put_ind[0,0,0,0,0], (slice(None), slice(None), slice(None), slice(None), slice(None))) try: - elem=(Ellipsis, [15,16,17,18,19], slice(None)) + elem=(Ellipsis, [15,16,17,18,19], slice(None)) # type: ignore # mypy doesn't like redefinition of elem here start, count, stride, put_ind = _StartCountStride(elem, (2,10,20,10,10)) assert_equal(None, 'Should throw an exception') except IndexError as e: assert_equal(str(e), "integer index exceeds dimension size") try: - elem=(Ellipsis, [15,16,17,18,19], Ellipsis) + elem=(Ellipsis, [15,16,17,18,19], Ellipsis) # type: ignore # mypy doesn't like redefinition of elem here start, count, stride, put_ind = _StartCountStride(elem, (2,10, 20,10,10)) assert_equal(None, 'Should throw an exception') except IndexError as e: @@ -303,7 +303,7 @@ def test_ellipsis(self): assert_equal(take_ind[0,0,0,0,0], (slice(None), slice(None), slice(None), slice(None), slice(None))) try: - elem=(Ellipsis, [15,16,17,18,19], slice(None)) + elem=(Ellipsis, [15,16,17,18,19], slice(None)) # type: ignore # mypy doesn't like redefinition of elem here start, count, stride, take_ind = _StartCountStride(elem, (2,10,20,10,10),\ ['time', 'z', 'y', 'x'], grp, (2,10,5,10,10), put=True) assert_equal(None, 'Should throw an exception') @@ -312,7 +312,7 @@ def test_ellipsis(self): assert_equal(str(e), "list index out of range") try: - elem=(Ellipsis, [15,16,17,18,19], Ellipsis) + elem=(Ellipsis, [15,16,17,18,19], Ellipsis) # type: ignore # mypy doesn't like redefinition of elem here start, count, stride, take_ind = _StartCountStride(elem, (2,10, 20,10,10),\ ['time', 'z', 'y', 'x'], grp, (2,10,5,10,10), put=True) assert_equal(None, 'Should throw an exception') From d5f87dab3b600d1e47621ebff92025bf8dda6b87 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 09:18:57 -0700 Subject: [PATCH 36/81] Make mypy more thorough by checking contents of untyped functions --- .github/workflows/build_master.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index f8f841636..aab37eefb 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -83,5 +83,5 @@ jobs: - name: Stubtest run: | stubtest netCDF4 --allowlist .github/stubtest-allowlist --mypy-config-file=pyproject.toml - mypy test - mypy examples + mypy test --check-untyped-defs --allow-redefinition + mypy examples --check-untyped-defs --allow-redefinition From aee46f8f91208afefb3f739c6f5db638b46ee7a7 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 09:20:01 -0700 Subject: [PATCH 37/81] Add faux __iter__ method to Variable so type-checkers believe that it's iterable. --- src/netCDF4/__init__.pyi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 4c007356c..9fd408187 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -36,6 +36,7 @@ from typing import ( Final, Generic, Iterable, + Iterator, Literal, Mapping, NoReturn, @@ -611,6 +612,7 @@ class Variable(Generic[VarT]): def __setitem__(self, elem: GetSetItemKey, data: npt.ArrayLike) -> None: ... def __array__(self) -> np.ndarray: ... def __len__(self) -> int: ... + def __iter__(self) -> Iterator[Any]: ... # faux method so mypy believes Variable is iterable class CompoundType: dtype: np.dtype From d41ea1572c8ea2f194e6f88aa6bce293165488c1 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 09:25:35 -0700 Subject: [PATCH 38/81] Actually test for expected warning --- test/test_masked.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_masked.py b/test/test_masked.py index 0775f29ad..3cb2bf786 100644 --- a/test/test_masked.py +++ b/test/test_masked.py @@ -93,7 +93,8 @@ def setUp(self): # of raising an exception when auto-converted slice to a # masked array with netCDF4.Dataset(pathlib.Path(__file__).parent / "issue1152.nc") as dataset: - data = dataset['v'][:] + with self.assertWarns(UserWarning): + data = dataset['v'][:] # issue #1271 (mask is ignored when assigning bool array to uint8 var) ds = netCDF4.Dataset(self.file3, "w") From 19c736ffbed86a7b8a280f13623b1b7b3ba8a6de Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 09:52:42 -0700 Subject: [PATCH 39/81] Updated allowlist with faux __iter__ method --- .github/stubtest-allowlist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/stubtest-allowlist b/.github/stubtest-allowlist index 44fa8ce8d..c362121e0 100644 --- a/.github/stubtest-allowlist +++ b/.github/stubtest-allowlist @@ -31,11 +31,13 @@ netCDF4.NumericVarT netCDF4.Dimension.__reduce_cython__ netCDF4.Dimension.__setstate_cython__ netCDF4.Variable.auto_complex +netCDF4.Variable.__iter__ netCDF4._netCDF4.Dimension.__reduce_cython__ netCDF4._netCDF4.Dimension.__setstate_cython__ netCDF4._netCDF4.NC_DISKLESS netCDF4._netCDF4.NC_PERSIST netCDF4._netCDF4.Variable.auto_complex +netCDF4._netCDF4.Variable.__iter__ netCDF4._netCDF4.__reduce_cython__ netCDF4._netCDF4.__setstate_cython__ netCDF4._netCDF4.__test__ From cfb5fcb5447fd5033f78e3b9146d423f5f4c9312 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 11:16:07 -0700 Subject: [PATCH 40/81] Allow specifying more verbosity in test/run_all.py --- test/run_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_all.py b/test/run_all.py index deff8c6af..b99565963 100755 --- a/test/run_all.py +++ b/test/run_all.py @@ -24,7 +24,7 @@ sys.stdout.write('netcdf lib version: %s\n' % __netcdf4libversion__) sys.stdout.write('numpy version %s\n' % numpy.__version__) sys.stdout.write('cython version %s\n' % cython.__version__) - runner = unittest.TextTestRunner(verbosity=1) + runner = unittest.TextTestRunner(verbosity=(2 if "-v" in sys.argv else 1)) result = runner.run(testsuite) if not result.wasSuccessful(): sys.exit(1) From fbcb79f0b2079f3182913e7293aac1cb5254b0be Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 11:19:04 -0700 Subject: [PATCH 41/81] Fix EnumDictTestCase not called due to bad indentation --- test/test_enum.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/test/test_enum.py b/test/test_enum.py index ca59cff49..90761d849 100644 --- a/test/test_enum.py +++ b/test/test_enum.py @@ -66,6 +66,7 @@ def runTest(self): f.close() class EnumDictTestCase(unittest.TestCase): + # issue 1128 def setUp(self): DT = np.int16; BITS = 8 @@ -79,14 +80,16 @@ def setUp(self): ev = nc.createVariable('evar', et) # Succeeds because the created EnumType does keep the correct dict ev[...] = self.STORED_VAL - def tearDown(self): - os.remove(self.file) - def runTest(self): - with netCDF4.Dataset(self.file, 'r') as nc: - read_var = nc['evar'] - read_et = nc.enumtypes["etype"] - assert read_var[...] == self.STORED_VAL - assert read_et.enum_dict == self.VAL_MAP + + def tearDown(self): + os.remove(self.file) + + def runTest(self): + with netCDF4.Dataset(self.file, 'r') as nc: + read_var = nc['evar'] + read_et = nc.enumtypes["etype"] + assert read_var[...] == self.STORED_VAL + assert read_et.enum_dict == self.VAL_MAP if __name__ == '__main__': unittest.main() From ac985501ccd5332ce33d4c43baa95487cfc72e37 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 12:13:35 -0700 Subject: [PATCH 42/81] Allow str as key for variable --- src/netCDF4/__init__.pyi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 9fd408187..7b12dd809 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -149,13 +149,14 @@ DateTimeArray: TypeAlias = npt.NDArray[np.object_] GetSetItemKey: TypeAlias = ( int | float + | str # strs that can be cooerced to ints | np.number | slice | ellipsis | list[int] | list[float] | list[bool] - | list[str] # strs that can be cooerced to ints + | list[str] | list[int | float] | list[int | bool] | list[int | str] From 866ae476fde9829d939868da15b3b8af6fb20ed6 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 17 Jul 2024 12:55:35 -0700 Subject: [PATCH 43/81] minor comment update --- src/netCDF4/__init__.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 7b12dd809..683a446f8 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -435,7 +435,7 @@ class Dimension: def __len__(self) -> int: ... class _VarDatatypeProperty: - # A descriptor definition of the property to allow overloads + # A faux descriptor definition of the property to allow overloads @overload def __get__(self, instance: Variable[RealVarT], owner: Any) -> RealVarT: ... @overload @@ -448,7 +448,7 @@ class _VarDatatypeProperty: ) -> Any: ... # actual return type np.dtype | CompoundType | VLType | EnumType class _VarDtypeProperty: - # A descriptor definition of the property to allow overloads + # A faux descriptor definition of the property to allow overloads @overload def __get__(self, instance: Variable[NumericVarT], owner: Any) -> np.dtype[NumericVarT]: ... @overload From 252deab1ab9cfe81b5e2083863cff4b2654fc330 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 24 Jul 2024 11:56:18 -0700 Subject: [PATCH 44/81] Moved mypy flags to pyproject.toml --- .github/workflows/build_master.yml | 4 ++-- pyproject.toml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index aab37eefb..f8f841636 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -83,5 +83,5 @@ jobs: - name: Stubtest run: | stubtest netCDF4 --allowlist .github/stubtest-allowlist --mypy-config-file=pyproject.toml - mypy test --check-untyped-defs --allow-redefinition - mypy examples --check-untyped-defs --allow-redefinition + mypy test + mypy examples diff --git a/pyproject.toml b/pyproject.toml index 8c8c8a327..2c058ccb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,8 @@ pythonpath = ["test"] [tool.mypy] files = ["src/netCDF4"] +check_untyped_defs = true +allow_redefinition = true [[tool.mypy.overrides]] ignore_missing_imports = true From 620faf73d9bc7905d367731af3b3b32197716cd3 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 24 Jul 2024 12:18:33 -0700 Subject: [PATCH 45/81] Rename variables rather than mypy ignore to avoid redefinition errors --- test/test_utils.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index 65360c42a..12000aa6b 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -53,14 +53,14 @@ def test_fancy(self): try: - elem = ( np.arange(6).reshape((3,2)), slice(None), slice(None) ) # type: ignore # mypy doesn't like redefinition of elem here - start, count, stride, put_ind = _StartCountStride(elem, (3,4,5)) + elem2 = ( np.arange(6).reshape((3,2)), slice(None), slice(None) ) + start, count, stride, put_ind = _StartCountStride(elem2, (3,4,5)) except IndexError: pass # this one should be converted to a slice - elem = [slice(None), [1,3,5], 8] - start, count, stride, put_ind = _StartCountStride(elem, (50, 6, 10)) + elem3 = [slice(None), [1,3,5], 8] + start, count, stride, put_ind = _StartCountStride(elem3, (50, 6, 10)) # pull request #683 now does not convert integer sequences to strided # slices. PR #1224 reverts this behavior. assert_equal(put_ind[...,1].squeeze(), slice(None,None,None)) @@ -81,8 +81,8 @@ def test_multiple_sequences(self): assert_equal(count[...,2], 10) i = [1,3,4] - elem = (i, i, i) - start, count, stride, put_ind = _StartCountStride(elem, (50, 5, 10)) + elem2 = (i, i, i) + start, count, stride, put_ind = _StartCountStride(elem2, (50, 5, 10)) assert_equal(_out_array_shape(count), (3,3,3)) def test_put_indices(self): @@ -198,15 +198,15 @@ def test_ellipsis(self): assert_equal(put_ind[0,0,0,0,0], (slice(None), slice(None), slice(None), slice(None), slice(None))) try: - elem=(Ellipsis, [15,16,17,18,19], slice(None)) # type: ignore # mypy doesn't like redefinition of elem here - start, count, stride, put_ind = _StartCountStride(elem, (2,10,20,10,10)) + elem2=(Ellipsis, [15,16,17,18,19], slice(None)) + start, count, stride, put_ind = _StartCountStride(elem2, (2,10,20,10,10)) assert_equal(None, 'Should throw an exception') except IndexError as e: assert_equal(str(e), "integer index exceeds dimension size") try: - elem=(Ellipsis, [15,16,17,18,19], Ellipsis) # type: ignore # mypy doesn't like redefinition of elem here - start, count, stride, put_ind = _StartCountStride(elem, (2,10, 20,10,10)) + elem3=(Ellipsis, [15,16,17,18,19], Ellipsis) + start, count, stride, put_ind = _StartCountStride(elem3, (2,10, 20,10,10)) assert_equal(None, 'Should throw an exception') except IndexError as e: assert_equal(str(e), "At most one ellipsis allowed in a slicing expression") @@ -303,8 +303,8 @@ def test_ellipsis(self): assert_equal(take_ind[0,0,0,0,0], (slice(None), slice(None), slice(None), slice(None), slice(None))) try: - elem=(Ellipsis, [15,16,17,18,19], slice(None)) # type: ignore # mypy doesn't like redefinition of elem here - start, count, stride, take_ind = _StartCountStride(elem, (2,10,20,10,10),\ + elem2=(Ellipsis, [15,16,17,18,19], slice(None)) + start, count, stride, take_ind = _StartCountStride(elem2, (2,10,20,10,10),\ ['time', 'z', 'y', 'x'], grp, (2,10,5,10,10), put=True) assert_equal(None, 'Should throw an exception') except IndexError as e: @@ -312,8 +312,8 @@ def test_ellipsis(self): assert_equal(str(e), "list index out of range") try: - elem=(Ellipsis, [15,16,17,18,19], Ellipsis) # type: ignore # mypy doesn't like redefinition of elem here - start, count, stride, take_ind = _StartCountStride(elem, (2,10, 20,10,10),\ + elem3=(Ellipsis, [15,16,17,18,19], Ellipsis) + start, count, stride, take_ind = _StartCountStride(elem3, (2,10, 20,10,10),\ ['time', 'z', 'y', 'x'], grp, (2,10,5,10,10), put=True) assert_equal(None, 'Should throw an exception') except IndexError as e: From 31513eec323c102e83e5e98cba6190af6a0b279a Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 24 Jul 2024 12:25:26 -0700 Subject: [PATCH 46/81] Restore bool type on truthy inputs, related fix on Variable.__init__ --- src/netCDF4/__init__.pyi | 62 ++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 4e74eaaea..a94b9b6f2 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -312,14 +312,14 @@ class Dataset: datatype: np.dtype[NumericVarT] | type[NumericVarT], dimensions: DimensionsSpecifier = (), compression: CompressionType | str | None = None, - zlib: Any = False, + zlib: bool = False, complevel: CompressionLevel | int | None = 4, - shuffle: Any = True, + shuffle: bool = True, szip_coding: Literal["nn", "ec"] | str = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, blosc_shuffle: Literal[0, 1, 2] | int = 1, - fletcher32: Any = False, - contiguous: Any = False, + fletcher32: bool = False, + contiguous: bool = False, chunksizes: int | None = None, endian: EndianType | str = "native", least_significant_digit: int | None = None, @@ -335,14 +335,14 @@ class Dataset: datatype: np.dtype[np.str_] | type[str | np.str_], dimensions: DimensionsSpecifier = (), compression: CompressionType | str | None = None, - zlib: Any = False, + zlib: bool = False, complevel: CompressionLevel | int | None = 4, - shuffle: Any = True, + shuffle: bool = True, szip_coding: Literal["nn", "ec"] | str = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, blosc_shuffle: Literal[0, 1, 2] | int = 1, - fletcher32: Any = False, - contiguous: Any = False, + fletcher32: bool = False, + contiguous: bool = False, chunksizes: int | None = None, endian: EndianType | str = "native", least_significant_digit: int | None = None, @@ -358,14 +358,14 @@ class Dataset: datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), compression: CompressionType | str | None = None, - zlib: Any = False, + zlib: bool = False, complevel: CompressionLevel | int | None = 4, - shuffle: Any = True, + shuffle: bool = True, szip_coding: Literal["nn", "ec"] | str = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, blosc_shuffle: Literal[0, 1, 2] | int = 1, - fletcher32: Any = False, - contiguous: Any = False, + fletcher32: bool = False, + contiguous: bool = False, chunksizes: int | None = None, endian: EndianType | str = "native", least_significant_digit: int | None = None, @@ -467,14 +467,14 @@ class Variable(Generic[VarT]): datatype: np.dtype[NumericVarT] | type[NumericVarT], dimensions: DimensionsSpecifier = (), compression: CompressionType | str | None = None, - zlib: Any = False, + zlib: bool = False, complevel: CompressionLevel | int | None = 4, - shuffle: Any = True, + shuffle: bool = True, szip_coding: Literal["nn", "ec"] | str = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, blosc_shuffle: Literal[0, 1, 2] | int = 1, - fletcher32: Any = False, - contiguous: Any = False, + fletcher32: bool = False, + contiguous: bool = False, chunksizes: Sequence[int] | None = None, endian: EndianType | str = "native", least_significant_digit: int | None = None, @@ -492,14 +492,14 @@ class Variable(Generic[VarT]): datatype: np.dtype[np.str_] | type[str | np.str_], dimensions: DimensionsSpecifier = (), compression: CompressionType | str | None = None, - zlib: Any = False, + zlib: bool = False, complevel: CompressionLevel | int | None = 4, - shuffle: Any = True, + shuffle: bool = True, szip_coding: Literal["nn", "ec"] | str = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, blosc_shuffle: Literal[0, 1, 2] | int = 1, - fletcher32: Any = False, - contiguous: Any = False, + fletcher32: bool = False, + contiguous: bool = False, chunksizes: Sequence[int] | None = None, endian: EndianType | str = "native", least_significant_digit: int | None = None, @@ -517,14 +517,14 @@ class Variable(Generic[VarT]): datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), compression: CompressionType | str | None = None, - zlib: Any = False, + zlib: bool = False, complevel: CompressionLevel | int | None = 4, - shuffle: Any = True, + shuffle: bool = True, szip_coding: Literal["nn", "ec"] | str = "nn", szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, blosc_shuffle: Literal[0, 1, 2] | int = 1, - fletcher32: Any = False, - contiguous: Any = False, + fletcher32: bool = False, + contiguous: bool = False, chunksizes: Sequence[int] | None = None, endian: EndianType | str = "native", least_significant_digit: int | None = None, @@ -541,14 +541,14 @@ class Variable(Generic[VarT]): datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), compression: CompressionType | str | None = None, - zlib: Any = False, + zlib: bool = False, complevel: CompressionLevel | int | None = 4, - shuffle: Any = True, - szip_coding: Literal["nn", "ec"] = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, - blosc_shuffle: Literal[0, 1, 2] = 1, - fletcher32: Any = False, - contiguous: Any = False, + shuffle: bool = True, + szip_coding: Literal["nn", "ec"] | str = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, + blosc_shuffle: Literal[0, 1, 2] | int = 1, + fletcher32: bool = False, + contiguous: bool = False, chunksizes: Sequence[int] | None = None, endian: EndianType | str = "native", least_significant_digit: int | None = None, From 15151d2fe376738eed0ad19ba67229af1aeb570a Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 24 Jul 2024 12:53:21 -0700 Subject: [PATCH 47/81] Fix type for shuffle argument in test --- test/test_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_types.py b/test/test_types.py index 7e2a998ee..1f65fc5fc 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -14,7 +14,7 @@ n1dim = 5 n2dim = 10 ranarr = 100.*uniform(size=(n1dim,n2dim)) -zlib=False;complevel=0;shuffle=0;least_significant_digit=None +zlib=False;complevel=0;shuffle=False;least_significant_digit=None datatypes = ['f8','f4','i1','i2','i4','i8','u1','u2','u4','u8','S1'] FillValue = 1.0 issue273_data = np.ma.array(['z']*10,dtype='S1',\ @@ -82,7 +82,7 @@ def runTest(self): v2 = f.variables['issue273'] assert type(v2._FillValue) == bytes assert v2._FillValue == b'\x00' - assert str(issue273_data) == str(v2[:]) + assert str(issue273_data) == str(v2[:]) # issue 707 (don't apply missing_value if cast to variable type is # unsafe) v3 = f.variables['issue707'] From d982b4c7fb79ede403ceb08167cb3f00f1076476 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 24 Jul 2024 15:10:00 -0700 Subject: [PATCH 48/81] Make sure rc_get and rc_set are exported --- src/netCDF4/__init__.pyi | 2 ++ src/netCDF4/_netCDF4.pyi | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index a94b9b6f2..dd42c9103 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -63,6 +63,8 @@ __all__ = [ "VLType", "date2num", "num2date", + "rc_get", + "rc_set", "date2index", "stringtochar", "chartostring", diff --git a/src/netCDF4/_netCDF4.pyi b/src/netCDF4/_netCDF4.pyi index 759dc3004..d0bdb389f 100644 --- a/src/netCDF4/_netCDF4.pyi +++ b/src/netCDF4/_netCDF4.pyi @@ -43,6 +43,8 @@ from . import ( is_native_big, is_native_little, num2date, + rc_get, + rc_set, set_alignment, set_chunk_cache, stringtoarr, From f85bbfdaf8f6ac5d562223c807aefa21e575ee23 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 24 Jul 2024 15:10:17 -0700 Subject: [PATCH 49/81] Make sure mypy and stubtest exclude utils --- .github/stubtest-allowlist | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/stubtest-allowlist b/.github/stubtest-allowlist index c362121e0..4f0e7a94d 100644 --- a/.github/stubtest-allowlist +++ b/.github/stubtest-allowlist @@ -41,4 +41,4 @@ netCDF4._netCDF4.Variable.__iter__ netCDF4._netCDF4.__reduce_cython__ netCDF4._netCDF4.__setstate_cython__ netCDF4._netCDF4.__test__ -netCDF4.utils.bytes \ No newline at end of file +netCDF4.utils \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8b4feb8b1..fb245117b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ filterwarnings = [ [tool.mypy] files = ["src/netCDF4"] +exclude = "utils.py" check_untyped_defs = true allow_redefinition = true From 40000e27aa78ce14a5be1f41c7950044d58174a7 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 25 Jul 2024 11:23:26 -0700 Subject: [PATCH 50/81] Add missing bzip2 to CompressionType --- src/netCDF4/__init__.pyi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index dd42c9103..0bfd2f41d 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -130,7 +130,9 @@ ComplexVarT = TypeVar("ComplexVarT", bound=NumPyComplexType) NumericVarT = TypeVar("NumericVarT", bound=NumPyNumericType) DimensionsSpecifier: TypeAlias = Union[str, bytes, Dimension, Iterable[Union[str, bytes, Dimension]]] -CompressionType: TypeAlias = Literal["zlib", "szip", "zstd", "blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd"] +CompressionType: TypeAlias = Literal[ + "zlib", "szip", "zstd", "bzip2", "blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd" +] CompressionLevel: TypeAlias = Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] AccessMode: TypeAlias = Literal["r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"] Format: TypeAlias = Literal["NETCDF4", "NETCDF4_CLASSIC", "NETCDF3_CLASSIC", "NETCDF3_64BIT_OFFSET", "NETCDF3_64BIT_DATA"] From f24edef263e8c0afeee3ef52f216a6d603dee83b Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 25 Jul 2024 11:29:17 -0700 Subject: [PATCH 51/81] Restore limiting types to literals when creating datasets and variables --- src/netCDF4/__init__.pyi | 112 +++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 0bfd2f41d..834d77375 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -255,9 +255,9 @@ class Dataset: def __init__( self, filename: str | os.PathLike, - mode: AccessMode | str = "r", + mode: AccessMode = "r", clobber: bool = True, - format: Format | str = "NETCDF4", + format: Format = "NETCDF4", diskless: bool = False, persist: bool = False, keepweakref: bool = False, @@ -315,20 +315,20 @@ class Dataset: varname: str, datatype: np.dtype[NumericVarT] | type[NumericVarT], dimensions: DimensionsSpecifier = (), - compression: CompressionType | str | None = None, + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevel | int | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal["nn", "ec"] | str = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, - blosc_shuffle: Literal[0, 1, 2] | int = 1, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: int | None = None, - endian: EndianType | str = "native", + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode | str = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable[NumericVarT]: ... @@ -338,20 +338,20 @@ class Dataset: varname: str, datatype: np.dtype[np.str_] | type[str | np.str_], dimensions: DimensionsSpecifier = (), - compression: CompressionType | str | None = None, + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevel | int | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal["nn", "ec"] | str = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, - blosc_shuffle: Literal[0, 1, 2] | int = 1, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: int | None = None, - endian: EndianType | str = "native", + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode | str = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable[str]: ... @@ -361,20 +361,20 @@ class Dataset: varname: str, datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), - compression: CompressionType | str | None = None, + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevel | int | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal["nn", "ec"] | str = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, - blosc_shuffle: Literal[0, 1, 2] | int = 1, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, chunksizes: int | None = None, - endian: EndianType | str = "native", + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode | str = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable: ... @@ -407,7 +407,7 @@ class Dataset: def get_variables_by_attributes(self, **kwargs: Callable[[Any], bool] | Any) -> list[Variable]: ... @staticmethod def fromcdl( - cdlfilename: str, ncfilename: str | None = None, mode: AccessMode | str = "a", format: Format | str = "NETCDF4" + cdlfilename: str, ncfilename: str | None = None, mode: AccessMode = "a", format: Format = "NETCDF4" ) -> Dataset: ... @overload def tocdl(self, coordvars: bool = False, data: bool = False, outfile: None = None) -> str: ... @@ -470,20 +470,20 @@ class Variable(Generic[VarT]): name: str, datatype: np.dtype[NumericVarT] | type[NumericVarT], dimensions: DimensionsSpecifier = (), - compression: CompressionType | str | None = None, + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevel | int | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal["nn", "ec"] | str = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, - blosc_shuffle: Literal[0, 1, 2] | int = 1, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: Sequence[int] | None = None, - endian: EndianType | str = "native", + chunksizes: int | None = None, + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode | str = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, @@ -495,20 +495,20 @@ class Variable(Generic[VarT]): name: str, datatype: np.dtype[np.str_] | type[str | np.str_], dimensions: DimensionsSpecifier = (), - compression: CompressionType | str | None = None, + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevel | int | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal["nn", "ec"] | str = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, - blosc_shuffle: Literal[0, 1, 2] | int = 1, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: Sequence[int] | None = None, - endian: EndianType | str = "native", + chunksizes: int | None = None, + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode | str = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, @@ -520,20 +520,20 @@ class Variable(Generic[VarT]): name: str, datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), - compression: CompressionType | str | None = None, + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevel | int | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal["nn", "ec"] | str = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, - blosc_shuffle: Literal[0, 1, 2] | int = 1, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: Sequence[int] | None = None, - endian: EndianType | str = "native", + chunksizes: int | None = None, + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode | str = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, @@ -544,20 +544,20 @@ class Variable(Generic[VarT]): name: str, datatype: DatatypeSpecifier, dimensions: DimensionsSpecifier = (), - compression: CompressionType | str | None = None, + compression: CompressionType | None = None, zlib: bool = False, - complevel: CompressionLevel | int | None = 4, + complevel: CompressionLevel | None = 4, shuffle: bool = True, - szip_coding: Literal["nn", "ec"] | str = "nn", - szip_pixels_per_block: Literal[4, 8, 16, 32] | int = 8, - blosc_shuffle: Literal[0, 1, 2] | int = 1, + szip_coding: Literal["nn", "ec"] = "nn", + szip_pixels_per_block: Literal[4, 8, 16, 32] = 8, + blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: Sequence[int] | None = None, - endian: EndianType | str = "native", + chunksizes: int | None = None, + endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, - quantize_mode: QuantizeMode | str = "BitGroom", + quantize_mode: QuantizeMode = "BitGroom", fill_value: int | float | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, From 7eebd5a70f6c8f6d52c0a723dc72777ade9bc536 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 25 Jul 2024 11:30:45 -0700 Subject: [PATCH 52/81] Add np.number as valid type for fill_value --- src/netCDF4/__init__.pyi | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 834d77375..734fa83f7 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -329,7 +329,7 @@ class Dataset: least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable[NumericVarT]: ... @overload @@ -352,7 +352,7 @@ class Dataset: least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable[str]: ... @overload @@ -375,7 +375,7 @@ class Dataset: least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, ) -> Variable: ... def renameVariable(self, oldname: str, newname: str) -> None: ... @@ -484,7 +484,7 @@ class Variable(Generic[VarT]): least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, ) -> Variable[NumericVarT]: ... @@ -509,7 +509,7 @@ class Variable(Generic[VarT]): least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, ) -> Variable[str]: ... @@ -534,7 +534,7 @@ class Variable(Generic[VarT]): least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, ) -> Variable: ... @@ -558,7 +558,7 @@ class Variable(Generic[VarT]): least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, chunk_cache: int | None = None, **kwargs: Any, ) -> None: ... From 97273dd47a0798c50900965f03f431841070d317 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 25 Jul 2024 13:41:00 -0700 Subject: [PATCH 53/81] Test updates to deal with only allowing Literals for some parameters --- examples/bench_compress.py | 8 ++++++- examples/mpi_example.py | 31 +++++++++++++------------ pyproject.toml | 3 +++ test/test_compression.py | 4 ++++ test/test_compression_blosc.py | 2 ++ test/test_compression_bzip2.py | 2 ++ test/test_endian.py | 2 ++ test/test_enum.py | 2 +- test/test_stringarr.py | 2 ++ test/test_types.py | 2 ++ test/test_utils.py | 2 +- test/type_guards.py | 41 ++++++++++++++++++++++++++++++++++ 12 files changed, 84 insertions(+), 17 deletions(-) create mode 100644 test/type_guards.py diff --git a/examples/bench_compress.py b/examples/bench_compress.py index 39bffe906..06a343391 100644 --- a/examples/bench_compress.py +++ b/examples/bench_compress.py @@ -1,7 +1,9 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. from numpy.random.mtrand import uniform +from typing_extensions import TypeGuard import netCDF4 +import netCDF4.utils from timeit import Timer import os, sys @@ -13,8 +15,10 @@ ntrials = 10 sys.stdout.write('reading and writing a %s by %s by %s by %s random array ..\n'%(n1dim,n2dim,n3dim,n4dim)) sys.stdout.write('(average of %s trials)\n' % ntrials) -array = netCDF4.utils._quantize(uniform(size=(n1dim,n2dim,n3dim,n4dim)),4) # type: ignore +array = netCDF4.utils._quantize(uniform(size=(n1dim,n2dim,n3dim,n4dim)),4) +def valid_complevel(complevel) -> TypeGuard[netCDF4.CompressionLevel | None]: + return complevel is None or isinstance(complevel, int) and 0 <= complevel <= 6 def write_netcdf(filename,zlib=False,shuffle=False,complevel=6): file = netCDF4.Dataset(filename,'w',format='NETCDF4') @@ -22,6 +26,8 @@ def write_netcdf(filename,zlib=False,shuffle=False,complevel=6): file.createDimension('n2', n2dim) file.createDimension('n3', n3dim) file.createDimension('n4', n4dim) + if not valid_complevel(complevel): + raise ValueError("Invalid compression level") foo = file.createVariable('data',\ 'f8',('n1','n2','n3','n4'),zlib=zlib,shuffle=shuffle,complevel=complevel) foo[:] = array diff --git a/examples/mpi_example.py b/examples/mpi_example.py index 7966bdafe..1e781b461 100644 --- a/examples/mpi_example.py +++ b/examples/mpi_example.py @@ -1,27 +1,30 @@ # to run: mpirun -np 4 python mpi_example.py import sys -from typing import Literal +from typing_extensions import TypeGuard from mpi4py import MPI import numpy as np from netCDF4 import Dataset +import netCDF4 -format: Literal[ - 'NETCDF4', - 'NETCDF4_CLASSIC', - 'NETCDF3_CLASSIC', - 'NETCDF3_64BIT_OFFSET', - 'NETCDF3_64BIT_DATA' -] -if len(sys.argv) == 2: - format = sys.argv[1] # type: ignore -else: - format = 'NETCDF4_CLASSIC' + +def is_nc_format(fmt: str) -> TypeGuard[netCDF4.Format]: + return fmt in { + 'NETCDF4', + 'NETCDF4_CLASSIC', + 'NETCDF3_CLASSIC', + 'NETCDF3_64BIT_OFFSET', + 'NETCDF3_64BIT_DATA' + } + +nc_format = 'NETCDF4_CLASSIC' if len(sys.argv) < 2 else sys.argv[1] +if not is_nc_format(nc_format): + raise ValueError("Invalid file format") rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run) if rank == 0: - print('Creating file with format {}'.format(format)) + print('Creating file with format {}'.format(nc_format)) nc = Dataset('parallel_test.nc', 'w', parallel=True, comm=MPI.COMM_WORLD, - info=MPI.Info(), format=format) + info=MPI.Info(), format=nc_format) # below should work also - MPI_COMM_WORLD and MPI_INFO_NULL will be used. #nc = Dataset('parallel_test.nc', 'w', parallel=True) d = nc.createDimension('dim',4) diff --git a/pyproject.toml b/pyproject.toml index fb245117b..c577e761a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,9 @@ files = ["src/netCDF4"] exclude = "utils.py" check_untyped_defs = true allow_redefinition = true +# next 2 lines workarounds for mypy dealing with type_guards.py +mypy_path = "test" +explicit_package_bases = true [[tool.mypy.overrides]] ignore_missing_imports = true diff --git a/test/test_compression.py b/test/test_compression.py index 78827ddff..eab808227 100644 --- a/test/test_compression.py +++ b/test/test_compression.py @@ -3,6 +3,7 @@ from netCDF4.utils import _quantize from numpy.testing import assert_almost_equal import os, tempfile, unittest +from type_guards import valid_complevel, valid_comptype ndim = 100000 ndim2 = 100 @@ -15,6 +16,7 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,contiguous=False,\ chunksizes=None,complevel=6,fletcher32=False): + assert valid_complevel(complevel) file = Dataset(filename,'w') file.createDimension('n', ndim) foo = file.createVariable('data',\ @@ -30,6 +32,7 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=F #compression='' #compression=0 #compression='gzip' # should fail + assert valid_comptype(compression) foo2 = file.createVariable('data2',\ dtype,('n'),compression=compression,least_significant_digit=least_significant_digit,\ shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) @@ -46,6 +49,7 @@ def write_netcdf2(filename,zlib,least_significant_digit,data,dtype='f8',shuffle= file = Dataset(filename,'w') file.createDimension('n', ndim) file.createDimension('n2', ndim2) + assert valid_complevel(complevel) foo = file.createVariable('data2',\ dtype,('n','n2'),zlib=zlib,least_significant_digit=least_significant_digit,\ shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) diff --git a/test/test_compression_blosc.py b/test/test_compression_blosc.py index 57d8e0f56..6915544c8 100644 --- a/test/test_compression_blosc.py +++ b/test/test_compression_blosc.py @@ -3,6 +3,7 @@ from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_blosc_filter +from type_guards import valid_complevel, valid_bloscshuffle ndim = 100000 @@ -12,6 +13,7 @@ datarr = uniform(size=(ndim,)) def write_netcdf(filename,dtype='f8',blosc_shuffle=1,complevel=6): + assert valid_complevel(complevel) and valid_bloscshuffle(blosc_shuffle) nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ diff --git a/test/test_compression_bzip2.py b/test/test_compression_bzip2.py index 75c0fced1..9a58b0668 100644 --- a/test/test_compression_bzip2.py +++ b/test/test_compression_bzip2.py @@ -3,6 +3,7 @@ from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_bzip2_filter +from type_guards import valid_complevel ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -10,6 +11,7 @@ array = uniform(size=(ndim,)) def write_netcdf(filename,dtype='f8',complevel=6): + assert valid_complevel(complevel) nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ diff --git a/test/test_endian.py b/test/test_endian.py index 99d6b44f8..3a8aecebb 100644 --- a/test/test_endian.py +++ b/test/test_endian.py @@ -2,6 +2,7 @@ import numpy as np import unittest, os, tempfile from numpy.testing import assert_array_equal, assert_array_almost_equal +from type_guards import valid_endian data = np.arange(12,dtype='f4').reshape(3,4) FILE_NAME = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -73,6 +74,7 @@ def issue310(file): endian='little' else: raise ValueError('cannot determine native endianness') + assert valid_endian(endian) # mypy fails to narrow endian on its own var_big_endian = nc.createVariable(\ 'obs_big_endian', '>f8', ('obs', ),\ endian=endian,fill_value=fval) diff --git a/test/test_enum.py b/test/test_enum.py index 90761d849..945905058 100644 --- a/test/test_enum.py +++ b/test/test_enum.py @@ -27,7 +27,7 @@ def setUp(self): cloud_type = f.createEnumType(ENUM_BASETYPE,ENUM_NAME,ENUM_DICT) # make sure KeyError raised if non-integer basetype used. try: - cloud_typ2 = f.createEnumType(np.float32,ENUM_NAME,ENUM_DICT) # type: ignore # mypy correctly doesn't like float32 + cloud_typ2 = f.createEnumType(np.float32,ENUM_NAME,ENUM_DICT) # type: ignore[arg-type] # mypy correctly doesn't like float32 except KeyError: pass f.createDimension('time',None) diff --git a/test/test_stringarr.py b/test/test_stringarr.py index 24c890fb2..597e40096 100644 --- a/test/test_stringarr.py +++ b/test/test_stringarr.py @@ -3,6 +3,7 @@ import unittest import os from numpy.testing import assert_array_equal, assert_array_almost_equal +from type_guards import valid_ncformat def generateString(length, alphabet=string.ascii_letters + string.digits + string.punctuation): return(''.join([random.choice(alphabet) for i in range(length)])) @@ -24,6 +25,7 @@ class StringArrayTestCase(unittest.TestCase): def setUp(self): self.file = FILE_NAME + assert valid_ncformat(FILE_FORMAT) nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT) nc.createDimension('n1',None) nc.createDimension('n2',n2) diff --git a/test/test_types.py b/test/test_types.py index 1f65fc5fc..b2c608bcf 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -6,6 +6,7 @@ from numpy.testing import assert_array_equal, assert_array_almost_equal from numpy.random.mtrand import uniform import netCDF4 +from type_guards import valid_complevel # test primitive data types. @@ -28,6 +29,7 @@ def setUp(self): f.createDimension('n1', None) f.createDimension('n2', n2dim) for typ in datatypes: + assert valid_complevel(complevel) foo = f.createVariable('data_'+typ, typ, ('n1','n2',),zlib=zlib,complevel=complevel,shuffle=shuffle,least_significant_digit=least_significant_digit,fill_value=FillValue) #foo._FillValue = FillValue # test writing of _FillValue attribute for diff types diff --git a/test/test_utils.py b/test/test_utils.py index 12000aa6b..88ee12c62 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -90,7 +90,7 @@ def test_put_indices(self): start, count, stride, put_ind = _StartCountStride(elem, (3,4,5)) orig = np.arange(60).reshape((3,4,5)) dest = np.empty(_out_array_shape(count)) - dest[tuple(put_ind[0,0,0])] = orig[tuple(elem)] # type: ignore # mypy doesn't like indexing of orig, but it's OK + dest[tuple(put_ind[0,0,0])] = orig[tuple(elem)] def test_boolean(self): elem = (1, slice(None), np.array([True, True, False, False, True])) diff --git a/test/type_guards.py b/test/type_guards.py new file mode 100644 index 000000000..ee8181ef9 --- /dev/null +++ b/test/type_guards.py @@ -0,0 +1,41 @@ +"""_type_guards - Helpers for input type-checking""" +from typing import Literal, TYPE_CHECKING + +from typing_extensions import TypeGuard +if TYPE_CHECKING: + # in stubs only + from netCDF4 import CompressionLevel, EndianType, CompressionType + from netCDF4 import Format as NCFormat +else: + CompressionLevel = EndianType = CompressionType = NCFormat = object + +def valid_complevel(complevel) -> TypeGuard[CompressionLevel | None]: + return complevel is None or isinstance(complevel, int) and 0 <= complevel <= 9 + +def valid_endian(endian) -> TypeGuard[EndianType]: + return endian in {"native", "big", "little"} + +def valid_comptype(comptype) -> TypeGuard[CompressionType | None]: + return comptype is None or comptype in { + "zlib", + "szip", + "zstd", + "bzip2", + "blosc_lz", + "blosc_lz4", + "blosc_lz4hc", + "blosc_zlib", + "blosc_zstd", + } + +def valid_bloscshuffle(bloscshuffle) -> TypeGuard[Literal[0, 1, 2]]: + return bloscshuffle in {0, 1, 2} + +def valid_ncformat(ncformat) -> TypeGuard[NCFormat]: + return ncformat in [ + "NETCDF4", + "NETCDF4_CLASSIC", + "NETCDF3_CLASSIC", + "NETCDF3_64BIT_OFFSET", + "NETCDF3_64BIT_DATA" + ] From 13a79a1b38c9fc6d7f737fdf3167ca8f8643007a Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 25 Jul 2024 13:43:08 -0700 Subject: [PATCH 54/81] Revamp GetSetItemKey --- src/netCDF4/__init__.pyi | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 734fa83f7..cbebd1990 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -153,24 +153,33 @@ DateTimeArray: TypeAlias = npt.NDArray[np.object_] GetSetItemKey: TypeAlias = ( int | float - | str # strs that can be cooerced to ints | np.number + | str | slice | ellipsis | list[int] | list[float] - | list[bool] + | list[np.number] | list[str] - | list[int | float] - | list[int | bool] - | list[int | str] - | list[float | bool] - | list[float | str] - | list[bool | str] - | list[int | float | bool | str] - | npt.NDArray[np.number | np.bool_] + | list[bool] + | list[np.bool_] + | npt.NDArray[np.number] + | npt.NDArray[np.bool_] | tuple[ - int | float | str | slice | ellipsis | Sequence[int] | Sequence[bool] | npt.NDArray[np.number | np.bool_], + int + | float + | np.number + | str + | slice + | ellipsis + | Sequence[int] + | Sequence[float] + | Sequence[np.number] + | Sequence[str] + | Sequence[bool] + | Sequence[np.bool_] + | npt.NDArray[np.number] + | npt.NDArray[np.bool_], ..., ] ) From a33f631bad27accbb9b98741036ad3a0f56754e0 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 25 Jul 2024 13:51:09 -0700 Subject: [PATCH 55/81] Fix runtime behavior of examples --- examples/bench_compress.py | 7 ++++++- examples/mpi_example.py | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/bench_compress.py b/examples/bench_compress.py index 06a343391..99053aac8 100644 --- a/examples/bench_compress.py +++ b/examples/bench_compress.py @@ -1,11 +1,16 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. +from typing import TYPE_CHECKING from numpy.random.mtrand import uniform from typing_extensions import TypeGuard import netCDF4 import netCDF4.utils from timeit import Timer import os, sys +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = object # create an n1dim by n2dim by n3dim random array. n1dim = 30 @@ -17,7 +22,7 @@ sys.stdout.write('(average of %s trials)\n' % ntrials) array = netCDF4.utils._quantize(uniform(size=(n1dim,n2dim,n3dim,n4dim)),4) -def valid_complevel(complevel) -> TypeGuard[netCDF4.CompressionLevel | None]: +def valid_complevel(complevel) -> TypeGuard[CompressionLevel | None]: return complevel is None or isinstance(complevel, int) and 0 <= complevel <= 6 def write_netcdf(filename,zlib=False,shuffle=False,complevel=6): diff --git a/examples/mpi_example.py b/examples/mpi_example.py index 1e781b461..71d8b8eb2 100644 --- a/examples/mpi_example.py +++ b/examples/mpi_example.py @@ -1,13 +1,17 @@ # to run: mpirun -np 4 python mpi_example.py import sys +from typing import TYPE_CHECKING from typing_extensions import TypeGuard from mpi4py import MPI import numpy as np from netCDF4 import Dataset -import netCDF4 +if TYPE_CHECKING: + from netCDF4 import Format as NCFormat +else: + NCFormat = object -def is_nc_format(fmt: str) -> TypeGuard[netCDF4.Format]: +def is_nc_format(fmt: str) -> TypeGuard[NCFormat]: return fmt in { 'NETCDF4', 'NETCDF4_CLASSIC', From 64420742504f7b2c82a171f7ff977f5903af5689 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 25 Jul 2024 14:02:21 -0700 Subject: [PATCH 56/81] add typing-extensions so tests can pass --- .github/workflows/build_latest.yml | 18 +++++++++--------- .github/workflows/build_old.yml | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index 079774710..8f536624e 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -53,21 +53,21 @@ jobs: # if: ${{ failure() }} # run: | # cd netcdf-c-${NETCDF_VERSION} -# cat config.log +# cat config.log - name: Install python dependencies via pip run: | python -m pip install --upgrade pip - pip install numpy cython cftime pytest twine wheel check-manifest mpi4py + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py typing-extensions - name: Install netcdf4-python run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir python setup.py install - name: Test run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} python checkversion.py # serial cd test @@ -98,9 +98,9 @@ jobs: # - name: Tarball # run: | -# export PATH=${NETCDF_DIR}/bin:${PATH} -# python setup.py --version +# export PATH=${NETCDF_DIR}/bin:${PATH} +# python setup.py --version # check-manifest --version -# check-manifest --verbose -# pip wheel . -w dist --no-deps -# twine check dist/* +# check-manifest --verbose +# pip wheel . -w dist --no-deps +# twine check dist/* diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index eeaa61c3b..b10977d9d 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -53,21 +53,21 @@ jobs: # if: ${{ failure() }} # run: | # cd netcdf-c-${NETCDF_VERSION} -# cat config.log +# cat config.log - name: Install python dependencies via pip run: | python -m pip install --upgrade pip - pip install numpy cython cftime pytest twine wheel check-manifest mpi4py + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py typing-extensions - name: Install netcdf4-python run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir python setup.py install - name: Test run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} python checkversion.py # serial cd test @@ -98,9 +98,9 @@ jobs: # - name: Tarball # run: | -# export PATH=${NETCDF_DIR}/bin:${PATH} -# python setup.py --version +# export PATH=${NETCDF_DIR}/bin:${PATH} +# python setup.py --version # check-manifest --version -# check-manifest --verbose -# pip wheel . -w dist --no-deps -# twine check dist/* +# check-manifest --verbose +# pip wheel . -w dist --no-deps +# twine check dist/* From c4e6e72a28e48d14d3f95a05afc32d98a9f19e10 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 30 Jul 2024 10:59:11 -0700 Subject: [PATCH 57/81] tests and examples are py311, don't need typing_extensions --- .github/workflows/build_latest.yml | 2 +- .github/workflows/build_old.yml | 2 +- examples/bench_compress.py | 3 +-- examples/mpi_example.py | 3 +-- test/type_guards.py | 3 +-- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index 8f536624e..852354d18 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -58,7 +58,7 @@ jobs: - name: Install python dependencies via pip run: | python -m pip install --upgrade pip - pip install numpy cython cftime pytest twine wheel check-manifest mpi4py typing-extensions + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py - name: Install netcdf4-python run: | diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index b10977d9d..d93eba15e 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -58,7 +58,7 @@ jobs: - name: Install python dependencies via pip run: | python -m pip install --upgrade pip - pip install numpy cython cftime pytest twine wheel check-manifest mpi4py typing-extensions + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py - name: Install netcdf4-python run: | diff --git a/examples/bench_compress.py b/examples/bench_compress.py index 99053aac8..56b9cb285 100644 --- a/examples/bench_compress.py +++ b/examples/bench_compress.py @@ -1,8 +1,7 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeGuard from numpy.random.mtrand import uniform -from typing_extensions import TypeGuard import netCDF4 import netCDF4.utils from timeit import Timer diff --git a/examples/mpi_example.py b/examples/mpi_example.py index 71d8b8eb2..0a89b28df 100644 --- a/examples/mpi_example.py +++ b/examples/mpi_example.py @@ -1,7 +1,6 @@ # to run: mpirun -np 4 python mpi_example.py import sys -from typing import TYPE_CHECKING -from typing_extensions import TypeGuard +from typing import TYPE_CHECKING, TypeGuard from mpi4py import MPI import numpy as np from netCDF4 import Dataset diff --git a/test/type_guards.py b/test/type_guards.py index ee8181ef9..e8121485e 100644 --- a/test/type_guards.py +++ b/test/type_guards.py @@ -1,7 +1,6 @@ """_type_guards - Helpers for input type-checking""" -from typing import Literal, TYPE_CHECKING +from typing import Literal, TYPE_CHECKING, TypeGuard -from typing_extensions import TypeGuard if TYPE_CHECKING: # in stubs only from netCDF4 import CompressionLevel, EndianType, CompressionType From 7c9eb602e79f294613fd3fea9f317f801035bec5 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 30 Jul 2024 12:00:54 -0700 Subject: [PATCH 58/81] disable some cicd --- .github/workflows/build_latest.yml | 3 ++- .github/workflows/build_master.yml | 11 ++++++----- .github/workflows/build_old.yml | 3 ++- .github/workflows/cibuildwheel.yml | 17 +++++++++-------- .github/workflows/miniconda.yml | 16 +++++++++------- 5 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index 852354d18..965cf2b18 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -1,5 +1,6 @@ name: Build and Test Linux with latest netcdf-c -on: [push, pull_request] +# on: [push, pull_request] +on: [] # disable jobs: build-linux: name: Python (${{ matrix.python-version }}) diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index f8f841636..a1455a945 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -1,5 +1,6 @@ name: Build and Test on Linux with netcdf-c github master -on: [push, pull_request] +# on: [push, pull_request] +on: [] # disable jobs: build-linux: name: Python (${{ matrix.python-version }}) @@ -33,7 +34,7 @@ jobs: export LDFLAGS="-L${NETCDF_DIR}/lib" export LIBS="-lhdf5_mpich_hl -lhdf5_mpich -lm -lz" autoreconf -i - ./configure --prefix $NETCDF_DIR --enable-netcdf-4 --enable-shared --enable-dap --enable-parallel4 + ./configure --prefix $NETCDF_DIR --enable-netcdf-4 --enable-shared --enable-dap --enable-parallel4 make -j 2 make install popd @@ -42,7 +43,7 @@ jobs: # if: ${{ failure() }} # run: | # cd netcdf-c-${NETCDF_VERSION} -# cat config.log +# cat config.log - name: Install python dependencies via pip run: | @@ -51,13 +52,13 @@ jobs: - name: Install netcdf4-python run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir python setup.py install - name: Test run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} #export HDF5_PLUGIN_PATH=${NETCDF_DIR}/plugins/plugindir python checkversion.py # serial diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index d93eba15e..228aa9cc7 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -1,5 +1,6 @@ name: Build and Test Linux with older netcdf-c -on: [push, pull_request] +# on: [push, pull_request] +on: [] # disable jobs: build-linux: name: Python (${{ matrix.python-version }}) diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index 5eb8bb975..0fbaf4fb1 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -1,13 +1,14 @@ name: Wheels -on: - pull_request: - push: - tags: - - "v*" - release: - types: - - published +on: ["push"] +# on: +# pull_request: +# push: +# tags: +# - "v*" +# release: +# types: +# - published permissions: contents: read diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index a79c8e572..c930be65f 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -1,9 +1,10 @@ name: Build and Test -on: - pull_request: - push: - branches: [master] +# on: +# pull_request: +# push: +# branches: [master] +on: ["push"] jobs: run-serial: @@ -12,11 +13,12 @@ jobs: # NO_NET: 1 strategy: matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + # python-version: [ "3.9", "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10"] os: [windows-latest, ubuntu-latest, macos-latest] platform: [x64, x32] exclude: - - os: macos-latest + - os: macos-latest platform: x32 fail-fast: false defaults: @@ -82,7 +84,7 @@ jobs: run: | cd test && python run_all.py cd ../examples - export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" + export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" which mpirun mpirun --version mpirun -np 4 --oversubscribe python mpi_example.py # for openmpi From ea84857502f40906aa533e0ee860917168623665 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 30 Jul 2024 12:06:54 -0700 Subject: [PATCH 59/81] Always get and use typing-extensions in cast py<3.10 --- .github/workflows/build_latest.yml | 2 +- .github/workflows/build_master.yml | 2 +- .github/workflows/build_old.yml | 2 +- .github/workflows/cibuildwheel.yml | 4 ++-- .github/workflows/miniconda.yml | 4 ++-- examples/bench_compress.py | 3 ++- examples/mpi_example.py | 3 ++- examples/parallel_test.nc | Bin 0 -> 6176 bytes examples/parallel_test_compressed.nc | Bin 0 -> 10288 bytes test/test_compression_quant.py | 30 ++++++++++++++------------- test/test_compression_zstd.py | 4 +++- test/type_guards.py | 10 ++++++--- 12 files changed, 37 insertions(+), 27 deletions(-) create mode 100644 examples/parallel_test.nc create mode 100644 examples/parallel_test_compressed.nc diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index 965cf2b18..28f65c7a8 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -59,7 +59,7 @@ jobs: - name: Install python dependencies via pip run: | python -m pip install --upgrade pip - pip install numpy cython cftime pytest twine wheel check-manifest mpi4py + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py typing-extensions - name: Install netcdf4-python run: | diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index a1455a945..af16c3eec 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -48,7 +48,7 @@ jobs: - name: Install python dependencies via pip run: | python -m pip install --upgrade pip - pip install numpy cython cftime pytest twine wheel check-manifest mpi4py mypy types-setuptools + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py mypy types-setuptools typing-extensions - name: Install netcdf4-python run: | diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index 228aa9cc7..67d141eef 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -59,7 +59,7 @@ jobs: - name: Install python dependencies via pip run: | python -m pip install --upgrade pip - pip install numpy cython cftime pytest twine wheel check-manifest mpi4py + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py typing-extensions - name: Install netcdf4-python run: | diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index 0fbaf4fb1..09ca9b752 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -104,7 +104,7 @@ jobs: CIBW_TEST_SKIP: "cp38-*_aarch64 cp39-*_aarch64 cp310-*_aarch64 cp311-*_aarch64" CIBW_ENVIRONMENT: ${{ matrix.CIBW_ENVIRONMENT }} CIBW_BEFORE_BUILD_MACOS: brew install hdf5 netcdf - CIBW_TEST_REQUIRES: pytest cython packaging + CIBW_TEST_REQUIRES: pytest cython packaging typing-extensions CIBW_TEST_COMMAND: > python -c "import netCDF4; print(f'netCDF4 v{netCDF4.__version__}')" && pytest -s -rxs -v {project}/test @@ -158,7 +158,7 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: > delvewheel show {wheel} && delvewheel repair -w {dest_dir} {wheel} - CIBW_TEST_REQUIRES: pytest cython packaging + CIBW_TEST_REQUIRES: pytest cython packaging typing-extensions CIBW_TEST_COMMAND: > python -c "import netCDF4; print(f'netCDF4 v{netCDF4.__version__}')" && pytest -s -rxs -v {project}\\test diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index c930be65f..fb291fdc0 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -37,7 +37,7 @@ jobs: init-shell: bash create-args: >- python=${{ matrix.python-version }} - numpy cython pip pytest hdf5 libnetcdf cftime zlib certifi + numpy cython pip pytest hdf5 libnetcdf cftime zlib certifi typing-extensions --channel conda-forge - name: Install netcdf4-python @@ -71,7 +71,7 @@ jobs: init-shell: bash create-args: >- python=${{ matrix.python-version }} - numpy cython pip pytest mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib certifi + numpy cython pip pytest mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib certifi typing-extensions --channel conda-forge - name: Install netcdf4-python with mpi diff --git a/examples/bench_compress.py b/examples/bench_compress.py index 56b9cb285..845cf333b 100644 --- a/examples/bench_compress.py +++ b/examples/bench_compress.py @@ -1,11 +1,12 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. -from typing import TYPE_CHECKING, TypeGuard +from typing import TYPE_CHECKING from numpy.random.mtrand import uniform import netCDF4 import netCDF4.utils from timeit import Timer import os, sys +from typing_extensions import TypeGuard if TYPE_CHECKING: from netCDF4 import CompressionLevel else: diff --git a/examples/mpi_example.py b/examples/mpi_example.py index 0a89b28df..3f8cb0f87 100644 --- a/examples/mpi_example.py +++ b/examples/mpi_example.py @@ -1,9 +1,10 @@ # to run: mpirun -np 4 python mpi_example.py import sys -from typing import TYPE_CHECKING, TypeGuard +from typing import TYPE_CHECKING from mpi4py import MPI import numpy as np from netCDF4 import Dataset +from typing_extensions import TypeGuard if TYPE_CHECKING: from netCDF4 import Format as NCFormat else: diff --git a/examples/parallel_test.nc b/examples/parallel_test.nc new file mode 100644 index 0000000000000000000000000000000000000000..ec95803e081f032b7314eaf74434c9916246e2bd GIT binary patch literal 6176 zcmeH{KX21O6u_VDCNjZo0t5mGf$0JjAxhhTTA@gd9EYT&35q+hL~iV=My-Pq(-j7U z#7KnNiEqHj%11y#o%jk2z{U#io$r#WKw)4(x|i5@zI*rn{NA%))T_1W)Wwvl>mcMi z7xR3)Qsf(Z5B+?*UhSyHTXmMLX)50mM~;pu6%ec{;@OxZJq--}p=vkm zEp~^|Am~Mq!O-Y5fC=P~EM_wFJksb6u)^+mW6t>IA`i~to6B#m@3ZiymlP#Si>jh1 zv0>o=Bgm4GQ&+EyUFLMiJwfdR%SAg8yUz22BW_(}IQ9DsZ;+}6Nzlo*)vP!UoWUhZ zg<`EX*6fzkXt%nKRcYE_JfMU$NW=$Pl{K5`uWka~UW7{v7LP}s@Kq}gw;mBKl)bln=)nF&^eu9oZQC5pDzqGCTJ;RPe zn~6i~wygVu{v&@71^$r5IE%`Z)2=_DfXWMbj1jGuONHgaLVnBJyj7kr%ohtwAKo8) zi^~GXmBpa49PWffidM8assm1*>ZjSWLOLX@L2wj>Ii~e0q_061IbdIOIR~rXA9z9N zMwrrJ$J90Gkck-SA?}L<-jZ0mSgwi5UyQ3a^*e+&d5)CFy$Kpt_A5Ux_JnHn0~j^LLkDpCrqIGCPRtg0itq z1T{!Qkvw+nn%%Kmmfd9@Z)1F<>A8_R^rMOEamRxGC&VI0G{fZC$bBW|a+CxlfqzAS z-go+-B=NdywVUiq!(VhZ$94W`0P$7AB_Ch3k@wGBuq~HL{VB=R6vm$ISwhL4vKr?KyYGLwP+KRmgQz$5g=V{icpap!=z^;W^m+S`OG>@O=h;~@bEwfBz>OP#Y1tXJ2Y0`zBK*v%qGV}YRg|NUWN2EJB0hWd zif{~I!h#M9uTeWCa?#Gj9-k#1-zN@aq!TAbYTP7M4JJV++g82ox-f@BlmX>iS>Lc7 zcfH}XT&rBS!FY_wa6u|w;FLFPp}&^sZ*GSJ6MvrB4I-;jH4$pi9fZBE*?tk3-Ci`D zLZ+eD-U$k(lx3iRRJ1=01_jh|f~ezfEmy+b(1%&9hnR^efsK?-EFak6)+?V>gy9pOjgCbkNi(Z;>R>)fM0^U%l|M}q4 zO2eu>)X&i)mUHs9B^iRTLzSVy3>3*@)2`Z0+p+AH@OT@mgEW0F@&-XPbbYkpppViX z7m<9UC?b-3KfljWM21aIR0N_bTaNXp-+LbPqYz=yPUP^S>f8(ZR9dB_JXRalNol!o zr?8aY_P16_i-pBvVfDt>+&ffcVFFCxloELPr1P}xb=L5W--pZtHFZk!;INqh6JP>N zfC(^xlL>4#ZJQ2W@;EKqG3RGtgpL`b5+4!eQE_yJ`%HidFaajO1en18Lg2pDsEbdf w$(Kvf$K>SS0{Ky2=KEzv_PAvNOn?b60Vco%m;e)C0!)AjFaajO1WpTqKPK($K>z>% literal 0 HcmV?d00001 diff --git a/test/test_compression_quant.py b/test/test_compression_quant.py index 3654bf9d5..7cdd11e2b 100644 --- a/test/test_compression_quant.py +++ b/test/test_compression_quant.py @@ -3,6 +3,7 @@ from numpy.testing import assert_almost_equal import numpy as np import os, tempfile, unittest +from type_guards import valid_complevel, valid_quantize_mode ndim = 100000 nfiles = 7 @@ -16,6 +17,7 @@ def write_netcdf(filename,zlib,significant_digits,data,dtype='f8',shuffle=False, complevel=6,quantize_mode="BitGroom"): file = Dataset(filename,'w') file.createDimension('n', ndim) + assert valid_complevel(complevel) and valid_quantize_mode(quantize_mode) foo = file.createVariable('data',\ dtype,('n'),zlib=zlib,significant_digits=significant_digits,\ shuffle=shuffle,complevel=complevel,quantize_mode=quantize_mode) @@ -61,7 +63,7 @@ def runTest(self): assert_almost_equal(data_array,f.variables['data'][:]) assert f.variables['data'].filters() ==\ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':complevel,'fletcher32':False} - assert size < 0.95*uncompressed_size + assert size < 0.95*uncompressed_size f.close() # check compression with shuffle f = Dataset(self.files[2]) @@ -70,43 +72,43 @@ def runTest(self): assert_almost_equal(data_array,f.variables['data'][:]) assert f.variables['data'].filters() ==\ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':complevel,'fletcher32':False} - assert size < 0.85*uncompressed_size + assert size < 0.85*uncompressed_size f.close() # check lossy compression without shuffle f = Dataset(self.files[3]) size = os.stat(self.files[3]).st_size errmax = (np.abs(data_array-f.variables['data'][:])).max() #print('compressed lossy no shuffle = ',size,' max err = ',errmax) - assert f.variables['data'].quantization() == (nsd,'BitGroom') - assert errmax < 1.e-3 - assert size < 0.35*uncompressed_size + assert f.variables['data'].quantization() == (nsd,'BitGroom') + assert errmax < 1.e-3 + assert size < 0.35*uncompressed_size f.close() # check lossy compression with shuffle f = Dataset(self.files[4]) size = os.stat(self.files[4]).st_size errmax = (np.abs(data_array-f.variables['data'][:])).max() print('compressed lossy with shuffle and standard quantization = ',size,' max err = ',errmax) - assert f.variables['data'].quantization() == (nsd,'BitGroom') - assert errmax < 1.e-3 - assert size < 0.24*uncompressed_size + assert f.variables['data'].quantization() == (nsd,'BitGroom') + assert errmax < 1.e-3 + assert size < 0.24*uncompressed_size f.close() # check lossy compression with shuffle and alternate quantization f = Dataset(self.files[5]) size = os.stat(self.files[5]).st_size errmax = (np.abs(data_array-f.variables['data'][:])).max() print('compressed lossy with shuffle and alternate quantization = ',size,' max err = ',errmax) - assert f.variables['data'].quantization() == (nsd,'GranularBitRound') - assert errmax < 1.e-3 - assert size < 0.24*uncompressed_size + assert f.variables['data'].quantization() == (nsd,'GranularBitRound') + assert errmax < 1.e-3 + assert size < 0.24*uncompressed_size f.close() # check lossy compression with shuffle and alternate quantization f = Dataset(self.files[6]) size = os.stat(self.files[6]).st_size errmax = (np.abs(data_array-f.variables['data'][:])).max() print('compressed lossy with shuffle and alternate quantization = ',size,' max err = ',errmax) - assert f.variables['data'].quantization() == (nsb,'BitRound') - assert errmax < 1.e-3 - assert size < 0.24*uncompressed_size + assert f.variables['data'].quantization() == (nsb,'BitRound') + assert errmax < 1.e-3 + assert size < 0.24*uncompressed_size f.close() if __name__ == '__main__': diff --git a/test/test_compression_zstd.py b/test/test_compression_zstd.py index 9f4259fd0..b8242e4cb 100644 --- a/test/test_compression_zstd.py +++ b/test/test_compression_zstd.py @@ -3,6 +3,7 @@ from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_zstd_filter +from type_guards import valid_complevel ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -12,6 +13,7 @@ def write_netcdf(filename,dtype='f8',complevel=6): nc = Dataset(filename,'w') nc.createDimension('n', ndim) + assert valid_complevel(complevel) foo = nc.createVariable('data',\ dtype,('n'),compression='zstd',complevel=complevel) foo[:] = array @@ -47,7 +49,7 @@ def runTest(self): assert_almost_equal(array,f.variables['data'][:]) assert f.variables['data'].filters() ==\ {'zlib':False,'szip':False,'zstd':True,'bzip2':False,'blosc':False,'shuffle':False,'complevel':4,'fletcher32':False} - assert size < 0.96*uncompressed_size + assert size < 0.96*uncompressed_size f.close() if __name__ == '__main__': diff --git a/test/type_guards.py b/test/type_guards.py index e8121485e..09b5e0080 100644 --- a/test/type_guards.py +++ b/test/type_guards.py @@ -1,12 +1,13 @@ """_type_guards - Helpers for input type-checking""" -from typing import Literal, TYPE_CHECKING, TypeGuard +from typing import Literal, TYPE_CHECKING +from typing_extensions import TypeGuard if TYPE_CHECKING: # in stubs only - from netCDF4 import CompressionLevel, EndianType, CompressionType + from netCDF4 import CompressionLevel, EndianType, CompressionType, QuantizeMode from netCDF4 import Format as NCFormat else: - CompressionLevel = EndianType = CompressionType = NCFormat = object + CompressionLevel = EndianType = CompressionType = NCFormat = QuantizeMode = object def valid_complevel(complevel) -> TypeGuard[CompressionLevel | None]: return complevel is None or isinstance(complevel, int) and 0 <= complevel <= 9 @@ -38,3 +39,6 @@ def valid_ncformat(ncformat) -> TypeGuard[NCFormat]: "NETCDF3_64BIT_OFFSET", "NETCDF3_64BIT_DATA" ] + +def valid_quantize_mode(quantize_mode) -> TypeGuard[QuantizeMode]: + return quantize_mode in ["BitGroom", "BitRound", "GranularBitRound"] \ No newline at end of file From 303e8b1a35181d77b3a3c74b432dd0943a70dc5e Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 30 Jul 2024 12:18:45 -0700 Subject: [PATCH 60/81] Remove None union from typeguards --- examples/bench_compress.py | 4 ++-- test/test_compression.py | 18 +++++++++--------- test/test_compression_blosc.py | 2 +- test/test_compression_bzip2.py | 4 ++-- test/test_compression_quant.py | 2 +- test/test_compression_zstd.py | 2 +- test/test_types.py | 2 +- test/type_guards.py | 6 +++--- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/examples/bench_compress.py b/examples/bench_compress.py index 845cf333b..84e209e92 100644 --- a/examples/bench_compress.py +++ b/examples/bench_compress.py @@ -22,7 +22,7 @@ sys.stdout.write('(average of %s trials)\n' % ntrials) array = netCDF4.utils._quantize(uniform(size=(n1dim,n2dim,n3dim,n4dim)),4) -def valid_complevel(complevel) -> TypeGuard[CompressionLevel | None]: +def valid_complevel(complevel) -> TypeGuard[CompressionLevel]: return complevel is None or isinstance(complevel, int) and 0 <= complevel <= 6 def write_netcdf(filename,zlib=False,shuffle=False,complevel=6): @@ -31,7 +31,7 @@ def write_netcdf(filename,zlib=False,shuffle=False,complevel=6): file.createDimension('n2', n2dim) file.createDimension('n3', n3dim) file.createDimension('n4', n4dim) - if not valid_complevel(complevel): + if not (valid_complevel(complevel) or complevel is None): raise ValueError("Invalid compression level") foo = file.createVariable('data',\ 'f8',('n1','n2','n3','n4'),zlib=zlib,shuffle=shuffle,complevel=complevel) diff --git a/test/test_compression.py b/test/test_compression.py index eab808227..7972609be 100644 --- a/test/test_compression.py +++ b/test/test_compression.py @@ -16,7 +16,7 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,contiguous=False,\ chunksizes=None,complevel=6,fletcher32=False): - assert valid_complevel(complevel) + assert valid_complevel(complevel) or complevel is None file = Dataset(filename,'w') file.createDimension('n', ndim) foo = file.createVariable('data',\ @@ -32,7 +32,7 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=F #compression='' #compression=0 #compression='gzip' # should fail - assert valid_comptype(compression) + assert valid_comptype(compression) or compression is None foo2 = file.createVariable('data2',\ dtype,('n'),compression=compression,least_significant_digit=least_significant_digit,\ shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) @@ -49,7 +49,7 @@ def write_netcdf2(filename,zlib,least_significant_digit,data,dtype='f8',shuffle= file = Dataset(filename,'w') file.createDimension('n', ndim) file.createDimension('n2', ndim2) - assert valid_complevel(complevel) + assert valid_complevel(complevel) or complevel is None foo = file.createVariable('data2',\ dtype,('n','n2'),zlib=zlib,least_significant_digit=least_significant_digit,\ shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) @@ -107,7 +107,7 @@ def runTest(self): {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':6,'fletcher32':False} assert f.variables['data2'].filters() ==\ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':6,'fletcher32':False} - assert size < 0.95*uncompressed_size + assert size < 0.95*uncompressed_size f.close() # check compression with shuffle f = Dataset(self.files[2]) @@ -118,7 +118,7 @@ def runTest(self): {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':False} assert f.variables['data2'].filters() ==\ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':False} - assert size < 0.85*uncompressed_size + assert size < 0.85*uncompressed_size f.close() # check lossy compression without shuffle f = Dataset(self.files[3]) @@ -126,14 +126,14 @@ def runTest(self): checkarray = _quantize(array,lsd) assert_almost_equal(checkarray,f.variables['data'][:]) assert_almost_equal(checkarray,f.variables['data2'][:]) - assert size < 0.27*uncompressed_size + assert size < 0.27*uncompressed_size f.close() # check lossy compression with shuffle f = Dataset(self.files[4]) size = os.stat(self.files[4]).st_size assert_almost_equal(checkarray,f.variables['data'][:]) assert_almost_equal(checkarray,f.variables['data2'][:]) - assert size < 0.20*uncompressed_size + assert size < 0.20*uncompressed_size size_save = size f.close() # check lossy compression with shuffle and fletcher32 checksum. @@ -145,9 +145,9 @@ def runTest(self): {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':True} assert f.variables['data2'].filters() ==\ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':True} - assert size < 0.20*uncompressed_size + assert size < 0.20*uncompressed_size # should be slightly larger than without fletcher32 - assert size > size_save + assert size > size_save # check chunksizes f.close() f = Dataset(self.files[6]) diff --git a/test/test_compression_blosc.py b/test/test_compression_blosc.py index 6915544c8..bbfc08a39 100644 --- a/test/test_compression_blosc.py +++ b/test/test_compression_blosc.py @@ -13,7 +13,7 @@ datarr = uniform(size=(ndim,)) def write_netcdf(filename,dtype='f8',blosc_shuffle=1,complevel=6): - assert valid_complevel(complevel) and valid_bloscshuffle(blosc_shuffle) + assert (valid_complevel(complevel) or complevel is None) and valid_bloscshuffle(blosc_shuffle) nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ diff --git a/test/test_compression_bzip2.py b/test/test_compression_bzip2.py index 9a58b0668..1b008f190 100644 --- a/test/test_compression_bzip2.py +++ b/test/test_compression_bzip2.py @@ -11,7 +11,7 @@ array = uniform(size=(ndim,)) def write_netcdf(filename,dtype='f8',complevel=6): - assert valid_complevel(complevel) + assert valid_complevel(complevel) or complevel is None nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ @@ -49,7 +49,7 @@ def runTest(self): assert_almost_equal(array,f.variables['data'][:]) assert f.variables['data'].filters() ==\ {'zlib':False,'szip':False,'zstd':False,'bzip2':True,'blosc':False,'shuffle':False,'complevel':4,'fletcher32':False} - assert size < 0.96*uncompressed_size + assert size < 0.96*uncompressed_size f.close() diff --git a/test/test_compression_quant.py b/test/test_compression_quant.py index 7cdd11e2b..dd124c348 100644 --- a/test/test_compression_quant.py +++ b/test/test_compression_quant.py @@ -17,7 +17,7 @@ def write_netcdf(filename,zlib,significant_digits,data,dtype='f8',shuffle=False, complevel=6,quantize_mode="BitGroom"): file = Dataset(filename,'w') file.createDimension('n', ndim) - assert valid_complevel(complevel) and valid_quantize_mode(quantize_mode) + assert (valid_complevel(complevel) or complevel is None) and valid_quantize_mode(quantize_mode) foo = file.createVariable('data',\ dtype,('n'),zlib=zlib,significant_digits=significant_digits,\ shuffle=shuffle,complevel=complevel,quantize_mode=quantize_mode) diff --git a/test/test_compression_zstd.py b/test/test_compression_zstd.py index b8242e4cb..4ee16daff 100644 --- a/test/test_compression_zstd.py +++ b/test/test_compression_zstd.py @@ -13,7 +13,7 @@ def write_netcdf(filename,dtype='f8',complevel=6): nc = Dataset(filename,'w') nc.createDimension('n', ndim) - assert valid_complevel(complevel) + assert valid_complevel(complevel) or complevel is None foo = nc.createVariable('data',\ dtype,('n'),compression='zstd',complevel=complevel) foo[:] = array diff --git a/test/test_types.py b/test/test_types.py index b2c608bcf..b8eb74b2e 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -29,7 +29,7 @@ def setUp(self): f.createDimension('n1', None) f.createDimension('n2', n2dim) for typ in datatypes: - assert valid_complevel(complevel) + assert valid_complevel(complevel) or complevel is None foo = f.createVariable('data_'+typ, typ, ('n1','n2',),zlib=zlib,complevel=complevel,shuffle=shuffle,least_significant_digit=least_significant_digit,fill_value=FillValue) #foo._FillValue = FillValue # test writing of _FillValue attribute for diff types diff --git a/test/type_guards.py b/test/type_guards.py index 09b5e0080..0b2ca8ff8 100644 --- a/test/type_guards.py +++ b/test/type_guards.py @@ -1,5 +1,5 @@ """_type_guards - Helpers for input type-checking""" -from typing import Literal, TYPE_CHECKING +from typing import Literal, TYPE_CHECKING, Union from typing_extensions import TypeGuard if TYPE_CHECKING: @@ -9,13 +9,13 @@ else: CompressionLevel = EndianType = CompressionType = NCFormat = QuantizeMode = object -def valid_complevel(complevel) -> TypeGuard[CompressionLevel | None]: +def valid_complevel(complevel) -> TypeGuard[CompressionLevel]: return complevel is None or isinstance(complevel, int) and 0 <= complevel <= 9 def valid_endian(endian) -> TypeGuard[EndianType]: return endian in {"native", "big", "little"} -def valid_comptype(comptype) -> TypeGuard[CompressionType | None]: +def valid_comptype(comptype) -> TypeGuard[CompressionType]: return comptype is None or comptype in { "zlib", "szip", From 111d3473aa2fbd128db1c25d7198d070648a4ca3 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 30 Jul 2024 15:06:04 -0700 Subject: [PATCH 61/81] Revamp type_guards a bit --- examples/bench_compress.py | 5 +- test/test_compression.py | 8 +-- test/test_compression_blosc.py | 5 +- test/test_compression_bzip2.py | 4 +- test/test_compression_quant.py | 4 +- test/test_compression_zstd.py | 4 +- test/test_endian.py | 4 +- test/test_stringarr.py | 12 ++--- test/test_types.py | 4 +- test/type_guards.py | 93 ++++++++++++++++++++++++++++------ 10 files changed, 103 insertions(+), 40 deletions(-) diff --git a/examples/bench_compress.py b/examples/bench_compress.py index 84e209e92..be8915c53 100644 --- a/examples/bench_compress.py +++ b/examples/bench_compress.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from netCDF4 import CompressionLevel else: - CompressionLevel = object + CompressionLevel = Any # create an n1dim by n2dim by n3dim random array. n1dim = 30 @@ -23,7 +23,8 @@ array = netCDF4.utils._quantize(uniform(size=(n1dim,n2dim,n3dim,n4dim)),4) def valid_complevel(complevel) -> TypeGuard[CompressionLevel]: - return complevel is None or isinstance(complevel, int) and 0 <= complevel <= 6 + """Check for a valid `complevel` argument for creating a Variable""" + return isinstance(complevel, int) and 0 <= complevel <= 9 def write_netcdf(filename,zlib=False,shuffle=False,complevel=6): file = netCDF4.Dataset(filename,'w',format='NETCDF4') diff --git a/test/test_compression.py b/test/test_compression.py index 7972609be..7d53fe436 100644 --- a/test/test_compression.py +++ b/test/test_compression.py @@ -3,7 +3,7 @@ from netCDF4.utils import _quantize from numpy.testing import assert_almost_equal import os, tempfile, unittest -from type_guards import valid_complevel, valid_comptype +import type_guards as n4t ndim = 100000 ndim2 = 100 @@ -16,7 +16,7 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,contiguous=False,\ chunksizes=None,complevel=6,fletcher32=False): - assert valid_complevel(complevel) or complevel is None + assert n4t.valid_complevel(complevel) or complevel is None file = Dataset(filename,'w') file.createDimension('n', ndim) foo = file.createVariable('data',\ @@ -32,7 +32,7 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=F #compression='' #compression=0 #compression='gzip' # should fail - assert valid_comptype(compression) or compression is None + assert n4t.valid_compression(compression) or compression is None foo2 = file.createVariable('data2',\ dtype,('n'),compression=compression,least_significant_digit=least_significant_digit,\ shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) @@ -49,7 +49,7 @@ def write_netcdf2(filename,zlib,least_significant_digit,data,dtype='f8',shuffle= file = Dataset(filename,'w') file.createDimension('n', ndim) file.createDimension('n2', ndim2) - assert valid_complevel(complevel) or complevel is None + assert n4t.valid_complevel(complevel) or complevel is None foo = file.createVariable('data2',\ dtype,('n','n2'),zlib=zlib,least_significant_digit=least_significant_digit,\ shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) diff --git a/test/test_compression_blosc.py b/test/test_compression_blosc.py index bbfc08a39..f8f13e48d 100644 --- a/test/test_compression_blosc.py +++ b/test/test_compression_blosc.py @@ -1,9 +1,10 @@ from numpy.random.mtrand import uniform from netCDF4 import Dataset +import type_guards as n4t from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_blosc_filter -from type_guards import valid_complevel, valid_bloscshuffle +import type_guards as n4t ndim = 100000 @@ -13,7 +14,7 @@ datarr = uniform(size=(ndim,)) def write_netcdf(filename,dtype='f8',blosc_shuffle=1,complevel=6): - assert (valid_complevel(complevel) or complevel is None) and valid_bloscshuffle(blosc_shuffle) + assert (n4t.valid_complevel(complevel) or complevel is None) and n4t.valid_bloscshuffle(blosc_shuffle) nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ diff --git a/test/test_compression_bzip2.py b/test/test_compression_bzip2.py index 1b008f190..b207288df 100644 --- a/test/test_compression_bzip2.py +++ b/test/test_compression_bzip2.py @@ -3,7 +3,7 @@ from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_bzip2_filter -from type_guards import valid_complevel +import type_guards as n4t ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -11,7 +11,7 @@ array = uniform(size=(ndim,)) def write_netcdf(filename,dtype='f8',complevel=6): - assert valid_complevel(complevel) or complevel is None + assert n4t.valid_complevel(complevel) or complevel is None nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ diff --git a/test/test_compression_quant.py b/test/test_compression_quant.py index dd124c348..bbb2b2676 100644 --- a/test/test_compression_quant.py +++ b/test/test_compression_quant.py @@ -3,7 +3,7 @@ from numpy.testing import assert_almost_equal import numpy as np import os, tempfile, unittest -from type_guards import valid_complevel, valid_quantize_mode +import type_guards as n4t ndim = 100000 nfiles = 7 @@ -17,7 +17,7 @@ def write_netcdf(filename,zlib,significant_digits,data,dtype='f8',shuffle=False, complevel=6,quantize_mode="BitGroom"): file = Dataset(filename,'w') file.createDimension('n', ndim) - assert (valid_complevel(complevel) or complevel is None) and valid_quantize_mode(quantize_mode) + assert (n4t.valid_complevel(complevel) or complevel is None) and n4t.valid_quantize_mode(quantize_mode) foo = file.createVariable('data',\ dtype,('n'),zlib=zlib,significant_digits=significant_digits,\ shuffle=shuffle,complevel=complevel,quantize_mode=quantize_mode) diff --git a/test/test_compression_zstd.py b/test/test_compression_zstd.py index 4ee16daff..820b3df37 100644 --- a/test/test_compression_zstd.py +++ b/test/test_compression_zstd.py @@ -3,7 +3,7 @@ from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_zstd_filter -from type_guards import valid_complevel +import type_guards as n4t ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -13,7 +13,7 @@ def write_netcdf(filename,dtype='f8',complevel=6): nc = Dataset(filename,'w') nc.createDimension('n', ndim) - assert valid_complevel(complevel) or complevel is None + assert n4t.valid_complevel(complevel) or complevel is None foo = nc.createVariable('data',\ dtype,('n'),compression='zstd',complevel=complevel) foo[:] = array diff --git a/test/test_endian.py b/test/test_endian.py index 3a8aecebb..12f1bef7f 100644 --- a/test/test_endian.py +++ b/test/test_endian.py @@ -2,7 +2,7 @@ import numpy as np import unittest, os, tempfile from numpy.testing import assert_array_equal, assert_array_almost_equal -from type_guards import valid_endian +import type_guards as n4t data = np.arange(12,dtype='f4').reshape(3,4) FILE_NAME = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -74,7 +74,7 @@ def issue310(file): endian='little' else: raise ValueError('cannot determine native endianness') - assert valid_endian(endian) # mypy fails to narrow endian on its own + assert n4t.valid_endian(endian) # mypy fails to narrow endian on its own var_big_endian = nc.createVariable(\ 'obs_big_endian', '>f8', ('obs', ),\ endian=endian,fill_value=fval) diff --git a/test/test_stringarr.py b/test/test_stringarr.py index 597e40096..ef05b0b38 100644 --- a/test/test_stringarr.py +++ b/test/test_stringarr.py @@ -3,7 +3,7 @@ import unittest import os from numpy.testing import assert_array_equal, assert_array_almost_equal -from type_guards import valid_ncformat +import type_guards as n4t def generateString(length, alphabet=string.ascii_letters + string.digits + string.punctuation): return(''.join([random.choice(alphabet) for i in range(length)])) @@ -25,7 +25,7 @@ class StringArrayTestCase(unittest.TestCase): def setUp(self): self.file = FILE_NAME - assert valid_ncformat(FILE_FORMAT) + assert n4t.valid_ncformat(FILE_FORMAT) nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT) nc.createDimension('n1',None) nc.createDimension('n2',n2) @@ -72,19 +72,19 @@ def runTest(self): assert_array_equal(data3,datau) # these slices should return a char array, not a string array data4 = v2[:,:,0] - assert data4.dtype.itemsize == 1 + assert data4.dtype.itemsize == 1 assert_array_equal(data4, datac[:,:,0]) data5 = v2[0,0:nchar,0] - assert data5.dtype.itemsize == 1 + assert data5.dtype.itemsize == 1 assert_array_equal(data5, datac[0,0:nchar,0]) # test turning auto-conversion off. v2.set_auto_chartostring(False) data6 = v2[:] - assert data6.dtype.itemsize == 1 + assert data6.dtype.itemsize == 1 assert_array_equal(data6, datac) nc.set_auto_chartostring(False) data7 = v3[:] - assert data7.dtype.itemsize == 1 + assert data7.dtype.itemsize == 1 assert_array_equal(data7, datac) nc.close() diff --git a/test/test_types.py b/test/test_types.py index b8eb74b2e..1ddb292a4 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -6,7 +6,7 @@ from numpy.testing import assert_array_equal, assert_array_almost_equal from numpy.random.mtrand import uniform import netCDF4 -from type_guards import valid_complevel +import type_guards as n4t # test primitive data types. @@ -29,7 +29,7 @@ def setUp(self): f.createDimension('n1', None) f.createDimension('n2', n2dim) for typ in datatypes: - assert valid_complevel(complevel) or complevel is None + assert n4t.valid_complevel(complevel) or complevel is None foo = f.createVariable('data_'+typ, typ, ('n1','n2',),zlib=zlib,complevel=complevel,shuffle=shuffle,least_significant_digit=least_significant_digit,fill_value=FillValue) #foo._FillValue = FillValue # test writing of _FillValue attribute for diff types diff --git a/test/type_guards.py b/test/type_guards.py index 0b2ca8ff8..02e226fa9 100644 --- a/test/type_guards.py +++ b/test/type_guards.py @@ -1,22 +1,60 @@ -"""_type_guards - Helpers for input type-checking""" -from typing import Literal, TYPE_CHECKING, Union +"""type_guards.py - Helpers for static and runtime type-checking of initialization arguments +for Dataset and Variable""" + +from typing import TYPE_CHECKING, Any, Literal + from typing_extensions import TypeGuard if TYPE_CHECKING: # in stubs only - from netCDF4 import CompressionLevel, EndianType, CompressionType, QuantizeMode + from netCDF4 import ( + AccessMode, + CalendarType, + CompressionLevel, + CompressionType, + EndianType, + QuantizeMode, + ) from netCDF4 import Format as NCFormat else: - CompressionLevel = EndianType = CompressionType = NCFormat = QuantizeMode = object + AccessMode = Any + CalendarType = Any + CompressionLevel = Any + DiskFormat = Any + EndianType = Any + CompressionType = Any + NCFormat = Any + QuantizeMode = Any + + +def valid_access_mode(mode) -> TypeGuard[AccessMode]: + """Check for a valid `mode` argument for opening a Dataset""" + return mode in {"r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"} + + +def valid_calendar(calendar) -> TypeGuard[CalendarType]: + """Check for a valid `calendar` argument for cftime functions""" + return calendar in { + "standard", + "gregorian", + "proleptic_gregorian", + "noleap", + "365_day", + "360_day", + "julian", + "all_leap", + "366_day", + } + def valid_complevel(complevel) -> TypeGuard[CompressionLevel]: - return complevel is None or isinstance(complevel, int) and 0 <= complevel <= 9 + """Check for a valid `complevel` argument for creating a Variable""" + return isinstance(complevel, int) and 0 <= complevel <= 9 -def valid_endian(endian) -> TypeGuard[EndianType]: - return endian in {"native", "big", "little"} -def valid_comptype(comptype) -> TypeGuard[CompressionType]: - return comptype is None or comptype in { +def valid_compression(compression) -> TypeGuard[CompressionType]: + """Check for a valid `compression` argument for creating a Variable""" + return compression in { "zlib", "szip", "zstd", @@ -28,17 +66,40 @@ def valid_comptype(comptype) -> TypeGuard[CompressionType]: "blosc_zstd", } -def valid_bloscshuffle(bloscshuffle) -> TypeGuard[Literal[0, 1, 2]]: - return bloscshuffle in {0, 1, 2} -def valid_ncformat(ncformat) -> TypeGuard[NCFormat]: - return ncformat in [ +def valid_ncformat(format) -> TypeGuard[NCFormat]: + """Check for a valid `format` argument for opening a Dataset""" + return format in { "NETCDF4", "NETCDF4_CLASSIC", "NETCDF3_CLASSIC", "NETCDF3_64BIT_OFFSET", - "NETCDF3_64BIT_DATA" - ] + "NETCDF3_64BIT_DATA", + } + + +def valid_endian(endian) -> TypeGuard[EndianType]: + """Check for a valid `endian` argument for creating a Variable""" + return endian in {"native", "big", "little"} + + +def valid_bloscshuffle(blosc_shuffle) -> TypeGuard[Literal[0, 1, 2]]: + """Check for a valid `blosc_shuffle` argument for creating a Variable""" + return blosc_shuffle in {0, 1, 2} + def valid_quantize_mode(quantize_mode) -> TypeGuard[QuantizeMode]: - return quantize_mode in ["BitGroom", "BitRound", "GranularBitRound"] \ No newline at end of file + """Check for a valid `quantize_mode` argument for creating a Variable""" + return quantize_mode in {"BitGroom", "BitRound", "GranularBitRound"} + + +def valid_szip_coding(szip_coding) -> TypeGuard[Literal["nn", "ec"]]: + """Check for a valid `szip_coding` argument for creating a Variable""" + return szip_coding in {"nn", "ec"} + + +def valid_szip_pixels_per_block( + szip_pixels_per_block, +) -> TypeGuard[Literal[4, 8, 16, 32]]: + """Check for a valid `szip_pixels_per_block` argument for creating a Variable""" + return szip_pixels_per_block in {4, 8, 16, 32} From fab4050505bbd60b14fa40bf37a7261abe260d57 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 30 Jul 2024 15:13:38 -0700 Subject: [PATCH 62/81] Revert "disable some cicd" This reverts commit 7c9eb602e79f294613fd3fea9f317f801035bec5. --- .github/workflows/build_latest.yml | 3 +-- .github/workflows/build_master.yml | 11 +++++------ .github/workflows/build_old.yml | 3 +-- .github/workflows/cibuildwheel.yml | 17 ++++++++--------- .github/workflows/miniconda.yml | 16 +++++++--------- 5 files changed, 22 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index 28f65c7a8..8f536624e 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -1,6 +1,5 @@ name: Build and Test Linux with latest netcdf-c -# on: [push, pull_request] -on: [] # disable +on: [push, pull_request] jobs: build-linux: name: Python (${{ matrix.python-version }}) diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index af16c3eec..9e0cf70f3 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -1,6 +1,5 @@ name: Build and Test on Linux with netcdf-c github master -# on: [push, pull_request] -on: [] # disable +on: [push, pull_request] jobs: build-linux: name: Python (${{ matrix.python-version }}) @@ -34,7 +33,7 @@ jobs: export LDFLAGS="-L${NETCDF_DIR}/lib" export LIBS="-lhdf5_mpich_hl -lhdf5_mpich -lm -lz" autoreconf -i - ./configure --prefix $NETCDF_DIR --enable-netcdf-4 --enable-shared --enable-dap --enable-parallel4 + ./configure --prefix $NETCDF_DIR --enable-netcdf-4 --enable-shared --enable-dap --enable-parallel4 make -j 2 make install popd @@ -43,7 +42,7 @@ jobs: # if: ${{ failure() }} # run: | # cd netcdf-c-${NETCDF_VERSION} -# cat config.log +# cat config.log - name: Install python dependencies via pip run: | @@ -52,13 +51,13 @@ jobs: - name: Install netcdf4-python run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir python setup.py install - name: Test run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} #export HDF5_PLUGIN_PATH=${NETCDF_DIR}/plugins/plugindir python checkversion.py # serial diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index 67d141eef..b10977d9d 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -1,6 +1,5 @@ name: Build and Test Linux with older netcdf-c -# on: [push, pull_request] -on: [] # disable +on: [push, pull_request] jobs: build-linux: name: Python (${{ matrix.python-version }}) diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index 09ca9b752..cfbc6823d 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -1,14 +1,13 @@ name: Wheels -on: ["push"] -# on: -# pull_request: -# push: -# tags: -# - "v*" -# release: -# types: -# - published +on: + pull_request: + push: + tags: + - "v*" + release: + types: + - published permissions: contents: read diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index fb291fdc0..ba790d589 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -1,10 +1,9 @@ name: Build and Test -# on: -# pull_request: -# push: -# branches: [master] -on: ["push"] +on: + pull_request: + push: + branches: [master] jobs: run-serial: @@ -13,12 +12,11 @@ jobs: # NO_NET: 1 strategy: matrix: - # python-version: [ "3.9", "3.10", "3.11", "3.12" ] - python-version: [ "3.9", "3.10"] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] os: [windows-latest, ubuntu-latest, macos-latest] platform: [x64, x32] exclude: - - os: macos-latest + - os: macos-latest platform: x32 fail-fast: false defaults: @@ -84,7 +82,7 @@ jobs: run: | cd test && python run_all.py cd ../examples - export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" + export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" which mpirun mpirun --version mpirun -np 4 --oversubscribe python mpi_example.py # for openmpi From e0719dac3c35c7e7f1fac49ba42e1bf934abd3b7 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 30 Jul 2024 15:20:54 -0700 Subject: [PATCH 63/81] (minor) restore whitespace to clean diff --- .github/workflows/build_latest.yml | 16 ++++++++-------- .github/workflows/build_old.yml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index 8f536624e..4a80cf01a 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -53,7 +53,7 @@ jobs: # if: ${{ failure() }} # run: | # cd netcdf-c-${NETCDF_VERSION} -# cat config.log +# cat config.log - name: Install python dependencies via pip run: | @@ -62,12 +62,12 @@ jobs: - name: Install netcdf4-python run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir python setup.py install - name: Test run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} python checkversion.py # serial cd test @@ -98,9 +98,9 @@ jobs: # - name: Tarball # run: | -# export PATH=${NETCDF_DIR}/bin:${PATH} -# python setup.py --version +# export PATH=${NETCDF_DIR}/bin:${PATH} +# python setup.py --version # check-manifest --version -# check-manifest --verbose -# pip wheel . -w dist --no-deps -# twine check dist/* +# check-manifest --verbose +# pip wheel . -w dist --no-deps +# twine check dist/* diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index b10977d9d..063f0a87f 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -53,7 +53,7 @@ jobs: # if: ${{ failure() }} # run: | # cd netcdf-c-${NETCDF_VERSION} -# cat config.log +# cat config.log - name: Install python dependencies via pip run: | @@ -62,12 +62,12 @@ jobs: - name: Install netcdf4-python run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir python setup.py install - name: Test run: | - export PATH=${NETCDF_DIR}/bin:${PATH} + export PATH=${NETCDF_DIR}/bin:${PATH} python checkversion.py # serial cd test @@ -98,9 +98,9 @@ jobs: # - name: Tarball # run: | -# export PATH=${NETCDF_DIR}/bin:${PATH} -# python setup.py --version +# export PATH=${NETCDF_DIR}/bin:${PATH} +# python setup.py --version # check-manifest --version -# check-manifest --verbose -# pip wheel . -w dist --no-deps -# twine check dist/* +# check-manifest --verbose +# pip wheel . -w dist --no-deps +# twine check dist/* From d42667d66a532e566cb2d115262cc201c10b933b Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 30 Jul 2024 15:22:22 -0700 Subject: [PATCH 64/81] (minor) restore whitespace again to clean diff --- .github/workflows/build_latest.yml | 2 +- .github/workflows/build_old.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index 4a80cf01a..0f3bb7d3e 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -99,7 +99,7 @@ jobs: # - name: Tarball # run: | # export PATH=${NETCDF_DIR}/bin:${PATH} -# python setup.py --version +# python setup.py --version # check-manifest --version # check-manifest --verbose # pip wheel . -w dist --no-deps diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index 063f0a87f..7f958f17a 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -99,7 +99,7 @@ jobs: # - name: Tarball # run: | # export PATH=${NETCDF_DIR}/bin:${PATH} -# python setup.py --version +# python setup.py --version # check-manifest --version # check-manifest --verbose # pip wheel . -w dist --no-deps From 3b1de4bf18cbbe88f05b8860282e5b5498ba59ae Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 30 Jul 2024 15:58:24 -0700 Subject: [PATCH 65/81] update examples and tests type_guards --- examples/bench.py | 2 + examples/bench_compress4.py | 2 + examples/bench_diskless.py | 2 + examples/type_guards.py | 105 +++++++++++++++++++++++++++++++++ test/test_compression.py | 8 +-- test/test_compression_blosc.py | 6 +- test/test_compression_bzip2.py | 4 +- test/test_compression_quant.py | 4 +- test/test_compression_zstd.py | 4 +- test/test_endian.py | 4 +- test/test_stringarr.py | 4 +- test/test_types.py | 4 +- test/type_guards.py | 2 +- 13 files changed, 131 insertions(+), 20 deletions(-) create mode 100644 examples/type_guards.py diff --git a/examples/bench.py b/examples/bench.py index f3ad75246..1d18ed8e1 100644 --- a/examples/bench.py +++ b/examples/bench.py @@ -4,6 +4,7 @@ import netCDF4 from timeit import Timer import os, sys +import type_guards # create an n1dim by n2dim by n3dim random array. n1dim = 30 @@ -15,6 +16,7 @@ array = uniform(size=(n1dim,n2dim,n3dim,n4dim)) def write_netcdf(filename,zlib=False,least_significant_digit=None,format='NETCDF4'): + assert type_guards.valid_format(format) file = netCDF4.Dataset(filename,'w',format=format) file.createDimension('n1', n1dim) file.createDimension('n2', n2dim) diff --git a/examples/bench_compress4.py b/examples/bench_compress4.py index a7d0e1033..05b9ade80 100644 --- a/examples/bench_compress4.py +++ b/examples/bench_compress4.py @@ -4,6 +4,7 @@ import netCDF4 from timeit import Timer import os, sys +import type_guards # use real data. URL="http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep.reanalysis/pressure/hgt.1990.nc" @@ -24,6 +25,7 @@ def write_netcdf(filename,nsd,quantize_mode='BitGroom'): file.createDimension('n1', None) file.createDimension('n3', n3dim) file.createDimension('n4', n4dim) + assert type_guards.valid_quantize_mode(quantize_mode) foo = file.createVariable('data',\ 'f4',('n1','n3','n4'),\ zlib=True,shuffle=True,\ diff --git a/examples/bench_diskless.py b/examples/bench_diskless.py index dd7a78315..ae92e037b 100644 --- a/examples/bench_diskless.py +++ b/examples/bench_diskless.py @@ -4,6 +4,7 @@ import netCDF4 from timeit import Timer import os, sys +import type_guards # create an n1dim by n2dim by n3dim random array. n1dim = 30 @@ -15,6 +16,7 @@ array = uniform(size=(n1dim,n2dim,n3dim,n4dim)) def write_netcdf(filename,zlib=False,least_significant_digit=None,format='NETCDF4',closeit=False): + assert type_guards.valid_format(format) file = netCDF4.Dataset(filename,'w',format=format,diskless=True,persist=True) file.createDimension('n1', n1dim) file.createDimension('n2', n2dim) diff --git a/examples/type_guards.py b/examples/type_guards.py new file mode 100644 index 000000000..2baabda68 --- /dev/null +++ b/examples/type_guards.py @@ -0,0 +1,105 @@ +"""type_guards.py - Helpers for static and runtime type-checking of initialization arguments +for Dataset and Variable""" + +from typing import TYPE_CHECKING, Any, Literal + +from typing_extensions import TypeGuard + +if TYPE_CHECKING: + # in stubs only + from netCDF4 import ( + AccessMode, + CalendarType, + CompressionLevel, + CompressionType, + EndianType, + QuantizeMode, + ) + from netCDF4 import Format as NCFormat +else: + AccessMode = Any + CalendarType = Any + CompressionLevel = Any + DiskFormat = Any + EndianType = Any + CompressionType = Any + NCFormat = Any + QuantizeMode = Any + + +def valid_access_mode(mode) -> TypeGuard[AccessMode]: + """Check for a valid `mode` argument for opening a Dataset""" + return mode in {"r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"} + + +def valid_calendar(calendar) -> TypeGuard[CalendarType]: + """Check for a valid `calendar` argument for cftime functions""" + return calendar in { + "standard", + "gregorian", + "proleptic_gregorian", + "noleap", + "365_day", + "360_day", + "julian", + "all_leap", + "366_day", + } + + +def valid_complevel(complevel) -> TypeGuard[CompressionLevel]: + """Check for a valid `complevel` argument for creating a Variable""" + return isinstance(complevel, int) and 0 <= complevel <= 9 + + +def valid_compression(compression) -> TypeGuard[CompressionType]: + """Check for a valid `compression` argument for creating a Variable""" + return compression in { + "zlib", + "szip", + "zstd", + "bzip2", + "blosc_lz", + "blosc_lz4", + "blosc_lz4hc", + "blosc_zlib", + "blosc_zstd", + } + + +def valid_format(format) -> TypeGuard[NCFormat]: + """Check for a valid `format` argument for opening a Dataset""" + return format in { + "NETCDF4", + "NETCDF4_CLASSIC", + "NETCDF3_CLASSIC", + "NETCDF3_64BIT_OFFSET", + "NETCDF3_64BIT_DATA", + } + + +def valid_endian(endian) -> TypeGuard[EndianType]: + """Check for a valid `endian` argument for creating a Variable""" + return endian in {"native", "big", "little"} + + +def valid_bloscshuffle(blosc_shuffle) -> TypeGuard[Literal[0, 1, 2]]: + """Check for a valid `blosc_shuffle` argument for creating a Variable""" + return blosc_shuffle in {0, 1, 2} + + +def valid_quantize_mode(quantize_mode) -> TypeGuard[QuantizeMode]: + """Check for a valid `quantize_mode` argument for creating a Variable""" + return quantize_mode in {"BitGroom", "BitRound", "GranularBitRound"} + + +def valid_szip_coding(szip_coding) -> TypeGuard[Literal["nn", "ec"]]: + """Check for a valid `szip_coding` argument for creating a Variable""" + return szip_coding in {"nn", "ec"} + + +def valid_szip_pixels_per_block( + szip_pixels_per_block, +) -> TypeGuard[Literal[4, 8, 16, 32]]: + """Check for a valid `szip_pixels_per_block` argument for creating a Variable""" + return szip_pixels_per_block in {4, 8, 16, 32} diff --git a/test/test_compression.py b/test/test_compression.py index 7d53fe436..1d4b59076 100644 --- a/test/test_compression.py +++ b/test/test_compression.py @@ -3,7 +3,7 @@ from netCDF4.utils import _quantize from numpy.testing import assert_almost_equal import os, tempfile, unittest -import type_guards as n4t +import type_guards ndim = 100000 ndim2 = 100 @@ -16,7 +16,7 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,contiguous=False,\ chunksizes=None,complevel=6,fletcher32=False): - assert n4t.valid_complevel(complevel) or complevel is None + assert type_guards.valid_complevel(complevel) or complevel is None file = Dataset(filename,'w') file.createDimension('n', ndim) foo = file.createVariable('data',\ @@ -32,7 +32,7 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=F #compression='' #compression=0 #compression='gzip' # should fail - assert n4t.valid_compression(compression) or compression is None + assert type_guards.valid_compression(compression) or compression is None foo2 = file.createVariable('data2',\ dtype,('n'),compression=compression,least_significant_digit=least_significant_digit,\ shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) @@ -49,7 +49,7 @@ def write_netcdf2(filename,zlib,least_significant_digit,data,dtype='f8',shuffle= file = Dataset(filename,'w') file.createDimension('n', ndim) file.createDimension('n2', ndim2) - assert n4t.valid_complevel(complevel) or complevel is None + assert type_guards.valid_complevel(complevel) or complevel is None foo = file.createVariable('data2',\ dtype,('n','n2'),zlib=zlib,least_significant_digit=least_significant_digit,\ shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) diff --git a/test/test_compression_blosc.py b/test/test_compression_blosc.py index f8f13e48d..c532fef32 100644 --- a/test/test_compression_blosc.py +++ b/test/test_compression_blosc.py @@ -1,10 +1,10 @@ from numpy.random.mtrand import uniform from netCDF4 import Dataset -import type_guards as n4t +import type_guards from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_blosc_filter -import type_guards as n4t +import type_guards ndim = 100000 @@ -14,7 +14,7 @@ datarr = uniform(size=(ndim,)) def write_netcdf(filename,dtype='f8',blosc_shuffle=1,complevel=6): - assert (n4t.valid_complevel(complevel) or complevel is None) and n4t.valid_bloscshuffle(blosc_shuffle) + assert (type_guards.valid_complevel(complevel) or complevel is None) and type_guards.valid_bloscshuffle(blosc_shuffle) nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ diff --git a/test/test_compression_bzip2.py b/test/test_compression_bzip2.py index b207288df..ec32ba248 100644 --- a/test/test_compression_bzip2.py +++ b/test/test_compression_bzip2.py @@ -3,7 +3,7 @@ from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_bzip2_filter -import type_guards as n4t +import type_guards ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -11,7 +11,7 @@ array = uniform(size=(ndim,)) def write_netcdf(filename,dtype='f8',complevel=6): - assert n4t.valid_complevel(complevel) or complevel is None + assert type_guards.valid_complevel(complevel) or complevel is None nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ diff --git a/test/test_compression_quant.py b/test/test_compression_quant.py index bbb2b2676..24267615a 100644 --- a/test/test_compression_quant.py +++ b/test/test_compression_quant.py @@ -3,7 +3,7 @@ from numpy.testing import assert_almost_equal import numpy as np import os, tempfile, unittest -import type_guards as n4t +import type_guards ndim = 100000 nfiles = 7 @@ -17,7 +17,7 @@ def write_netcdf(filename,zlib,significant_digits,data,dtype='f8',shuffle=False, complevel=6,quantize_mode="BitGroom"): file = Dataset(filename,'w') file.createDimension('n', ndim) - assert (n4t.valid_complevel(complevel) or complevel is None) and n4t.valid_quantize_mode(quantize_mode) + assert (type_guards.valid_complevel(complevel) or complevel is None) and type_guards.valid_quantize_mode(quantize_mode) foo = file.createVariable('data',\ dtype,('n'),zlib=zlib,significant_digits=significant_digits,\ shuffle=shuffle,complevel=complevel,quantize_mode=quantize_mode) diff --git a/test/test_compression_zstd.py b/test/test_compression_zstd.py index 820b3df37..123397f12 100644 --- a/test/test_compression_zstd.py +++ b/test/test_compression_zstd.py @@ -3,7 +3,7 @@ from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_zstd_filter -import type_guards as n4t +import type_guards ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -13,7 +13,7 @@ def write_netcdf(filename,dtype='f8',complevel=6): nc = Dataset(filename,'w') nc.createDimension('n', ndim) - assert n4t.valid_complevel(complevel) or complevel is None + assert type_guards.valid_complevel(complevel) or complevel is None foo = nc.createVariable('data',\ dtype,('n'),compression='zstd',complevel=complevel) foo[:] = array diff --git a/test/test_endian.py b/test/test_endian.py index 12f1bef7f..1a8f58681 100644 --- a/test/test_endian.py +++ b/test/test_endian.py @@ -2,7 +2,7 @@ import numpy as np import unittest, os, tempfile from numpy.testing import assert_array_equal, assert_array_almost_equal -import type_guards as n4t +import type_guards data = np.arange(12,dtype='f4').reshape(3,4) FILE_NAME = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -74,7 +74,7 @@ def issue310(file): endian='little' else: raise ValueError('cannot determine native endianness') - assert n4t.valid_endian(endian) # mypy fails to narrow endian on its own + assert type_guards.valid_endian(endian) # mypy fails to narrow endian on its own var_big_endian = nc.createVariable(\ 'obs_big_endian', '>f8', ('obs', ),\ endian=endian,fill_value=fval) diff --git a/test/test_stringarr.py b/test/test_stringarr.py index ef05b0b38..f3e140c0a 100644 --- a/test/test_stringarr.py +++ b/test/test_stringarr.py @@ -3,7 +3,7 @@ import unittest import os from numpy.testing import assert_array_equal, assert_array_almost_equal -import type_guards as n4t +import type_guards def generateString(length, alphabet=string.ascii_letters + string.digits + string.punctuation): return(''.join([random.choice(alphabet) for i in range(length)])) @@ -25,7 +25,7 @@ class StringArrayTestCase(unittest.TestCase): def setUp(self): self.file = FILE_NAME - assert n4t.valid_ncformat(FILE_FORMAT) + assert type_guards.valid_format(FILE_FORMAT) nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT) nc.createDimension('n1',None) nc.createDimension('n2',n2) diff --git a/test/test_types.py b/test/test_types.py index 1ddb292a4..3cc7bbc3b 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -6,7 +6,7 @@ from numpy.testing import assert_array_equal, assert_array_almost_equal from numpy.random.mtrand import uniform import netCDF4 -import type_guards as n4t +import type_guards # test primitive data types. @@ -29,7 +29,7 @@ def setUp(self): f.createDimension('n1', None) f.createDimension('n2', n2dim) for typ in datatypes: - assert n4t.valid_complevel(complevel) or complevel is None + assert type_guards.valid_complevel(complevel) or complevel is None foo = f.createVariable('data_'+typ, typ, ('n1','n2',),zlib=zlib,complevel=complevel,shuffle=shuffle,least_significant_digit=least_significant_digit,fill_value=FillValue) #foo._FillValue = FillValue # test writing of _FillValue attribute for diff types diff --git a/test/type_guards.py b/test/type_guards.py index 02e226fa9..2baabda68 100644 --- a/test/type_guards.py +++ b/test/type_guards.py @@ -67,7 +67,7 @@ def valid_compression(compression) -> TypeGuard[CompressionType]: } -def valid_ncformat(format) -> TypeGuard[NCFormat]: +def valid_format(format) -> TypeGuard[NCFormat]: """Check for a valid `format` argument for opening a Dataset""" return format in { "NETCDF4", From 9f7698d6abc372e3513d03c3bbe8e12d913611c7 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 31 Jul 2024 09:42:55 -0700 Subject: [PATCH 66/81] disallow lists of floats or strings in GetSetItemKey --- src/netCDF4/__init__.pyi | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index cbebd1990..790305579 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -147,9 +147,9 @@ BoolInt: TypeAlias = Literal[0, 1] DateTimeArray: TypeAlias = npt.NDArray[np.object_] """numpy array of datetime.datetime or cftime.datetime""" -# netCDF accepts floats that can be cooerced to an integer value, and sometimes strings that can be coerced to an integer value. -# There's currently no way to specify this statically, so we allow all floats and strings. -# Also, we have to specify all the possible combinations of list[...] because lists are invariant. +# - Allow singular float or str keys, but not lists (or sequences) of them +# - `list` and `ndarray` are specifically listed for the 1-D case as Sequence is too general -- it includes +# tuples which are rather for multi-dimensional indexing. GetSetItemKey: TypeAlias = ( int | float @@ -158,12 +158,8 @@ GetSetItemKey: TypeAlias = ( | slice | ellipsis | list[int] - | list[float] - | list[np.number] - | list[str] | list[bool] - | list[np.bool_] - | npt.NDArray[np.number] + | npt.NDArray[np.integer] | npt.NDArray[np.bool_] | tuple[ int @@ -173,12 +169,10 @@ GetSetItemKey: TypeAlias = ( | slice | ellipsis | Sequence[int] - | Sequence[float] - | Sequence[np.number] - | Sequence[str] + | Sequence[np.integer] | Sequence[bool] | Sequence[np.bool_] - | npt.NDArray[np.number] + | npt.NDArray[np.integer] | npt.NDArray[np.bool_], ..., ] From 1975e12daa7baf7b0a53f80bdef09673937bf6d1 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 31 Jul 2024 09:43:14 -0700 Subject: [PATCH 67/81] disallow float dimension size --- src/netCDF4/__init__.pyi | 4 ++-- test/test_multifile2.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 790305579..e9c22ed8a 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -310,7 +310,7 @@ class Dataset: def sync(self) -> None: ... def set_fill_on(self) -> None: ... def set_fill_off(self) -> None: ... - def createDimension(self, dimname: str, size: int | float | None = None) -> Dimension: ... + def createDimension(self, dimname: str, size: int | None = None) -> Dimension: ... def renameDimension(self, oldname: str, newname: str) -> None: ... @overload def createVariable( @@ -433,7 +433,7 @@ class Group(Dataset): def close(self) -> NoReturn: ... class Dimension: - def __init__(self, grp: Dataset, name: str, size: int | float | None = None, **kwargs: Any) -> None: ... + def __init__(self, grp: Dataset, name: str, size: int | None = None, **kwargs: Any) -> None: ... @property def name(self) -> str: ... @property diff --git a/test/test_multifile2.py b/test/test_multifile2.py index 3ab3d482b..2c37a6f83 100644 --- a/test/test_multifile2.py +++ b/test/test_multifile2.py @@ -23,7 +23,7 @@ def setUp(self): for nfile,file in enumerate(self.files): f = Dataset(file,'w',format='NETCDF4_CLASSIC') #f.createDimension('x',None) - f.createDimension('x',ninc) + f.createDimension('x',int(ninc)) f.createDimension('y',ydim) f.createDimension('z',zdim) f.history = 'created today' From cd002c8a61738df0e5debd12f7d4a2f8ea60349f Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 31 Jul 2024 11:09:02 -0700 Subject: [PATCH 68/81] Add TypeGuard for variable type --- examples/type_guards.py | 16 +++++++++++++++- test/type_guards.py | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/examples/type_guards.py b/examples/type_guards.py index 2baabda68..565801dd9 100644 --- a/examples/type_guards.py +++ b/examples/type_guards.py @@ -1,8 +1,9 @@ """type_guards.py - Helpers for static and runtime type-checking of initialization arguments for Dataset and Variable""" -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, Type, TypeVar, Union, overload +import netCDF4 from typing_extensions import TypeGuard if TYPE_CHECKING: @@ -27,6 +28,19 @@ QuantizeMode = Any +T = TypeVar("T") + + +def var_isof_type( + var: netCDF4.Variable, type_: Type[T] +) -> TypeGuard[netCDF4.Variable[T]]: + """Check that a variable is of some type. This function does not support CompoundType, + EnumType, or VLType""" + if isinstance(type_, (netCDF4.EnumType, netCDF4.VLType, netCDF4.CompoundType)): + raise TypeError("This function is not valid for user-defined types.") + return (type_ is str and var.dtype is type_) or var.dtype.type is type_ + + def valid_access_mode(mode) -> TypeGuard[AccessMode]: """Check for a valid `mode` argument for opening a Dataset""" return mode in {"r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"} diff --git a/test/type_guards.py b/test/type_guards.py index 2baabda68..565801dd9 100644 --- a/test/type_guards.py +++ b/test/type_guards.py @@ -1,8 +1,9 @@ """type_guards.py - Helpers for static and runtime type-checking of initialization arguments for Dataset and Variable""" -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, Type, TypeVar, Union, overload +import netCDF4 from typing_extensions import TypeGuard if TYPE_CHECKING: @@ -27,6 +28,19 @@ QuantizeMode = Any +T = TypeVar("T") + + +def var_isof_type( + var: netCDF4.Variable, type_: Type[T] +) -> TypeGuard[netCDF4.Variable[T]]: + """Check that a variable is of some type. This function does not support CompoundType, + EnumType, or VLType""" + if isinstance(type_, (netCDF4.EnumType, netCDF4.VLType, netCDF4.CompoundType)): + raise TypeError("This function is not valid for user-defined types.") + return (type_ is str and var.dtype is type_) or var.dtype.type is type_ + + def valid_access_mode(mode) -> TypeGuard[AccessMode]: """Check for a valid `mode` argument for opening a Dataset""" return mode in {"r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"} From 3f2e8aaa8aa10e291938fae1b9bb2cef5dd05b24 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 31 Jul 2024 11:09:22 -0700 Subject: [PATCH 69/81] Update type stub notes --- src/netCDF4/__init__.pyi | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index e9c22ed8a..7c138d5b4 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -2,26 +2,27 @@ # Notes: # # - The stubs in this file are manually-generated and must be updated if and when the API is changed. -# - The following **ruff** commands may be used to properly format this file according to +# - The following **ruff** commands may be used to format this file according to # https://typing.readthedocs.io/en/latest/source/stubs.html # # ruff format --line-length 130 src/netCDF4/__init__.pyi # format code # ruff check --line-length 130 --select I --fix src/netCDF4/__init__.pyi # sort imports # # - The Variable class is a generic and may thus be statically typed, but this has limited utility for the following reasons: -# - The return type of `Variable.__getitem__()` (and `Variable.getValue()`) depends on a number of factors (e.g. variable shape, -# key shape, whether masking is enabled) that cannot be easily determined statically. -# - Similarly, the types and shapes of data that `Variable.__setitem__()` may accept varies widely depending on many factors and -# is intractable to determine statically. -# - Automatic typing of a Variable on variable creation is tedious due to the large number of ways to specify a variable's type -# (in particular all the type literals). +# - The return type of `Variable.__getitem__()` (and `Variable.getValue()`) depends on a number of factors (e.g. variable +# shape, key shape, whether masking is enabled) that cannot be easily determined statically. +# - Similarly, the types and shapes of data that `Variable.__setitem__()` may accept varies widely depending on many factors +# and is intractable to determine statically. +# - Some facility for automatically typing a Variable on creation has been provided, however it is not exhaustive as a variable +# may created with a string literal indicating its type and it would require an excessive number of overloads to enumerate +# each of these cases. # - It is not possible to statically type a Variable of any user-defined type (CompoundType, EnumType, VLType) as these types # are created dynamically. -# It is thus best left to the user to implement TypeGuards and/or perform other mixed static/runtime type-checking to ensure the -# type and shape of data retrieved from this library. -# - `Dataset.__getitem__()` may return either a Variable or a Group, depending on the string passed to it. Rather than return a -# Union of Variable and Group, the authors of these stubs have elected to to return Any, leaving it up to users to determine the -# type of the returned value. +# Thus it is most often best for the user to implement TypeGuards and/or perform other mixed static/runtime type-checking to +# ensure the type and shape of data retrieved from this library. +# - The return type of some functions or properties (such as `Dataset.__getitem__()`) may one of a number of types. Where it is +# not possible to narrow the type with overloads, the authors of these stubs have generally chosen to use `Any` as the return +# type rather than a union of the possible types. # - `MFDataset.dimensions` returns `dict[str, Dimension]` and `MFDataset.variables` returns `dict[str, Variable]` even though the # dict value types may actually be `_Dimension` and `_Variable`, respectively. The original authors of this stubfile have # elected to do this for simplicity's sake, but it may make sense to change this in the future, or just return `dict[str, Any]`. From 4400567c69ef48e9fcf30b1ce9b750104056413f Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Wed, 31 Jul 2024 11:58:37 -0700 Subject: [PATCH 70/81] Revert "Add TypeGuard for variable type" This reverts commit cd002c8a61738df0e5debd12f7d4a2f8ea60349f. --- examples/type_guards.py | 16 +--------------- test/type_guards.py | 16 +--------------- 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/examples/type_guards.py b/examples/type_guards.py index 565801dd9..2baabda68 100644 --- a/examples/type_guards.py +++ b/examples/type_guards.py @@ -1,9 +1,8 @@ """type_guards.py - Helpers for static and runtime type-checking of initialization arguments for Dataset and Variable""" -from typing import TYPE_CHECKING, Any, Literal, Type, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, Literal -import netCDF4 from typing_extensions import TypeGuard if TYPE_CHECKING: @@ -28,19 +27,6 @@ QuantizeMode = Any -T = TypeVar("T") - - -def var_isof_type( - var: netCDF4.Variable, type_: Type[T] -) -> TypeGuard[netCDF4.Variable[T]]: - """Check that a variable is of some type. This function does not support CompoundType, - EnumType, or VLType""" - if isinstance(type_, (netCDF4.EnumType, netCDF4.VLType, netCDF4.CompoundType)): - raise TypeError("This function is not valid for user-defined types.") - return (type_ is str and var.dtype is type_) or var.dtype.type is type_ - - def valid_access_mode(mode) -> TypeGuard[AccessMode]: """Check for a valid `mode` argument for opening a Dataset""" return mode in {"r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"} diff --git a/test/type_guards.py b/test/type_guards.py index 565801dd9..2baabda68 100644 --- a/test/type_guards.py +++ b/test/type_guards.py @@ -1,9 +1,8 @@ """type_guards.py - Helpers for static and runtime type-checking of initialization arguments for Dataset and Variable""" -from typing import TYPE_CHECKING, Any, Literal, Type, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, Literal -import netCDF4 from typing_extensions import TypeGuard if TYPE_CHECKING: @@ -28,19 +27,6 @@ QuantizeMode = Any -T = TypeVar("T") - - -def var_isof_type( - var: netCDF4.Variable, type_: Type[T] -) -> TypeGuard[netCDF4.Variable[T]]: - """Check that a variable is of some type. This function does not support CompoundType, - EnumType, or VLType""" - if isinstance(type_, (netCDF4.EnumType, netCDF4.VLType, netCDF4.CompoundType)): - raise TypeError("This function is not valid for user-defined types.") - return (type_ is str and var.dtype is type_) or var.dtype.type is type_ - - def valid_access_mode(mode) -> TypeGuard[AccessMode]: """Check for a valid `mode` argument for opening a Dataset""" return mode in {"r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"} From 50ed873b0dee6efc3e77a32f6d2058170918f8b0 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Fri, 23 Aug 2024 10:41:33 -0700 Subject: [PATCH 71/81] Revert erroneous change in chunksizes type --- src/netCDF4/__init__.pyi | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 7c138d5b4..1feb1871a 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -328,7 +328,7 @@ class Dataset: blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: int | None = None, + chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, @@ -351,7 +351,7 @@ class Dataset: blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: int | None = None, + chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, @@ -374,7 +374,7 @@ class Dataset: blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: int | None = None, + chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, @@ -483,7 +483,7 @@ class Variable(Generic[VarT]): blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: int | None = None, + chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, @@ -508,7 +508,7 @@ class Variable(Generic[VarT]): blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: int | None = None, + chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, @@ -533,7 +533,7 @@ class Variable(Generic[VarT]): blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: int | None = None, + chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, @@ -557,7 +557,7 @@ class Variable(Generic[VarT]): blosc_shuffle: Literal[0, 1, 2] = 1, fletcher32: bool = False, contiguous: bool = False, - chunksizes: int | None = None, + chunksizes: Sequence[int] | None = None, endian: EndianType = "native", least_significant_digit: int | None = None, significant_digits: int | None = None, From 0cf4d6d2d1b167dc94cd629a975f7a56355d4fba Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Fri, 23 Aug 2024 11:10:45 -0700 Subject: [PATCH 72/81] Fix incorrect type of datatype for Real types --- src/netCDF4/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 1feb1871a..59415c146 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -446,7 +446,7 @@ class Dimension: class _VarDatatypeProperty: # A faux descriptor definition of the property to allow overloads @overload - def __get__(self, instance: Variable[RealVarT], owner: Any) -> RealVarT: ... + def __get__(self, instance: Variable[RealVarT], owner: Any) -> np.dtype[RealVarT]: ... @overload def __get__(self, instance: Variable[ComplexVarT], owner: Any) -> CompoundType: ... @overload From 55e88e76e42a92fb8a0d2fa00d5585f8f46ba3b7 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Fri, 23 Aug 2024 11:13:21 -0700 Subject: [PATCH 73/81] Fixed inconsistent naming of DimensionsSpecifier >> DimensionsType --- src/netCDF4/__init__.pyi | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 59415c146..0a1af0fa5 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -130,7 +130,7 @@ RealVarT = TypeVar("RealVarT", bound=NumPyRealType) ComplexVarT = TypeVar("ComplexVarT", bound=NumPyComplexType) NumericVarT = TypeVar("NumericVarT", bound=NumPyNumericType) -DimensionsSpecifier: TypeAlias = Union[str, bytes, Dimension, Iterable[Union[str, bytes, Dimension]]] +DimensionsType: TypeAlias = Union[str, bytes, Dimension, Iterable[Union[str, bytes, Dimension]]] CompressionType: TypeAlias = Literal[ "zlib", "szip", "zstd", "bzip2", "blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd" ] @@ -318,7 +318,7 @@ class Dataset: self, varname: str, datatype: np.dtype[NumericVarT] | type[NumericVarT], - dimensions: DimensionsSpecifier = (), + dimensions: DimensionsType = (), compression: CompressionType | None = None, zlib: bool = False, complevel: CompressionLevel | None = 4, @@ -341,7 +341,7 @@ class Dataset: self, varname: str, datatype: np.dtype[np.str_] | type[str | np.str_], - dimensions: DimensionsSpecifier = (), + dimensions: DimensionsType = (), compression: CompressionType | None = None, zlib: bool = False, complevel: CompressionLevel | None = 4, @@ -364,7 +364,7 @@ class Dataset: self, varname: str, datatype: DatatypeSpecifier, - dimensions: DimensionsSpecifier = (), + dimensions: DimensionsType = (), compression: CompressionType | None = None, zlib: bool = False, complevel: CompressionLevel | None = 4, @@ -473,7 +473,7 @@ class Variable(Generic[VarT]): grp: Dataset, name: str, datatype: np.dtype[NumericVarT] | type[NumericVarT], - dimensions: DimensionsSpecifier = (), + dimensions: DimensionsType = (), compression: CompressionType | None = None, zlib: bool = False, complevel: CompressionLevel | None = 4, @@ -498,7 +498,7 @@ class Variable(Generic[VarT]): grp: Dataset, name: str, datatype: np.dtype[np.str_] | type[str | np.str_], - dimensions: DimensionsSpecifier = (), + dimensions: DimensionsType = (), compression: CompressionType | None = None, zlib: bool = False, complevel: CompressionLevel | None = 4, @@ -523,7 +523,7 @@ class Variable(Generic[VarT]): grp: Dataset, name: str, datatype: DatatypeSpecifier, - dimensions: DimensionsSpecifier = (), + dimensions: DimensionsType = (), compression: CompressionType | None = None, zlib: bool = False, complevel: CompressionLevel | None = 4, @@ -547,7 +547,7 @@ class Variable(Generic[VarT]): grp: Dataset, name: str, datatype: DatatypeSpecifier, - dimensions: DimensionsSpecifier = (), + dimensions: DimensionsType = (), compression: CompressionType | None = None, zlib: bool = False, complevel: CompressionLevel | None = 4, From 00d3c61d42ac9627ff0cff9af8373ad1e073686a Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Fri, 23 Aug 2024 11:34:49 -0700 Subject: [PATCH 74/81] Also rename DatatypeSpecifier >> DatatypeType --- src/netCDF4/__init__.pyi | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 0a1af0fa5..d808b2a6e 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -117,7 +117,7 @@ NetCDFUDTClass: TypeAlias = CompoundType | VLType | EnumType # Possible argument types for the datatype argument used in Variable creation. At this time, it is not possible to allow unknown # strings arguments in the datatype field but exclude and string literals that are not one of `TypeLiteral`, so really # `TypeLiteral` is made irrelevant, except for anyone who looks at this file. -DatatypeSpecifier: TypeAlias = ( +DatatypeType: TypeAlias = ( TypeLiteral | str | np.dtype[NumPyNumericType | np.str_] @@ -363,7 +363,7 @@ class Dataset: def createVariable( self, varname: str, - datatype: DatatypeSpecifier, + datatype: DatatypeType, dimensions: DimensionsType = (), compression: CompressionType | None = None, zlib: bool = False, @@ -522,7 +522,7 @@ class Variable(Generic[VarT]): cls, grp: Dataset, name: str, - datatype: DatatypeSpecifier, + datatype: DatatypeType, dimensions: DimensionsType = (), compression: CompressionType | None = None, zlib: bool = False, @@ -546,7 +546,7 @@ class Variable(Generic[VarT]): self, grp: Dataset, name: str, - datatype: DatatypeSpecifier, + datatype: DatatypeType, dimensions: DimensionsType = (), compression: CompressionType | None = None, zlib: bool = False, From 29fd359648cd1536fcfa1d3841a82daac44cf452 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Fri, 23 Aug 2024 14:13:35 -0700 Subject: [PATCH 75/81] Don't need the type_guards module --- examples/bench.py | 9 ++- examples/bench_compress.py | 11 +--- examples/bench_compress4.py | 9 ++- examples/bench_diskless.py | 13 ++-- examples/type_guards.py | 105 --------------------------------- src/netCDF4/__init__.pyi | 4 +- test/test_compression.py | 17 +++--- test/test_compression_blosc.py | 12 ++-- test/test_compression_bzip2.py | 9 ++- test/test_compression_quant.py | 10 +++- test/test_compression_zstd.py | 9 ++- test/test_endian.py | 14 ++--- test/test_stringarr.py | 4 +- test/test_types.py | 20 +++++-- test/type_guards.py | 105 --------------------------------- 15 files changed, 82 insertions(+), 269 deletions(-) delete mode 100644 examples/type_guards.py delete mode 100644 test/type_guards.py diff --git a/examples/bench.py b/examples/bench.py index 1d18ed8e1..08f95d48f 100644 --- a/examples/bench.py +++ b/examples/bench.py @@ -1,10 +1,14 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform import netCDF4 from timeit import Timer import os, sys -import type_guards +if TYPE_CHECKING: + from netCDF4 import Format as NCFormat +else: + NCFormat = Any # create an n1dim by n2dim by n3dim random array. n1dim = 30 @@ -15,8 +19,7 @@ sys.stdout.write('reading and writing a %s by %s by %s by %s random array ..\n'%(n1dim,n2dim,n3dim,n4dim)) array = uniform(size=(n1dim,n2dim,n3dim,n4dim)) -def write_netcdf(filename,zlib=False,least_significant_digit=None,format='NETCDF4'): - assert type_guards.valid_format(format) +def write_netcdf(filename,zlib=False,least_significant_digit=None,format: NCFormat='NETCDF4'): file = netCDF4.Dataset(filename,'w',format=format) file.createDimension('n1', n1dim) file.createDimension('n2', n2dim) diff --git a/examples/bench_compress.py b/examples/bench_compress.py index be8915c53..f094a6ffa 100644 --- a/examples/bench_compress.py +++ b/examples/bench_compress.py @@ -1,12 +1,11 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform import netCDF4 import netCDF4.utils from timeit import Timer import os, sys -from typing_extensions import TypeGuard if TYPE_CHECKING: from netCDF4 import CompressionLevel else: @@ -22,18 +21,12 @@ sys.stdout.write('(average of %s trials)\n' % ntrials) array = netCDF4.utils._quantize(uniform(size=(n1dim,n2dim,n3dim,n4dim)),4) -def valid_complevel(complevel) -> TypeGuard[CompressionLevel]: - """Check for a valid `complevel` argument for creating a Variable""" - return isinstance(complevel, int) and 0 <= complevel <= 9 - -def write_netcdf(filename,zlib=False,shuffle=False,complevel=6): +def write_netcdf(filename,zlib=False,shuffle=False,complevel: CompressionLevel = 6): file = netCDF4.Dataset(filename,'w',format='NETCDF4') file.createDimension('n1', n1dim) file.createDimension('n2', n2dim) file.createDimension('n3', n3dim) file.createDimension('n4', n4dim) - if not (valid_complevel(complevel) or complevel is None): - raise ValueError("Invalid compression level") foo = file.createVariable('data',\ 'f8',('n1','n2','n3','n4'),zlib=zlib,shuffle=shuffle,complevel=complevel) foo[:] = array diff --git a/examples/bench_compress4.py b/examples/bench_compress4.py index 05b9ade80..d8f643935 100644 --- a/examples/bench_compress4.py +++ b/examples/bench_compress4.py @@ -1,10 +1,10 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. +from typing import Literal from numpy.random.mtrand import uniform import netCDF4 from timeit import Timer import os, sys -import type_guards # use real data. URL="http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep.reanalysis/pressure/hgt.1990.nc" @@ -20,12 +20,15 @@ array = nc.variables['hgt'][0:n1dim,5,:,:] -def write_netcdf(filename,nsd,quantize_mode='BitGroom'): +def write_netcdf( + filename, + nsd, + quantize_mode: Literal["BitGroom", "BitRound", "GranularBitRound"] = "BitGroom" + ): file = netCDF4.Dataset(filename,'w',format='NETCDF4') file.createDimension('n1', None) file.createDimension('n3', n3dim) file.createDimension('n4', n4dim) - assert type_guards.valid_quantize_mode(quantize_mode) foo = file.createVariable('data',\ 'f4',('n1','n3','n4'),\ zlib=True,shuffle=True,\ diff --git a/examples/bench_diskless.py b/examples/bench_diskless.py index ae92e037b..076f446b4 100644 --- a/examples/bench_diskless.py +++ b/examples/bench_diskless.py @@ -1,10 +1,14 @@ # benchmark reads and writes, with and without compression. # tests all four supported file formats. +from typing import TYPE_CHECKING, Any, Literal from numpy.random.mtrand import uniform import netCDF4 from timeit import Timer import os, sys -import type_guards +if TYPE_CHECKING: + from netCDF4 import Format as NCFormat +else: + NCFormat = Any # create an n1dim by n2dim by n3dim random array. n1dim = 30 @@ -15,8 +19,7 @@ sys.stdout.write('reading and writing a %s by %s by %s by %s random array ..\n'%(n1dim,n2dim,n3dim,n4dim)) array = uniform(size=(n1dim,n2dim,n3dim,n4dim)) -def write_netcdf(filename,zlib=False,least_significant_digit=None,format='NETCDF4',closeit=False): - assert type_guards.valid_format(format) +def write_netcdf(filename, zlib=False, least_significant_digit=None, format: NCFormat='NETCDF4',closeit=False): file = netCDF4.Dataset(filename,'w',format=format,diskless=True,persist=True) file.createDimension('n1', n1dim) file.createDimension('n2', n2dim) @@ -44,13 +47,13 @@ def read_netcdf(ncfile): sys.stdout.write('writing took %s seconds\n' %\ repr(sum(t.repeat(ntrials,1))/ntrials)) # test reading. - ncfile = write_netcdf('test1.nc',format=format) + ncfile = write_netcdf('test1.nc',format=format) # type: ignore t = Timer("read_netcdf(ncfile)","from __main__ import read_netcdf,ncfile") sys.stdout.write('reading took %s seconds\n' % repr(sum(t.repeat(ntrials,1))/ntrials)) # test diskless=True in nc_open -format='NETCDF3_CLASSIC' +format: Literal["NETCDF3_CLASSIC"] = 'NETCDF3_CLASSIC' # mypy should know this but it needs help... trials=50 sys.stdout.write('test caching of file in memory on open for %s\n' % format) sys.stdout.write('testing file format %s ...\n' % format) diff --git a/examples/type_guards.py b/examples/type_guards.py deleted file mode 100644 index 2baabda68..000000000 --- a/examples/type_guards.py +++ /dev/null @@ -1,105 +0,0 @@ -"""type_guards.py - Helpers for static and runtime type-checking of initialization arguments -for Dataset and Variable""" - -from typing import TYPE_CHECKING, Any, Literal - -from typing_extensions import TypeGuard - -if TYPE_CHECKING: - # in stubs only - from netCDF4 import ( - AccessMode, - CalendarType, - CompressionLevel, - CompressionType, - EndianType, - QuantizeMode, - ) - from netCDF4 import Format as NCFormat -else: - AccessMode = Any - CalendarType = Any - CompressionLevel = Any - DiskFormat = Any - EndianType = Any - CompressionType = Any - NCFormat = Any - QuantizeMode = Any - - -def valid_access_mode(mode) -> TypeGuard[AccessMode]: - """Check for a valid `mode` argument for opening a Dataset""" - return mode in {"r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"} - - -def valid_calendar(calendar) -> TypeGuard[CalendarType]: - """Check for a valid `calendar` argument for cftime functions""" - return calendar in { - "standard", - "gregorian", - "proleptic_gregorian", - "noleap", - "365_day", - "360_day", - "julian", - "all_leap", - "366_day", - } - - -def valid_complevel(complevel) -> TypeGuard[CompressionLevel]: - """Check for a valid `complevel` argument for creating a Variable""" - return isinstance(complevel, int) and 0 <= complevel <= 9 - - -def valid_compression(compression) -> TypeGuard[CompressionType]: - """Check for a valid `compression` argument for creating a Variable""" - return compression in { - "zlib", - "szip", - "zstd", - "bzip2", - "blosc_lz", - "blosc_lz4", - "blosc_lz4hc", - "blosc_zlib", - "blosc_zstd", - } - - -def valid_format(format) -> TypeGuard[NCFormat]: - """Check for a valid `format` argument for opening a Dataset""" - return format in { - "NETCDF4", - "NETCDF4_CLASSIC", - "NETCDF3_CLASSIC", - "NETCDF3_64BIT_OFFSET", - "NETCDF3_64BIT_DATA", - } - - -def valid_endian(endian) -> TypeGuard[EndianType]: - """Check for a valid `endian` argument for creating a Variable""" - return endian in {"native", "big", "little"} - - -def valid_bloscshuffle(blosc_shuffle) -> TypeGuard[Literal[0, 1, 2]]: - """Check for a valid `blosc_shuffle` argument for creating a Variable""" - return blosc_shuffle in {0, 1, 2} - - -def valid_quantize_mode(quantize_mode) -> TypeGuard[QuantizeMode]: - """Check for a valid `quantize_mode` argument for creating a Variable""" - return quantize_mode in {"BitGroom", "BitRound", "GranularBitRound"} - - -def valid_szip_coding(szip_coding) -> TypeGuard[Literal["nn", "ec"]]: - """Check for a valid `szip_coding` argument for creating a Variable""" - return szip_coding in {"nn", "ec"} - - -def valid_szip_pixels_per_block( - szip_pixels_per_block, -) -> TypeGuard[Literal[4, 8, 16, 32]]: - """Check for a valid `szip_pixels_per_block` argument for creating a Variable""" - return szip_pixels_per_block in {4, 8, 16, 32} diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index d808b2a6e..a7db85fc4 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -453,7 +453,7 @@ class _VarDatatypeProperty: def __get__(self, instance: Variable[str], owner: Any) -> VLType: ... @overload def __get__( - self, instance: Variable[Any], owner: Any + self, instance: Variable, owner: Any ) -> Any: ... # actual return type np.dtype | CompoundType | VLType | EnumType class _VarDtypeProperty: @@ -463,7 +463,7 @@ class _VarDtypeProperty: @overload def __get__(self, instance: Variable[str], owner: Any) -> type[str]: ... @overload - def __get__(self, instance: Variable[Any], owner: Any) -> Any: ... # actual return type np.dtype | Type[str] + def __get__(self, instance: Variable, owner: Any) -> Any: ... # actual return type np.dtype | Type[str] class Variable(Generic[VarT]): # Overloads of __new__ are provided for some cases where the Variable's type may be statically inferred from the datatype arg diff --git a/test/test_compression.py b/test/test_compression.py index 1d4b59076..6c9351af6 100644 --- a/test/test_compression.py +++ b/test/test_compression.py @@ -1,9 +1,13 @@ +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform from netCDF4 import Dataset from netCDF4.utils import _quantize from numpy.testing import assert_almost_equal import os, tempfile, unittest -import type_guards +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any ndim = 100000 ndim2 = 100 @@ -15,8 +19,7 @@ lsd = 3 def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,contiguous=False,\ - chunksizes=None,complevel=6,fletcher32=False): - assert type_guards.valid_complevel(complevel) or complevel is None + chunksizes=None, complevel: CompressionLevel = 6, fletcher32=False): file = Dataset(filename,'w') file.createDimension('n', ndim) foo = file.createVariable('data',\ @@ -32,9 +35,8 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=F #compression='' #compression=0 #compression='gzip' # should fail - assert type_guards.valid_compression(compression) or compression is None - foo2 = file.createVariable('data2',\ - dtype,('n'),compression=compression,least_significant_digit=least_significant_digit,\ + foo2 = file.createVariable('data2', + dtype,('n'),compression=compression,least_significant_digit=least_significant_digit, # type: ignore # mypy doesn't like compression shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) foo[:] = data foo2[:] = data @@ -45,11 +47,10 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=F file.close() def write_netcdf2(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,contiguous=False,\ - chunksizes=None,complevel=6,fletcher32=False): + chunksizes=None, complevel: CompressionLevel = 6, fletcher32=False): file = Dataset(filename,'w') file.createDimension('n', ndim) file.createDimension('n2', ndim2) - assert type_guards.valid_complevel(complevel) or complevel is None foo = file.createVariable('data2',\ dtype,('n','n2'),zlib=zlib,least_significant_digit=least_significant_digit,\ shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes) diff --git a/test/test_compression_blosc.py b/test/test_compression_blosc.py index c532fef32..fda01e216 100644 --- a/test/test_compression_blosc.py +++ b/test/test_compression_blosc.py @@ -1,10 +1,13 @@ +from typing import TYPE_CHECKING, Any, Literal from numpy.random.mtrand import uniform from netCDF4 import Dataset -import type_guards from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_blosc_filter -import type_guards +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any ndim = 100000 @@ -13,8 +16,7 @@ filename = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name datarr = uniform(size=(ndim,)) -def write_netcdf(filename,dtype='f8',blosc_shuffle=1,complevel=6): - assert (type_guards.valid_complevel(complevel) or complevel is None) and type_guards.valid_bloscshuffle(blosc_shuffle) +def write_netcdf(filename, dtype='f8', blosc_shuffle: Literal[0, 1, 2] = 1, complevel: CompressionLevel = 6): nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ @@ -41,7 +43,7 @@ def write_netcdf(filename,dtype='f8',blosc_shuffle=1,complevel=6): class CompressionTestCase(unittest.TestCase): def setUp(self): self.filename = filename - write_netcdf(self.filename,complevel=iblosc_complevel,blosc_shuffle=iblosc_shuffle) + write_netcdf(self.filename,complevel=iblosc_complevel,blosc_shuffle=iblosc_shuffle) # type: ignore def tearDown(self): # Remove the temporary files diff --git a/test/test_compression_bzip2.py b/test/test_compression_bzip2.py index ec32ba248..15ef8eaec 100644 --- a/test/test_compression_bzip2.py +++ b/test/test_compression_bzip2.py @@ -1,17 +1,20 @@ +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform from netCDF4 import Dataset from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_bzip2_filter -import type_guards +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name filename2 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name array = uniform(size=(ndim,)) -def write_netcdf(filename,dtype='f8',complevel=6): - assert type_guards.valid_complevel(complevel) or complevel is None +def write_netcdf(filename,dtype='f8',complevel: CompressionLevel = 6): nc = Dataset(filename,'w') nc.createDimension('n', ndim) foo = nc.createVariable('data',\ diff --git a/test/test_compression_quant.py b/test/test_compression_quant.py index 24267615a..b56739624 100644 --- a/test/test_compression_quant.py +++ b/test/test_compression_quant.py @@ -1,9 +1,14 @@ +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform from netCDF4 import Dataset, __has_quantization_support__ from numpy.testing import assert_almost_equal import numpy as np import os, tempfile, unittest -import type_guards +if TYPE_CHECKING: + from netCDF4 import CompressionLevel, QuantizeMode +else: + CompressionLevel = Any + QuantizeMode = Any ndim = 100000 nfiles = 7 @@ -14,10 +19,9 @@ complevel = 6 def write_netcdf(filename,zlib,significant_digits,data,dtype='f8',shuffle=False,\ - complevel=6,quantize_mode="BitGroom"): + complevel: CompressionLevel = 6, quantize_mode: QuantizeMode = "BitGroom"): file = Dataset(filename,'w') file.createDimension('n', ndim) - assert (type_guards.valid_complevel(complevel) or complevel is None) and type_guards.valid_quantize_mode(quantize_mode) foo = file.createVariable('data',\ dtype,('n'),zlib=zlib,significant_digits=significant_digits,\ shuffle=shuffle,complevel=complevel,quantize_mode=quantize_mode) diff --git a/test/test_compression_zstd.py b/test/test_compression_zstd.py index 123397f12..3557d5d0c 100644 --- a/test/test_compression_zstd.py +++ b/test/test_compression_zstd.py @@ -1,19 +1,22 @@ +from typing import TYPE_CHECKING, Any from numpy.random.mtrand import uniform from netCDF4 import Dataset from numpy.testing import assert_almost_equal import os, tempfile, unittest, sys from filter_availability import no_plugins, has_zstd_filter -import type_guards +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any ndim = 100000 filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name filename2 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name array = uniform(size=(ndim,)) -def write_netcdf(filename,dtype='f8',complevel=6): +def write_netcdf(filename,dtype='f8',complevel: CompressionLevel = 6): nc = Dataset(filename,'w') nc.createDimension('n', ndim) - assert type_guards.valid_complevel(complevel) or complevel is None foo = nc.createVariable('data',\ dtype,('n'),compression='zstd',complevel=complevel) foo[:] = array diff --git a/test/test_endian.py b/test/test_endian.py index 1a8f58681..716d1a958 100644 --- a/test/test_endian.py +++ b/test/test_endian.py @@ -2,7 +2,6 @@ import numpy as np import unittest, os, tempfile from numpy.testing import assert_array_equal, assert_array_almost_equal -import type_guards data = np.arange(12,dtype='f4').reshape(3,4) FILE_NAME = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name @@ -74,14 +73,13 @@ def issue310(file): endian='little' else: raise ValueError('cannot determine native endianness') - assert type_guards.valid_endian(endian) # mypy fails to narrow endian on its own - var_big_endian = nc.createVariable(\ - 'obs_big_endian', '>f8', ('obs', ),\ - endian=endian,fill_value=fval) + var_big_endian = nc.createVariable( + 'obs_big_endian', '>f8', ('obs', ), endian=endian, fill_value=fval, # type: ignore # mypy is bad at narrowing endian + ) # use default _FillValue - var_big_endian2 = nc.createVariable(\ - 'obs_big_endian2', '>f8', ('obs', ),\ - endian=endian) + var_big_endian2 = nc.createVariable( + 'obs_big_endian2', '>f8', ('obs', ), endian=endian, # type: ignore # mypy is bad at narrowing endian + ) # NOTE: missing_value be written in same byte order # as variable, or masked array won't be masked correctly # when data is read in. diff --git a/test/test_stringarr.py b/test/test_stringarr.py index f3e140c0a..9d4fcd909 100644 --- a/test/test_stringarr.py +++ b/test/test_stringarr.py @@ -3,7 +3,6 @@ import unittest import os from numpy.testing import assert_array_equal, assert_array_almost_equal -import type_guards def generateString(length, alphabet=string.ascii_letters + string.digits + string.punctuation): return(''.join([random.choice(alphabet) for i in range(length)])) @@ -25,8 +24,7 @@ class StringArrayTestCase(unittest.TestCase): def setUp(self): self.file = FILE_NAME - assert type_guards.valid_format(FILE_FORMAT) - nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT) + nc = Dataset(FILE_NAME,'w',format=FILE_FORMAT) # type: ignore # FILE_FORMAT nc.createDimension('n1',None) nc.createDimension('n2',n2) nc.createDimension('nchar',nchar) diff --git a/test/test_types.py b/test/test_types.py index 3cc7bbc3b..3c5054bbd 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -1,4 +1,5 @@ import sys +from typing import TYPE_CHECKING, Any import unittest import os import tempfile @@ -6,7 +7,10 @@ from numpy.testing import assert_array_equal, assert_array_almost_equal from numpy.random.mtrand import uniform import netCDF4 -import type_guards +if TYPE_CHECKING: + from netCDF4 import CompressionLevel +else: + CompressionLevel = Any # test primitive data types. @@ -15,7 +19,7 @@ n1dim = 5 n2dim = 10 ranarr = 100.*uniform(size=(n1dim,n2dim)) -zlib=False;complevel=0;shuffle=False;least_significant_digit=None +zlib=False; complevel=0; shuffle=False; least_significant_digit=None datatypes = ['f8','f4','i1','i2','i4','i8','u1','u2','u4','u8','S1'] FillValue = 1.0 issue273_data = np.ma.array(['z']*10,dtype='S1',\ @@ -29,8 +33,16 @@ def setUp(self): f.createDimension('n1', None) f.createDimension('n2', n2dim) for typ in datatypes: - assert type_guards.valid_complevel(complevel) or complevel is None - foo = f.createVariable('data_'+typ, typ, ('n1','n2',),zlib=zlib,complevel=complevel,shuffle=shuffle,least_significant_digit=least_significant_digit,fill_value=FillValue) + foo = f.createVariable( + f"data_{typ}", + typ, + ('n1','n2',), + zlib=zlib, + complevel=complevel, # type: ignore # type checkers bad at narrowing + shuffle=shuffle, + least_significant_digit=least_significant_digit, + fill_value=FillValue, + ) #foo._FillValue = FillValue # test writing of _FillValue attribute for diff types # (should be cast to type of variable silently) diff --git a/test/type_guards.py b/test/type_guards.py deleted file mode 100644 index 2baabda68..000000000 --- a/test/type_guards.py +++ /dev/null @@ -1,105 +0,0 @@ -"""type_guards.py - Helpers for static and runtime type-checking of initialization arguments -for Dataset and Variable""" - -from typing import TYPE_CHECKING, Any, Literal - -from typing_extensions import TypeGuard - -if TYPE_CHECKING: - # in stubs only - from netCDF4 import ( - AccessMode, - CalendarType, - CompressionLevel, - CompressionType, - EndianType, - QuantizeMode, - ) - from netCDF4 import Format as NCFormat -else: - AccessMode = Any - CalendarType = Any - CompressionLevel = Any - DiskFormat = Any - EndianType = Any - CompressionType = Any - NCFormat = Any - QuantizeMode = Any - - -def valid_access_mode(mode) -> TypeGuard[AccessMode]: - """Check for a valid `mode` argument for opening a Dataset""" - return mode in {"r", "w", "r+", "a", "x", "rs", "ws", "r+s", "as"} - - -def valid_calendar(calendar) -> TypeGuard[CalendarType]: - """Check for a valid `calendar` argument for cftime functions""" - return calendar in { - "standard", - "gregorian", - "proleptic_gregorian", - "noleap", - "365_day", - "360_day", - "julian", - "all_leap", - "366_day", - } - - -def valid_complevel(complevel) -> TypeGuard[CompressionLevel]: - """Check for a valid `complevel` argument for creating a Variable""" - return isinstance(complevel, int) and 0 <= complevel <= 9 - - -def valid_compression(compression) -> TypeGuard[CompressionType]: - """Check for a valid `compression` argument for creating a Variable""" - return compression in { - "zlib", - "szip", - "zstd", - "bzip2", - "blosc_lz", - "blosc_lz4", - "blosc_lz4hc", - "blosc_zlib", - "blosc_zstd", - } - - -def valid_format(format) -> TypeGuard[NCFormat]: - """Check for a valid `format` argument for opening a Dataset""" - return format in { - "NETCDF4", - "NETCDF4_CLASSIC", - "NETCDF3_CLASSIC", - "NETCDF3_64BIT_OFFSET", - "NETCDF3_64BIT_DATA", - } - - -def valid_endian(endian) -> TypeGuard[EndianType]: - """Check for a valid `endian` argument for creating a Variable""" - return endian in {"native", "big", "little"} - - -def valid_bloscshuffle(blosc_shuffle) -> TypeGuard[Literal[0, 1, 2]]: - """Check for a valid `blosc_shuffle` argument for creating a Variable""" - return blosc_shuffle in {0, 1, 2} - - -def valid_quantize_mode(quantize_mode) -> TypeGuard[QuantizeMode]: - """Check for a valid `quantize_mode` argument for creating a Variable""" - return quantize_mode in {"BitGroom", "BitRound", "GranularBitRound"} - - -def valid_szip_coding(szip_coding) -> TypeGuard[Literal["nn", "ec"]]: - """Check for a valid `szip_coding` argument for creating a Variable""" - return szip_coding in {"nn", "ec"} - - -def valid_szip_pixels_per_block( - szip_pixels_per_block, -) -> TypeGuard[Literal[4, 8, 16, 32]]: - """Check for a valid `szip_pixels_per_block` argument for creating a Variable""" - return szip_pixels_per_block in {4, 8, 16, 32} From ac8c6fa81048e309e690e0d11c119ac296b479d2 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Fri, 23 Aug 2024 14:25:28 -0700 Subject: [PATCH 76/81] Don't bother with TypeGuard --- examples/mpi_example.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/examples/mpi_example.py b/examples/mpi_example.py index 3f8cb0f87..93ca57bc7 100644 --- a/examples/mpi_example.py +++ b/examples/mpi_example.py @@ -1,34 +1,23 @@ # to run: mpirun -np 4 python mpi_example.py import sys -from typing import TYPE_CHECKING from mpi4py import MPI import numpy as np from netCDF4 import Dataset -from typing_extensions import TypeGuard -if TYPE_CHECKING: - from netCDF4 import Format as NCFormat -else: - NCFormat = object - -def is_nc_format(fmt: str) -> TypeGuard[NCFormat]: - return fmt in { - 'NETCDF4', - 'NETCDF4_CLASSIC', - 'NETCDF3_CLASSIC', - 'NETCDF3_64BIT_OFFSET', - 'NETCDF3_64BIT_DATA' - } nc_format = 'NETCDF4_CLASSIC' if len(sys.argv) < 2 else sys.argv[1] -if not is_nc_format(nc_format): - raise ValueError("Invalid file format") rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run) if rank == 0: print('Creating file with format {}'.format(nc_format)) -nc = Dataset('parallel_test.nc', 'w', parallel=True, comm=MPI.COMM_WORLD, - info=MPI.Info(), format=nc_format) +nc = Dataset( + "parallel_test.nc", + "w", + parallel=True, + comm=MPI.COMM_WORLD, + info=MPI.Info(), + format=nc_format, # type: ignore # we'll assume it's OK +) # below should work also - MPI_COMM_WORLD and MPI_INFO_NULL will be used. #nc = Dataset('parallel_test.nc', 'w', parallel=True) d = nc.createDimension('dim',4) From ba2e99a99e6499b65ec82eebc3c367fbc0bc4eda Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Thu, 5 Sep 2024 09:27:35 -0700 Subject: [PATCH 77/81] Removed GetSetItemKey and replaced with typing.Any --- src/netCDF4/__init__.pyi | 46 ++++------------------------------------ 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index a7db85fc4..443762f25 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -79,13 +79,6 @@ __all__ = [ ] __pdoc__ = {"utils": False} -if sys.version_info >= (3, 10): - from types import EllipsisType - - ellipsis = EllipsisType -elif not TYPE_CHECKING: - ellipsis = type(Ellipsis) # keeps ruff happy until ruff uses typeshed - # string type specifiers # fmt: off RealTypeLiteral: TypeAlias = Literal[ @@ -148,37 +141,6 @@ BoolInt: TypeAlias = Literal[0, 1] DateTimeArray: TypeAlias = npt.NDArray[np.object_] """numpy array of datetime.datetime or cftime.datetime""" -# - Allow singular float or str keys, but not lists (or sequences) of them -# - `list` and `ndarray` are specifically listed for the 1-D case as Sequence is too general -- it includes -# tuples which are rather for multi-dimensional indexing. -GetSetItemKey: TypeAlias = ( - int - | float - | np.number - | str - | slice - | ellipsis - | list[int] - | list[bool] - | npt.NDArray[np.integer] - | npt.NDArray[np.bool_] - | tuple[ - int - | float - | np.number - | str - | slice - | ellipsis - | Sequence[int] - | Sequence[np.integer] - | Sequence[bool] - | Sequence[np.bool_] - | npt.NDArray[np.integer] - | npt.NDArray[np.bool_], - ..., - ] -) - class BloscInfo(TypedDict): compressor: Literal["blosc_lz", "blosc_lz4", "blosc_lz4hc", "blosc_zlib", "blosc_zstd"] shuffle: Literal[0, 1, 2] @@ -618,8 +580,8 @@ class Variable(Generic[VarT]): def __delattr__(self, name: str) -> None: ... def __setattr__(self, name: str, value: Any) -> None: ... def __getattr__(self, name: str) -> Any: ... - def __getitem__(self, elem: GetSetItemKey) -> Any: ... - def __setitem__(self, elem: GetSetItemKey, data: npt.ArrayLike) -> None: ... + def __getitem__(self, elem: Any) -> Any: ... + def __setitem__(self, elem: Any, data: npt.ArrayLike) -> None: ... def __array__(self) -> np.ndarray: ... def __len__(self) -> int: ... def __iter__(self) -> Iterator[Any]: ... # faux method so mypy believes Variable is iterable @@ -700,7 +662,7 @@ class _Variable: def set_auto_scale(self, val: bool) -> None: ... def set_always_mask(self, val: bool) -> None: ... def __getattr__(self, name: str) -> Any: ... - def __getitem__(self, elem: GetSetItemKey) -> Any: ... + def __getitem__(self, elem: Any) -> Any: ... def __len__(self) -> int: ... class MFTime(_Variable): @@ -708,7 +670,7 @@ class MFTime(_Variable): units: str | None def __init__(self, time: Variable, units: str | None = None, calendar: CalendarType | str | None = None): ... - def __getitem__(self, elem: GetSetItemKey) -> np.ndarray: ... + def __getitem__(self, elem: Any) -> np.ndarray: ... @overload def stringtoarr( From b26da50a660505688ae2f702f6bfb2df11aa6801 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Mon, 21 Oct 2024 12:02:40 -0700 Subject: [PATCH 78/81] Remove accidentally-commited nc files. --- examples/parallel_test.nc | Bin 6176 -> 0 bytes examples/parallel_test_compressed.nc | Bin 10288 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 examples/parallel_test.nc delete mode 100644 examples/parallel_test_compressed.nc diff --git a/examples/parallel_test.nc b/examples/parallel_test.nc deleted file mode 100644 index ec95803e081f032b7314eaf74434c9916246e2bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6176 zcmeH{KX21O6u_VDCNjZo0t5mGf$0JjAxhhTTA@gd9EYT&35q+hL~iV=My-Pq(-j7U z#7KnNiEqHj%11y#o%jk2z{U#io$r#WKw)4(x|i5@zI*rn{NA%))T_1W)Wwvl>mcMi z7xR3)Qsf(Z5B+?*UhSyHTXmMLX)50mM~;pu6%ec{;@OxZJq--}p=vkm zEp~^|Am~Mq!O-Y5fC=P~EM_wFJksb6u)^+mW6t>IA`i~to6B#m@3ZiymlP#Si>jh1 zv0>o=Bgm4GQ&+EyUFLMiJwfdR%SAg8yUz22BW_(}IQ9DsZ;+}6Nzlo*)vP!UoWUhZ zg<`EX*6fzkXt%nKRcYE_JfMU$NW=$Pl{K5`uWka~UW7{v7LP}s@Kq}gw;mBKl)bln=)nF&^eu9oZQC5pDzqGCTJ;RPe zn~6i~wygVu{v&@71^$r5IE%`Z)2=_DfXWMbj1jGuONHgaLVnBJyj7kr%ohtwAKo8) zi^~GXmBpa49PWffidM8assm1*>ZjSWLOLX@L2wj>Ii~e0q_061IbdIOIR~rXA9z9N zMwrrJ$J90Gkck-SA?}L<-jZ0mSgwi5UyQ3a^*e+&d5)CFy$Kpt_A5Ux_JnHn0~j^LLkDpCrqIGCPRtg0itq z1T{!Qkvw+nn%%Kmmfd9@Z)1F<>A8_R^rMOEamRxGC&VI0G{fZC$bBW|a+CxlfqzAS z-go+-B=NdywVUiq!(VhZ$94W`0P$7AB_Ch3k@wGBuq~HL{VB=R6vm$ISwhL4vKr?KyYGLwP+KRmgQz$5g=V{icpap!=z^;W^m+S`OG>@O=h;~@bEwfBz>OP#Y1tXJ2Y0`zBK*v%qGV}YRg|NUWN2EJB0hWd zif{~I!h#M9uTeWCa?#Gj9-k#1-zN@aq!TAbYTP7M4JJV++g82ox-f@BlmX>iS>Lc7 zcfH}XT&rBS!FY_wa6u|w;FLFPp}&^sZ*GSJ6MvrB4I-;jH4$pi9fZBE*?tk3-Ci`D zLZ+eD-U$k(lx3iRRJ1=01_jh|f~ezfEmy+b(1%&9hnR^efsK?-EFak6)+?V>gy9pOjgCbkNi(Z;>R>)fM0^U%l|M}q4 zO2eu>)X&i)mUHs9B^iRTLzSVy3>3*@)2`Z0+p+AH@OT@mgEW0F@&-XPbbYkpppViX z7m<9UC?b-3KfljWM21aIR0N_bTaNXp-+LbPqYz=yPUP^S>f8(ZR9dB_JXRalNol!o zr?8aY_P16_i-pBvVfDt>+&ffcVFFCxloELPr1P}xb=L5W--pZtHFZk!;INqh6JP>N zfC(^xlL>4#ZJQ2W@;EKqG3RGtgpL`b5+4!eQE_yJ`%HidFaajO1en18Lg2pDsEbdf w$(Kvf$K>SS0{Ky2=KEzv_PAvNOn?b60Vco%m;e)C0!)AjFaajO1WpTqKPK($K>z>% From d81d4fad2fc33f927ee6616ba3dfc2e52a822ad4 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Mon, 21 Oct 2024 15:32:28 -0700 Subject: [PATCH 79/81] Update stubtest-allowlist for changes in recent commits --- .github/stubtest-allowlist | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/stubtest-allowlist b/.github/stubtest-allowlist index 4f0e7a94d..0352fdcf7 100644 --- a/.github/stubtest-allowlist +++ b/.github/stubtest-allowlist @@ -10,20 +10,18 @@ netCDF4.NetCDFUDTClass netCDF4.AccessMode netCDF4.CompressionLevel netCDF4.CompressionType -netCDF4.DatatypeSpecifier -netCDF4.DimensionsSpecifier +netCDF4.DatatypeType +netCDF4.DimensionsType netCDF4.DiskFormat netCDF4.EndianType netCDF4.Format netCDF4.QuantizeMode netCDF4.CalendarType -netCDF4.ellipsis netCDF4.DateTimeArray netCDF4.FiltersDict netCDF4.SzipInfo netCDF4.BloscInfo netCDF4.BoolInt -netCDF4.GetSetItemKey netCDF4.VarT netCDF4.RealVarT netCDF4.ComplexVarT From 6a31be49b850baf6c00b0c242072399c2e8cced0 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 22 Oct 2024 09:12:03 -0700 Subject: [PATCH 80/81] post-merge mypy/stubtest fixes. Allow any numpy generic or array in fillvalue. --- .github/stubtest-allowlist | 2 ++ src/netCDF4/__init__.pyi | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/stubtest-allowlist b/.github/stubtest-allowlist index 0352fdcf7..aa1b1d69b 100644 --- a/.github/stubtest-allowlist +++ b/.github/stubtest-allowlist @@ -29,12 +29,14 @@ netCDF4.NumericVarT netCDF4.Dimension.__reduce_cython__ netCDF4.Dimension.__setstate_cython__ netCDF4.Variable.auto_complex +netCDF4.Variable.get_fill_value netCDF4.Variable.__iter__ netCDF4._netCDF4.Dimension.__reduce_cython__ netCDF4._netCDF4.Dimension.__setstate_cython__ netCDF4._netCDF4.NC_DISKLESS netCDF4._netCDF4.NC_PERSIST netCDF4._netCDF4.Variable.auto_complex +netCDF4._netCDF4.Variable.get_fill_value netCDF4._netCDF4.Variable.__iter__ netCDF4._netCDF4.__reduce_cython__ netCDF4._netCDF4.__setstate_cython__ diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index 5bdc58d00..6ed415f7a 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -295,7 +295,7 @@ class Dataset: least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, chunk_cache: int | None = None, ) -> Variable[NumericVarT]: ... @overload @@ -318,7 +318,7 @@ class Dataset: least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, chunk_cache: int | None = None, ) -> Variable[str]: ... @overload @@ -341,7 +341,7 @@ class Dataset: least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, chunk_cache: int | None = None, ) -> Variable: ... def renameVariable(self, oldname: str, newname: str) -> None: ... @@ -450,7 +450,7 @@ class Variable(Generic[VarT]): least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, chunk_cache: int | None = None, **kwargs: Any, ) -> Variable[NumericVarT]: ... @@ -475,7 +475,7 @@ class Variable(Generic[VarT]): least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, chunk_cache: int | None = None, **kwargs: Any, ) -> Variable[str]: ... @@ -500,7 +500,7 @@ class Variable(Generic[VarT]): least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, chunk_cache: int | None = None, **kwargs: Any, ) -> Variable: ... @@ -524,7 +524,7 @@ class Variable(Generic[VarT]): least_significant_digit: int | None = None, significant_digits: int | None = None, quantize_mode: QuantizeMode = "BitGroom", - fill_value: int | float | np.number | str | bytes | Literal[False] | None = None, + fill_value: int | float | np.generic | str | bytes | Literal[False] | np.ndarray | None = None, chunk_cache: int | None = None, **kwargs: Any, ) -> None: ... From 1004467517b44e1fd5832f2ea904653a67021a09 Mon Sep 17 00:00:00 2001 From: Randall Pittman Date: Tue, 22 Oct 2024 09:36:13 -0700 Subject: [PATCH 81/81] Revert stubtest change -- mypy v1.12 differs from 1.11 --- .github/stubtest-allowlist | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/stubtest-allowlist b/.github/stubtest-allowlist index aa1b1d69b..0352fdcf7 100644 --- a/.github/stubtest-allowlist +++ b/.github/stubtest-allowlist @@ -29,14 +29,12 @@ netCDF4.NumericVarT netCDF4.Dimension.__reduce_cython__ netCDF4.Dimension.__setstate_cython__ netCDF4.Variable.auto_complex -netCDF4.Variable.get_fill_value netCDF4.Variable.__iter__ netCDF4._netCDF4.Dimension.__reduce_cython__ netCDF4._netCDF4.Dimension.__setstate_cython__ netCDF4._netCDF4.NC_DISKLESS netCDF4._netCDF4.NC_PERSIST netCDF4._netCDF4.Variable.auto_complex -netCDF4._netCDF4.Variable.get_fill_value netCDF4._netCDF4.Variable.__iter__ netCDF4._netCDF4.__reduce_cython__ netCDF4._netCDF4.__setstate_cython__