ENH: add NA scalar for missing value indicator, use in StringArray.

pandas/__init__.py

@@ -68,6 +68,7 @@
     DatetimeTZDtype,
     StringDtype,
     # missing
+    NA,
     isna,
     isnull,
     notna,

pandas/_libs/lib.pyx

@@ -58,10 +58,9 @@ from pandas._libs.tslibs.timedeltas cimport convert_to_timedelta64
 from pandas._libs.tslibs.timezones cimport get_timezone, tz_compare
 
 from pandas._libs.missing cimport (
-    checknull, isnaobj, is_null_datetime64, is_null_timedelta64, is_null_period
+    checknull, isnaobj, is_null_datetime64, is_null_timedelta64, is_null_period, C_NA
 )
 
-
 # constants that will be compared to potentially arbitrarily large
 # python int
 cdef:
@@ -161,6 +160,7 @@ def is_scalar(val: object) -> bool:
             or PyTime_Check(val)
             # We differ from numpy, which claims that None is not scalar;
             # see np.isscalar
+            or val is C_NA
             or val is None
             or isinstance(val, (Fraction, Number))
             or util.is_period_object(val)

pandas/_libs/missing.pxd

@@ -9,3 +9,8 @@ cpdef ndarray[uint8_t] isnaobj(ndarray arr)
 cdef bint is_null_datetime64(v)
 cdef bint is_null_timedelta64(v)
 cdef bint is_null_period(v)
+
+cdef class C_NAType:
+    pass
+
+cdef C_NAType C_NA

pandas/_libs/missing.pyx

@@ -44,7 +44,7 @@ cpdef bint checknull(object val):
     The difference between `checknull` and `checknull_old` is that `checknull`
     does *not* consider INF or NEGINF to be NA.
     """
-    return is_null_datetimelike(val, inat_is_null=False)
+    return val is C_NA or is_null_datetimelike(val, inat_is_null=False)
 
 
 cpdef bint checknull_old(object val):
@@ -278,3 +278,126 @@ cdef inline bint is_null_period(v):
     # determine if we have a null for a Period (or integer versions),
     # excluding np.datetime64('nat') and np.timedelta64('nat')
     return checknull_with_nat(v)
+
+
+# -----------------------------------------------------------------------------
+# Implementation of NA singleton
+
+
+def _create_binary_propagating_op(name):
+    import numbers
+
+    def method(self, other):
+        if isinstance(other, numbers.Number) or other is NA or isinstance(other, str):
+            return NA
+
+        return NotImplemented
+
+    method.__name__ = name
+    return method
+
+
+def _create_unary_propagating_op(name):
+    def method(self):
+        return NA
+
+    method.__name__ = name
+    return method
+
+
+cdef class C_NAType:
+    pass
+
+
+class NAType(C_NAType):
+    """
+    NA ("not available") missing value indicator.
+    """
+
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if NAType._instance is None:
+            NAType._instance = C_NAType.__new__(cls, *args, **kwargs)
+        return NAType._instance
+
+    def __repr__(self) -> str:
+        return "NA"
+
+    def __str__(self) -> str:
+        return "NA"
+
+    def __bool__(self):
+        raise TypeError("boolean value of NA is ambiguous")
+
+    def __hash__(self):
+        return id(self)
+
+    # Binary arithmetic and comparison ops -> propagate
+
+    __add__ = _create_binary_propagating_op("__add__")
+    __radd__ = _create_binary_propagating_op("__radd__")
+    __sub__ = _create_binary_propagating_op("__sub__")
+    __rsub__ = _create_binary_propagating_op("__rsub__")
+    __mul__ = _create_binary_propagating_op("__mul__")
+    __rmul__ = _create_binary_propagating_op("__rmul__")
+    __matmul__ = _create_binary_propagating_op("__matmul__")
+    __rmatmul__ = _create_binary_propagating_op("__rmatmul__")
+    __truediv__ = _create_binary_propagating_op("__truediv__")
+    __rtruediv__ = _create_binary_propagating_op("__rtruediv__")
+    __floordiv__ = _create_binary_propagating_op("__floordiv__")
+    __rfloordiv__ = _create_binary_propagating_op("__rfloordiv__")
+    __mod__ = _create_binary_propagating_op("__mod__")
+    __rmod__ = _create_binary_propagating_op("__rmod__")
+    __divmod__ = _create_binary_propagating_op("__divmod__")
+    __rdivmod__ = _create_binary_propagating_op("__rdivmod__")
+    __pow__ = _create_binary_propagating_op("__pow__")
+    __rpow__ = _create_binary_propagating_op("__rpow__")
+    # __lshift__ and __rshift__ are not implemented
+
+    __eq__ = _create_binary_propagating_op("__eq__")
+    __ne__ = _create_binary_propagating_op("__ne__")
+    __le__ = _create_binary_propagating_op("__le__")
+    __lt__ = _create_binary_propagating_op("__lt__")
+    __gt__ = _create_binary_propagating_op("__gt__")
+    __ge__ = _create_binary_propagating_op("__ge__")
+
+    # Unary ops
+
+    __neg__ = _create_unary_propagating_op("__neg__")
+    __pos__ = _create_unary_propagating_op("__pos__")
+    __abs__ = _create_unary_propagating_op("__abs__")
+    __invert__ = _create_unary_propagating_op("__invert__")
+
+    # Logical ops using Kleene logic
+
+    def __and__(self, other):
+        if other is False:
+            return False
+        elif other is True or other is NA:
+            return NA
+        else:
+            return NotImplemented
+
+    __rand__ = __and__
+
+    def __or__(self, other):
+        if other is True:
+            return True
+        elif other is False or other is NA:
+            return NA
+        else:
+            return NotImplemented
+
+    __ror__ = __or__
+
+    def __xor__(self, other):
+        if other is False or other is True or other is NA:
+            return NA
+        return NotImplemented
+
+    __rxor__ = __xor__
+
+
+C_NA = NAType()   # C-visible
+NA = C_NA         # Python-visible

pandas/core/api.py

@@ -56,3 +56,5 @@
 
 # DataFrame needs to be imported after NamedAgg to avoid a circular import
 from pandas.core.frame import DataFrame  # isort:skip
+
+from pandas._libs.missing import NA

pandas/tests/api/test_api.py

@@ -46,7 +46,7 @@ class TestPDApi(Base):
     deprecated_modules = []  # type: List[str]
 
     # misc
-    misc = ["IndexSlice", "NaT"]
+    misc = ["IndexSlice", "NaT", "NA"]
 
     # top-level classes
     classes = [

pandas/tests/scalar/test_na_scalar.py

@@ -0,0 +1,128 @@
+import numpy as np
+import pytest
+
+from pandas._libs.missing import NA
+
+from pandas.core.dtypes.common import is_scalar
+
+import pandas as pd
+import pandas.util.testing as tm
+
+
+def test_singleton():
+    assert NA is NA
+    new_NA = type(NA)()
+    assert new_NA is NA
+
+
+def test_repr():
+    assert repr(NA) == "NA"
+    assert str(NA) == "NA"
+
+
+def test_truthiness():
+    with pytest.raises(TypeError):
+        bool(NA)
+
+    with pytest.raises(TypeError):
+        not NA
+
+
+def test_hashable():
+    assert hash(NA) == hash(NA)
+    d = {NA: "test"}
+    assert d[NA] == "test"
+
+
+def test_arithmetic_ops(all_arithmetic_functions):
+    op = all_arithmetic_functions
+
+    for other in [NA, 1, 1.0, "a", np.int64(1), np.nan]:
+        if op.__name__ == "rmod" and isinstance(other, str):
+            continue
+        assert op(NA, other) is NA
+
+
+def test_comparison_ops():
+
+    for other in [NA, 1, 1.0, "a", np.int64(1), np.nan]:
+        assert (NA == other) is NA
+        assert (NA != other) is NA
+        assert (NA > other) is NA
+        assert (NA >= other) is NA
+        assert (NA < other) is NA
+        assert (NA <= other) is NA
+
+        if isinstance(other, np.int64):
+            # for numpy scalars we get a deprecation warning and False as result
+            # for equality or error for larger/lesser than
+            continue
+
+        assert (other == NA) is NA
+        assert (other != NA) is NA
+        assert (other > NA) is NA
+        assert (other >= NA) is NA
+        assert (other < NA) is NA
+        assert (other <= NA) is NA
+
+
+def test_unary_ops():
+    assert +NA is NA
+    assert -NA is NA
+    assert abs(NA) is NA
+    assert ~NA is NA
+
+
+def test_logical_and():
+
+    assert NA & True is NA
+    assert True & NA is NA
+    assert NA & False is False
+    assert False & NA is False
+    assert NA & NA is NA
+
+    with pytest.raises(TypeError):
+        NA & 5
+
+
+def test_logical_or():
+
+    assert NA | True is True
+    assert True | NA is True
+    assert NA | False is NA
+    assert False | NA is NA
+    assert NA | NA is NA
+
+    with pytest.raises(TypeError):
+        NA | 5
+
+
+def test_logical_xor():
+
+    assert NA ^ True is NA
+    assert True ^ NA is NA
+    assert NA ^ False is NA
+    assert False ^ NA is NA
+    assert NA ^ NA is NA
+
+    with pytest.raises(TypeError):
+        NA ^ 5
+
+
+def test_logical_not():
+    assert ~NA is NA
+
+
+def test_is_scalar():
+    assert is_scalar(NA) is True
+
+
+def test_isna():
+    assert pd.isna(NA) is True
+    assert pd.notna(NA) is False
+
+
+def test_series_isna():
+    s = pd.Series([1, NA], dtype=object)
+    expected = pd.Series([False, True])
+    tm.assert_series_equal(s.isna(), expected)