Merge branch 'development' into dependabot/pip/docs/jinja2-3.1.5

.github/workflows/nbval.yaml

@@ -29,7 +29,7 @@ jobs:
     - name: Run notebook and check output
       run: |
         # --sanitize-with:  pre-process text to remove irrelevant differences (e.g. warning filepaths)
-        pytest --nbval --sanitize-with docs/nbval_sanitization_rules.cfg docs/${{ matrix.notebook-file }}
+        pytest --nbval docs/${{ matrix.notebook-file }} --sanitize-with docs/nbval_sanitization_rules.cfg
     - name: Run notebooks again, save files
       run: |
         pip install nbconvert[webpdf]

.gitignore

@@ -27,6 +27,7 @@ docs/sphinx/source/generated
 .eggs/
 build/
 dist/
+tmp/
 rdtools.egg-info*
 
 # emacs temp files

docs/sphinx/source/changelog/pending.rst

@@ -12,7 +12,9 @@ Bug fixes
 ---------
 * Set marker linewidth to zero in `rdtools.plotting.degradation_summary_plots` (:pull:`433`)
 * Fix :py:func:`~rdtools.normalization.energy_from_power` returns incorrect index for shifted hourly data (:issue:`370`, :pull:`437`)
-* Add warning to clearsky workflow when power_expected is passed by user (:pull:`439`)
+* Add warning to clearsky workflow when ``power_expected`` is passed by user (:pull:`439`)
+* Fix different results with Nan's and Zeros in power series (:issue:`313`, :pull:`442`)
+* Fix pandas deprecation warnings in tests (:pull:`444`)
 
 
 Requirements

rdtools/__init__.py

@@ -36,6 +36,9 @@
 # from rdtools.plotting import soiling_rate_histogram
 # from rdtools.plotting import availability_summary_plots
 # from rdtools.availability import AvailabilityAnalysis
+from rdtools.utilities import robust_quantile
+from rdtools.utilities import robust_median
+from rdtools.utilities import robust_mean
 
 from . import _version
 __version__ = _version.get_versions()['version']

rdtools/analysis_chains.py

@@ -8,7 +8,7 @@
 import numpy as np
 import matplotlib.pyplot as plt
 from rdtools import normalization, filtering, aggregation, degradation
-from rdtools import clearsky_temperature, plotting
+from rdtools import clearsky_temperature, plotting, utilities
 import warnings
 
 
@@ -493,8 +493,9 @@ def _pvwatts_norm(self, poa_global, temperature_cell):
         if renorm:
             # Normalize to the 95th percentile for convenience, this is renormalized out
             # in the calculations but is relevant to normalized_filter()
-            x = energy_normalized[np.isfinite(energy_normalized)]
-            energy_normalized = energy_normalized / x.quantile(0.95)
+            q = utilities.robust_quantile(energy_normalized[np.isfinite(energy_normalized)], 0.95)
+
+            energy_normalized = energy_normalized / q
 
         return energy_normalized, insolation
 
@@ -618,17 +619,16 @@ def _call_clearsky_filter(filter_string):
                 warnings.warn(
                     "ad_hoc_filter contains NaN values; setting to False (excluding)"
                 )
-                ad_hoc_filter = ad_hoc_filter.fillna(False)
+                ad_hoc_filter.loc[ad_hoc_filter.isnull()] = False
 
             if not filter_components.index.equals(ad_hoc_filter.index):
                 warnings.warn(
                     "ad_hoc_filter index does not match index of other filters; missing "
                     "values will be set to True (kept). Align the index with the index "
                     "of the filter_components attribute to prevent this warning"
                 )
-                ad_hoc_filter = ad_hoc_filter.reindex(filter_components.index).fillna(
-                    True
-                )
+                ad_hoc_filter = ad_hoc_filter.reindex(filter_components.index)
+                ad_hoc_filter.loc[ad_hoc_filter.isnull()] = True
 
             filter_components["ad_hoc_filter"] = ad_hoc_filter
 
@@ -709,7 +709,7 @@ def _aggregated_filter(self, aggregated, case):
                 warnings.warn(
                     "aggregated ad_hoc_filter contains NaN values; setting to False (excluding)"
                 )
-                ad_hoc_filter_aggregated = ad_hoc_filter_aggregated.fillna(False)
+                ad_hoc_filter_aggregated.loc[ad_hoc_filter_aggregated.isnull()] = False
 
             if not filter_components_aggregated.index.equals(
                 ad_hoc_filter_aggregated.index
@@ -722,7 +722,8 @@ def _aggregated_filter(self, aggregated, case):
                 )
                 ad_hoc_filter_aggregated = ad_hoc_filter_aggregated.reindex(
                     filter_components_aggregated.index
-                ).fillna(True)
+                )
+                ad_hoc_filter_aggregated.loc[ad_hoc_filter_aggregated.isnull()] = True
 
             filter_components_aggregated["ad_hoc_filter"] = ad_hoc_filter_aggregated
 
@@ -1012,7 +1013,6 @@ def sensor_analysis(
         -------
         None
         """
-
         self._sensor_preprocess()
         sensor_results = {}
 

rdtools/availability.py

@@ -269,7 +269,7 @@ def _calc_loss_subsystem(self, low_threshold, relative_sizes,
         subsystem_fraction = relative_sizes / relative_sizes.sum()
         smallest_delta = (
             power_subsystem.le(low_threshold)
-            .replace(False, np.nan)
+            .replace(False, None)
             .multiply(subsystem_fraction)
             .min(axis=1)
             .astype(float)

rdtools/degradation.py

@@ -5,6 +5,7 @@
 import statsmodels.api as sm
 from rdtools.bootstrap import _make_time_series_bootstrap_samples, \
     _construct_confidence_intervals
+from rdtools import utilities
 
 
 def degradation_ols(energy_normalized, confidence_level=68.2):
@@ -129,7 +130,9 @@ def degradation_classical_decomposition(energy_normalized,
     # Compute yearly rolling mean to isolate trend component using
     # moving average
     energy_ma = df['energy_normalized'].rolling('365d', center=True).mean()
-    has_full_year = (df['years'] >= df['years'][0] + 0.5) & (df['years'] <= df['years'][-1] - 0.5)
+    has_full_year = (df["years"] >= df["years"].iloc[0] + 0.5) & (
+        df["years"] <= df["years"].iloc[-1] - 0.5
+    )
     energy_ma[~has_full_year] = np.nan
     df['energy_ma'] = energy_ma
 
@@ -259,7 +262,7 @@ def degradation_year_on_year(energy_normalized, recenter=True,
     if recenter:
         start = energy_normalized.index[0]
         oneyear = start + pd.Timedelta('364d')
-        renorm = energy_normalized[start:oneyear].median()
+        renorm = utilities.robust_median(energy_normalized[start:oneyear])
     else:
         renorm = 1.0
 

rdtools/filtering.py

@@ -8,6 +8,7 @@
 from scipy.interpolate import interp1d
 import rdtools
 import xgboost as xgb
+from rdtools import utilities
 
 # Load in the XGBoost clipping model using joblib.
 xgboost_clipping_model = None
@@ -294,7 +295,7 @@ def pvlib_clearsky_filter(
 def clip_filter(power_ac, model="logic", **kwargs):
     """
     Master wrapper for running one of the desired clipping filters.
-    The default filter run is the quantile clipping filter.
+    The default filter run is the logic clipping filter.
 
     Parameters
     ----------
@@ -350,7 +351,7 @@ def quantile_clip_filter(power_ac, quantile=0.98):
         Boolean Series of whether the given measurement is below 99% of the
         quantile filter.
     """
-    v = power_ac.quantile(quantile)
+    v = utilities.robust_quantile(power_ac, quantile)
     return power_ac < v * 0.99
 
 
@@ -510,13 +511,15 @@ def _apply_overall_clipping_threshold(power_ac, clipping_mask, clipped_power_ac)
         periods are labeled as True and non-clipping periods are
         labeled as False. Has a pandas datetime index.
     """
+    q_power_ac = utilities.robust_quantile(power_ac, 0.99)
+    q_clipped_power_ac = utilities.robust_quantile(clipped_power_ac, 0.99)
+
     upper_bound_pdiff = abs(
-        (power_ac.quantile(0.99) - clipped_power_ac.quantile(0.99))
-        / ((power_ac.quantile(0.99) + clipped_power_ac.quantile(0.99)) / 2)
+        (q_power_ac - q_clipped_power_ac) / ((q_power_ac + q_clipped_power_ac) / 2)
     )
     percent_clipped = len(clipped_power_ac) / len(power_ac) * 100
     if (upper_bound_pdiff < 0.005) & (percent_clipped > 4):
-        max_clip = power_ac >= power_ac.quantile(0.99)
+        max_clip = power_ac >= q_power_ac
         clipping_mask = clipping_mask | max_clip
     return clipping_mask
 
@@ -644,9 +647,11 @@ def logic_clip_filter(
         # for high frequency data sets.
         daily_mean = clip_pwr.resample("D").mean()
         df_daily = daily_mean.to_frame(name="mean")
-        df_daily["clipping_max"] = clip_pwr.groupby(pd.Grouper(freq="D")).quantile(0.99)
-        df_daily["clipping_min"] = clip_pwr.groupby(pd.Grouper(freq="D")).quantile(
-            0.075
+        df_daily["clipping_max"] = clip_pwr.groupby(pd.Grouper(freq="D")).agg(
+            utilities.robust_quantile, q=0.99
+        )
+        df_daily["clipping_min"] = clip_pwr.groupby(pd.Grouper(freq="D")).agg(
+            utilities.robust_quantile, q=0.075
         )
         daily_clipping_max = df_daily["clipping_max"].reindex(
             index=power_ac_copy.index, method="ffill"

rdtools/test/analysis_chains_test.py

@@ -42,7 +42,7 @@ def degradation_trend(basic_parameters, cs_input):
     from degradation_test import DegradationTestCase
 
     rd = -0.05
-    input_freq = "H"
+    input_freq = "h"
     degradation_trend = DegradationTestCase.get_corr_energy(rd, input_freq)
     tz = cs_input["pvlib_location"].tz
     return degradation_trend.tz_localize(tz)
@@ -57,7 +57,7 @@ def sensor_parameters(basic_parameters, degradation_trend):
     basic_parameters["pv"] = power
     basic_parameters["poa_global"] = poa_global
     basic_parameters["temperature_ambient"] = temperature_ambient
-    basic_parameters["interp_freq"] = "H"
+    basic_parameters["interp_freq"] = "h"
     return basic_parameters
 
 
@@ -68,6 +68,48 @@ def sensor_analysis(sensor_parameters):
     return rd_analysis
 
 
+@pytest.fixture
+def sensor_analysis_nans(sensor_parameters):
+    def randomly_replace_with(series, replace_with=0, fraction=0.1, seed=None):
+        """
+        Randomly replace a fraction of entries in a pandas Series with input value `replace_with`.
+
+        Parameters:
+        series (pd.Series): The input pandas Series.
+        fraction (float): The fraction of entries to replace with 0. Default is 0.1 (10%).
+        seed (int, optional): Seed for the random number generator for reproducibility.
+
+        Returns:
+        pd.Series: The modified pandas Series with some entries replaced by 0.
+        """
+        if seed is not None:
+            np.random.seed(seed)
+
+        # Determine the number of entries to replace
+        n_replace = int(len(series) * fraction)
+
+        # Randomly select indices to replace
+        replace_indices = np.random.choice(series.index, size=n_replace, replace=False)
+
+        # Replace selected entries with
+        series.loc[replace_indices] = replace_with
+
+        return series
+
+    sensor_parameters_zeros = sensor_parameters.copy()
+    sensor_parameters_nans = sensor_parameters.copy()
+
+    sensor_parameters_zeros["pv"] = randomly_replace_with(sensor_parameters["pv"], seed=0)
+    sensor_parameters_nans["pv"] = sensor_parameters_zeros["pv"].replace(0, np.nan)
+
+    rd_analysis_zeros = TrendAnalysis(**sensor_parameters_zeros)
+    rd_analysis_zeros.sensor_analysis(analyses=["yoy_degradation"])
+
+    rd_analysis_nans = TrendAnalysis(**sensor_parameters_nans)
+    rd_analysis_nans.sensor_analysis(analyses=["yoy_degradation"])
+    return rd_analysis_zeros, rd_analysis_nans
+
+
 @pytest.fixture
 def sensor_analysis_exp_power(sensor_parameters):
     power_expected = normalization.pvwatts_dc_power(
@@ -155,7 +197,7 @@ def test_interpolation(basic_parameters, degradation_trend):
     basic_parameters["temperature_cell"] = dummy_series
     basic_parameters["windspeed"] = dummy_series
     basic_parameters["power_expected"] = dummy_series
-    basic_parameters["interp_freq"] = "H"
+    basic_parameters["interp_freq"] = "h"
 
     rd_analysis = TrendAnalysis(**basic_parameters)
 
@@ -209,6 +251,21 @@ def test_sensor_analysis(sensor_analysis):
     assert [-1, -1] == pytest.approx(ci, abs=1e-2)
 
 
+def test_sensor_analysis_nans(sensor_analysis_nans):
+    rd_analysis_zeros, rd_analysis_nans = sensor_analysis_nans
+
+    yoy_results_zeros = rd_analysis_zeros.results["sensor"]["yoy_degradation"]
+    rd_zeros = yoy_results_zeros["p50_rd"]
+    ci_zeros = yoy_results_zeros["rd_confidence_interval"]
+
+    yoy_results_nans = rd_analysis_nans.results["sensor"]["yoy_degradation"]
+    rd_nans = yoy_results_nans["p50_rd"]
+    ci_nans = yoy_results_nans["rd_confidence_interval"]
+
+    assert rd_zeros == pytest.approx(rd_nans, abs=1e-2)
+    assert ci_zeros == pytest.approx(ci_nans, abs=1e-1)
+
+
 def test_sensor_analysis_filter_components(sensor_analysis):
     columns = sensor_analysis.sensor_filter_components_aggregated.columns
     assert {'two_way_window_filter'} == set(columns)
@@ -400,8 +457,8 @@ def test_filter_ad_hoc_warnings(workflow, sensor_parameters):
     assert components["ad_hoc_filter"].all()
 
     # warning about NaNs
-    ad_hoc_filter = pd.Series(True, index=sensor_parameters["pv"].index)
-    ad_hoc_filter.iloc[10] = np.nan
+    ad_hoc_filter = pd.Series(True, index=sensor_parameters["pv"].index, dtype="boolean")
+    ad_hoc_filter.iloc[10] = pd.NA
     rd_analysis.filter_params["ad_hoc_filter"] = ad_hoc_filter
     with pytest.warns(
         UserWarning, match="ad_hoc_filter contains NaN values; setting to False"
@@ -451,8 +508,10 @@ def test_aggregated_filter_ad_hoc_warnings(workflow, sensor_parameters):
     # disable all filters outside of CSI
     rd_analysis_2.filter_params = {"clearsky_filter": {"model": "csi"}}
     daily_ad_hoc_filter = pd.Series(True, index=sensor_parameters["pv"].index)
-    daily_ad_hoc_filter = daily_ad_hoc_filter.resample("1D").first().dropna(how="all")
-    daily_ad_hoc_filter.iloc[10] = np.nan
+    daily_ad_hoc_filter = (
+        daily_ad_hoc_filter.resample("1D").first().dropna(how="all").astype("boolean")
+    )
+    daily_ad_hoc_filter.iloc[10] = pd.NA
     rd_analysis_2.filter_params_aggregated["ad_hoc_filter"] = daily_ad_hoc_filter
     with pytest.warns(
         UserWarning, match="ad_hoc_filter contains NaN values; setting to False"