Add an option to skip existing intermediate variables when aggregating recursivly

RELEASE_NOTES.md

@@ -1,5 +1,6 @@
 # Next Release
 
+- [#532](https://github.com/IAMconsortium/pyam/pull/532) Add an option to skip existing intermediate variables when aggregating recursivly
 - [#527](https://github.com/IAMconsortium/pyam/pull/527) Add an in-dataframe basic mathematical operations `subtract`, `add`, `multiply`, `divide`
 - [#519](https://github.com/IAMconsortium/pyam/pull/519) Enable explicit `label` and fix for non-string items in plot legend 
 

pyam/_aggregate.py

@@ -57,7 +57,7 @@ def _aggregate(df, variable, components=None, method=np.sum):
     return _group_and_agg(_df, [], method)
 
 
-def _aggregate_recursive(df, variable):
+def _aggregate_recursive(df, variable, skip_intermediate=False):
     """Recursive aggregation along the variable tree"""
 
     # downselect to components of `variable`, initialize list for aggregated (new) data
@@ -69,6 +69,9 @@ def _aggregate_recursive(df, variable):
         components = compress(_df.variable, find_depth(_df.variable, level=d + 1))
         var_list = set([reduce_hierarchy(v, -1) for v in components])
 
+        if skip_intermediate:
+            # skip aggregating variables that already exist in dataframe _df
+            var_list = var_list - set(_df.variable)
         # a temporary dataframe allows to distinguish between full data and new data
         temp_df = _df.aggregate(variable=var_list)
         _df.append(temp_df, inplace=True)

pyam/core.py

@@ -1163,7 +1163,13 @@ def normalize(self, inplace=False, **kwargs):
             return ret
 
     def aggregate(
-        self, variable, components=None, method="sum", recursive=False, append=False
+        self,
+        variable,
+        components=None,
+        method="sum",
+        recursive=False,
+        append=False,
+        skip_intermediate=False
     ):
         """Aggregate timeseries by components or subcategories within each region
 
@@ -1179,6 +1185,9 @@ def aggregate(
             Iterate recursively (bottom-up) over all subcategories of `variable`.
         append : bool, optional
             Whether to append aggregated timeseries data to this instance.
+        skip_intermediate : bool, optional
+            Skip aggregating already existing intermediate variables. Only has an
+            effect if recursive=True
 
         Returns
         -------
@@ -1199,7 +1208,9 @@ def aggregate(
                     "Recursive aggregation only supported with `method='sum'`!"
                 )
 
-            _df = IamDataFrame(_aggregate_recursive(self, variable), meta=self.meta)
+            _df = IamDataFrame(
+                _aggregate_recursive(self, variable, skip_intermediate), meta=self.meta
+            )
         else:
             _df = _aggregate(self, variable, components=components, method=method)
 

tests/test_feature_aggregate.py

@@ -160,6 +160,35 @@ def test_aggregate_recursive(time_col):
     assert_iamframe_equal(df_minimal, df)
 
 
+@pytest.mark.parametrize("time_col", (("year"), ("time")))
+def test_aggregate_skip_intermediate(time_col):
+    # use the feature `recursive=True` and `skip_intermediate=True`
+    data = (
+        RECURSIVE_DF
+        if time_col == "year"
+        else RECURSIVE_DF.rename(DTS_MAPPING, axis="columns")
+    )
+    df = IamDataFrame(data, model="model_a", scenario="scen_a", region="World")
+    df2 = df.rename(scenario={"scen_a": "scen_b"})
+    df2.data.value *= 2
+    df.append(df2, inplace=True)
+
+    # create object without variables to be aggregated, but with intermediate variables
+    v = "Secondary Energy|Electricity"
+    agg_vars = [f"{v}{i}" for i in [""]]
+    df_minimal = df.filter(variable=agg_vars, keep=False)
+
+    # return recursively aggregated data as new object
+    obs = df_minimal.aggregate(variable=v, recursive=True, skip_intermediate=True)
+    assert_iamframe_equal(obs, df.filter(variable=agg_vars))
+
+    # append to `self`
+    df_minimal.aggregate(
+        variable=v, recursive=True, append=True, skip_intermediate=True
+    )
+    assert_iamframe_equal(df_minimal, df)
+
+
 @pytest.mark.parametrize(
     "variable, append", (("Primary Energy|Coal", "foo"), (False, True))
 )