From 74555086c65655dcc3f9221e546d842a3a7c0ba8 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 29 Apr 2024 16:36:27 +0200 Subject: [PATCH 01/53] add test file for power transformer --- tests/unit/test_power_transformer.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/unit/test_power_transformer.py diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py new file mode 100644 index 00000000..e69de29b From 7dae2b8f906e9efee0121f926a162a33e8720263 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 29 Apr 2024 17:56:44 +0200 Subject: [PATCH 02/53] test lambda function --- tests/unit/test_power_transformer.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index e69de29b..406a497e 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -0,0 +1,27 @@ +import numpy as np +import pytest +import xarray as xr + +import mesmer +from mesmer.mesmer_m.power_transformer import ( + PowerTransformerVariableLambda, + lambda_function +) + +def test_lambda_function(): + # Note that we test with normally distributed data + # which should make lambda close to 1 and + # the coefficients close to 1 and 0 + # but for the sake of testing, we set the coefficients slighly differently + coeffs = [1, 0.001] + local_yearly_T_test_data = np.random.rand(10)*100 + + # even for random numbers, the lambdas should always be between 0 and 2 + lambdas = lambda_function(coeffs, local_yearly_T_test_data) + + assert np.all(lambdas >= 0) and np.all(lambdas <= 2) + + local_yearly_T_test_data = np.array([-3,-2,-1,0,1,2,3]) + lambdas = lambda_function(coeffs, local_yearly_T_test_data) + expected_lambdas = np.array([1.0015, 1.001 , 1.0005, 1. , 0.9995, 0.999 , 0.9985]) + From 1a49b59132e0d07ee2edfeb83bac341e40bf3ad8 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 30 Apr 2024 15:23:53 +0200 Subject: [PATCH 03/53] add test for fit --- tests/unit/test_power_transformer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 406a497e..4d8fd977 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -25,3 +25,17 @@ def test_lambda_function(): lambdas = lambda_function(coeffs, local_yearly_T_test_data) expected_lambdas = np.array([1.0015, 1.001 , 1.0005, 1. , 0.9995, 0.999 , 0.9985]) + +def test_fit_power_transformer(): + # with enough random data points the fit should be close to 1 and 0 + gridcells = 1 + years = 100000 + monthly_residuals = np.random.rand(years, gridcells)*10 + yearly_T = np.ones((years, gridcells)) + pt = PowerTransformerVariableLambda() + pt.fit(monthly_residuals, yearly_T, gridcells) + pt.coeffs_ + + result = pt.coeffs_[0] + expected = np.array([1, 0]) + np.testing.assert_allclose(result, expected, rtol=1e-10, atol=1e-10) From 31868eac1d26515b1f12eec9ae5957fb81791e14 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 13:29:50 +0000 Subject: [PATCH 04/53] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/test_power_transformer.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 4d8fd977..9dc98d97 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -5,32 +5,33 @@ import mesmer from mesmer.mesmer_m.power_transformer import ( PowerTransformerVariableLambda, - lambda_function + lambda_function, ) + def test_lambda_function(): # Note that we test with normally distributed data # which should make lambda close to 1 and # the coefficients close to 1 and 0 - # but for the sake of testing, we set the coefficients slighly differently + # but for the sake of testing, we set the coefficients slighly differently coeffs = [1, 0.001] - local_yearly_T_test_data = np.random.rand(10)*100 + local_yearly_T_test_data = np.random.rand(10) * 100 # even for random numbers, the lambdas should always be between 0 and 2 lambdas = lambda_function(coeffs, local_yearly_T_test_data) assert np.all(lambdas >= 0) and np.all(lambdas <= 2) - local_yearly_T_test_data = np.array([-3,-2,-1,0,1,2,3]) + local_yearly_T_test_data = np.array([-3, -2, -1, 0, 1, 2, 3]) lambdas = lambda_function(coeffs, local_yearly_T_test_data) - expected_lambdas = np.array([1.0015, 1.001 , 1.0005, 1. , 0.9995, 0.999 , 0.9985]) + expected_lambdas = np.array([1.0015, 1.001, 1.0005, 1.0, 0.9995, 0.999, 0.9985]) def test_fit_power_transformer(): # with enough random data points the fit should be close to 1 and 0 gridcells = 1 years = 100000 - monthly_residuals = np.random.rand(years, gridcells)*10 + monthly_residuals = np.random.rand(years, gridcells) * 10 yearly_T = np.ones((years, gridcells)) pt = PowerTransformerVariableLambda() pt.fit(monthly_residuals, yearly_T, gridcells) From b5d4a174b67e8fcd30fb2f5dd99a1834a95aaf78 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 30 Apr 2024 15:35:34 +0200 Subject: [PATCH 05/53] increase tolerance --- tests/unit/test_power_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 9dc98d97..fc7172bc 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -39,4 +39,4 @@ def test_fit_power_transformer(): result = pt.coeffs_[0] expected = np.array([1, 0]) - np.testing.assert_allclose(result, expected, rtol=1e-10, atol=1e-10) + np.testing.assert_allclose(result, expected, rtol=1e-5, atol=1e-5) From ec939a30c359f871f8dd91ce984c9bd7d0d72133 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 30 Apr 2024 15:39:42 +0200 Subject: [PATCH 06/53] flaking --- tests/unit/test_power_transformer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index fc7172bc..16f1d6bd 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -1,8 +1,5 @@ import numpy as np -import pytest -import xarray as xr -import mesmer from mesmer.mesmer_m.power_transformer import ( PowerTransformerVariableLambda, lambda_function, @@ -25,6 +22,7 @@ def test_lambda_function(): local_yearly_T_test_data = np.array([-3, -2, -1, 0, 1, 2, 3]) lambdas = lambda_function(coeffs, local_yearly_T_test_data) expected_lambdas = np.array([1.0015, 1.001, 1.0005, 1.0, 0.9995, 0.999, 0.9985]) + np.testing.assert_equal(lambdas, expected_lambdas) def test_fit_power_transformer(): From ae2ac44ae041f5f9e64cc2853ea471f0e4dcf2f2 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 30 Apr 2024 15:42:51 +0200 Subject: [PATCH 07/53] decrease tolerance again --- tests/unit/test_power_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 16f1d6bd..f0c30a54 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -37,4 +37,4 @@ def test_fit_power_transformer(): result = pt.coeffs_[0] expected = np.array([1, 0]) - np.testing.assert_allclose(result, expected, rtol=1e-5, atol=1e-5) + np.testing.assert_allclose(result, expected, rtol=1e-10, atol=1e-10) From 2fa5b645df93d22f3e9dd0320ab457cca104d028 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 30 Apr 2024 16:09:44 +0200 Subject: [PATCH 08/53] test transform --- tests/unit/test_power_transformer.py | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index f0c30a54..2c1ed00b 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -20,8 +20,10 @@ def test_lambda_function(): assert np.all(lambdas >= 0) and np.all(lambdas <= 2) local_yearly_T_test_data = np.array([-3, -2, -1, 0, 1, 2, 3]) + lambdas = lambda_function(coeffs, local_yearly_T_test_data) expected_lambdas = np.array([1.0015, 1.001, 1.0005, 1.0, 0.9995, 0.999, 0.9985]) + np.testing.assert_equal(lambdas, expected_lambdas) @@ -31,10 +33,37 @@ def test_fit_power_transformer(): years = 100000 monthly_residuals = np.random.rand(years, gridcells) * 10 yearly_T = np.ones((years, gridcells)) + pt = PowerTransformerVariableLambda() pt.fit(monthly_residuals, yearly_T, gridcells) pt.coeffs_ result = pt.coeffs_[0] expected = np.array([1, 0]) + np.testing.assert_allclose(result, expected, rtol=1e-10, atol=1e-10) + +def test_transform(): + pt = PowerTransformerVariableLambda() + pt.standardize = False + pt.coeffs_ = np.tile([1,0], (10, 1)) + + monthly_residuals = np.ones((10,10)) + yearly_T = np.zeros((10,10)) + + result = pt.transform(monthly_residuals, yearly_T) + expected = np.ones((10,10)) + + np.testing.assert_equal(result, expected) + +def test_yeo_johnson_transform(): + pt = PowerTransformerVariableLambda() + + # test all possible combinations of local_monthly_residuals and lambdas + local_monthly_residuals = np.array([0,1,0,1,-1,-1]) + lambdas = np.array([1,1,0,0,1,2]) + + result = pt._yeo_johnson_transform(local_monthly_residuals, lambdas) + expected = np.array([0,1,0,0,-1,0]) + + np.testing.assert_equal(result, expected) From f960e4460e66043402c6f412e3ffda935bb32b0c Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 30 Apr 2024 16:10:29 +0200 Subject: [PATCH 09/53] test inverse transform --- tests/unit/test_power_transformer.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 2c1ed00b..bba1aa4d 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -67,3 +67,18 @@ def test_yeo_johnson_transform(): expected = np.array([0,1,0,0,-1,0]) np.testing.assert_equal(result, expected) + +def test_inverse_transform(): + monthly_residuals = np.ones((10,10)) + yearly_T = np.zeros((10,10)) + + pt = PowerTransformerVariableLambda() + pt.standardize = False + pt.coeffs_ = np.tile([1,0], (10, 1)) + pt.mins_ = np.amin(monthly_residuals, axis=0) + pt.maxs_ = np.amax(monthly_residuals, axis=0) + + transformed = pt.transform(monthly_residuals, yearly_T) + result = pt.inverse_transform(transformed, yearly_T) + + np.testing.assert_equal(result, monthly_residuals) \ No newline at end of file From e4bc840c61cb684c1f586e779158c7b33c826e1d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 14:11:13 +0000 Subject: [PATCH 10/53] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/test_power_transformer.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index bba1aa4d..dd15377f 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -43,42 +43,45 @@ def test_fit_power_transformer(): np.testing.assert_allclose(result, expected, rtol=1e-10, atol=1e-10) + def test_transform(): pt = PowerTransformerVariableLambda() pt.standardize = False - pt.coeffs_ = np.tile([1,0], (10, 1)) + pt.coeffs_ = np.tile([1, 0], (10, 1)) - monthly_residuals = np.ones((10,10)) - yearly_T = np.zeros((10,10)) + monthly_residuals = np.ones((10, 10)) + yearly_T = np.zeros((10, 10)) result = pt.transform(monthly_residuals, yearly_T) - expected = np.ones((10,10)) + expected = np.ones((10, 10)) np.testing.assert_equal(result, expected) + def test_yeo_johnson_transform(): pt = PowerTransformerVariableLambda() # test all possible combinations of local_monthly_residuals and lambdas - local_monthly_residuals = np.array([0,1,0,1,-1,-1]) - lambdas = np.array([1,1,0,0,1,2]) + local_monthly_residuals = np.array([0, 1, 0, 1, -1, -1]) + lambdas = np.array([1, 1, 0, 0, 1, 2]) result = pt._yeo_johnson_transform(local_monthly_residuals, lambdas) - expected = np.array([0,1,0,0,-1,0]) + expected = np.array([0, 1, 0, 0, -1, 0]) np.testing.assert_equal(result, expected) + def test_inverse_transform(): - monthly_residuals = np.ones((10,10)) - yearly_T = np.zeros((10,10)) + monthly_residuals = np.ones((10, 10)) + yearly_T = np.zeros((10, 10)) pt = PowerTransformerVariableLambda() pt.standardize = False - pt.coeffs_ = np.tile([1,0], (10, 1)) + pt.coeffs_ = np.tile([1, 0], (10, 1)) pt.mins_ = np.amin(monthly_residuals, axis=0) pt.maxs_ = np.amax(monthly_residuals, axis=0) transformed = pt.transform(monthly_residuals, yearly_T) result = pt.inverse_transform(transformed, yearly_T) - np.testing.assert_equal(result, monthly_residuals) \ No newline at end of file + np.testing.assert_equal(result, monthly_residuals) From c4927af9ddb2e35803c49bee0e1846f01840440b Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 30 Apr 2024 16:25:33 +0200 Subject: [PATCH 11/53] remove hard coded test for lambda function --- tests/unit/test_power_transformer.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index dd15377f..8eee539a 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -10,21 +10,14 @@ def test_lambda_function(): # Note that we test with normally distributed data # which should make lambda close to 1 and # the coefficients close to 1 and 0 - # but for the sake of testing, we set the coefficients slighly differently - coeffs = [1, 0.001] + # but for the sake of testing, we set the coefficients differently + coeffs = [1, 0.1] local_yearly_T_test_data = np.random.rand(10) * 100 # even for random numbers, the lambdas should always be between 0 and 2 lambdas = lambda_function(coeffs, local_yearly_T_test_data) - assert np.all(lambdas >= 0) and np.all(lambdas <= 2) - - local_yearly_T_test_data = np.array([-3, -2, -1, 0, 1, 2, 3]) - - lambdas = lambda_function(coeffs, local_yearly_T_test_data) - expected_lambdas = np.array([1.0015, 1.001, 1.0005, 1.0, 0.9995, 0.999, 0.9985]) - - np.testing.assert_equal(lambdas, expected_lambdas) + assert np.all(lambdas > 0) and np.all(lambdas < 2) def test_fit_power_transformer(): From f717f66f154a42736c9688a9c7ca72b6adb1a3b0 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 6 May 2024 10:43:38 +0200 Subject: [PATCH 12/53] change from uniform to normal test data --- tests/unit/test_power_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 8eee539a..4ba958fa 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -12,7 +12,7 @@ def test_lambda_function(): # the coefficients close to 1 and 0 # but for the sake of testing, we set the coefficients differently coeffs = [1, 0.1] - local_yearly_T_test_data = np.random.rand(10) * 100 + local_yearly_T_test_data = np.random.normal(0, 1, 10) # even for random numbers, the lambdas should always be between 0 and 2 lambdas = lambda_function(coeffs, local_yearly_T_test_data) From 62b2fa9f3dad78c9fa19c42078c19a736856f6cc Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 6 May 2024 10:43:52 +0200 Subject: [PATCH 13/53] update comment in fit --- tests/unit/test_power_transformer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 4ba958fa..f0ce077e 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -22,6 +22,10 @@ def test_lambda_function(): def test_fit_power_transformer(): # with enough random data points the fit should be close to 1 and 0 + # here we test with uniform random data because it is quicker to fit + # Uniform data is also symmetrically distributed so coefficients + # should be close to 1 and 0 as well + gridcells = 1 years = 100000 monthly_residuals = np.random.rand(years, gridcells) * 10 From 044f62e1815bba4ba1ff7fe1964c9c869714946f Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 6 May 2024 10:44:03 +0200 Subject: [PATCH 14/53] relax tolerance --- tests/unit/test_power_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index f0ce077e..370a1899 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -38,7 +38,7 @@ def test_fit_power_transformer(): result = pt.coeffs_[0] expected = np.array([1, 0]) - np.testing.assert_allclose(result, expected, rtol=1e-10, atol=1e-10) + np.testing.assert_allclose(result, expected, atol=1e-7) def test_transform(): From feca6017ba62318c327c3cab5ed8e61c0a05bf36 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 6 May 2024 10:54:13 +0200 Subject: [PATCH 15/53] remove superfluous line --- tests/unit/test_power_transformer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 370a1899..7f989ec3 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -33,7 +33,6 @@ def test_fit_power_transformer(): pt = PowerTransformerVariableLambda() pt.fit(monthly_residuals, yearly_T, gridcells) - pt.coeffs_ result = pt.coeffs_[0] expected = np.array([1, 0]) From 2980592424bf989a902923afdc8d18e89bb77d3d Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 6 May 2024 10:59:03 +0200 Subject: [PATCH 16/53] readability of pt_coeffs result --- tests/unit/test_power_transformer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 7f989ec3..808dde77 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -34,8 +34,8 @@ def test_fit_power_transformer(): pt = PowerTransformerVariableLambda() pt.fit(monthly_residuals, yearly_T, gridcells) - result = pt.coeffs_[0] - expected = np.array([1, 0]) + result = pt.coeffs_ + expected = np.array([[1, 0]]) np.testing.assert_allclose(result, expected, atol=1e-7) From 186b6670a4cb30643f7ee826dc4195b3b2440da1 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 6 May 2024 11:04:33 +0200 Subject: [PATCH 17/53] explicit n_years and n_gridcells in test_transform --- tests/unit/test_power_transformer.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 808dde77..a86fb34a 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -41,15 +41,18 @@ def test_fit_power_transformer(): def test_transform(): + n_years = 20 + n_gridcells = 10 + pt = PowerTransformerVariableLambda() pt.standardize = False - pt.coeffs_ = np.tile([1, 0], (10, 1)) + pt.coeffs_ = np.tile([1, 0], (n_gridcells, 1)) - monthly_residuals = np.ones((10, 10)) - yearly_T = np.zeros((10, 10)) + monthly_residuals = np.ones((n_years, n_gridcells)) + yearly_T = np.zeros((n_years, n_gridcells)) result = pt.transform(monthly_residuals, yearly_T) - expected = np.ones((10, 10)) + expected = np.ones((n_years, n_gridcells)) np.testing.assert_equal(result, expected) From 4d06fa4ac5427c985513a3ded004236235a6d8fb Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 6 May 2024 11:54:54 +0200 Subject: [PATCH 18/53] change from ints to float for transform --- tests/unit/test_power_transformer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index a86fb34a..c0176f82 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -61,11 +61,11 @@ def test_yeo_johnson_transform(): pt = PowerTransformerVariableLambda() # test all possible combinations of local_monthly_residuals and lambdas - local_monthly_residuals = np.array([0, 1, 0, 1, -1, -1]) - lambdas = np.array([1, 1, 0, 0, 1, 2]) + local_monthly_residuals = np.array([0., 1., 0., 1., -1., -1.]) + lambdas = np.array([1., 1., 0., 0., 1., 2.]) result = pt._yeo_johnson_transform(local_monthly_residuals, lambdas) - expected = np.array([0, 1, 0, 0, -1, 0]) + expected = np.array([0., 1., 0., np.log1p(1.), -1., -np.log1p(1.)]) np.testing.assert_equal(result, expected) From 0ec13cb327906fa76de4960d1c937204e6fdd609 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 6 May 2024 12:12:40 +0200 Subject: [PATCH 19/53] extend inverse transform tes --- tests/unit/test_power_transformer.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index c0176f82..5aeb5525 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -71,16 +71,22 @@ def test_yeo_johnson_transform(): def test_inverse_transform(): - monthly_residuals = np.ones((10, 10)) - yearly_T = np.zeros((10, 10)) + n_years = 20 + n_gridcells = 5 + # dummy seasonal cylce, having negative and positive values + monthly_residuals = np.sin(np.linspace(0, 2 * np.pi, n_years * n_gridcells)).reshape(n_years, n_gridcells) + yearly_T = np.zeros((n_years, n_gridcells)) pt = PowerTransformerVariableLambda() pt.standardize = False - pt.coeffs_ = np.tile([1, 0], (10, 1)) + # dummy lambdas (since yearly_T is zero lambda comes out to be second coefficient) + # we have all cases for lambdas 0 and 2 (special cases), 1 (identity case) + # lambda between 1 and 1 and lambda between 1 and 2 for concave and convex cases + pt.coeffs_ = np.array([[0, 0], [0, 1], [0, 2], [0, 0.5], [0, 1.5]]) pt.mins_ = np.amin(monthly_residuals, axis=0) pt.maxs_ = np.amax(monthly_residuals, axis=0) transformed = pt.transform(monthly_residuals, yearly_T) result = pt.inverse_transform(transformed, yearly_T) - np.testing.assert_equal(result, monthly_residuals) + np.testing.assert_allclose(result, monthly_residuals, atol=1e-7) From e4af1b42655fbb2deaf76320c1647e42461e0c2a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 10:13:37 +0000 Subject: [PATCH 20/53] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/test_power_transformer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 5aeb5525..3e11a875 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -23,7 +23,7 @@ def test_lambda_function(): def test_fit_power_transformer(): # with enough random data points the fit should be close to 1 and 0 # here we test with uniform random data because it is quicker to fit - # Uniform data is also symmetrically distributed so coefficients + # Uniform data is also symmetrically distributed so coefficients # should be close to 1 and 0 as well gridcells = 1 @@ -61,11 +61,11 @@ def test_yeo_johnson_transform(): pt = PowerTransformerVariableLambda() # test all possible combinations of local_monthly_residuals and lambdas - local_monthly_residuals = np.array([0., 1., 0., 1., -1., -1.]) - lambdas = np.array([1., 1., 0., 0., 1., 2.]) + local_monthly_residuals = np.array([0.0, 1.0, 0.0, 1.0, -1.0, -1.0]) + lambdas = np.array([1.0, 1.0, 0.0, 0.0, 1.0, 2.0]) result = pt._yeo_johnson_transform(local_monthly_residuals, lambdas) - expected = np.array([0., 1., 0., np.log1p(1.), -1., -np.log1p(1.)]) + expected = np.array([0.0, 1.0, 0.0, np.log1p(1.0), -1.0, -np.log1p(1.0)]) np.testing.assert_equal(result, expected) @@ -74,7 +74,9 @@ def test_inverse_transform(): n_years = 20 n_gridcells = 5 # dummy seasonal cylce, having negative and positive values - monthly_residuals = np.sin(np.linspace(0, 2 * np.pi, n_years * n_gridcells)).reshape(n_years, n_gridcells) + monthly_residuals = np.sin( + np.linspace(0, 2 * np.pi, n_years * n_gridcells) + ).reshape(n_years, n_gridcells) yearly_T = np.zeros((n_years, n_gridcells)) pt = PowerTransformerVariableLambda() From 1273b39a54aaa59ed0e88423956035e1e7e754bc Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 6 May 2024 17:01:44 +0200 Subject: [PATCH 21/53] change n_years to n_ts --- tests/unit/test_power_transformer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 3e11a875..61f1f978 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -71,13 +71,13 @@ def test_yeo_johnson_transform(): def test_inverse_transform(): - n_years = 20 + n_ts = 20 n_gridcells = 5 # dummy seasonal cylce, having negative and positive values monthly_residuals = np.sin( - np.linspace(0, 2 * np.pi, n_years * n_gridcells) - ).reshape(n_years, n_gridcells) - yearly_T = np.zeros((n_years, n_gridcells)) + np.linspace(0, 2 * np.pi, n_ts * n_gridcells) + ).reshape(n_ts, n_gridcells) + yearly_T = np.zeros((n_ts, n_gridcells)) pt = PowerTransformerVariableLambda() pt.standardize = False From 242c4936a971596f0c9d911303e471ba142485b1 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Mon, 6 May 2024 17:03:47 +0200 Subject: [PATCH 22/53] rename other timesteps --- tests/unit/test_power_transformer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 61f1f978..fb8074a7 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -27,9 +27,9 @@ def test_fit_power_transformer(): # should be close to 1 and 0 as well gridcells = 1 - years = 100000 - monthly_residuals = np.random.rand(years, gridcells) * 10 - yearly_T = np.ones((years, gridcells)) + n_months = 100000 + monthly_residuals = np.random.rand(n_months, gridcells) * 10 + yearly_T = np.ones((n_months, gridcells)) pt = PowerTransformerVariableLambda() pt.fit(monthly_residuals, yearly_T, gridcells) @@ -41,18 +41,18 @@ def test_fit_power_transformer(): def test_transform(): - n_years = 20 + n_ts = 20 n_gridcells = 10 pt = PowerTransformerVariableLambda() pt.standardize = False pt.coeffs_ = np.tile([1, 0], (n_gridcells, 1)) - monthly_residuals = np.ones((n_years, n_gridcells)) - yearly_T = np.zeros((n_years, n_gridcells)) + monthly_residuals = np.ones((n_ts, n_gridcells)) + yearly_T = np.zeros((n_ts, n_gridcells)) result = pt.transform(monthly_residuals, yearly_T) - expected = np.ones((n_years, n_gridcells)) + expected = np.ones((n_ts, n_gridcells)) np.testing.assert_equal(result, expected) From d47be705526f5a9b78b4e074095e83134bb84c76 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 7 May 2024 09:06:10 +0200 Subject: [PATCH 23/53] add test for optimize lambda --- tests/unit/test_power_transformer.py | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index fb8074a7..cf9752ec 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -1,4 +1,5 @@ import numpy as np +import scipy from mesmer.mesmer_m.power_transformer import ( PowerTransformerVariableLambda, @@ -40,6 +41,35 @@ def test_fit_power_transformer(): np.testing.assert_allclose(result, expected, atol=1e-7) +def test_yeo_johnson_optimize_lambda(): + np.random.seed(0) + n_years = 10_000 + yearly_T = np.repeat(np.random.randn(n_years), 12) + + skew = -2 + local_monthly_residuals = scipy.stats.skewnorm.rvs(skew, size=n_years * 12) + + pt = PowerTransformerVariableLambda() + pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) + lmbda = lambda_function(pt.coeffs_, yearly_T) + transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) + + assert (lmbda > 1).all() & (lmbda <= 2).all() + np.testing.assert_allclose(scipy.stats.skew(transformed), 0, atol=0.01) + + # this fails, need to investigate more + # skew = 2 + # local_monthly_residuals = scipy.stats.skewnorm.rvs(skew, size=n_years * 12) + + # pt = PowerTransformerVariableLambda() + # pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) + # lmbda = lambda_function(pt.coeffs_, yearly_T) + # transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) + + # assert (lmbda >= 0).all() & (lmbda <= 1).all() + # np.testing.assert_allclose(scipy.stats.skew(transformed), 0, atol=0.01) + + def test_transform(): n_ts = 20 n_gridcells = 10 From f79201c837026029a4bcf41d5dd7e43147ce5451 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 7 May 2024 09:15:04 +0200 Subject: [PATCH 24/53] remove years*12 --- tests/unit/test_power_transformer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index cf9752ec..e2665679 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -44,10 +44,10 @@ def test_fit_power_transformer(): def test_yeo_johnson_optimize_lambda(): np.random.seed(0) n_years = 10_000 - yearly_T = np.repeat(np.random.randn(n_years), 12) + yearly_T = np.random.randn(n_years) skew = -2 - local_monthly_residuals = scipy.stats.skewnorm.rvs(skew, size=n_years * 12) + local_monthly_residuals = scipy.stats.skewnorm.rvs(skew, size=n_years) pt = PowerTransformerVariableLambda() pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) @@ -59,7 +59,7 @@ def test_yeo_johnson_optimize_lambda(): # this fails, need to investigate more # skew = 2 - # local_monthly_residuals = scipy.stats.skewnorm.rvs(skew, size=n_years * 12) + # local_monthly_residuals = scipy.stats.skewnorm.rvs(skew, size=n_years) # pt = PowerTransformerVariableLambda() # pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) From 3455103fd54a858d788abe5bbf6e4710a7e1e6d9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 07:17:05 +0000 Subject: [PATCH 25/53] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/test_power_transformer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index e2665679..293454d5 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -53,7 +53,7 @@ def test_yeo_johnson_optimize_lambda(): pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) lmbda = lambda_function(pt.coeffs_, yearly_T) transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) - + assert (lmbda > 1).all() & (lmbda <= 2).all() np.testing.assert_allclose(scipy.stats.skew(transformed), 0, atol=0.01) @@ -65,7 +65,7 @@ def test_yeo_johnson_optimize_lambda(): # pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) # lmbda = lambda_function(pt.coeffs_, yearly_T) # transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) - + # assert (lmbda >= 0).all() & (lmbda <= 1).all() # np.testing.assert_allclose(scipy.stats.skew(transformed), 0, atol=0.01) @@ -104,9 +104,9 @@ def test_inverse_transform(): n_ts = 20 n_gridcells = 5 # dummy seasonal cylce, having negative and positive values - monthly_residuals = np.sin( - np.linspace(0, 2 * np.pi, n_ts * n_gridcells) - ).reshape(n_ts, n_gridcells) + monthly_residuals = np.sin(np.linspace(0, 2 * np.pi, n_ts * n_gridcells)).reshape( + n_ts, n_gridcells + ) yearly_T = np.zeros((n_ts, n_gridcells)) pt = PowerTransformerVariableLambda() From 405f5a0c581fe62016a3d3cbaabf1678644c9dc4 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 7 May 2024 09:28:02 +0200 Subject: [PATCH 26/53] rewording in test_labda_function --- tests/unit/test_power_transformer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 293454d5..17edbb71 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -8,10 +8,10 @@ def test_lambda_function(): - # Note that we test with normally distributed data - # which should make lambda close to 1 and - # the coefficients close to 1 and 0 - # but for the sake of testing, we set the coefficients differently + # Note that we test with normally distributed data without skewness + # which would yield coefficients close to 1 and 0 + # and a constant lambda of about 1 + # but for the sake of testing, we use coefficients which respresent skewness coeffs = [1, 0.1] local_yearly_T_test_data = np.random.normal(0, 1, 10) From 294b158e5f1abbf0e142bf92198ee5f88d01de72 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 7 May 2024 09:29:27 +0200 Subject: [PATCH 27/53] 100_000 --- tests/unit/test_power_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 17edbb71..3fb3caf7 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -28,7 +28,7 @@ def test_fit_power_transformer(): # should be close to 1 and 0 as well gridcells = 1 - n_months = 100000 + n_months = 100_000 monthly_residuals = np.random.rand(n_months, gridcells) * 10 yearly_T = np.ones((n_months, gridcells)) From 8dfebb6746f49e70b393dc116574d84e8dc3878d Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Tue, 7 May 2024 09:29:59 +0200 Subject: [PATCH 28/53] add comment in test lambda function --- tests/unit/test_power_transformer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 3fb3caf7..326b588e 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -16,6 +16,7 @@ def test_lambda_function(): local_yearly_T_test_data = np.random.normal(0, 1, 10) # even for random numbers, the lambdas should always be between 0 and 2 + # because the function is logistic between 0 and 2 lambdas = lambda_function(coeffs, local_yearly_T_test_data) assert np.all(lambdas > 0) and np.all(lambdas < 2) From c26aaac18d2415473b7915a7363f1ab7e4a6293b Mon Sep 17 00:00:00 2001 From: Victoria <112418493+veni-vidi-vici-dormivi@users.noreply.github.com> Date: Wed, 15 May 2024 15:43:27 +0200 Subject: [PATCH 29/53] Update tests/unit/test_power_transformer.pyAdd Note f Co-authored-by: Mathias Hauser --- tests/unit/test_power_transformer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 326b588e..c10504f5 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -72,6 +72,7 @@ def test_yeo_johnson_optimize_lambda(): def test_transform(): + # NOTE: testing trivial transform with lambda = 1 n_ts = 20 n_gridcells = 10 From d6445e4e057aaccb43a73b22191a36890bff2d02 Mon Sep 17 00:00:00 2001 From: Victoria <112418493+veni-vidi-vici-dormivi@users.noreply.github.com> Date: Wed, 15 May 2024 15:45:11 +0200 Subject: [PATCH 30/53] typo Co-authored-by: Mathias Hauser --- tests/unit/test_power_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index c10504f5..e013e01d 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -115,7 +115,7 @@ def test_inverse_transform(): pt.standardize = False # dummy lambdas (since yearly_T is zero lambda comes out to be second coefficient) # we have all cases for lambdas 0 and 2 (special cases), 1 (identity case) - # lambda between 1 and 1 and lambda between 1 and 2 for concave and convex cases + # lambda between 0 and 1 and lambda between 1 and 2 for concave and convex cases pt.coeffs_ = np.array([[0, 0], [0, 1], [0, 2], [0, 0.5], [0, 1.5]]) pt.mins_ = np.amin(monthly_residuals, axis=0) pt.maxs_ = np.amax(monthly_residuals, axis=0) From ec79d122b4960bc87369bfdc87e27e7a64228af9 Mon Sep 17 00:00:00 2001 From: Victoria <112418493+veni-vidi-vici-dormivi@users.noreply.github.com> Date: Wed, 15 May 2024 15:48:15 +0200 Subject: [PATCH 31/53] Renaming inverse transform Co-authored-by: Mathias Hauser --- tests/unit/test_power_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index e013e01d..2bd90c6a 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -102,7 +102,7 @@ def test_yeo_johnson_transform(): np.testing.assert_equal(result, expected) -def test_inverse_transform(): +def test_transform_roundtrip(): n_ts = 20 n_gridcells = 5 # dummy seasonal cylce, having negative and positive values From c3964b69254d04e11bdc64c49c014ec77704d440 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Wed, 15 May 2024 15:41:10 +0200 Subject: [PATCH 32/53] deterministic lambda func tests --- tests/unit/test_power_transformer.py | 31 ++++++++++++++++------------ 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 2bd90c6a..f69e1b41 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -1,25 +1,30 @@ import numpy as np import scipy +import pytest from mesmer.mesmer_m.power_transformer import ( PowerTransformerVariableLambda, lambda_function, ) +@pytest.mark.parametrize( + "coeffs, t, expected", + [ + ([1, 0.1], -np.inf, 2.0), + ([1, 0.1], np.inf, 0.0), + ([1, -0.1], -np.inf, 0.0), + ([1, -0.1], np.inf, 2.0), + ([0, 0], 1, 2), + ([0, 1], 1, 2), + ([1, 0], 1, 1), + ([2, 0], 1, 2 / 3), + ([1, 1], np.log(9), 2 / 10), + ] +) +def test_lambda_function(coeffs, t, expected): -def test_lambda_function(): - # Note that we test with normally distributed data without skewness - # which would yield coefficients close to 1 and 0 - # and a constant lambda of about 1 - # but for the sake of testing, we use coefficients which respresent skewness - coeffs = [1, 0.1] - local_yearly_T_test_data = np.random.normal(0, 1, 10) - - # even for random numbers, the lambdas should always be between 0 and 2 - # because the function is logistic between 0 and 2 - lambdas = lambda_function(coeffs, local_yearly_T_test_data) - - assert np.all(lambdas > 0) and np.all(lambdas < 2) + result = lambda_function(coeffs, t) + np.testing.assert_allclose(result, expected) def test_fit_power_transformer(): From 1b8c82fb7291dc47f58cf803856c0a9366fc60ce Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Wed, 15 May 2024 15:41:58 +0200 Subject: [PATCH 33/53] scipy as sp --- tests/unit/test_power_transformer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index f69e1b41..322b7a5f 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -1,5 +1,5 @@ import numpy as np -import scipy +import scipy as sp import pytest from mesmer.mesmer_m.power_transformer import ( @@ -53,7 +53,7 @@ def test_yeo_johnson_optimize_lambda(): yearly_T = np.random.randn(n_years) skew = -2 - local_monthly_residuals = scipy.stats.skewnorm.rvs(skew, size=n_years) + local_monthly_residuals = sp.stats.skewnorm.rvs(skew, size=n_years) pt = PowerTransformerVariableLambda() pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) @@ -61,11 +61,11 @@ def test_yeo_johnson_optimize_lambda(): transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) assert (lmbda > 1).all() & (lmbda <= 2).all() - np.testing.assert_allclose(scipy.stats.skew(transformed), 0, atol=0.01) + np.testing.assert_allclose(sp.stats.skew(transformed), 0, atol=0.01) # this fails, need to investigate more # skew = 2 - # local_monthly_residuals = scipy.stats.skewnorm.rvs(skew, size=n_years) + # local_monthly_residuals = sp.stats.skewnorm.rvs(skew, size=n_years) # pt = PowerTransformerVariableLambda() # pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) @@ -73,7 +73,7 @@ def test_yeo_johnson_optimize_lambda(): # transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) # assert (lmbda >= 0).all() & (lmbda <= 1).all() - # np.testing.assert_allclose(scipy.stats.skew(transformed), 0, atol=0.01) + # np.testing.assert_allclose(sp.stats.skew(transformed), 0, atol=0.01) def test_transform(): From 7ddab6abf47de92ad979404f37ebc2fcb2d78ba0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 14:42:59 +0000 Subject: [PATCH 34/53] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/test_power_transformer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 322b7a5f..d5c6c673 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -1,12 +1,13 @@ import numpy as np -import scipy as sp import pytest +import scipy as sp from mesmer.mesmer_m.power_transformer import ( PowerTransformerVariableLambda, lambda_function, ) + @pytest.mark.parametrize( "coeffs, t, expected", [ @@ -19,7 +20,7 @@ ([1, 0], 1, 1), ([2, 0], 1, 2 / 3), ([1, 1], np.log(9), 2 / 10), - ] + ], ) def test_lambda_function(coeffs, t, expected): From cdd7501e01b74173992a3ad66a4e1a9ed03469db Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Fri, 17 May 2024 11:49:05 +0200 Subject: [PATCH 35/53] switch from uniform to normal data --- tests/unit/test_power_transformer.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index d5c6c673..60bf1b0b 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -29,14 +29,10 @@ def test_lambda_function(coeffs, t, expected): def test_fit_power_transformer(): - # with enough random data points the fit should be close to 1 and 0 - # here we test with uniform random data because it is quicker to fit - # Uniform data is also symmetrically distributed so coefficients - # should be close to 1 and 0 as well - + # with enough random normal data points the fit should be close to 1 and 0 gridcells = 1 n_months = 100_000 - monthly_residuals = np.random.rand(n_months, gridcells) * 10 + monthly_residuals = np.random.standard_normal((n_months, gridcells)) * 10 yearly_T = np.ones((n_months, gridcells)) pt = PowerTransformerVariableLambda() @@ -45,7 +41,7 @@ def test_fit_power_transformer(): result = pt.coeffs_ expected = np.array([[1, 0]]) - np.testing.assert_allclose(result, expected, atol=1e-7) + np.testing.assert_allclose(result, expected, atol=1e-2) def test_yeo_johnson_optimize_lambda(): From 65afa40ebace5e195cb3fb4c329c157f3588e2a1 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Fri, 17 May 2024 11:50:12 +0200 Subject: [PATCH 36/53] add right skewed test --- tests/unit/test_power_transformer.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 60bf1b0b..dcd151c5 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -46,7 +46,7 @@ def test_fit_power_transformer(): def test_yeo_johnson_optimize_lambda(): np.random.seed(0) - n_years = 10_000 + n_years = 100_000 yearly_T = np.random.randn(n_years) skew = -2 @@ -60,17 +60,16 @@ def test_yeo_johnson_optimize_lambda(): assert (lmbda > 1).all() & (lmbda <= 2).all() np.testing.assert_allclose(sp.stats.skew(transformed), 0, atol=0.01) - # this fails, need to investigate more - # skew = 2 - # local_monthly_residuals = sp.stats.skewnorm.rvs(skew, size=n_years) + skew = 2 + local_monthly_residuals = sp.stats.skewnorm.rvs(skew, size=n_years) - # pt = PowerTransformerVariableLambda() - # pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) - # lmbda = lambda_function(pt.coeffs_, yearly_T) - # transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) + pt = PowerTransformerVariableLambda() + pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) + lmbda = lambda_function(pt.coeffs_, yearly_T) + transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) - # assert (lmbda >= 0).all() & (lmbda <= 1).all() - # np.testing.assert_allclose(sp.stats.skew(transformed), 0, atol=0.01) + assert (lmbda >= 0).all() & (lmbda <= 1).all() + np.testing.assert_allclose(sp.stats.skew(transformed), 0, atol=0.01) def test_transform(): From 267f61d65deef305d6abb659d09248ffc5cb37b8 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Fri, 17 May 2024 11:50:32 +0200 Subject: [PATCH 37/53] add comments --- tests/unit/test_power_transformer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index dcd151c5..00c08f7b 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -49,6 +49,7 @@ def test_yeo_johnson_optimize_lambda(): n_years = 100_000 yearly_T = np.random.randn(n_years) + # test with left skewed data skew = -2 local_monthly_residuals = sp.stats.skewnorm.rvs(skew, size=n_years) @@ -60,6 +61,7 @@ def test_yeo_johnson_optimize_lambda(): assert (lmbda > 1).all() & (lmbda <= 2).all() np.testing.assert_allclose(sp.stats.skew(transformed), 0, atol=0.01) + # test with right skewed data skew = 2 local_monthly_residuals = sp.stats.skewnorm.rvs(skew, size=n_years) From 7dbb58365d82c0e082f58a7781b4dc3175c24e54 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Fri, 17 May 2024 11:56:25 +0200 Subject: [PATCH 38/53] parameterize optimize --- tests/unit/test_power_transformer.py | 29 +++++++++++----------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 00c08f7b..61ba4c1d 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -44,25 +44,18 @@ def test_fit_power_transformer(): np.testing.assert_allclose(result, expected, atol=1e-2) -def test_yeo_johnson_optimize_lambda(): +@pytest.mark.parametrize( + "skew, bounds", + [ + (-2, [1, 2]), # left skewed data + (2, [0, 1]), # right skewed data + ], +) +def test_yeo_johnson_optimize_lambda(skew, bounds): np.random.seed(0) n_years = 100_000 - yearly_T = np.random.randn(n_years) - - # test with left skewed data - skew = -2 - local_monthly_residuals = sp.stats.skewnorm.rvs(skew, size=n_years) - pt = PowerTransformerVariableLambda() - pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) - lmbda = lambda_function(pt.coeffs_, yearly_T) - transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) - - assert (lmbda > 1).all() & (lmbda <= 2).all() - np.testing.assert_allclose(sp.stats.skew(transformed), 0, atol=0.01) - - # test with right skewed data - skew = 2 + yearly_T = np.random.randn(n_years) local_monthly_residuals = sp.stats.skewnorm.rvs(skew, size=n_years) pt = PowerTransformerVariableLambda() @@ -70,8 +63,8 @@ def test_yeo_johnson_optimize_lambda(): lmbda = lambda_function(pt.coeffs_, yearly_T) transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) - assert (lmbda >= 0).all() & (lmbda <= 1).all() - np.testing.assert_allclose(sp.stats.skew(transformed), 0, atol=0.01) + assert (lmbda >= bounds[0]).all() & (lmbda <= bounds[1]).all() + np.testing.assert_allclose(sp.stats.skew(transformed), 0, atol=0.1) def test_transform(): From 8f198adcd4e7bf6494caf6c1533ceabaa4c49f86 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Fri, 17 May 2024 11:58:08 +0200 Subject: [PATCH 39/53] add more optimize tests --- tests/unit/test_power_transformer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 61ba4c1d..df6edbf4 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -30,6 +30,7 @@ def test_lambda_function(coeffs, t, expected): def test_fit_power_transformer(): # with enough random normal data points the fit should be close to 1 and 0 + np.random.seed(0) gridcells = 1 n_months = 100_000 monthly_residuals = np.random.standard_normal((n_months, gridcells)) * 10 @@ -49,6 +50,11 @@ def test_fit_power_transformer(): [ (-2, [1, 2]), # left skewed data (2, [0, 1]), # right skewed data + (-5, [1, 2]), # more skew + (5, [0, 1]), # more skew + (-0.5, [1, 2]), # less skew + (0.5, [0, 1]), # less skew + (0, [0.9, 1.1]), # no skew ], ) def test_yeo_johnson_optimize_lambda(skew, bounds): From 8bdad5cb11e6dbbf2e1e721e17590e5389a36c4f Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Fri, 17 May 2024 12:00:58 +0200 Subject: [PATCH 40/53] test numerical stability --- tests/unit/test_power_transformer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index df6edbf4..92cad496 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -40,10 +40,14 @@ def test_fit_power_transformer(): pt.fit(monthly_residuals, yearly_T, gridcells) result = pt.coeffs_ + # to test viability expected = np.array([[1, 0]]) - np.testing.assert_allclose(result, expected, atol=1e-2) + # to test numerical stability + expected_exact = np.array([[0.99589859, 0.00178623]]) + np.testing.assert_allclose(result, expected_exact, atol=1e-7) + @pytest.mark.parametrize( "skew, bounds", From 8ee4f9d56b56477a1bd319ec7439db2f2a9391e7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 10:03:54 +0000 Subject: [PATCH 41/53] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/test_power_transformer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 92cad496..f8551001 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -52,13 +52,13 @@ def test_fit_power_transformer(): @pytest.mark.parametrize( "skew, bounds", [ - (-2, [1, 2]), # left skewed data - (2, [0, 1]), # right skewed data - (-5, [1, 2]), # more skew - (5, [0, 1]), # more skew - (-0.5, [1, 2]), # less skew - (0.5, [0, 1]), # less skew - (0, [0.9, 1.1]), # no skew + (-2, [1, 2]), # left skewed data + (2, [0, 1]), # right skewed data + (-5, [1, 2]), # more skew + (5, [0, 1]), # more skew + (-0.5, [1, 2]), # less skew + (0.5, [0, 1]), # less skew + (0, [0.9, 1.1]), # no skew ], ) def test_yeo_johnson_optimize_lambda(skew, bounds): From f22a20ae651fdbb07b89a8c1157fe001846344df Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Thu, 23 May 2024 11:05:02 +0200 Subject: [PATCH 42/53] fit power transformer with standardize = False --- tests/unit/test_power_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index f8551001..87849cbc 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -36,7 +36,7 @@ def test_fit_power_transformer(): monthly_residuals = np.random.standard_normal((n_months, gridcells)) * 10 yearly_T = np.ones((n_months, gridcells)) - pt = PowerTransformerVariableLambda() + pt = PowerTransformerVariableLambda(standardize=False) pt.fit(monthly_residuals, yearly_T, gridcells) result = pt.coeffs_ From abc63bdef580fa2f48e5eabf0b3744010aa19c7e Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Thu, 23 May 2024 11:44:39 +0200 Subject: [PATCH 43/53] test random data --- tests/unit/test_power_transformer.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 87849cbc..30ce37c8 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -34,6 +34,16 @@ def test_fit_power_transformer(): gridcells = 1 n_months = 100_000 monthly_residuals = np.random.standard_normal((n_months, gridcells)) * 10 + np.testing.assert_allclose(monthly_residuals[0:10], np.array([[17.64052346], + [ 4.00157208], + [ 9.78737984], + [22.40893199], + [18.6755799 ], + [-9.7727788 ], + [ 9.50088418], + [-1.51357208], + [-1.03218852], + [ 4.10598502]]), atol = 1e-7) yearly_T = np.ones((n_months, gridcells)) pt = PowerTransformerVariableLambda(standardize=False) From 3b3036276bc133fe0d1be3df12c9a446952a4b43 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 09:45:59 +0000 Subject: [PATCH 44/53] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/test_power_transformer.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 30ce37c8..123472ce 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -34,16 +34,24 @@ def test_fit_power_transformer(): gridcells = 1 n_months = 100_000 monthly_residuals = np.random.standard_normal((n_months, gridcells)) * 10 - np.testing.assert_allclose(monthly_residuals[0:10], np.array([[17.64052346], - [ 4.00157208], - [ 9.78737984], - [22.40893199], - [18.6755799 ], - [-9.7727788 ], - [ 9.50088418], - [-1.51357208], - [-1.03218852], - [ 4.10598502]]), atol = 1e-7) + np.testing.assert_allclose( + monthly_residuals[0:10], + np.array( + [ + [17.64052346], + [4.00157208], + [9.78737984], + [22.40893199], + [18.6755799], + [-9.7727788], + [9.50088418], + [-1.51357208], + [-1.03218852], + [4.10598502], + ] + ), + atol=1e-7, + ) yearly_T = np.ones((n_months, gridcells)) pt = PowerTransformerVariableLambda(standardize=False) From fcaab1ae085633f90ae5b72cfcf0c655fa9ba8fa Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Thu, 23 May 2024 11:52:38 +0200 Subject: [PATCH 45/53] remove random data testing again --- tests/unit/test_power_transformer.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 123472ce..87849cbc 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -34,24 +34,6 @@ def test_fit_power_transformer(): gridcells = 1 n_months = 100_000 monthly_residuals = np.random.standard_normal((n_months, gridcells)) * 10 - np.testing.assert_allclose( - monthly_residuals[0:10], - np.array( - [ - [17.64052346], - [4.00157208], - [9.78737984], - [22.40893199], - [18.6755799], - [-9.7727788], - [9.50088418], - [-1.51357208], - [-1.03218852], - [4.10598502], - ] - ), - atol=1e-7, - ) yearly_T = np.ones((n_months, gridcells)) pt = PowerTransformerVariableLambda(standardize=False) From 2bd00c27777b1e19f8b11653996d0e80408042d1 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Thu, 23 May 2024 11:58:43 +0200 Subject: [PATCH 46/53] test eps equality --- tests/unit/test_power_transformer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 87849cbc..15d752d3 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -36,6 +36,8 @@ def test_fit_power_transformer(): monthly_residuals = np.random.standard_normal((n_months, gridcells)) * 10 yearly_T = np.ones((n_months, gridcells)) + assert np.finfo(np.float64).eps == 2.220446049250313e-16 + pt = PowerTransformerVariableLambda(standardize=False) pt.fit(monthly_residuals, yearly_T, gridcells) From a69a8bd3dce09eb77700d89c561d7565c3da75ce Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Thu, 23 May 2024 12:08:23 +0200 Subject: [PATCH 47/53] remove eps testing again --- tests/unit/test_power_transformer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 15d752d3..87849cbc 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -36,8 +36,6 @@ def test_fit_power_transformer(): monthly_residuals = np.random.standard_normal((n_months, gridcells)) * 10 yearly_T = np.ones((n_months, gridcells)) - assert np.finfo(np.float64).eps == 2.220446049250313e-16 - pt = PowerTransformerVariableLambda(standardize=False) pt.fit(monthly_residuals, yearly_T, gridcells) From 3b418c1a271a0b88a171824269d9d5b40a682841 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Thu, 23 May 2024 12:13:15 +0200 Subject: [PATCH 48/53] try with Nelder Mead --- mesmer/mesmer_m/power_transformer.py | 2 +- tests/unit/test_power_transformer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mesmer/mesmer_m/power_transformer.py b/mesmer/mesmer_m/power_transformer.py index d47c910d..c8dbfa91 100644 --- a/mesmer/mesmer_m/power_transformer.py +++ b/mesmer/mesmer_m/power_transformer.py @@ -153,7 +153,7 @@ def _neg_log_likelihood(coeffs): _neg_log_likelihood, first_guess, bounds=bounds, - method="SLSQP", + method="Nelder-Mead", ).x def _yeo_johnson_transform(self, local_monthly_residuals, lambdas): diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 87849cbc..356d51ba 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -45,7 +45,7 @@ def test_fit_power_transformer(): np.testing.assert_allclose(result, expected, atol=1e-2) # to test numerical stability - expected_exact = np.array([[0.99589859, 0.00178623]]) + expected_exact = np.array([[9.976913e-01, -1.998520e-05]]) np.testing.assert_allclose(result, expected_exact, atol=1e-7) From bf1acf4e05a3d9d65cceb9e7e0e9c0c6ae6e2401 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Thu, 23 May 2024 15:51:18 +0200 Subject: [PATCH 49/53] change back to SLSQP --- mesmer/mesmer_m/power_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesmer/mesmer_m/power_transformer.py b/mesmer/mesmer_m/power_transformer.py index c8dbfa91..d47c910d 100644 --- a/mesmer/mesmer_m/power_transformer.py +++ b/mesmer/mesmer_m/power_transformer.py @@ -153,7 +153,7 @@ def _neg_log_likelihood(coeffs): _neg_log_likelihood, first_guess, bounds=bounds, - method="Nelder-Mead", + method="SLSQP", ).x def _yeo_johnson_transform(self, local_monthly_residuals, lambdas): From 25b2e36aa267477893372e51721580751d5928f6 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Thu, 23 May 2024 16:24:11 +0200 Subject: [PATCH 50/53] set all standardize to false --- tests/unit/test_power_transformer.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 356d51ba..62756410 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -36,7 +36,8 @@ def test_fit_power_transformer(): monthly_residuals = np.random.standard_normal((n_months, gridcells)) * 10 yearly_T = np.ones((n_months, gridcells)) - pt = PowerTransformerVariableLambda(standardize=False) + pt = PowerTransformerVariableLambda(standardize=False) + # standardize false speeds up the fit and does not impact the coefficients pt.fit(monthly_residuals, yearly_T, gridcells) result = pt.coeffs_ @@ -68,7 +69,7 @@ def test_yeo_johnson_optimize_lambda(skew, bounds): yearly_T = np.random.randn(n_years) local_monthly_residuals = sp.stats.skewnorm.rvs(skew, size=n_years) - pt = PowerTransformerVariableLambda() + pt = PowerTransformerVariableLambda(standardize=False) pt.coeffs_ = pt._yeo_johnson_optimize_lambda(local_monthly_residuals, yearly_T) lmbda = lambda_function(pt.coeffs_, yearly_T) transformed = pt._yeo_johnson_transform(local_monthly_residuals, lmbda) @@ -82,8 +83,7 @@ def test_transform(): n_ts = 20 n_gridcells = 10 - pt = PowerTransformerVariableLambda() - pt.standardize = False + pt = PowerTransformerVariableLambda(standardize=False) pt.coeffs_ = np.tile([1, 0], (n_gridcells, 1)) monthly_residuals = np.ones((n_ts, n_gridcells)) @@ -96,7 +96,7 @@ def test_transform(): def test_yeo_johnson_transform(): - pt = PowerTransformerVariableLambda() + pt = PowerTransformerVariableLambda(standardize=False) # test all possible combinations of local_monthly_residuals and lambdas local_monthly_residuals = np.array([0.0, 1.0, 0.0, 1.0, -1.0, -1.0]) @@ -117,8 +117,7 @@ def test_transform_roundtrip(): ) yearly_T = np.zeros((n_ts, n_gridcells)) - pt = PowerTransformerVariableLambda() - pt.standardize = False + pt = PowerTransformerVariableLambda(standardize = False) # dummy lambdas (since yearly_T is zero lambda comes out to be second coefficient) # we have all cases for lambdas 0 and 2 (special cases), 1 (identity case) # lambda between 0 and 1 and lambda between 1 and 2 for concave and convex cases From 7b8ba9c274b5d5b0e431f65a5eba71a48be51ea6 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Thu, 23 May 2024 16:39:38 +0200 Subject: [PATCH 51/53] add test for scaler --- tests/unit/test_power_transformer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index 62756410..d24bff12 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -129,3 +129,20 @@ def test_transform_roundtrip(): result = pt.inverse_transform(transformed, yearly_T) np.testing.assert_allclose(result, monthly_residuals, atol=1e-7) + +def test_standard_scaler(): + # generate random data with mean different from 0 and std different from one + np.random.seed(0) + n_ts = 100_000 + n_gridcells = 1 + monthly_residuals = np.random.randn(n_ts, n_gridcells) * 10 + 5 + yearly_T = np.zeros((n_ts, n_gridcells)) + + # fit the transformer + pt = PowerTransformerVariableLambda(standardize=True) + pt.fit(monthly_residuals, yearly_T, n_gridcells) + transformed = pt.transform(monthly_residuals, yearly_T) + + # the transformed data should have mean close to 0 and std close to 1 + np.testing.assert_allclose(np.mean(transformed), 0, atol=1e-2) + np.testing.assert_allclose(np.std(transformed), 1, atol=1e-2) From 7691916a7295e4be511f149b06f451b684c71dee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 16:02:21 +0000 Subject: [PATCH 52/53] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/test_power_transformer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index d24bff12..b671a850 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -36,7 +36,7 @@ def test_fit_power_transformer(): monthly_residuals = np.random.standard_normal((n_months, gridcells)) * 10 yearly_T = np.ones((n_months, gridcells)) - pt = PowerTransformerVariableLambda(standardize=False) + pt = PowerTransformerVariableLambda(standardize=False) # standardize false speeds up the fit and does not impact the coefficients pt.fit(monthly_residuals, yearly_T, gridcells) @@ -117,7 +117,7 @@ def test_transform_roundtrip(): ) yearly_T = np.zeros((n_ts, n_gridcells)) - pt = PowerTransformerVariableLambda(standardize = False) + pt = PowerTransformerVariableLambda(standardize=False) # dummy lambdas (since yearly_T is zero lambda comes out to be second coefficient) # we have all cases for lambdas 0 and 2 (special cases), 1 (identity case) # lambda between 0 and 1 and lambda between 1 and 2 for concave and convex cases @@ -130,8 +130,9 @@ def test_transform_roundtrip(): np.testing.assert_allclose(result, monthly_residuals, atol=1e-7) + def test_standard_scaler(): - # generate random data with mean different from 0 and std different from one + # generate random data with mean different from 0 and std different from one np.random.seed(0) n_ts = 100_000 n_gridcells = 1 From c21905f3cefa3f6f474d7885ce83a33207e51883 Mon Sep 17 00:00:00 2001 From: veni-vidi-vici-dormivi Date: Thu, 23 May 2024 18:24:50 +0200 Subject: [PATCH 53/53] add inverse trasform to scaler test --- tests/unit/test_power_transformer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_power_transformer.py b/tests/unit/test_power_transformer.py index b671a850..c7f242cd 100644 --- a/tests/unit/test_power_transformer.py +++ b/tests/unit/test_power_transformer.py @@ -147,3 +147,7 @@ def test_standard_scaler(): # the transformed data should have mean close to 0 and std close to 1 np.testing.assert_allclose(np.mean(transformed), 0, atol=1e-2) np.testing.assert_allclose(np.std(transformed), 1, atol=1e-2) + + # inverse transform should give back the original data + result = pt.inverse_transform(transformed, yearly_T) + np.testing.assert_allclose(result, monthly_residuals, atol=1e-7)