From b08fe9d568bfd0af3af5dfadf07e91cdd335dfab Mon Sep 17 00:00:00 2001 From: anujsinha3 Date: Fri, 27 Oct 2023 11:49:16 -0700 Subject: [PATCH] refactor: change freq_norm from string to enum (#248) * refactor: change freq_norm from string to enum * refactor: pre-commit hook format changes * refactor: change freq_norm from string to enum * refactor: correct import * refactor: add parameterized test cases * refactor: fix import * refactor: fix variable type * refactor: fix data sphinx error * refactor: revert the change * refactor: change initialization of freq_norm from string to enum * refactor: change initialization of freq_norm from string to enum * refactor: change initialization of freq_norm from string to enum * refactor: Fix navigation_with_keys warning. Set navigation_with_keys to False to get rid of the warning. Setting it to True has negative accessibility implications: pydata/pydata-sphinx-theme#1492 * refactor: Fix navigation_with_keys warning. Set navigation_with_keys to True * refactor: revert change * refactor: set navigation_with_keys as false to prevent sphinx validation error * refactor: style consistency with \n --- script/test_whiten.py | 11 ++--- src/noisepy/seis/datatypes.py | 10 ++++- src/noisepy/seis/noise_module.py | 4 +- tests/test_whiten.py | 45 ++++++++++++------- tutorials/_config.yml | 7 +++ tutorials/get_started.ipynb | 6 +-- tutorials/noisepy_pnwstore_tutorial.ipynb | 4 +- tutorials/noisepy_scedc_tutorial.ipynb | 4 +- .../old/cross_correlation_from_sac.ipynb | 3 +- .../download_toASDF_cross_correlation.ipynb | 4 +- tutorials/run_mpi_scedc.ipynb | 4 +- 11 files changed, 65 insertions(+), 37 deletions(-) diff --git a/script/test_whiten.py b/script/test_whiten.py index 456b787c..49b08d35 100644 --- a/script/test_whiten.py +++ b/script/test_whiten.py @@ -3,6 +3,7 @@ import scipy from scipy.fftpack import next_fast_len +from noisepy.seis.datatypes import FreqNorm from noisepy.seis.noise_module import moving_ave, whiten @@ -62,9 +63,9 @@ def whiten_original(data, fft_para): 1j * np.angle(FFTRawSign[:, low:left]) ) # Pass band: - if freq_norm == "phase_only": + if freq_norm == FreqNorm.PHASE_ONLY: FFTRawSign[:, left:right] = np.exp(1j * np.angle(FFTRawSign[:, left:right])) - elif freq_norm == "rma": + elif freq_norm == FreqNorm.RMA: for ii in range(data.shape[0]): tave = moving_ave(np.abs(FFTRawSign[ii, left:right]), smooth_N) FFTRawSign[ii, left:right] = FFTRawSign[ii, left:right] / tave @@ -82,9 +83,9 @@ def whiten_original(data, fft_para): 1j * np.angle(FFTRawSign[low:left]) ) # Pass band: - if freq_norm == "phase_only": + if freq_norm == FreqNorm.PHASE_ONLY: FFTRawSign[left:right] = np.exp(1j * np.angle(FFTRawSign[left:right])) - elif freq_norm == "rma": + elif freq_norm == FreqNorm.RMA: tave = moving_ave(np.abs(FFTRawSign[left:right]), smooth_N) FFTRawSign[left:right] = FFTRawSign[left:right] / tave # Right tapering: @@ -109,7 +110,7 @@ def whiten_original(data, fft_para): "freqmin": 0.01, "freqmax": 0.2, "smooth_N": 1, - "freq_norm": "phase_only", + "freq_norm": FreqNorm.PHASE_ONLY, } # 1 D case diff --git a/src/noisepy/seis/datatypes.py b/src/noisepy/seis/datatypes.py index 9e14da7d..ac5f799d 100644 --- a/src/noisepy/seis/datatypes.py +++ b/src/noisepy/seis/datatypes.py @@ -110,6 +110,12 @@ class StackMethod(Enum): ALL = "all" +class FreqNorm(Enum): + RMA = "rma" + NO = "no" + PHASE_ONLY = "phase_only" + + class ConfigParameters(BaseModel): model_config = ConfigDict(validate_default=True) @@ -138,8 +144,8 @@ class ConfigParameters(BaseModel): step: float = Field(default=450.0, description="overlapping between each cc_len (sec)") freqmin: float = Field(default=0.05) freqmax: float = Field(default=2.0) - freq_norm: str = Field( - default="rma", description="choose between 'rma' for a soft whitenning or 'no' for no whitening" + freq_norm: FreqNorm = Field( + default=FreqNorm.RMA.value, description="choose between 'rma' for a soft whitenning or 'no' for no whitening" ) # TODO: change "no"for "None", and add "one_bit"as an option # TODO: change time_norm option from "no"to "None" diff --git a/src/noisepy/seis/noise_module.py b/src/noisepy/seis/noise_module.py index eb38be71..02ef4630 100644 --- a/src/noisepy/seis/noise_module.py +++ b/src/noisepy/seis/noise_module.py @@ -25,7 +25,7 @@ from scipy.fftpack import next_fast_len from scipy.signal import hilbert -from .datatypes import ChannelData, ConfigParameters, StackMethod +from .datatypes import ChannelData, ConfigParameters, FreqNorm, StackMethod logger = logging.getLogger(__name__) """ @@ -559,7 +559,7 @@ def noise_processing(fft_para: ConfigParameters, dataS): white = dataS # -----to whiten or not------ - if fft_para.freq_norm != "no": + if fft_para.freq_norm != FreqNorm.NO: source_white = whiten(white, fft_para) # whiten and return FFT else: Nfft = int(next_fast_len(int(dataS.shape[1]))) diff --git a/tests/test_whiten.py b/tests/test_whiten.py index 622e29e6..3bfbda2e 100644 --- a/tests/test_whiten.py +++ b/tests/test_whiten.py @@ -1,9 +1,11 @@ import matplotlib.pyplot as plt import numpy as np +import pytest import scipy from scipy.fftpack import next_fast_len from noisepy.seis.correlate import ConfigParameters +from noisepy.seis.datatypes import FreqNorm from noisepy.seis.noise_module import moving_ave, whiten @@ -56,9 +58,9 @@ def whiten_original(data, fft_para: ConfigParameters): 1j * np.angle(FFTRawSign[:, low:left]) ) # Pass band: - if fft_para.freq_norm == "phase_only": + if fft_para.freq_norm == FreqNorm.PHASE_ONLY: FFTRawSign[:, left:right] = np.exp(1j * np.angle(FFTRawSign[:, left:right])) - elif fft_para.freq_norm == "rma": + elif fft_para.freq_norm == FreqNorm.RMA: for ii in range(data.shape[0]): tave = moving_ave(np.abs(FFTRawSign[ii, left:right]), fft_para.smooth_N) FFTRawSign[ii, left:right] = FFTRawSign[ii, left:right] / tave @@ -76,9 +78,9 @@ def whiten_original(data, fft_para: ConfigParameters): 1j * np.angle(FFTRawSign[low:left]) ) # Pass band: - if fft_para.freq_norm == "phase_only": + if fft_para.freq_norm == FreqNorm.PHASE_ONLY: FFTRawSign[left:right] = np.exp(1j * np.angle(FFTRawSign[left:right])) - elif fft_para.freq_norm == "rma": + elif fft_para.freq_norm == FreqNorm.RMA: tave = moving_ave(np.abs(FFTRawSign[left:right]), fft_para.smooth_N) FFTRawSign[left:right] = FFTRawSign[left:right] / tave # Right tapering: @@ -98,16 +100,16 @@ def whiten_original(data, fft_para: ConfigParameters): # it is not expected that the smoothed version returns the same, so currently no test for that # (would be good to add one based on some expected outcome) -fft_para = ConfigParameters() -fft_para.samp_freq = 1.0 -fft_para.freqmin = 0.01 -fft_para.freqmax = 0.2 -fft_para.smooth_N = 1 -fft_para.freq_norm = "phase_only" - -def whiten1d(): +def whiten1d(freq_norm: FreqNorm): # 1 D case + fft_para = ConfigParameters() + fft_para.samp_freq = 1.0 + fft_para.freqmin = 0.01 + fft_para.freqmax = 0.2 + fft_para.smooth_N = 1 + fft_para.freq_norm = freq_norm + data = np.random.random(1000) white_original = whiten_original(data, fft_para) white_new = whiten(data, fft_para) @@ -118,8 +120,15 @@ def whiten1d(): return white_original, white_new -def whiten2d(): +def whiten2d(freq_norm: FreqNorm): # 2 D case + fft_para = ConfigParameters() + fft_para.samp_freq = 1.0 + fft_para.freqmin = 0.01 + fft_para.freqmax = 0.2 + fft_para.smooth_N = 1 + fft_para.freq_norm = freq_norm + data = np.random.random((5, 1000)) white_original = whiten_original(data, fft_para) white_new = whiten(data, fft_para) @@ -157,12 +166,14 @@ def plot_2d(white_original, white_new): # Use wrappers since test functions are not supposed to return values -def test_whiten1d(): - _, _ = whiten1d() +@pytest.mark.parametrize("freqNorm", [FreqNorm.PHASE_ONLY, FreqNorm.RMA]) +def test_whiten1d(freqNorm: FreqNorm): + _, _ = whiten1d(freqNorm) -def test_whiten2d(): - _, _ = whiten2d() +@pytest.mark.parametrize("freqNorm", [FreqNorm.PHASE_ONLY, FreqNorm.RMA]) +def test_whiten2d(freqNorm: FreqNorm): + _, _ = whiten2d(freqNorm) if __name__ == "__main__": diff --git a/tutorials/_config.yml b/tutorials/_config.yml index 163ed962..6c745a57 100644 --- a/tutorials/_config.yml +++ b/tutorials/_config.yml @@ -32,3 +32,10 @@ repository: html: use_issues_button: true use_repository_button: true + +# Set navigation_with_keys: false to prevent pydata-sphinx-theme validation warning/error +# See https://github.com/pydata/pydata-sphinx-theme/issues/1492 +sphinx: + config: + html_theme_options: + navigation_with_keys: false diff --git a/tutorials/get_started.ipynb b/tutorials/get_started.ipynb index 0357a214..609b5945 100644 --- a/tutorials/get_started.ipynb +++ b/tutorials/get_started.ipynb @@ -60,7 +60,7 @@ "source": [ "from noisepy.seis import download, cross_correlate, stack, plotting_modules, __version__\n", "from noisepy.seis.asdfstore import ASDFRawDataStore, ASDFCCStore, ASDFStackStore\n", - "from noisepy.seis.datatypes import ConfigParameters\n", + "from noisepy.seis.datatypes import ConfigParameters, FreqNorm\n", "from dateutil.parser import isoparse\n", "import os\n", "print(f\"Using NoisePy version {__version__}\")\n", @@ -119,7 +119,7 @@ "config.max_over_std = 10 # threshold to remove window of bad signals: set it to 10*9 if prefer not to remove them\n", "\n", "# TEMPORAL and SPECTRAL NORMALISATION\n", - "config.freq_norm= \"rma\" # choose between \"rma\" for a soft whitenning or \"no\" for no whitening. Pure whitening is not implemented correctly at this point.\n", + "config.freq_norm= FreqNorm.RMA # choose between \"rma\" for a soft whitenning or \"no\" for no whitening. Pure whitening is not implemented correctly at this point.\n", "config.smoothspect_N = 10 # moving window length to smooth spectrum amplitude (points)\n", " # here, choose smoothspect_N for the case of a strict whitening (e.g., phase_only)\n", "\n", @@ -248,7 +248,7 @@ }, "outputs": [], "source": [ - "config.freq_norm = \"rma\"\n", + "config.freq_norm = FreqNorm.RMA\n", "raw_store = ASDFRawDataStore(raw_data_path) # Store for reading raw data\n", "cc_store = ASDFCCStore(cc_data_path) # Store for writing CC data\n", "\n", diff --git a/tutorials/noisepy_pnwstore_tutorial.ipynb b/tutorials/noisepy_pnwstore_tutorial.ipynb index 3977f089..6cd93733 100644 --- a/tutorials/noisepy_pnwstore_tutorial.ipynb +++ b/tutorials/noisepy_pnwstore_tutorial.ipynb @@ -77,7 +77,7 @@ "from noisepy.seis.asdfstore import ASDFCCStore, ASDFStackStore # Object to store ASDF data within noisepy\n", "from noisepy.seis.scedc_s3store import channel_filter\n", "from noisepy.seis.pnwstore import PNWDataStore\n", - "from noisepy.seis.datatypes import ConfigParameters, Channel, ChannelData, ChannelType, Station # Main configuration object\n", + "from noisepy.seis.datatypes import ConfigParameters, Channel, ChannelData, ChannelType, FreqNorm, Station # Main configuration object\n", "from noisepy.seis.channelcatalog import XMLStationChannelCatalog # Required stationXML handling object\n", "import os\n", "from datetime import datetime\n", @@ -185,7 +185,7 @@ "config.max_over_std = 10 # threshold to remove window of bad signals: set it to 10*9 if prefer not to remove them\n", "\n", "# TEMPORAL and SPECTRAL NORMALISATION\n", - "config.freq_norm= \"rma\" # choose between \"rma\" for a soft whitenning or \"no\" for no whitening. Pure whitening is not implemented correctly at this point.\n", + "config.freq_norm= FreqNorm.RMA # choose between \"rma\" for a soft whitenning or \"no\" for no whitening. Pure whitening is not implemented correctly at this point.\n", "config.smoothspect_N = 10 # moving window length to smooth spectrum amplitude (points)\n", " # here, choose smoothspect_N for the case of a strict whitening (e.g., phase_only)\n", "\n", diff --git a/tutorials/noisepy_scedc_tutorial.ipynb b/tutorials/noisepy_scedc_tutorial.ipynb index e1a93080..7b2be15b 100644 --- a/tutorials/noisepy_scedc_tutorial.ipynb +++ b/tutorials/noisepy_scedc_tutorial.ipynb @@ -67,7 +67,7 @@ "from noisepy.seis import cross_correlate, stack, plotting_modules, __version__ # noisepy core functions\n", "from noisepy.seis.asdfstore import ASDFCCStore, ASDFStackStore # Object to store ASDF data within noisepy\n", "from noisepy.seis.scedc_s3store import SCEDCS3DataStore, channel_filter # Object to query SCEDC data from on S3\n", - "from noisepy.seis.datatypes import ConfigParameters, StackMethod # Main configuration object\n", + "from noisepy.seis.datatypes import ConfigParameters, FreqNorm, StackMethod # Main configuration object\n", "from noisepy.seis.channelcatalog import XMLStationChannelCatalog # Required stationXML handling object\n", "import os\n", "from datetime import datetime\n", @@ -201,7 +201,7 @@ "config.max_over_std = 10 # threshold to remove window of bad signals: set it to 10*9 if prefer not to remove them\n", "\n", "################### SPECTRAL NORMALIZATION ############\n", - "config.freq_norm= \"rma\" # choose between \"rma\" for a soft whitening or \"no\" for no whitening. Pure whitening is not implemented correctly at this point.\n", + "config.freq_norm= FreqNorm.RMA # choose between \"rma\" for a soft whitening or \"no\" for no whitening. Pure whitening is not implemented correctly at this point.\n", "config.smoothspect_N = 10 # moving window length to smooth spectrum amplitude (points)\n", " # here, choose smoothspect_N for the case of a strict whitening (e.g., phase_only)\n", "\n", diff --git a/tutorials/old/cross_correlation_from_sac.ipynb b/tutorials/old/cross_correlation_from_sac.ipynb index e96f902d..e702963a 100644 --- a/tutorials/old/cross_correlation_from_sac.ipynb +++ b/tutorials/old/cross_correlation_from_sac.ipynb @@ -72,6 +72,7 @@ "import matplotlib.pyplot as plt \n", "\n", "sys.path.insert(1,'../src')\n", + "from noisepy.seis.datatypes import FreqNorm\n", "from noisepy.seis import noise_module" ] }, @@ -105,7 +106,7 @@ "freqmin = 0.1 # frequency range\n", "freqmax = 8 \n", "\n", - "freq_norm = 'rma' # rma-> running mean average for frequency-domain normalization\n", + "freq_norm = FreqNorm.RMA # rma-> running mean average for frequency-domain normalization\n", "time_norm = 'no' # no-> no time-domain normalization; other options are 'rma' for running-mean and 'one-bit'\n", "cc_method = 'xcorr' # xcorr-> pure cross correlation; other option is 'decon'\n", "substack = False # sub-stack daily cross-correlation or not\n", diff --git a/tutorials/old/download_toASDF_cross_correlation.ipynb b/tutorials/old/download_toASDF_cross_correlation.ipynb index e59694ec..c12053ac 100644 --- a/tutorials/old/download_toASDF_cross_correlation.ipynb +++ b/tutorials/old/download_toASDF_cross_correlation.ipynb @@ -69,8 +69,10 @@ "import pandas as pd\n", "from obspy import UTCDateTime\n", "import matplotlib.pyplot as plt\n", + "from noisepy.seis.datatypes import FreqNorm\n", "from obspy.clients.fdsn import Client\n", "\n", + "\n", "sys.path.insert(1,'../src')\n", "import noise_module" ] @@ -206,7 +208,7 @@ "dt = 1/samp_freq # sampling time intervals of the data: in real case it reads from data directly\n", "inc_hours = 24 # basic length (hour) of the continous noise data \n", "\n", - "freq_norm = 'rma' # rma-> running mean average for frequency-domain normalization\n", + "freq_norm = FreqNorm.RMA # rma-> running mean average for frequency-domain normalization\n", "time_norm = 'no' # no-> no time-domain normalization; other options are 'rma' for running-mean and 'one-bit'\n", "cc_method = 'xcorr' # xcorr-> pure cross correlation; other option is 'decon'\n", "substack = False # sub-stack daily cross-correlation or not\n", diff --git a/tutorials/run_mpi_scedc.ipynb b/tutorials/run_mpi_scedc.ipynb index c17a841e..c837d5c7 100644 --- a/tutorials/run_mpi_scedc.ipynb +++ b/tutorials/run_mpi_scedc.ipynb @@ -9,7 +9,7 @@ "from noisepy.seis import cross_correlate, stack, plotting_modules # noisepy core functions\n", "from noisepy.seis.asdfstore import ASDFCCStore # Object to store ASDF data within noisepy\n", "from noisepy.seis.scedc_s3store import SCEDCS3DataStore, channel_filter # Object to query SCEDC data from on S3\n", - "from noisepy.seis.datatypes import ConfigParameters # Main configuration object\n", + "from noisepy.seis.datatypes import ConfigParameters, FreqNorm # Main configuration object\n", "from noisepy.seis.channelcatalog import XMLStationChannelCatalog # Required stationXML handling object\n", "import os\n", "import glob\n", @@ -27,7 +27,7 @@ "stack_data_path = os.path.join(path, \"STACK\")\n", "os.makedirs(cc_data_path, exist_ok=True)\n", "os.makedirs(stack_data_path, exist_ok=True)\n", - "freq_norm=\"rma\"\n", + "freq_norm = FreqNorm.RMA\n", "cc_store = ASDFCCStore(cc_data_path) # Store for writing CC data\n", "\n", "\n",