diff --git a/CHANGELOG.md b/CHANGELOG.md index 8544cd30..2a3c518d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ Changelog ========= +Version 0.82.0 +-------------- +* added nkuluflag module + +Version 0.81.7 +-------------- +* bugfixes +* added whisper feature extractor + Version 0.81.6 -------------- * updated documentation diff --git a/README.md b/README.md index 2ca0a2e5..eebbfda3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +usage: nkuluflag.py [-h] [--config CONFIG] [--data [DATA ...]] [--label [LABEL ...]] [--tuning_params [TUNING_PARAMS ...]] [--layers [LAYERS ...]] [--model MODEL] [--feat FEAT] [--set SET] + [--with_os WITH_OS] [--target TARGET] [--epochs EPOCHS] [--runs RUNS] [--learning_rate LEARNING_RATE] [--drop DROP] - [Overview](#overview) - [Confusion matrix](#confusion-matrix) - [Epoch progression](#epoch-progression) @@ -159,6 +161,14 @@ All of them take *--config * as an argument. * **nkululeko.predict**: [predict features](http://blog.syntheticspeech.de/2023/08/16/nkululeko-how-to-predict-labels-for-your-data-from-existing-models-and-check-them/) like SNR, MOS, arousal/valence, age/gender, with DNN models * **nkululeko.segment**: [segment a database](http://blog.syntheticspeech.de/2023/07/14/nkululeko-segmenting-a-database/) based on VAD (voice activity detection) * **nkululeko.resample**: check on all [sampling rates and change](http://blog.syntheticspeech.de/2023/08/31/how-to-fix-different-sampling-rates-in-a-dataset-with-nkululeko/) to 16kHz +* **nkululeko.nkuluflag**: a convenient module to specify configuration parameters on the command-line. + * usage: nkuluflag.py [-h] [--config CONFIG] [--data [DATA ...]] [--label [LABEL ...]] [--tuning_params [TUNING_PARAMS ...]] [--layers [LAYERS ...]] [--model MODEL] [--feat FEAT] [--set SET] + [--with_os WITH_OS] [--target TARGET] [--epochs EPOCHS] [--runs RUNS] [--learning_rate LEARNING_RATE] [--drop DROP] + + + + + There's my [blog](http://blog.syntheticspeech.de/?s=nkululeko) with tutorials: * [Introduction](http://blog.syntheticspeech.de/2021/08/04/machine-learning-experiment-framework/) diff --git a/ini_file.md b/ini_file.md index aa7b9b50..5047912b 100644 --- a/ini_file.md +++ b/ini_file.md @@ -193,6 +193,8 @@ * "hubert-base-ls960", "hubert-large-ll60k", "hubert-large-ls960-ft", hubert-xlarge-ll60k, "hubert-xlarge-ls960-ft" * **WavLM**: * "wavlm-base", "wavlm-base-plus", "wavlm-large" + * **Whisper**: [whisper models](https://huggingface.co/models?other=whisper) + * "whisper-base", "whisper-large", "whisper-medium", "whisper-tiny" * **audmodel**: [audEERING emotion model embeddings](https://arxiv.org/abs/2203.07378), wav2vec2.0 model finetuned on [MSPPodcast](https://ecs.utdallas.edu/research/researchlabs/msp-lab/MSP-Podcast.html) emotions, embeddings * **aud.model** = ./audmodel/ (*path to the audEERING model folder*) * **auddim**: [audEERING emotion model dimensions](https://arxiv.org/abs/2203.07378), wav2vec2.0 model finetuned on [MSPPodcast](https://ecs.utdallas.edu/research/researchlabs/msp-lab/MSP-Podcast.html) arousal, dominance, valence diff --git a/meta/demos/multiple_exeriments/do_experiments.py b/meta/demos/multiple_exeriments/do_experiments.py index 8931a6b7..70fcd3c0 100644 --- a/meta/demos/multiple_exeriments/do_experiments.py +++ b/meta/demos/multiple_exeriments/do_experiments.py @@ -1,48 +1,35 @@ import os -src_path = 'demos/multiple_exeriments/' classifiers = [ - {'--model': 'mlp', - '--layers': '\"{\'l1\':64,\'l2\':16}\"'}, - {'--model': 'mlp', - '--layers': '\"{\'l1\':64,\'l2\':16}\"', - '--learning_rate': '.1',}, - {'--model': 'mlp', - '--layers': '\"{\'l1\':64,\'l2\':16}\"', - '--learning_rate': '.0001', - '--drop': '.3', + {"--model": "mlp", "--layers": "\"{'l1':64,'l2':16}\"", "--epochs": 100}, + { + "--model": "mlp", + "--layers": "\"{'l1':128,'l2':64,'l3':16}\"", + "--learning_rate": ".01", + "--drop": ".3", + "--epochs": 100, }, - {'--model': 'xgb', - '--epochs':1}, - {'--model': 'svm', - '--epochs':1}, + {"--model": "xgb", "--epochs": 1}, + {"--model": "svm", "--epochs": 1}, ] features = [ - {'--feat': 'os'}, - # {'--feat': 'os', + {"--feat": "os"}, + # {'--feat': 'os', # '--set': 'ComParE_2016', # }, - # {'--feat': 'mld'}, - # {'--feat': 'mld', - # '--with_os': 'True', - # }, - # {'--feat': 'xbow'}, - # {'--feat': 'xbow', - # '--with_os': 'True', - # }, - # {'--feat': 'trill'}, - # {'--feat': 'wav2vec'}, + {"--feat": "audmodel"}, ] for c in classifiers: for f in features: - cmd = f'python {src_path}parse_nkulu.py ' + cmd = "python -m nkululeko.nkuluflag --config exp.ini " for item in c: - cmd += f'{item} {c[item]} ' + cmd += f"{item} {c[item]} " for item in f: - cmd += f'{item} {f[item]} ' + cmd += f"{item} {f[item]} " print(cmd) - os.system(cmd) \ No newline at end of file + os.system(cmd) + # print(f"results: {result}, {last_epoch}") diff --git a/meta/demos/multiple_exeriments/exp.ini b/meta/demos/multiple_exeriments/exp.ini index 7a5f1042..a50cf430 100644 --- a/meta/demos/multiple_exeriments/exp.ini +++ b/meta/demos/multiple_exeriments/exp.ini @@ -1,23 +1,22 @@ [EXP] root = ./ -name = exp_test +name = results runs = 1 epochs = 50 [DATA] -root_folders = data_roots.ini databases = ['emodb'] +emodb = ../../../data/emodb/emodb +emodb.split_strategy = specified +emodb.train_tables = ['emotion.categories.train.gold_standard'] +emodb.test_tables = ['emotion.categories.test.gold_standard'] target = emotion -labels = ['anger', 'boredom', 'disgust', 'fear', 'happiness', 'neutral', 'sadness'] +labels = ['anger', 'happiness'] [FEATS] -wav2vec.model = xxx/wav2vec2-large-robust-ft-swbd-300h -xbow.model = xxx/openXBOW/ -trill.model = xxx/trill_model -mld.model = xxx/mld/src -scale = standard [MODEL] C_val = .001 #drop = .5 learning_rate = 0.0001 store = True +patience = 5 [PLOT] -best_model = True \ No newline at end of file +best_model = True diff --git a/meta/demos/multiple_exeriments/parse_nkulu.py b/meta/demos/multiple_exeriments/parse_nkulu.py deleted file mode 100644 index 14bc537d..00000000 --- a/meta/demos/multiple_exeriments/parse_nkulu.py +++ /dev/null @@ -1,112 +0,0 @@ - -import sys -#sys.path.append("../../src") -sys.path.append("../nkululeko/src") -import constants -import numpy as np -import experiment as exp -import configparser -from util import Util -import argparse -import os.path -src_path = './demos/multiple_experiments/' - - -def main(): - parser = argparse.ArgumentParser(description='Call the nkululeko framework.') - parser.add_argument('--config', default='exp.ini', help='The base configuration') - parser.add_argument('--data', help='The databases', nargs='*', \ - action='append') - parser.add_argument('--label', nargs='*', help='The labels for the target', \ - action='append') - parser.add_argument('--tuning_params', nargs='*', help='parameters to be tuned', \ - action='append') - parser.add_argument('--layers', nargs='*', help='layer config for mlp, e.g. l1:128 ', \ - action='append') - parser.add_argument('--model', default='xgb', help='The model type') - parser.add_argument('--feat', default='os', help='The feature type') - parser.add_argument('--set', help='The opensmile set') - parser.add_argument('--with_os', help='To add os features') - parser.add_argument('--target', help='The target designation') - parser.add_argument('--epochs', help='The number of epochs') - parser.add_argument('--runs', help='The number of runs') - parser.add_argument('--learning_rate', help='The learning rate') - parser.add_argument('--drop', help='The dropout rate [0:1]') - - args = parser.parse_args() - - if args.config is not None: - config_file = args.config - else: - config_file = f'{src_path}exp.ini' - # test if config is there - if not os.path.isfile(config_file): - print(f'ERROR: no such file {config_file}') - - config = configparser.ConfigParser() - config.read(config_file) - # fill the config - - - if args.data is not None: - databases = [] - for t in args.data: - databases.append(t[0]) - print(f'got databases: {databases}') - config['DATA']['databases'] = str(databases) - if args.label is not None: - labels = [] - for l in args.label: - labels.append(l[0]) - print(f'got labels: {labels}') - config['DATA']['labels'] = str(labels) - if args.tuning_params is not None: - tuning_params = [] - for tp in args.tuning_params: - tuning_params.append(tp[0]) - config['MODEL']['tuning_params'] = str(tuning_params) - if args.layers is not None: - config['MODEL']['layers'] = args.layers[0][0] - if args.target is not None: - config['DATA']['target'] = args.target - if args.epochs is not None: - config['EXP']['epochs'] = args.epochs - if args.runs is not None: - config['EXP']['runs'] = args.runs - if args.learning_rate is not None: - config['MODEL']['learning_rate'] = args.learning_rate - if args.drop is not None: - config['MODEL']['drop'] = args.drop - if args.model is not None: - config['MODEL']['type'] = args.model - if args.feat is not None: - config['FEATS']['type'] = f'[\'{args.feat}\']' - if args.with_os is not None: - config['FEATS']['with_os'] = args.with_os - if args.set is not None: - config['FEATS']['set'] = args.set - - - name = config['EXP']['name'] - - # init the experiment - expr = exp.Experiment(config) - util = Util() - util.debug(f'running {name}, Nkululeko version {constants.VERSION}') - - # load the data - expr.load_datasets() - # split into train and test - expr.fill_train_and_tests() - # extract features - expr.extract_feats() - # initialize a run manager - expr.init_runmanager() - # run the experiment - reports = expr.run() - result = reports[-1].result.test - # report result - util.debug(f'result for {expr.get_name()} is {result}') - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/nkululeko/constants.py b/nkululeko/constants.py index 6124ca81..94325f65 100644 --- a/nkululeko/constants.py +++ b/nkululeko/constants.py @@ -1,2 +1,2 @@ -VERSION="0.81.6" +VERSION="0.82.0" SAMPLING_RATE = 16000 diff --git a/nkululeko/feat_extract/feats_import.py b/nkululeko/feat_extract/feats_import.py index 163f371a..26e818d8 100644 --- a/nkululeko/feat_extract/feats_import.py +++ b/nkululeko/feat_extract/feats_import.py @@ -11,8 +11,8 @@ class ImportSet(Featureset): """Class to import features that have been compiled elsewhere""" - def __init__(self, name, data_df): - super().__init__(name, data_df) + def __init__(self, name, data_df, feats_type): + super().__init__(name, data_df, feats_type) def extract(self): """Import the features.""" diff --git a/nkululeko/feat_extract/feats_mos.py b/nkululeko/feat_extract/feats_mos.py index abd8ea2e..d2c38f47 100644 --- a/nkululeko/feat_extract/feats_mos.py +++ b/nkululeko/feat_extract/feats_mos.py @@ -27,9 +27,9 @@ class MosSet(Featureset): """Class to predict MOS (mean opinion score)""" - def __init__(self, name, data_df): + def __init__(self, name, data_df, feats_type): """Constructor. is_train is needed to distinguish from test/dev sets, because they use the codebook from the training""" - super().__init__(name, data_df) + super().__init__(name, data_df, feats_type) self.device = self.util.config_val("MODEL", "device", "cpu") self.model_initialized = False diff --git a/nkululeko/feat_extract/feats_opensmile.py b/nkululeko/feat_extract/feats_opensmile.py index 26120ca0..8e407bc8 100644 --- a/nkululeko/feat_extract/feats_opensmile.py +++ b/nkululeko/feat_extract/feats_opensmile.py @@ -15,24 +15,14 @@ def __init__(self, name, data_df, feats_type=None, config_file=None): self.feature_set = eval(f"opensmile.FeatureSet.{self.featset}") # 'eGeMAPSv02, ComParE_2016, GeMAPSv01a, eGeMAPSv01a': except AttributeError: - self.util.error( - f"something is wrong with feature set: {self.featset}" - ) + self.util.error(f"something is wrong with feature set: {self.featset}") self.featlevel = self.util.config_val("FEATS", "level", "functionals") try: - self.featlevel = self.featlevel.replace( - "lld", "LowLevelDescriptors" - ) - self.featlevel = self.featlevel.replace( - "functionals", "Functionals" - ) - self.feature_level = eval( - f"opensmile.FeatureLevel.{self.featlevel}" - ) + self.featlevel = self.featlevel.replace("lld", "LowLevelDescriptors") + self.featlevel = self.featlevel.replace("functionals", "Functionals") + self.feature_level = eval(f"opensmile.FeatureLevel.{self.featlevel}") except AttributeError: - self.util.error( - f"something is wrong with feature level: {self.featlevel}" - ) + self.util.error(f"something is wrong with feature level: {self.featlevel}") def extract(self): """Extract the features based on the initialized dataset or re-open them when found on disk.""" @@ -44,9 +34,7 @@ def extract(self): ) no_reuse = eval(self.util.config_val("FEATS", "no_reuse", "False")) if extract or not os.path.isfile(storage) or no_reuse: - self.util.debug( - "extracting openSmile features, this might take a while..." - ) + self.util.debug("extracting openSmile features, this might take a while...") smile = opensmile.Smile( feature_set=self.feature_set, feature_level=self.feature_level, @@ -85,9 +73,7 @@ def filter(self): selected_features = ast.literal_eval( glob_conf.config["FEATS"]["os.features"] ) - self.util.debug( - f"selecting features from opensmile: {selected_features}" - ) + self.util.debug(f"selecting features from opensmile: {selected_features}") sel_feats_df = pd.DataFrame() hit = False for feat in selected_features: diff --git a/nkululeko/feat_extract/feats_spectra.py b/nkululeko/feat_extract/feats_spectra.py index d5e06a4e..691fec47 100644 --- a/nkululeko/feat_extract/feats_spectra.py +++ b/nkululeko/feat_extract/feats_spectra.py @@ -4,6 +4,7 @@ Inspired by code from Su Lei """ + import os import torchaudio import torchaudio.transforms as T @@ -23,9 +24,9 @@ class Spectraloader(Featureset): - def __init__(self, name, data_df): + def __init__(self, name, data_df, feat_type): """Constructor setting the name""" - Featureset.__init__(self, name, data_df) + super().__init__(name, data_df, feat_type) self.sampling_rate = SAMPLING_RATE self.num_bands = int(self.util.config_val("FEATS", "fft_nbands", "64")) self.win_dur = int(self.util.config_val("FEATS", "fft_win_dur", "25")) diff --git a/nkululeko/feat_extract/feats_squim.py b/nkululeko/feat_extract/feats_squim.py index 4fc9bd02..93baaf30 100644 --- a/nkululeko/feat_extract/feats_squim.py +++ b/nkululeko/feat_extract/feats_squim.py @@ -30,9 +30,9 @@ class SquimSet(Featureset): """Class to predict SQUIM features""" - def __init__(self, name, data_df): + def __init__(self, name, data_df, feats_type): """Constructor. is_train is needed to distinguish from test/dev sets, because they use the codebook from the training""" - super().__init__(name, data_df) + super().__init__(name, data_df, feats_type) self.device = self.util.config_val("MODEL", "device", "cpu") self.model_initialized = False diff --git a/nkululeko/feat_extract/feats_whisper.py b/nkululeko/feat_extract/feats_whisper.py new file mode 100644 index 00000000..b0333bec --- /dev/null +++ b/nkululeko/feat_extract/feats_whisper.py @@ -0,0 +1,110 @@ +# feats_whisper.py +import os + +import pandas as pd +import torch +from transformers import AutoFeatureExtractor +from transformers import WhisperModel + +import audeer +import audiofile + +from nkululeko.feat_extract.featureset import Featureset +import nkululeko.glob_conf as glob_conf + + +class Whisper(Featureset): + """Class to extract whisper embeddings.""" + + def __init__(self, name, data_df, feat_type): + super().__init__(name, data_df, feat_type) + cuda = "cuda" if torch.cuda.is_available() else "cpu" + self.device = self.util.config_val("MODEL", "device", cuda) + self.model_initialized = False + if feat_type == "whisper": + self.feat_type = "whisper-base" + else: + self.feat_type = feat_type + + def init_model(self): + # load model + self.util.debug("loading whisper model...") + model_name = f"openai/{self.feat_type}" + self.model = WhisperModel.from_pretrained(model_name).to(self.device) + print(f"intialized Whisper model on {self.device}") + self.feature_extractor = AutoFeatureExtractor.from_pretrained(model_name) + self.model_initialized = True + + def extract(self): + """Extract the features or load them from disk if present.""" + store = self.util.get_path("store") + storage = f"{store}{self.name}.pkl" + extract = self.util.config_val("FEATS", "needs_feature_extraction", False) + no_reuse = eval(self.util.config_val("FEATS", "no_reuse", "False")) + if extract or no_reuse or not os.path.isfile(storage): + if not self.model_initialized: + self.init_model() + self.util.debug("extracting whisper embeddings, this might take a while...") + emb_series = [] + for (file, start, end), _ in audeer.progress_bar( + self.data_df.iterrows(), + total=len(self.data_df), + desc=f"Running whisper on {len(self.data_df)} audiofiles", + ): + if end == pd.NaT: + signal, sr = audiofile.read(file, offset=start) + else: + signal, sr = audiofile.read( + file, duration=end - start, offset=start + ) + emb = self.get_embeddings(signal, sr, file) + emb_series.append(emb) + # print(f"emb_series shape: {emb_series.shape}") + self.df = pd.DataFrame(emb_series, index=self.data_df.index) + # print(f"df shape: {self.df.shape}") + self.df.to_pickle(storage) + try: + glob_conf.config["DATA"]["needs_feature_extraction"] = "false" + except KeyError: + pass + else: + self.util.debug("reusing extracted wav2vec2 embeddings") + self.df = pd.read_pickle(storage) + if self.df.isnull().values.any(): + nanrows = self.df.columns[self.df.isna().any()].tolist() + # print(nanrows) + self.util.error( + f"got nan: {self.df.shape} {self.df.isnull().sum().sum()}" + ) + + def get_embeddings(self, signal, sampling_rate, file): + r"""Extract embeddings from raw audio signal.""" + try: + with torch.no_grad(): + embed_size = self.model.config.hidden_size + embed_columns = [f"whisper_{i}" for i in range(embed_size)] + inputs = self.feature_extractor(signal, sampling_rate=16000)[ + "input_features" + ][0] + inputs = torch.from_numpy(inputs).to(self.device).unsqueeze(0) + decoder_input_ids = ( + torch.tensor([[1, 1]]).to(self.device) + * self.model.config.decoder_start_token_id + ) + full_outputs = self.model( + inputs, + decoder_input_ids=decoder_input_ids, + output_hidden_states=True, + ) + outputs = full_outputs.encoder_last_hidden_state[0] + average_embeds = outputs.squeeze().mean(axis=0).cpu().detach().numpy() + except RuntimeError as re: + print(str(re)) + self.util.error(f"couldn't extract file: {file}") + # print(f"y flattened shape: {y.ravel().shape}") + return average_embeds + + def extract_sample(self, signal, sr): + self.init_model() + feats = self.get_embeddings(signal, sr, "no file") + return feats diff --git a/nkululeko/feature_extractor.py b/nkululeko/feature_extractor.py index 9cc573c6..1a83ae5f 100644 --- a/nkululeko/feature_extractor.py +++ b/nkululeko/feature_extractor.py @@ -39,12 +39,10 @@ def extract(self): self.feats = pd.DataFrame() for feats_type in self.feats_types: store_name = f"{self.data_name}_{feats_type}" - self.feat_extractor = self._get_feat_extractor( - store_name, feats_type) + self.feat_extractor = self._get_feat_extractor(store_name, feats_type) self.feat_extractor.extract() self.feat_extractor.filter() - self.feats = pd.concat( - [self.feats, self.feat_extractor.df], axis=1) + self.feats = pd.concat([self.feats, self.feat_extractor.df], axis=1) return self.feats def extract_sample(self, signal, sr): @@ -61,22 +59,27 @@ def _get_feat_extractor(self, store_name, feats_type): def _get_feat_extractor_class(self, feats_type): if feats_type == "os": from nkululeko.feat_extract.feats_opensmile import Opensmileset + return Opensmileset elif feats_type == "spectra": from nkululeko.feat_extract.feats_spectra import Spectraloader + return Spectraloader elif feats_type == "trill": from nkululeko.feat_extract.feats_trill import TRILLset + return TRILLset elif feats_type.startswith( - ("wav2vec2", "hubert", "wavlm", "spkrec", "whisper")): + ("wav2vec2", "hubert", "wavlm", "spkrec", "whisper") + ): return self._get_feat_extractor_by_prefix(feats_type) elif feats_type == "xbow": from nkululeko.feat_extract.feats_oxbow import Openxbow + return Openxbow elif feats_type in ( @@ -100,15 +103,13 @@ def _get_feat_extractor_by_prefix(self, feats_type): prefix, _, ext = feats_type.partition("-") from importlib import import_module - module = import_module( - f"nkululeko.feat_extract.feats_{prefix.lower()}") + module = import_module(f"nkululeko.feat_extract.feats_{prefix.lower()}") class_name = f"{prefix.capitalize()}" return getattr(module, class_name) def _get_feat_extractor_by_name(self, feats_type): from importlib import import_module - module = import_module( - f"nkululeko.feat_extract.feats_{feats_type.lower()}") + module = import_module(f"nkululeko.feat_extract.feats_{feats_type.lower()}") class_name = f"{feats_type.capitalize()}Set" return getattr(module, class_name) diff --git a/nkululeko/multidb.py b/nkululeko/multidb.py index 23506b94..253141cb 100644 --- a/nkululeko/multidb.py +++ b/nkululeko/multidb.py @@ -12,18 +12,15 @@ import pandas as pd import seaborn as sn -import nkululeko.glob_conf as glob_conf from nkululeko.aug_train import doit as aug_train -from nkululeko.experiment import Experiment from nkululeko.nkululeko import doit as nkulu -from nkululeko.utils.util import Util def main(src_dir): parser = argparse.ArgumentParser( - description="Call the nkululeko MULTIDB framework.") - parser.add_argument("--config", default="exp.ini", - help="The base configuration") + description="Call the nkululeko MULTIDB framework." + ) + parser.add_argument("--config", default="exp.ini", help="The base configuration") args = parser.parse_args() if args.config is not None: config_file = args.config @@ -58,8 +55,7 @@ def main(src_dir): dataset = datasets[i] print(f"running {dataset}") if extra_trains: - extra_trains_1 = extra_trains.removeprefix( - "[").removesuffix("]") + extra_trains_1 = extra_trains.removeprefix("[").removesuffix("]") config["DATA"]["databases"] = f"['{dataset}', {extra_trains_1}]" extra_trains_2 = ast.literal_eval(extra_trains) for extra_train in extra_trains_2: @@ -72,8 +68,7 @@ def main(src_dir): test = datasets[j] print(f"running train: {train}, test: {test}") if extra_trains: - extra_trains_1 = extra_trains.removeprefix( - "[").removesuffix("]") + extra_trains_1 = extra_trains.removeprefix("[").removesuffix("]") config["DATA"][ "databases" ] = f"['{train}', '{test}', {extra_trains_1}]" diff --git a/nkululeko/nkuluflag.py b/nkululeko/nkuluflag.py new file mode 100644 index 00000000..5603bcff --- /dev/null +++ b/nkululeko/nkuluflag.py @@ -0,0 +1,95 @@ +import argparse +import configparser +import os +import os.path + +from nkululeko.nkululeko import doit as nkulu + + +def do_it(src_dir): + parser = argparse.ArgumentParser(description="Call the nkululeko framework.") + parser.add_argument("--config", help="The base configuration") + parser.add_argument("--data", help="The databases", nargs="*", action="append") + parser.add_argument( + "--label", nargs="*", help="The labels for the target", action="append" + ) + parser.add_argument( + "--tuning_params", nargs="*", help="parameters to be tuned", action="append" + ) + parser.add_argument( + "--layers", + nargs="*", + help="layer config for mlp, e.g. l1:128 ", + action="append", + ) + parser.add_argument("--model", default="xgb", help="The model type") + parser.add_argument("--feat", default="['os']", help="The feature type") + parser.add_argument("--set", help="The opensmile set") + parser.add_argument("--with_os", help="To add os features") + parser.add_argument("--target", help="The target designation") + parser.add_argument("--epochs", help="The number of epochs") + parser.add_argument("--runs", help="The number of runs") + parser.add_argument("--learning_rate", help="The learning rate") + parser.add_argument("--drop", help="The dropout rate [0:1]") + + args = parser.parse_args() + + if args.config is not None: + config_file = args.config + else: + print("ERROR: need config file") + quit(-1) + # test if config is there + if not os.path.isfile(config_file): + print(f"ERROR: no such file {config_file}") + + config = configparser.ConfigParser() + config.read(config_file) + # fill the config + + if args.data is not None: + databases = [] + for t in args.data: + databases.append(t[0]) + print(f"got databases: {databases}") + config["DATA"]["databases"] = str(databases) + if args.label is not None: + labels = [] + for label in args.label: + labels.append(label[0]) + print(f"got labels: {labels}") + config["DATA"]["labels"] = str(labels) + if args.tuning_params is not None: + tuning_params = [] + for tp in args.tuning_params: + tuning_params.append(tp[0]) + config["MODEL"]["tuning_params"] = str(tuning_params) + if args.layers is not None: + config["MODEL"]["layers"] = args.layers[0][0] + if args.target is not None: + config["DATA"]["target"] = args.target + if args.epochs is not None: + config["EXP"]["epochs"] = args.epochs + if args.runs is not None: + config["EXP"]["runs"] = args.runs + if args.learning_rate is not None: + config["MODEL"]["learning_rate"] = args.learning_rate + if args.drop is not None: + config["MODEL"]["drop"] = args.drop + if args.model is not None: + config["MODEL"]["type"] = args.model + if args.feat is not None: + config["FEATS"]["type"] = f"['{args.feat}']" + if args.set is not None: + config["FEATS"]["set"] = args.set + tmp_config = "tmp.ini" + with open(tmp_config, "w") as tmp_file: + config.write(tmp_file) + + result, last_epoch = nkulu(tmp_config) + return result, last_epoch + + +if __name__ == "__main__": + cwd = os.path.dirname(os.path.abspath(__file__)) + do_it(cwd) # sys.argv[1]) diff --git a/nkululeko/nkululeko.py b/nkululeko/nkululeko.py index 8adeaf04..d5c6db62 100644 --- a/nkululeko/nkululeko.py +++ b/nkululeko/nkululeko.py @@ -1,12 +1,14 @@ # nkululeko.py # Entry script to do a Nkululeko experiment -import numpy as np -import os.path -import configparser import argparse +import configparser +import os.path + +import numpy as np + +from nkululeko.constants import VERSION import nkululeko.experiment as exp from nkululeko.utils.util import Util -from nkululeko.constants import VERSION def doit(config_file): diff --git a/test_runs.sh b/test_runs.sh index 891d04fb..cf56051c 100755 --- a/test_runs.sh +++ b/test_runs.sh @@ -6,6 +6,7 @@ function Help { echo "Options:" echo " --Explore: test explore module" echo " --Nkulu: test basic nkululeko" + echo " --Feat: test selected acoustic features" echo " --Aug: test augmentation" echo " --Pred: test prediction" echo " --Demo: test demo" @@ -35,6 +36,11 @@ function Nkulu { python -m nkululeko.nkululeko --config tests/exp_ravdess_os_xgb.ini python -m nkululeko.nkululeko --config tests/exp_agedb_class_os_xgb.ini } +# test features +function Feat { + python -m nkululeko.nkululeko --config tests/exp_emodb_hubert_xgb.ini + python -m nkululeko.nkululeko --config tests/exp_emodb_whisper_xgb.ini +} # test augmentation function Aug { python -m nkululeko.augment --config tests/exp_emodb_augment_os_xgb.ini @@ -85,6 +91,9 @@ for arg in "$@"; do if [[ "$arg" = --Nkulu ]] || [[ "$arg" = --all ]]; then Nkulu fi + if [[ "$arg" = --Feat ]] || [[ "$arg" = --all ]]; then + Feat + fi if [[ "$arg" = --Aug ]] || [[ "$arg" = --all ]]; then Aug fi diff --git a/tests/exp_emodb_hubert_xgb.ini b/tests/exp_emodb_hubert_xgb.ini new file mode 100644 index 00000000..917b7f0a --- /dev/null +++ b/tests/exp_emodb_hubert_xgb.ini @@ -0,0 +1,18 @@ +[EXP] +root = ./tests/results/ +name = exp_emodb_feats +runs = 1 +epochs = 1 +save = True +[DATA] +databases = ['emodb'] +emodb = ./data/emodb/emodb +emodb.test_tables = ['emotion.categories.test.gold_standard'] +emodb.train_tables = ['emotion.categories.train.gold_standard'] +emodb.mapping = {'anger':'angry', 'happiness':'happy', 'sadness':'sad', 'neutral':'neutral'} +labels = ['angry', 'happy'] +target = emotion +[FEATS] +type = ['hubert-base-ls960'] +[MODEL] +type = xgb diff --git a/tests/exp_emodb_os_knn.ini b/tests/exp_emodb_os_knn.ini index 27ac2614..b36a7c57 100644 --- a/tests/exp_emodb_os_knn.ini +++ b/tests/exp_emodb_os_knn.ini @@ -1,6 +1,6 @@ [EXP] root = ./tests/results/ -name = exp_emodb +name = exp_emodb_classifiers runs = 1 epochs = 10 save = True diff --git a/tests/exp_emodb_os_mlp.ini b/tests/exp_emodb_os_mlp.ini index aa6edf5a..ffc349d5 100644 --- a/tests/exp_emodb_os_mlp.ini +++ b/tests/exp_emodb_os_mlp.ini @@ -1,6 +1,6 @@ [EXP] root = ./tests/results/ -name = exp_emodb +name = exp_emodb_classifiers runs = 1 epochs = 500 save = True diff --git a/tests/exp_emodb_os_svm.ini b/tests/exp_emodb_os_svm.ini index 5a1b6dd7..c24a1720 100644 --- a/tests/exp_emodb_os_svm.ini +++ b/tests/exp_emodb_os_svm.ini @@ -1,6 +1,6 @@ [EXP] root = ./tests/results/ -name = exp_emodb +name = exp_emodb_classifiers runs = 1 epochs = 10 save = True diff --git a/tests/exp_emodb_whisper_xgb.ini b/tests/exp_emodb_whisper_xgb.ini new file mode 100644 index 00000000..5316dc2f --- /dev/null +++ b/tests/exp_emodb_whisper_xgb.ini @@ -0,0 +1,18 @@ +[EXP] +root = ./tests/results/ +name = exp_emodb_feats +runs = 1 +epochs = 1 +save = True +[DATA] +databases = ['emodb'] +emodb = ./data/emodb/emodb +emodb.test_tables = ['emotion.categories.test.gold_standard'] +emodb.train_tables = ['emotion.categories.train.gold_standard'] +emodb.mapping = {'anger':'angry', 'happiness':'happy', 'sadness':'sad', 'neutral':'neutral'} +labels = ['angry', 'happy'] +target = emotion +[FEATS] +type = ['whisper'] +[MODEL] +type = xgb