Inner Evaluation 4#
Explore an approach based on sequential data (using MFCC time-varying features), with Recurrent Neural Networks.
Requirements#
import numpy as np
import polars as pl
from torch import nn
from skorch import NeuralNetClassifier
import torch
import sys
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.preprocessing import StandardScaler
import pickle
sys.path.insert(0, r"C:\Users\fscielzo\Documents\Packages\PyAudio_Package_Private")
from PyAudio.preprocessing import get_X_tensor_audio_features
sys.path.insert(0, r'C:\Users\fscielzo\Documents\Packages\PyML_Package_Private')
from PyML.evaluation import SimpleEvaluation
sys.path.insert(0, r'C:\Users\fscielzo\Documents\Packages\PyDL_Package_Private')
from PyDL.models import RNN
from PyDL.preprocessing import TensorStandardScaler
RNN_model = NeuralNetClassifier(
module=RNN(input_dim=10, output_dim=2, num_layers=3, hidden_size=128),
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.Adam,
lr=0.001,
batch_size=50,
max_epochs=15,
device='cuda' if torch.cuda.is_available() else 'cpu',
verbose=False
)
Data definition#
In this section we define the data to be used. Specifically we define the response variable and a set of predictors matrices to be used as different alternatives, each one associate to a combination of features extraction methods and statistics.
files_list_name = r'C:\Users\fscielzo\Documents\DataScience-GitHub\Audio Analysis\Parkinson_Severity_Classification\Data\Files_List.txt'
files_df = pl.read_csv(files_list_name, separator='\t', has_header=False, new_columns=['path', 'level'])
# Configuration variables for feature extraction
fs = 16000 # Sampling frequency
wst = 0.032 # Window size (seconds)
fpt = 0.008 # Frame period (seconds)
nfft = int(np.ceil(wst*fs)) # Window size (samples)
fp = int(np.ceil(fpt*fs)) # Frame period (samples)
nbands = 40 # Number of filters in the filterbank
ncomp = 20 # Number of MFCC components
We define the response and the tensor with the time-varying features extracted with the MFCC method.
Y = files_df['level'].to_numpy()
X_MFCC_tensor = get_X_tensor_audio_features(paths=files_df['path'], method='MFCC', sr=fs, n_fft=nfft, hop_length=fp, n_mels=nbands, n_mfcc=ncomp)
X_chroma_tensor = get_X_tensor_audio_features(paths=files_df['path'], method='chroma', sr=fs, n_fft=nfft, hop_length=fp, n_mels=nbands, n_mfcc=ncomp)
Outer validation method: train-test split#
We split our data (response and predictors) in two partitions, the training and the testing one. The training partition will be used in the inner evaluation for selecting the best approach to predict the PD level, and the test one will only be used at the very end for making an estimation of the future performance of the best approach, that is, and estimation of how this approach will classify the level of PD of new patients.
X_MFCC_tensor_train, X_MFCC_tensor_test, Y_train, Y_test = train_test_split(X_MFCC_tensor, Y, test_size=0.25, random_state=123, stratify=Y)
X_MFCC_tensor.shape
(240, 20, 4403)
X_MFCC_tensor_train.shape
(180, 20, 4403)
X_MFCC_tensor_test.shape
(60, 20, 4403)
Standardizing the data since it seems to work well with RNN
# Reshape data for standardization (from 3D to 2D)
n_samples, n_mfcc, max_length = X_MFCC_tensor_train.shape
X_MFCC_tensor_train_flatten = X_MFCC_tensor_train.reshape(-1, n_mfcc * max_length)
# Standardize the data
scaler = StandardScaler()
X_MFCC_train_standardized = scaler.fit_transform(X_MFCC_tensor_train_flatten)
# Reshape back to 3D
X_MFCC_tensor_train = X_MFCC_train_standardized.reshape(n_samples, n_mfcc, max_length)
# Reshape data for standardization (from 3D to 2D)
n_samples, n_mfcc, max_length = X_MFCC_tensor_test.shape
X_MFCC_tensor_test_flatten = X_MFCC_tensor_test.reshape(-1, n_mfcc * max_length)
# Standardize the data
X_MFCC_test_standardized = scaler.transform(X_MFCC_tensor_test_flatten)
# Reshape back to 3D
X_MFCC_tensor_test = X_MFCC_test_standardized.reshape(n_samples, n_mfcc, max_length)
A better approach to do this without falling in data leakage:
X_MFCC_tensor_train
array([[[ 0.77925549, 0.75966389, 0.75801119, ..., 0.07474351,
0.07474351, 0.07474351],
[ 0.49572165, 0.11072004, -0.31598378, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 1.34147712, 1.38633399, 1.4529877 , ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 0.48452161, -0.07294603, 0.47874218, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 1.382128 , 1.93450536, 2.12793082, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.97325322, -1.423122 , -1.50599919, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[-1.09754581, -1.2406114 , -1.43178652, ..., 0.07474351,
0.07474351, 0.07474351],
[-0.21227887, -0.53582309, -1.00587868, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.01043724, -0.08198521, -0.06441753, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 0.63225567, 0.53115129, 0.74958524, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.47306939, 0.46868749, 0.82995921, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 1.49799736, 1.56677656, 1.18140685, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[-0.93716879, -0.92827138, -0.89666019, ..., 0.07474351,
0.07474351, 0.07474351],
[-0.39703831, -0.31946003, -0.37899106, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.35987158, -0.70249106, -0.93452423, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[-0.28148851, -0.39495589, 0.04978402, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.34802631, -2.12577281, -1.47394572, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.49579407, -1.6952296 , -1.53697988, ..., -0.07474351,
-0.07474351, -0.07474351]],
...,
[[ 0.6565104 , 0.65331194, 0.70792749, ..., 0.07474351,
0.07474351, 0.07474351],
[ 0.72916882, 0.81260307, 0.70276269, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.70178372, 0.89056889, 0.78858701, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[-0.40537788, -0.64057784, -0.9321527 , ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.9375777 , -0.4679151 , 0.41955871, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.72704588, -0.15770425, 0.56146658, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[-1.26680811, -1.19761807, -1.21938432, ..., 0.07474351,
0.07474351, 0.07474351],
[-0.86848324, -0.76134084, -0.62641248, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.28175129, -0.54700239, -0.58084685, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[-0.38528358, 0.3547839 , 0.73461061, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 2.46115109, 1.7585346 , 1.48140808, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 1.38349091, 0.54929556, -0.42187174, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[ 0.04556678, -0.04351323, -0.12983973, ..., 0.07474351,
0.07474351, 0.07474351],
[ 1.99804549, 1.83888045, 1.60835686, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 2.35297563, 2.29635284, 2.03319404, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 0.30947655, 0.43012554, 0.22527788, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.12866233, 0.34554382, 0.82022089, ..., -0.07474351,
-0.07474351, -0.07474351],
[-1.43149724, -1.59260321, -1.39095793, ..., -0.07474351,
-0.07474351, -0.07474351]]])
X_MFCC_tensor_train.shape
(180, 20, 4403)
scaler = TensorStandardScaler(apply=True)
scaler.fit(X_MFCC_tensor_train)
scaler.transform(X_MFCC_tensor_train)
array([[[ 0.77925549, 0.75966389, 0.75801119, ..., 0.07474351,
0.07474351, 0.07474351],
[ 0.49572165, 0.11072004, -0.31598378, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 1.34147712, 1.38633399, 1.4529877 , ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 0.48452161, -0.07294603, 0.47874218, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 1.382128 , 1.93450536, 2.12793082, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.97325322, -1.423122 , -1.50599919, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[-1.09754581, -1.2406114 , -1.43178652, ..., 0.07474351,
0.07474351, 0.07474351],
[-0.21227887, -0.53582309, -1.00587868, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.01043724, -0.08198521, -0.06441753, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 0.63225567, 0.53115129, 0.74958524, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.47306939, 0.46868749, 0.82995921, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 1.49799736, 1.56677656, 1.18140685, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[-0.93716879, -0.92827138, -0.89666019, ..., 0.07474351,
0.07474351, 0.07474351],
[-0.39703831, -0.31946003, -0.37899106, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.35987158, -0.70249106, -0.93452423, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[-0.28148851, -0.39495589, 0.04978402, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.34802631, -2.12577281, -1.47394572, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.49579407, -1.6952296 , -1.53697988, ..., -0.07474351,
-0.07474351, -0.07474351]],
...,
[[ 0.6565104 , 0.65331194, 0.70792749, ..., 0.07474351,
0.07474351, 0.07474351],
[ 0.72916882, 0.81260307, 0.70276269, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.70178372, 0.89056889, 0.78858701, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[-0.40537788, -0.64057784, -0.9321527 , ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.9375777 , -0.4679151 , 0.41955871, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.72704588, -0.15770425, 0.56146658, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[-1.26680811, -1.19761807, -1.21938432, ..., 0.07474351,
0.07474351, 0.07474351],
[-0.86848324, -0.76134084, -0.62641248, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.28175129, -0.54700239, -0.58084685, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[-0.38528358, 0.3547839 , 0.73461061, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 2.46115109, 1.7585346 , 1.48140808, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 1.38349091, 0.54929556, -0.42187174, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[ 0.04556678, -0.04351323, -0.12983973, ..., 0.07474351,
0.07474351, 0.07474351],
[ 1.99804549, 1.83888045, 1.60835686, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 2.35297563, 2.29635284, 2.03319404, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 0.30947655, 0.43012554, 0.22527788, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.12866233, 0.34554382, 0.82022089, ..., -0.07474351,
-0.07474351, -0.07474351],
[-1.43149724, -1.59260321, -1.39095793, ..., -0.07474351,
-0.07474351, -0.07474351]]])
scaler.transform(X_MFCC_tensor_train).shape
(180, 20, 4403)
scaler.transform(X_MFCC_tensor_test)
array([[[-1.1088349 , -1.21068496, -1.24170116, ..., 0.07474351,
0.07474351, 0.07474351],
[-1.29952919, -1.45907204, -1.43475645, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.07323975, -0.41990087, -0.55769708, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 0.74406594, 0.77746824, 0.67277322, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.69785958, 0.69918349, 0.65714235, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.57196627, 0.64494054, 0.76531219, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[-0.3483037 , -0.15471678, 0.15473919, ..., 0.07474351,
0.07474351, 0.07474351],
[ 0.66910327, 0.9586072 , 1.04969775, ..., -0.07474351,
-0.07474351, -0.07474351],
[-1.33066569, -1.04541998, -0.74294924, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 2.24923983, 2.19150319, 1.54300361, ..., -0.07474351,
-0.07474351, -0.07474351],
[-1.78269042, -0.12646651, -0.44859062, ..., -0.07474351,
-0.07474351, -0.07474351],
[-2.07612607, 0.7965252 , 0.31931094, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[-1.03763687, -1.0695463 , -1.02396231, ..., 0.07474351,
0.07474351, 0.07474351],
[-1.32669176, -1.20835791, -0.94714662, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.32077486, -0.16320353, 0.13569455, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 0.8763195 , -0.01483926, -0.5642226 , ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.33603545, -0.5300002 , -0.87023331, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.17797754, -0.50627413, -0.62680684, ..., -0.07474351,
-0.07474351, -0.07474351]],
...,
[[ 0.79789592, 0.6718777 , 0.33046552, ..., 0.07474351,
0.07474351, 0.07474351],
[ 0.73263119, 0.99082343, 1.51836759, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.90628487, 1.27901072, 2.08666589, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 0.24868375, 0.24592557, 0.39300569, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.84737753, 0.36539538, -0.31156388, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 1.68994957, 1.07338474, 1.22594029, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[ 1.59412017, 1.53726977, 1.32176695, ..., 0.07474351,
0.07474351, 0.07474351],
[ 0.44501062, 0.12176085, -0.75543876, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.5138407 , -0.584392 , 0.04589857, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 1.73706153, 1.46862996, 3.56796765, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 1.13340098, 1.77850333, 1.6402638 , ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.24298456, 0.29429363, -1.11233773, ..., -0.07474351,
-0.07474351, -0.07474351]],
[[-1.17565817, -1.22614176, -1.23409303, ..., 0.07474351,
0.07474351, 0.07474351],
[-1.48596274, -1.36792995, -1.26105226, ..., -0.07474351,
-0.07474351, -0.07474351],
[-0.67358414, -0.27077418, -0.29346544, ..., -0.07474351,
-0.07474351, -0.07474351],
...,
[ 0.2830349 , 0.62647133, 0.50570222, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.31699058, 0.5937816 , 0.49041113, ..., -0.07474351,
-0.07474351, -0.07474351],
[ 0.19708623, 0.36830872, 0.51281274, ..., -0.07474351,
-0.07474351, -0.07474351]]])
Converting to tensor
PyTrochdata type
X_MFCC_tensor = torch.tensor(X_MFCC_tensor, dtype=torch.float32)
X_MFCC_tensor_train = torch.tensor(X_MFCC_tensor_train, dtype=torch.float32)
X_MFCC_tensor_test = torch.tensor(X_MFCC_tensor_test, dtype=torch.float32)
Y_tensor = torch.tensor(Y, dtype=torch.float32)
Y_train_tensor = torch.tensor(Y_train, dtype=torch.float32)
Y_test_tensor = torch.tensor(Y_test, dtype=torch.float32)
Applying Inner Evaluation#
In this section we are going to apply the round four of the inner evaluation.
Inner validation method: KFold Cross Validation#
We define the validation method to be used in the inner evaluation, that will be Stratified KFold Cross Validation.
inner = StratifiedKFold(n_splits=5, shuffle=True, random_state=123)
We define dictionaries to save important results that will be gathered in the inner evaluation.
inner_score, best_params, inner_results = {}, {}, {}
Grids for HPO#
Grid for RNN (PyTorch)#
def param_grid_RNN(trial, input_dim, output_dim):
param_grid = ({
'module__input_dim': trial.suggest_categorical('module__input_dim', [input_dim]),
'module__output_dim': trial.suggest_categorical('module__output_dim', [output_dim]),
'module__num_layers': trial.suggest_int('module__num_layers', 1, 10),
'module__hidden_size': trial.suggest_categorical('module__hidden_size', [50, 70, 100, 120, 150, 175, 200, 250]),
'module__dropout_rate': trial.suggest_float('module__dropout_rate', 0.05, 0.95, log=True),
'lr': trial.suggest_float('lr', 0.0001, 0.01, log=True),
'max_epochs': trial.suggest_categorical('max_epochs', [5, 7, 10, 15, 20, 25, 30, 40, 50, 75, 100]),
'batch_size': trial.suggest_categorical('batch_size', [15, 30, 50, 70, 100])
})
return param_grid
HPO#
Applying HPO over Recurrent Neural Networks using the MFCC sequencies of features.
HPO for RNN (PyTorch)#
model = 'RNN'
input_dim = X_MFCC_tensor_train.shape[2]
output_dim = len(np.unique(Y_train))
simple_eval = SimpleEvaluation(estimator=RNN_model, param_grid=param_grid_RNN,
inner=inner, search_method='optuna', scoring='balanced_accuracy', direction='maximize',
n_trials=5, random_state=123,
framework='PyTorch',
input_dim=input_dim,
output_dim=output_dim)
simple_eval.fit(X=X_MFCC_tensor_train, y=Y_train_tensor.long())
inner_score[model] = simple_eval.inner_score
best_params[model]= simple_eval.inner_best_params
inner_results[model] = simple_eval.inner_results
[I 2024-04-16 17:55:50,367] A new study created in memory with name: no-name-5987d063-91d8-4e87-bb8a-845f53459244
Saving the results#
'''
with open('results/best_params_4', 'wb') as file:
pickle.dump(best_params, file)
with open('results/inner_scores_4', 'wb') as file:
pickle.dump(inner_score, file)
with open('results/inner_results_4', 'wb') as file:
pickle.dump(inner_results, file)
'''
Opening the results#
with open(f'results/best_params_4', 'rb') as file:
best_params = pickle.load(file)
with open(f'results/inner_scores_4', 'rb') as file:
inner_score = pickle.load(file)
with open(f'results/inner_results_4', 'rb') as file:
inner_results = pickle.load(file)
Selecting the best pipeline#
In this case we don’t have several alternatives to compare, since we have only one, a RNN using sequential features extracted with MFCC method.
The inner obtained balanced accuracy for the optimal RNN is quite poor, which could be due to a bad specification of the model or the input data.
inner_score
{'RNN': 0.4431818181818182}