KeyError: metrics in talos.scan
Closed this issue · 7 comments
talos.version = 1.0.0
tf.version = 2.2.0
The concise demo fails with 'KeyError: 'metrics.'
The following code:
from keras.models import Sequential
from keras.layers import Dropout, Dense
import sys
import talos
x, y = talos.templates.datasets.breast_cancer()
x = talos.utils.rescale_meanzero(x)
def breast_cancer_model(x_train, y_train, x_val, y_val, params):
model = Sequential()
model.add(Dense(params['first_neuron'], input_dim=x_train.shape[1],
activation=params['activation'],
kernel_initializer=params['kernel_initializer']))
model.add(Dropout(params['dropout']))
model.add(Dense(1, activation=params['last_activation'],
kernel_initializer=params['kernel_initializer']))
model.compile(loss=params['losses'],
optimizer=params['optimizer'],
metrics=['acc', talos.utils.metrics.f1score])
history = model.fit(x_train, y_train,
validation_data=[x_val, y_val],
batch_size=params['batch_size'],
callbacks=[talos.utils.live()],
epochs=params['epochs'],
verbose=0)
return history, model
p = {'first_neuron':[9,10,11],
'hidden_layers':[0, 1, 2],
'batch_size': [30],
'epochs': [100],
'dropout': [0],
'kernel_initializer': ['uniform','normal'],
'optimizer': ['Nadam', 'Adam'],
'losses': ['binary_crossentropy'],
'activation':['relu', 'elu'],
'last_activation': ['sigmoid']}
t = talos.Scan(x=x,
y=y,
model=breast_cancer_model,
params=p,
experiment_name='breast_cancer',
round_limit=10)
...results in the following:
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-1-7a8ddf94f1cb> in <module>
47 'last_activation': ['sigmoid']}
48 # and run the experiment
---> 49 t = talos.Scan(x=x,
50 y=y,
51 model=breast_cancer_model,
/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/scan/Scan.py in __init__(self, x, y, params, model, experiment_name, x_val, y_val, val_split, random_method, seed, performance_target, fraction_limit, round_limit, time_limit, boolean_limit, reduction_method, reduction_interval, reduction_window, reduction_threshold, reduction_metric, minimize_loss, disable_progress_bar, print_params, clear_session, save_weights)
194 # start runtime
195 from .scan_run import scan_run
--> 196 scan_run(self)
/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/scan/scan_run.py in scan_run(self)
24 # otherwise proceed with next permutation
25 from .scan_round import scan_round
---> 26 self = scan_round(self)
27 self.pbar.update(1)
28
/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/scan/scan_round.py in scan_round(self)
17 # fit the model
18 from ..model.ingest_model import ingest_model
---> 19 self.model_history, self.round_model = ingest_model(self)
20 self.round_history.append(self.model_history.history)
21
/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/model/ingest_model.py in ingest_model(self)
4 through Scan() model paramater.'''
5
----> 6 return self.model(self.x_train,
7 self.y_train,
8 self.x_val,
<ipython-input-1-7a8ddf94f1cb> in breast_cancer_model(x_train, y_train, x_val, y_val, params)
27 metrics=['acc', talos.utils.metrics.f1score])
28
---> 29 history = model.fit(x_train, y_train,
30 validation_data=[x_val, y_val],
31 batch_size=params['batch_size'],
/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
64 def _method_wrapper(self, *args, **kwargs):
65 if not self._in_multi_worker_mode(): # pylint: disable=protected-access
---> 66 return method(self, *args, **kwargs)
67
68 # Running inside `run_distribute_coordinator` already.
/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
828 self.stop_training = False
829 train_function = self.make_train_function()
--> 830 callbacks.on_train_begin()
831 # Handle fault-tolerance for multi-worker.
832 # TODO(omalleyt): Fix the ordering issues that mean this has to
/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/tensorflow/python/keras/callbacks.py in on_train_begin(self, logs)
445 logs = self._process_logs(logs)
446 for callback in self.callbacks:
--> 447 callback.on_train_begin(logs)
448
449 def on_train_end(self, logs=None):
/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/kerasplotlib/traininglog.py in on_train_begin(self, logs)
31
32 def on_train_begin(self, logs={}):
---> 33 self.base_metrics = [metric for metric in self.params['metrics'] if not metric.startswith('val_')]
34 if self.figsize is None:
35 self.figsize = (
KeyError: 'metrics'
Welcome to Talos community! Thanks so much for creating your first issue :)
same issue exactly.
Use this instead:
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dropout, Dense
%matplotlib inline
import sys
sys.path.insert(0, '/Users/mikko/Documents/GitHub/talos')
import talos
# then we load the dataset
x, y = talos.templates.datasets.breast_cancer()
# and normalize every feature to mean 0, std 1
x = talos.utils.rescale_meanzero(x)
# first we have to make sure to input data and params into the function
def breast_cancer_model(x_train, y_train, x_val, y_val, params):
model = Sequential()
model.add(Dense(params['first_neuron'], input_dim=x_train.shape[1],
activation=params['activation'],
kernel_initializer=params['kernel_initializer']))
model.add(Dropout(params['dropout']))
model.add(Dense(1, activation=params['last_activation'],
kernel_initializer=params['kernel_initializer']))
model.compile(loss=params['losses'],
optimizer=params['optimizer'],
metrics=['acc', talos.utils.metrics.f1score])
history = model.fit(x_train, y_train,
validation_data=(x_val, y_val),
batch_size=params['batch_size'],
epochs=params['epochs'],
verbose=0)
return history, model
# then we can go ahead and set the parameter space
p = {'first_neuron':[9,10,11],
'hidden_layers':[0, 1, 2],
'batch_size': [30],
'epochs': [100],
'dropout': [0],
'kernel_initializer': ['uniform','normal'],
'optimizer': ['Nadam', 'Adam'],
'losses': ['binary_crossentropy'],
'activation':['relu', 'elu'],
'last_activation': ['sigmoid']}
# and run the experiment
t = talos.Scan(x=x,
y=y,
model=breast_cancer_model,
params=p,
experiment_name='breast_cancer',
round_limit=10)
Closing here. Feel free to open new issue if anything.
Sorry. The same issue persists. Here is my function, parameter space and experiment error. have you tested talos on the latest version of jupyter notebook?
first we have to make sure to input data and params into the function
def talos_model(x_train, y_train, x_val, y_val, params):
model = Sequential()
model.add(LSTM(params['first_neuron'], input_shape=(X_train_seq.shape[1], X_train_seq.shape[2]),
kernel_initializer=params['kernel_initializer']))
model.add(Dropout(params['dropout']))
model.add(Dense(params['second_neuron'], activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))
model.add(Dropout(params['dropout']))
model.add(Dense(params['third_neuron'], activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))
model.add(Dense(1, activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc', talos.utils.metrics.f1score])
history = model.fit(X_train_seq, y_train_seq, validation_data=[X_test_seq, y_test_seq], batch_size=params['batch_size'],
callbacks=[talos.utils.live()], epochs=params['epochs'], verbose=0, shuffle=False)
return history, model
then we can go ahead and set the parameter space
p = {'first_neuron':[20,30,50],
'second_neuron':[6,8,10],
'third_neuron':[2,4],
'hidden_layers':[2,3],
'batch_size': [31,45,60],
'epochs': [110,130,170],
'dropout': [0.1,0.3],
'kernel_initializer': ['normal'],
'activation':['relu', 'elu'],
'last_activation': ['sigmoid']}
and run the experiment
t = talos.Scan(x=X_train_seq,y=y_train_seq,model=talos_model, params=p, experiment_name='talos_test', round_limit=10)
ERROR:
KeyError Traceback (most recent call last)
in
1 # and run the experiment
----> 2 t = talos.Scan(x=X_train_seq,y=y_train_seq,model=talos_model, params=p, experiment_name='talos_test', round_limit=10)
/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/Scan.py in init(self, x, y, params, model, experiment_name, x_val, y_val, val_split, random_method, seed, performance_target, fraction_limit, round_limit, time_limit, boolean_limit, reduction_method, reduction_interval, reduction_window, reduction_threshold, reduction_metric, minimize_loss, disable_progress_bar, print_params, clear_session, save_weights)
194 # start runtime
195 from .scan_run import scan_run
--> 196 scan_run(self)
/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/scan_run.py in scan_run(self)
24 # otherwise proceed with next permutation
25 from .scan_round import scan_round
---> 26 self = scan_round(self)
27 self.pbar.update(1)
28
/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/scan_round.py in scan_round(self)
17 # fit the model
18 from ..model.ingest_model import ingest_model
---> 19 self.model_history, self.round_model = ingest_model(self)
20 self.round_history.append(self.model_history.history)
21
/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/model/ingest_model.py in ingest_model(self)
4 through Scan() model paramater.'''
5
----> 6 return self.model(self.x_train,
7 self.y_train,
8 self.x_val,
in talos_model(x_train, y_train, x_val, y_val, params)
18 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc', talos.utils.metrics.f1score])
19
---> 20 history = model.fit(X_train_seq, y_train_seq, validation_data=[X_test_seq, y_test_seq], batch_size=params['batch_size'],
21 callbacks=[talos.utils.live()], epochs=params['epochs'], verbose=0,shuffle=False)
22
~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
106 def _method_wrapper(self, *args, **kwargs):
107 if not self._in_multi_worker_mode(): # pylint: disable=protected-access
--> 108 return method(self, *args, **kwargs)
109
110 # Running inside run_distribute_coordinator already.
~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
1077 train_function = self.make_train_function()
1078 self._train_counter.assign(0)
-> 1079 callbacks.on_train_begin()
1080 training_logs = None
1081 # Handle fault-tolerance for multi-worker.
~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/callbacks.py in on_train_begin(self, logs)
499 if numpy_logs is None: # Only convert once.
500 numpy_logs = tf_utils.to_numpy_or_python_type(logs)
--> 501 callback.on_train_begin(numpy_logs)
502
503 def on_train_end(self, logs=None):
/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/kerasplotlib/traininglog.py in on_train_begin(self, logs)
31
32 def on_train_begin(self, logs={}):
---> 33 self.base_metrics = [metric for metric in self.params['metrics'] if not metric.startswith('val_')]
34 if self.figsize is None:
35 self.figsize = (
KeyError: 'metrics'
@mikkokotila Sorry. The same issue persists. Here is my function, parameter space and experiment error. have you tested talos on the latest version of jupyter notebook?
first we have to make sure to input data and params into the function
def talos_model(x_train, y_train, x_val, y_val, params):
model = Sequential()
model.add(LSTM(params['first_neuron'], input_shape=(X_train_seq.shape[1], X_train_seq.shape[2]),
kernel_initializer=params['kernel_initializer']))
model.add(Dropout(params['dropout']))
model.add(Dense(params['second_neuron'], activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))
model.add(Dropout(params['dropout']))
model.add(Dense(params['third_neuron'], activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))
model.add(Dense(1, activation=params['last_activation'],kernel_initializer=params['kernel_initializer']))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc', talos.utils.metrics.f1score])
history = model.fit(X_train_seq, y_train_seq, validation_data=[X_test_seq, y_test_seq], batch_size=params['batch_size'],
callbacks=[talos.utils.live()], epochs=params['epochs'], verbose=0, shuffle=False)
return history, model
then we can go ahead and set the parameter space
p = {'first_neuron':[20,30,50],
'second_neuron':[6,8,10],
'third_neuron':[2,4],
'hidden_layers':[2,3],
'batch_size': [31,45,60],
'epochs': [110,130,170],
'dropout': [0.1,0.3],
'kernel_initializer': ['normal'],
'optimizer': ['Adam'],
'losses': ['binary_crossentropy'],
'activation':['relu', 'elu'],
'last_activation': ['sigmoid']}
and run the experiment
t = talos.Scan(x=X_train_seq,y=y_train_seq,model=talos_model, params=p, experiment_name='talos_test', round_limit=10)
ERROR:
KeyError Traceback (most recent call last)
in
1 # and run the experiment
----> 2 t = talos.Scan(x=X_train_seq,y=y_train_seq,model=talos_model, params=p, experiment_name='talos_test', round_limit=10)
/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/Scan.py in init(self, x, y, params, model, experiment_name, x_val, y_val, val_split, random_method, seed, performance_target, fraction_limit, round_limit, time_limit, boolean_limit, reduction_method, reduction_interval, reduction_window, reduction_threshold, reduction_metric, minimize_loss, disable_progress_bar, print_params, clear_session, save_weights)
194 # start runtime
195 from .scan_run import scan_run
--> 196 scan_run(self)
/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/scan_run.py in scan_run(self)
24 # otherwise proceed with next permutation
25 from .scan_round import scan_round
---> 26 self = scan_round(self)
27 self.pbar.update(1)
28
/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/scan/scan_round.py in scan_round(self)
17 # fit the model
18 from ..model.ingest_model import ingest_model
---> 19 self.model_history, self.round_model = ingest_model(self)
20 self.round_history.append(self.model_history.history)
21
/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/talos/model/ingest_model.py in ingest_model(self)
4 through Scan() model paramater.'''
5
----> 6 return self.model(self.x_train,
7 self.y_train,
8 self.x_val,
in talos_model(x_train, y_train, x_val, y_val, params)
18 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc', talos.utils.metrics.f1score])
19
---> 20 history = model.fit(X_train_seq, y_train_seq, validation_data=[X_test_seq, y_test_seq], batch_size=params['batch_size'],
21 callbacks=[talos.utils.live()], epochs=params['epochs'], verbose=0, shuffle=False)
22
~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
106 def _method_wrapper(self, *args, **kwargs):
107 if not self._in_multi_worker_mode(): # pylint: disable=protected-access
--> 108 return method(self, *args, **kwargs)
109
110 # Running inside run_distribute_coordinator already.
~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
1077 train_function = self.make_train_function()
1078 self._train_counter.assign(0)
-> 1079 callbacks.on_train_begin()
1080 training_logs = None
1081 # Handle fault-tolerance for multi-worker.
~/Library/Python/3.8/lib/python/site-packages/tensorflow/python/keras/callbacks.py in on_train_begin(self, logs)
499 if numpy_logs is None: # Only convert once.
500 numpy_logs = tf_utils.to_numpy_or_python_type(logs)
--> 501 callback.on_train_begin(numpy_logs)
502
503 def on_train_end(self, logs=None):
/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/kerasplotlib/traininglog.py in on_train_begin(self, logs)
31
32 def on_train_begin(self, logs={}):
---> 33 self.base_metrics = [metric for metric in self.params['metrics'] if not metric.startswith('val_')]
34 if self.figsize is None:
35 self.figsize = (
KeyError: 'metrics'
I'm having this same issue with KeyError: metrics. Does anyone have solved this problem yet?
I'm having this same issue with KeyError: metrics. Does anyone have solved this problem yet?
did you solve it?