keras-team/autokeras

Bug: TypeError: can only concatenate str (not "NoneType") to st

qrdlgit opened this issue · 1 comments

Getting strange, non deterministic error when doing fit. This is on the kaggle environment, just using !pip install autokeras

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[3], line 92
     87     with CustomObjectScope({'custom_metric': custom_metric}):
     88         # Initialize and train the AutoKeras structured data classifier
     89         clf = StructuredDataClassifier(max_trials=mts,metrics=[custom_metric],
     90             objective = kerastuner.Objective("val_custom_metric", direction="min"),
     91             project_name = f'sdc30h_{split}')
---> 92         clf.fit(X_train, y_train, validation_data = [X_test, y_test], epochs = 1000, 
     93                 verbose=0, callbacks=[CustomCallback(), 
     94                 tf.keras.callbacks.EarlyStopping(
     95                         monitor="val_custom_metric",
     96                         min_delta=0,
     97                         patience=100,
     98                         verbose=1,
     99                         mode="min",
    100                         baseline=None,
    101                         restore_best_weights=True,
    102                         start_from_epoch=0,
    103 )])  # Set verbose to 1 or 2 to see the training progress
    104         print("y_test", list(y_test), list(test_index))
    105         metric = clf.evaluate(X_test, y_test, verbose=0)

File /opt/conda/lib/python3.10/site-packages/autokeras/tasks/structured_data.py:326, in StructuredDataClassifier.fit(self, x, y, epochs, callbacks, validation_split, validation_data, **kwargs)
    279 def fit(
    280     self,
    281     x=None,
   (...)
    287     **kwargs
    288 ):
    289     """Search for the best model and hyperparameters for the AutoModel.
    290 
    291     # Arguments
   (...)
    324             validation loss values and validation metrics values (if applicable).
    325     """
--> 326     history = super().fit(
    327         x=x,
    328         y=y,
    329         epochs=epochs,
    330         callbacks=callbacks,
    331         validation_split=validation_split,
    332         validation_data=validation_data,
    333         **kwargs
    334     )
    335     return history

File /opt/conda/lib/python3.10/site-packages/autokeras/tasks/structured_data.py:139, in BaseStructuredDataPipeline.fit(self, x, y, epochs, callbacks, validation_split, validation_data, **kwargs)
    135         validation_data = self._read_from_csv(x_val, y_val)
    137 self.check_in_fit(x)
--> 139 history = super().fit(
    140     x=x,
    141     y=y,
    142     epochs=epochs,
    143     callbacks=callbacks,
    144     validation_split=validation_split,
    145     validation_data=validation_data,
    146     **kwargs
    147 )
    148 return history

File /opt/conda/lib/python3.10/site-packages/autokeras/auto_model.py:292, in AutoModel.fit(self, x, y, batch_size, epochs, callbacks, validation_split, validation_data, verbose, **kwargs)
    287 if validation_data is None and validation_split:
    288     dataset, validation_data = data_utils.split_dataset(
    289         dataset, validation_split
    290     )
--> 292 history = self.tuner.search(
    293     x=dataset,
    294     epochs=epochs,
    295     callbacks=callbacks,
    296     validation_data=validation_data,
    297     validation_split=validation_split,
    298     verbose=verbose,
    299     **kwargs
    300 )
    302 return history

File /opt/conda/lib/python3.10/site-packages/autokeras/engine/tuner.py:193, in AutoTuner.search(self, epochs, callbacks, validation_split, verbose, **fit_kwargs)
    191 self._try_build(hp)
    192 self.oracle.update_space(hp)
--> 193 super().search(
    194     epochs=epochs, callbacks=new_callbacks, verbose=verbose, **fit_kwargs
    195 )
    197 # Train the best model use validation data.
    198 # Train the best model with enough number of epochs.
    199 if validation_split > 0 or early_stopping_inserted:

File /opt/conda/lib/python3.10/site-packages/keras_tuner/engine/base_tuner.py:231, in BaseTuner.search(self, *fit_args, **fit_kwargs)
    229     self.on_trial_begin(trial)
    230     self._try_run_and_update_trial(trial, *fit_args, **fit_kwargs)
--> 231     self.on_trial_end(trial)
    232 self.on_search_end()

File /opt/conda/lib/python3.10/site-packages/keras_tuner/engine/base_tuner.py:335, in BaseTuner.on_trial_end(self, trial)
    329 def on_trial_end(self, trial):
    330     """Called at the end of a trial.
    331 
    332     Args:
    333         trial: A `Trial` instance.
    334     """
--> 335     self.oracle.end_trial(trial)
    336     # Display needs the updated trial scored by the Oracle.
    337     self._display.on_trial_end(self.oracle.get_trial(trial.trial_id))

File /opt/conda/lib/python3.10/site-packages/keras_tuner/engine/oracle.py:107, in synchronized.<locals>.wrapped_func(*args, **kwargs)
    105     LOCKS[oracle].acquire()
    106     THREADS[oracle] = thread_name
--> 107 ret_val = func(*args, **kwargs)
    108 if need_acquire:
    109     THREADS[oracle] = None

File /opt/conda/lib/python3.10/site-packages/keras_tuner/engine/oracle.py:434, in Oracle.end_trial(self, trial)
    432 if not self._retry(trial):
    433     self.end_order.append(trial.trial_id)
--> 434     self._check_consecutive_failures()
    436 self._save_trial(trial)
    437 self.save()

File /opt/conda/lib/python3.10/site-packages/keras_tuner/engine/oracle.py:387, in Oracle._check_consecutive_failures(self)
    384     consecutive_failures = 0
    385 if consecutive_failures == self.max_consecutive_failed_trials:
    386     raise RuntimeError(
--> 387         "Number of consecutive failures excceeded the limit "
    388         f"of {self.max_consecutive_failed_trials}.\n"
    389         + trial.message
    390     )

TypeError: can only concatenate str (not "NoneType") to str

I am also encountering this issue.