CodeNet notebook broken after tensorflow image update to 2.7.0
ckadner opened this issue · 0 comments
ckadner commented
Bug Description:
The CodeNet Language Classification notebook fails with error AlreadyExistsError after tensorflow/tensorflow:latest image moved to version 2.7.0
Steps To Reproduce:
- Go to "Notebooks" > "Project CodeNet Language Classification"
- Select the "Launch" tab
- Click "Submit"
- Wait for the pipeline run graph, click on the notebook run task, and select the "Logs" tab
Error:
AlreadyExistsError: Another metric with the same name already exists.
Additional Information:
Analysis:
As a general rule, the keras Python package version must match the tensorflow version.
However, the MLX Codenet notebook's (still) specifies requirements: 'matplotlib==3.3.4,numpy==1.19.5,tensorflow==2.6.0' but the tensorflow==2.6.0 package requires keras~=2.6. This allows pip to install keras version 2.7.0 with tensorflow version 2.6.0.
Stacktrace:
...
pip install ...
...
Requirement already satisfied: keras~=2.6 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.6.0) (2.7.0)
...
Executing: 65%|██████▍ | 22/34 [00:14<00:21, 1.76s/cell][D 00:47:14.480] Skipping non-executing cell 22
[D 00:47:14.574] Executing cell:
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Activation, Dense, Dropout, Flatten, Input
from tensorflow.keras.layers import Conv1D, MaxPooling1D, Concatenate
# Model Hyperparameters
kernel_sizes = (3, 9, 19)
pooling_sizes = (3, 9, 19)
num_filters = 128
dropout_prob = 0.5
hidden_dims = 128
stage_in = Input(shape=(1024, 68))
convs = []
for i in range(0, len(kernel_sizes)):
conv = Conv1D(filters=num_filters,
kernel_size=kernel_sizes[i],
padding='valid',
activation='relu',
strides=1)(stage_in)
pool = MaxPooling1D(pool_size=pooling_sizes[i])(conv)
flatten = Flatten()(pool)
convs.append(flatten)
if len(kernel_sizes) > 1:
out = Concatenate()(convs)
else:
out = convs[0]
stages = Model(inputs=stage_in, outputs=out)
model = Sequential([
stages,
Dense(hidden_dims, activation='relu'),
Dropout(dropout_prob),
Dense(num_classes, activation='softmax')
])
model.summary()
# Note: also need pydot and GraphViz installed for this.
#from tensorflow.keras.utils import plot_model
#plot_model(model, show_shapes=True, expand_nested=True)
[D 00:47:14.591] msg_type: status
[D 00:47:14.591] content: {'execution_state': 'busy'}
[D 00:47:14.594] msg_type: execute_input
[D 00:47:14.594] content: {'code': "from tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Activation, Dense, Dropout, Flatten, Input\nfrom tensorflow.keras.layers import Conv1D, MaxPooling1D, Concatenate\n\n# Model Hyperparameters\nkernel_sizes = (3, 9, 19)\npooling_sizes = (3, 9, 19)\nnum_filters = 128\ndropout_prob = 0.5\nhidden_dims = 128\n\nstage_in = Input(shape=(1024, 68))\nconvs = []\nfor i in range(0, len(kernel_sizes)):\n conv = Conv1D(filters=num_filters,\n kernel_size=kernel_sizes[i],\n padding='valid',\n activation='relu',\n strides=1)(stage_in)\n pool = MaxPooling1D(pool_size=pooling_sizes[i])(conv)\n flatten = Flatten()(pool)\n convs.append(flatten)\n\nif len(kernel_sizes) > 1:\n out = Concatenate()(convs)\nelse:\n out = convs[0]\n\nstages = Model(inputs=stage_in, outputs=out)\n\nmodel = Sequential([\n stages,\n Dense(hidden_dims, activation='relu'),\n Dropout(dropout_prob),\n Dense(num_classes, activation='softmax')\n])\n\nmodel.summary()\n\n# Note: also need pydot and GraphViz installed for this.\n#from tensorflow.keras.utils import plot_model \n#plot_model(model, show_shapes=True, expand_nested=True)", 'execution_count': 13}
2021-11-12 00:47:14.758883: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory
2021-11-12 00:47:14.758919: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
2021-11-12 00:47:15.835249: E tensorflow/core/lib/monitoring/collection_registry.cc:77] Cannot register 2 metrics with the same name: /tensorflow/api/keras/optimizers
[D 00:47:16.006] msg_type: error
[D 00:47:16.006] content: {'traceback': ['\x1b[0;31m---------------------------------------------------------------------------\x1b[0m', '\x1b[0;31mAlreadyExistsError\x1b[0m Traceback (most recent call last)', ...: Another metric with the same name already exists.'], 'ename': 'AlreadyExistsError', 'evalue': 'Another metric with the same name already exists.'}
[D 00:47:16.009] msg_type: status
[D 00:47:16.010] content: {'execution_state': 'idle'}
Executing: 71%|███████ | 24/34 [00:15<00:14, 1.46s/cell]
Executing: 71%|███████ | 24/34 [00:17<00:07, 1.40cell/s]
[E 00:47:17.596] Unexpected error: <class 'papermill.exceptions.PapermillExecutionError'>
[I 00:47:17.666] 'notebooks/project-codenet-language-classification/':'Project_CodeNet_LangClass' - converting from Project_CodeNet_LangClass-output.ipynb to Project_CodeNet_LangClass.html
[D 00:47:17.730] Applying preprocessor: TagRemovePreprocessor
[D 00:47:17.781] Applying preprocessor: RegexRemovePreprocessor
[D 00:47:17.926] Applying preprocessor: coalesce_streams
[D 00:47:18.014] Applying preprocessor: CSSHTMLHeaderPreprocessor
[D 00:47:18.100] Applying preprocessor: HighlightMagicsPreprocessor
[D 00:47:18.225] Attempting to load template full.tpl
[D 00:47:18.225] template_path: .
[I 00:47:18.389] 'notebooks/project-codenet-language-classification/':'Project_CodeNet_LangClass' - Project_CodeNet_LangClass-output.ipynb converted to Project_CodeNet_LangClass.html (0.723 secs)
[D 00:47:18.404] http://172.21.223.16:9000 "PUT /mlpipeline/notebooks/project-codenet-language-classification/Project_CodeNet_LangClass.ipynb HTTP/1.1" 200 0
[I 00:47:18.405] 'notebooks/project-codenet-language-classification/':'Project_CodeNet_LangClass' - uploaded Project_CodeNet_LangClass-output.ipynb to bucket: mlpipeline object: notebooks/project-codenet-language-classification/Project_CodeNet_LangClass.ipynb (0.016 secs)
[D 00:47:18.419] http://172.21.223.16:9000 "PUT /mlpipeline/notebooks/project-codenet-language-classification/Project_CodeNet_LangClass.html HTTP/1.1" 200 0
[I 00:47:18.420] 'notebooks/project-codenet-language-classification/':'Project_CodeNet_LangClass' - uploaded Project_CodeNet_LangClass.html to bucket: mlpipeline object: notebooks/project-codenet-language-classification/Project_CodeNet_LangClass.html (0.015 secs)
Traceback (most recent call last):
File "bootstrapper.py", line 634, in <module>
main()
File "bootstrapper.py", line 624, in main
file_op.execute()
File "bootstrapper.py", line 369, in execute
raise ex
File "bootstrapper.py", line 354, in execute
papermill.execute_notebook(notebook, notebook_output, kernel_name=kernel_name)
File "/usr/local/lib/python3.8/dist-packages/papermill/execute.py", line 110, in execute_notebook
raise_for_execution_errors(nb, output_path)
File "/usr/local/lib/python3.8/dist-packages/papermill/execute.py", line 222, in raise_for_execution_errors
raise error
papermill.exceptions.PapermillExecutionError:
---------------------------------------------------------------------------
Exception encountered at "In [13]":
---------------------------------------------------------------------------
AlreadyExistsError Traceback (most recent call last)
<ipython-input-13-f486e58f3eab> in <module>
----> 1 from tensorflow.keras.models import Sequential, Model
2 from tensorflow.keras.layers import Activation, Dense, Dropout, Flatten, Input
3 from tensorflow.keras.layers import Conv1D, MaxPooling1D, Concatenate
4
5 # Model Hyperparameters
/usr/local/lib/python3.8/dist-packages/keras/api/_v2/keras/__init__.py in <module>
8 import sys as _sys
9
---> 10 from keras import __version__
11 from keras.api._v2.keras import __internal__
12 from keras.api._v2.keras import activations
/usr/local/lib/python3.8/dist-packages/keras/__init__.py in <module>
23
24 # See b/110718070#comment18 for more details about this import.
---> 25 from keras import models
26
27 from keras.engine.input_layer import Input
/usr/local/lib/python3.8/dist-packages/keras/models.py in <module>
18 import tensorflow.compat.v2 as tf
19 from keras import backend
---> 20 from keras import metrics as metrics_module
21 from keras import optimizer_v1
22 from keras.engine import functional
/usr/local/lib/python3.8/dist-packages/keras/metrics.py in <module>
24
25 import numpy as np
---> 26 from keras import activations
27 from keras import backend
28 from keras.engine import base_layer
/usr/local/lib/python3.8/dist-packages/keras/activations.py in <module>
18
19 from keras import backend
---> 20 from keras.layers import advanced_activations
21 from keras.utils.generic_utils import deserialize_keras_object
22 from keras.utils.generic_utils import serialize_keras_object
/usr/local/lib/python3.8/dist-packages/keras/layers/__init__.py in <module>
21
22 # Generic layers.
---> 23 from keras.engine.input_layer import Input
24 from keras.engine.input_layer import InputLayer
25 from keras.engine.input_spec import InputSpec
/usr/local/lib/python3.8/dist-packages/keras/engine/input_layer.py in <module>
19 from keras import backend
20 from keras.distribute import distributed_training_utils
---> 21 from keras.engine import base_layer
22 from keras.engine import keras_tensor
23 from keras.engine import node as node_module
/usr/local/lib/python3.8/dist-packages/keras/engine/base_layer.py in <module>
41 from keras.engine import node as node_module
42 from keras.mixed_precision import autocast_variable
---> 43 from keras.mixed_precision import loss_scale_optimizer
44 from keras.mixed_precision import policy
45 from keras.saving.saved_model import layer_serialization
/usr/local/lib/python3.8/dist-packages/keras/mixed_precision/loss_scale_optimizer.py in <module>
16
17 from keras import backend
---> 18 from keras import optimizers
19 from keras.mixed_precision import loss_scale as keras_loss_scale_module
20 from keras.optimizer_v2 import optimizer_v2
/usr/local/lib/python3.8/dist-packages/keras/optimizers.py in <module>
24 from keras.optimizer_v1 import Optimizer
25 from keras.optimizer_v1 import TFOptimizer
---> 26 from keras.optimizer_v2 import adadelta as adadelta_v2
27 from keras.optimizer_v2 import adagrad as adagrad_v2
28 from keras.optimizer_v2 import adam as adam_v2
/usr/local/lib/python3.8/dist-packages/keras/optimizer_v2/adadelta.py in <module>
20 import numpy as np
21 from keras import backend_config
---> 22 from keras.optimizer_v2 import optimizer_v2
23 from tensorflow.python.util.tf_export import keras_export
24
/usr/local/lib/python3.8/dist-packages/keras/optimizer_v2/optimizer_v2.py in <module>
34
35
---> 36 keras_optimizers_gauge = tf.__internal__.monitoring.BoolGauge(
37 "/tensorflow/api/keras/optimizers", "keras optimizer usage", "method")
38
/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/monitoring.py in __init__(self, name, description, *labels)
358 *labels: The label list of the new metric.
359 """
--> 360 super(BoolGauge, self).__init__('BoolGauge', _bool_gauge_methods,
361 len(labels), name, description, *labels)
362
/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/monitoring.py in __init__(self, metric_name, metric_methods, label_length, *args)
133 self._metric_name, len(self._metric_methods)))
134
--> 135 self._metric = self._metric_methods[self._label_length].create(*args)
136
137 def __del__(self):
AlreadyExistsError: Another metric with the same name already exists.