Version 0.9.1 makes saving a tf model fail with KeyError: "callable_inputs" #344
Description
When I add a debugger callback
try: debug_hook = smd.KerasHook.create_from_json_file() callbacks.append(debug_hook) except FileNotFoundError: log_dir = "tensorboard-log/" callbacks.append(tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1))
Then I try to save the model after training:
model.save(args.model_dir)
I get the following Error
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/save.py", line 134, in save_model signatures, options) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/save.py", line 80, in save save_lib.save(model, filepath, signatures, options) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/saved_model/save.py", line 976, in save obj, export_dir, signatures, options, meta_graph_def) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/saved_model/save.py", line 1047, in _build_meta_graph checkpoint_graph_view) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/saved_model/signature_serialization.py", line 75, in find_function_to_export functions = saveable_view.list_functions(saveable_view.root) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/saved_model/save.py", line 145, in list_functions self._serialization_cache) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py", line 2590, in _list_functions_for_serialization Model, self)._list_functions_for_serialization(serialization_cache) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py", line 3019, in _list_functions_for_serialization .list_functions_for_serialization(serialization_cache)) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/base_serialization.py", line 87, in list_functions_for_serialization fns = self.functions_to_serialize(serialization_cache) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/layer_serialization.py", line 79, in functions_to_serialize serialization_cache).functions_to_serialize) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/layer_serialization.py", line 95, in _get_serialized_attributes serialization_cache) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/model_serialization.py", line 57, in _get_serialized_attributes_internal serialization_cache)) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/layer_serialization.py", line 104, in _get_serialized_attributes_internal functions = save_impl.wrap_layer_functions(self.obj, serialization_cache) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/save_impl.py", line 155, in wrap_layer_functions original_fns = _replace_child_layer_functions(layer, serialization_cache) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/save_impl.py", line 274, in _replace_child_layer_functions serialization_cache).functions) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/layer_serialization.py", line 95, in _get_serialized_attributes serialization_cache) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/layer_serialization.py", line 161, in _get_serialized_attributes_internal serialization_cache)) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/layer_serialization.py", line 104, in _get_serialized_attributes_internal functions = save_impl.wrap_layer_functions(self.obj, serialization_cache) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/save_impl.py", line 165, in wrap_layer_functions '{}_layer_call_and_return_conditional_losses'.format(layer.name)) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/save_impl.py", line 505, in add_function self.add_trace(*self._input_signature) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/save_impl.py", line 420, in add_trace trace_with_training(True) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/save_impl.py", line 418, in trace_with_training fn.get_concrete_function(*args, **kwargs) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/save_impl.py", line 549, in get_concrete_function return super(LayerCall, self).get_concrete_function(*args, **kwargs) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py", line 1167, in get_concrete_function concrete = self._get_concrete_function_garbage_collected(*args, **kwargs) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py", line 1073, in _get_concrete_function_garbage_collected self._initialize(args, kwargs, add_initializers_to=initializers) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py", line 697, in _initialize *args, **kwds)) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py", line 2855, in _get_concrete_function_internal_garbage_collected graph_function, _, _ = self._maybe_define_function(args, kwargs) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py", line 3213, in _maybe_define_function graph_function = self._create_graph_function(args, kwargs) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py", line 3075, in _create_graph_function capture_by_value=self._capture_by_value), File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/func_graph.py", line 986, in func_graph_from_py_func func_outputs = python_func(*func_args, **func_kwargs) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py", line 600, in wrapped_fn return weak_wrapped_fn().__wrapped__(*args, **kwds) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/save_impl.py", line 515, in wrapper inputs = call_collection.get_input_arg_value(args, kwargs) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/saving/saved_model/save_impl.py", line 455, in get_input_arg_value self._input_arg_name, args, kwargs, inputs_in_args=True) File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py", line 2536, in _get_call_arg_value return args_dict[arg_name] KeyError: 'callable_inputs'
To make this work I would either downgrade to 0.8.1 or remove the debugger callback