Error when using 'model.predict_all_layers_full_cov'

[SaremS]

Hi,

at first thanks for sharing your work online - I am currently using this for my thesis and it's making
things much easier.

However, I am having trouble tracing back the following error:

• I have trained two DGP models using the Natural Gradient Optimization from this repository, one DGP has only one kernel, while the other one has a chain of three kernels

• model.predict_y() works fine for both

• model.predict_all_layers_full_cov() is only working with the one layer DGP, for the three-layer one (and I get the same error for a two-layer DGP) I get the following error:

InvalidArgumentError

Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1321 try:
-> 1322 return fn(*args)
1323 except errors.OpError as e:

/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1306 return self._call_tf_sessionrun(
-> 1307 options, feed_dict, fetch_list, target_list, run_metadata)
1308

/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1408 self._session, options, feed_dict, fetch_list, target_list,
-> 1409 run_metadata)
1410 else:

InvalidArgumentError: In[0].dim(1) and In[1].dim(1) must be the same: [30,1,30,30] vs [30,80,30,1]
[[Node: autoflow/DGP/predict_all_layers_full_cov_10/MatMul = BatchMatMul[T=DT_DOUBLE, adj_x=false, adj_y=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](autoflow/DGP/predict_all_layers_full_cov_10/Cholesky, autoflow/DGP/predict_all_layers_full_cov_10/strided_slice_4)]]

During handling of the above exception, another exception occurred:

InvalidArgumentError Traceback (most recent call last)
in ()
----> 1 full_c2 = m_dgp2.predict_all_layers_full_cov(X_test,30)

/usr/local/lib/python3.6/dist-packages/gpflow/decors.py in autoflow_wrapper(obj, *args, **kwargs)
163 _setup_storage(store, *af_args, **af_kwargs)
164 _build_method(method, obj, store)
--> 165 return _session_run(session, obj, store, *args, **kwargs)
166 return autoflow_wrapper
167 return autoflow_wrapper_decorator

/usr/local/lib/python3.6/dist-packages/gpflow/decors.py in _session_run(session, obj, store, *args, **kwargs)
201 initialize = kwargs.pop('initialize', False)
202 obj.initialize(session=session, force=initialize)
--> 203 return session.run(store['result'], **kwargs)
204
205

/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
898 try:
899 result = self._run(None, fetches, feed_dict, options_ptr,
--> 900 run_metadata_ptr)
901 if run_metadata:
902 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1133 if final_fetches or final_targets or (handle and feed_dict_tensor):
1134 results = self._do_run(handle, final_targets, final_fetches,
-> 1135 feed_dict_tensor, options, run_metadata)
1136 else:
1137 results = []

/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1314 if handle is None:
1315 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1316 run_metadata)
1317 else:
1318 return self._do_call(_prun_fn, handle, feeds, fetches)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1333 except KeyError:
1334 pass
-> 1335 raise type(e)(node_def, op, message)
1336
1337 def _extend_graph(self):

InvalidArgumentError: In[0].dim(1) and In[1].dim(1) must be the same: [30,1,30,30] vs [30,80,30,1]
[[Node: autoflow/DGP/predict_all_layers_full_cov_10/MatMul = BatchMatMul[T=DT_DOUBLE, adj_x=false, adj_y=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](autoflow/DGP/predict_all_layers_full_cov_10/Cholesky, autoflow/DGP/predict_all_layers_full_cov_10/strided_slice_4)]]

Caused by op 'autoflow/DGP/predict_all_layers_full_cov_10/MatMul', defined at:
File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py", line 16, in
app.launch_new_instance()
File "/usr/local/lib/python3.6/dist-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/usr/local/lib/python3.6/dist-packages/ipykernel/kernelapp.py", line 486, in start
self.io_loop.start()
File "/usr/local/lib/python3.6/dist-packages/tornado/platform/asyncio.py", line 127, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.6/asyncio/base_events.py", line 421, in run_forever
self._run_once()
File "/usr/lib/python3.6/asyncio/base_events.py", line 1426, in _run_once
handle._run()
File "/usr/lib/python3.6/asyncio/events.py", line 127, in _run
self._callback(*self._args)
File "/usr/local/lib/python3.6/dist-packages/tornado/platform/asyncio.py", line 117, in _handle_events
handler_func(fileobj, events)
File "/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py", line 276, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py", line 450, in _handle_events
self._handle_recv()
File "/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py", line 480, in _handle_recv
self._run_callback(callback, msg)
File "/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py", line 432, in _run_callback
callback(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py", line 276, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "/usr/local/lib/python3.6/dist-packages/ipykernel/ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.6/dist-packages/ipykernel/zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py", line 2903, in run_ast_nodes
if self.run_code(code, result):
File "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "", line 1, in
full_c2 = m_dgp2.predict_all_layers_full_cov(X_test,2)
File "/usr/local/lib/python3.6/dist-packages/gpflow/decors.py", line 164, in autoflow_wrapper
_build_method(method, obj, store)
File "/usr/local/lib/python3.6/dist-packages/gpflow/decors.py", line 207, in _build_method
store['result'] = method(obj, *store['arguments'])
File "/usr/local/lib/python3.6/dist-packages/doubly_stochastic_dgp/dgp.py", line 108, in predict_all_layers_full_cov
return self.propagate(Xnew, full_cov=True, S=num_samples)
File "/usr/local/lib/python3.6/dist-packages/gpflow/decors.py", line 67, in tensor_mode_wrapper
result = method(obj, *args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/doubly_stochastic_dgp/dgp.py", line 64, in propagate
F, Fmean, Fvar = layer.sample_from_conditional(F, z=z, full_cov=full_cov)
File "/usr/local/lib/python3.6/dist-packages/doubly_stochastic_dgp/layers.py", line 109, in sample_from_conditional
samples = reparameterize(mean, var, z, full_cov=full_cov)
File "/usr/local/lib/python3.6/dist-packages/doubly_stochastic_dgp/utils.py", line 47, in reparameterize
f = mean + tf.matmul(chol, z_SDN1)[:, :, :, 0] # SDN(1)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py", line 2084, in matmul
a, b, adj_x=adjoint_a, adj_y=adjoint_b, name=name)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_math_ops.py", line 1236, in batch_mat_mul
"BatchMatMul", x=x, y=y, adj_x=adj_x, adj_y=adj_y, name=name)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py", line 3392, in create_op
op_def=op_def)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

InvalidArgumentError (see above for traceback): In[0].dim(1) and In[1].dim(1) must be the same: [30,1,30,30] vs [30,80,30,1]
[[Node: autoflow/DGP/predict_all_layers_full_cov_10/MatMul = BatchMatMul[T=DT_DOUBLE, adj_x=false, adj_y=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](autoflow/DGP/predict_all_layers_full_cov_10/Cholesky, autoflow/DGP/predict_all_layers_full_cov_10/strided_slice_4)]]

Thanks for your help,

Sarem

[hughsalimbeni]

Could you show the details of how you construct the model? For some reason the 80 dimensions in the mean are not being matched in the covariance. Are you using the DGP class, or constructing the layers directly and using DGP_Base? The full code to reproduce the error (with dummy data) would be great.