Hello, world!
\n' - 'Check out my website!
') - - def test_table_formatting(self): - self._test( - textwrap.dedent( - u"""\ + def _test(self, markdown_string, expected): + actual = plugin_util.markdown_to_safe_html(markdown_string) + self.assertEqual(expected, actual) + + def test_empty_input(self): + self._test(u"", u"") + + def test_basic_formatting(self): + self._test( + u"# _Hello_, **world!**\n\n" + "Check out [my website](http://example.com)!", + u"Hello, world!
\n" + 'Check out my website!
', + ) + + def test_table_formatting(self): + self._test( + textwrap.dedent( + u"""\ Here is some data: TensorBoard usage | Happiness @@ -52,9 +53,10 @@ def test_table_formatting(self): 0.5 | 0.5 1.0 | 1.0 - Wouldn't you agree?"""), - textwrap.dedent( - u"""\ + Wouldn't you agree?""" + ), + textwrap.dedent( + u"""\Here is some data:
Wouldn't you agree?
""")) - - def test_whitelisted_tags_and_attributes_allowed(self): - s = (u'Check out ' - 'my website!') - self._test(s, u'%s
' % s) - - def test_arbitrary_tags_and_attributes_removed(self): - self._test(u'We should bring back the ; ' - '' - 'sign the petition!', - u'We should bring back the ' - '<blink>blink tag</blink>; ' - 'sign the petition!
') - - def test_javascript_hrefs_sanitized(self): - self._test(u'A sketchy link for you', - u'A sketchy link for you
') - - def test_byte_strings_interpreted_as_utf8(self): - s = u'> Look\u2014some UTF-8!'.encode('utf-8') - assert isinstance(s, six.binary_type), (type(s), six.binary_type) - self._test(s, - u'\n') - - def test_unicode_strings_passed_through(self): - s = u'> Look\u2014some UTF-8!' - assert not isinstance(s, six.binary_type), (type(s), six.binary_type) - self._test(s, - u'Look\u2014some UTF-8!
\n
\n') - - def test_null_bytes_stripped_before_markdown_processing(self): - # If this function is mistakenly called with UTF-16 or UTF-32 encoded text, - # there will probably be a bunch of null bytes. These would be stripped by - # the sanitizer no matter what, but make sure we remove them before markdown - # interpretation to avoid affecting output (e.g. middle-word underscores - # would generate erroneous tags like "underscore") and add an - # HTML comment with a warning. - s = u'un_der_score'.encode('utf-32-le') - # UTF-32 encoding of ASCII will have 3 null bytes per char. 36 = 3 * 12. - self._test(s, - u'\n' - 'Look\u2014some UTF-8!
\n
un_der_score
') +Wouldn't you agree?
""" + ), + ) + + def test_whitelisted_tags_and_attributes_allowed(self): + s = ( + u'Check out ' + "my website!" + ) + self._test(s, u"%s
" % s) + + def test_arbitrary_tags_and_attributes_removed(self): + self._test( + u"We should bring back the ; " + '' + "sign the petition!", + u"We should bring back the " + "<blink>blink tag</blink>; " + 'sign the petition!
', + ) + + def test_javascript_hrefs_sanitized(self): + self._test( + u'A sketchy link for you', + u"A sketchy link for you
", + ) + + def test_byte_strings_interpreted_as_utf8(self): + s = u"> Look\u2014some UTF-8!".encode("utf-8") + assert isinstance(s, six.binary_type), (type(s), six.binary_type) + self._test( + s, u"\n" + ) + + def test_unicode_strings_passed_through(self): + s = u"> Look\u2014some UTF-8!" + assert not isinstance(s, six.binary_type), (type(s), six.binary_type) + self._test( + s, u"Look\u2014some UTF-8!
\n
\n" + ) + + def test_null_bytes_stripped_before_markdown_processing(self): + # If this function is mistakenly called with UTF-16 or UTF-32 encoded text, + # there will probably be a bunch of null bytes. These would be stripped by + # the sanitizer no matter what, but make sure we remove them before markdown + # interpretation to avoid affecting output (e.g. middle-word underscores + # would generate erroneous tags like "underscore") and add an + # HTML comment with a warning. + s = u"un_der_score".encode("utf-32-le") + # UTF-32 encoding of ASCII will have 3 null bytes per char. 36 = 3 * 12. + self._test( + s, + u"\n" + "Look\u2014some UTF-8!
\n
un_der_score
", + ) class ExperimentIdTest(tb_test.TestCase): - """Tests for `plugin_util.experiment_id`.""" + """Tests for `plugin_util.experiment_id`.""" - def test_default(self): - # This shouldn't happen; the `ExperimentIdMiddleware` always set an - # experiment ID. In case something goes wrong, degrade gracefully. - environ = {} - self.assertEqual(plugin_util.experiment_id(environ), "") + def test_default(self): + # This shouldn't happen; the `ExperimentIdMiddleware` always set an + # experiment ID. In case something goes wrong, degrade gracefully. + environ = {} + self.assertEqual(plugin_util.experiment_id(environ), "") - def test_present(self): - environ = {experiment_id.WSGI_ENVIRON_KEY: "123"} - self.assertEqual(plugin_util.experiment_id(environ), "123") + def test_present(self): + environ = {experiment_id.WSGI_ENVIRON_KEY: "123"} + self.assertEqual(plugin_util.experiment_id(environ), "123") -if __name__ == '__main__': - tb_test.main() +if __name__ == "__main__": + tb_test.main() diff --git a/tensorboard/plugins/audio/audio_demo.py b/tensorboard/plugins/audio/audio_demo.py index 347ba77523..ec7df2c88a 100644 --- a/tensorboard/plugins/audio/audio_demo.py +++ b/tensorboard/plugins/audio/audio_demo.py @@ -30,157 +30,178 @@ FLAGS = flags.FLAGS -flags.DEFINE_string('logdir', '/tmp/audio_demo', - 'Directory into which to write TensorBoard data.') +flags.DEFINE_string( + "logdir", + "/tmp/audio_demo", + "Directory into which to write TensorBoard data.", +) -flags.DEFINE_integer('steps', 50, - 'Number of frequencies of each waveform to generate.') +flags.DEFINE_integer( + "steps", 50, "Number of frequencies of each waveform to generate." +) # Parameters for the audio output. -flags.DEFINE_integer('sample_rate', 44100, 'Sample rate, in Hz.') -flags.DEFINE_float('duration', 2.0, 'Duration of each waveform, in s.') +flags.DEFINE_integer("sample_rate", 44100, "Sample rate, in Hz.") +flags.DEFINE_float("duration", 2.0, "Duration of each waveform, in s.") def _samples(): - """Compute how many samples should be included in each waveform.""" - return int(FLAGS.sample_rate * FLAGS.duration) + """Compute how many samples should be included in each waveform.""" + return int(FLAGS.sample_rate * FLAGS.duration) def run(logdir, run_name, wave_name, wave_constructor): - """Generate wave data of the given form. - - The provided function `wave_constructor` should accept a scalar tensor - of type float32, representing the frequency (in Hz) at which to - construct a wave, and return a tensor of shape [1, _samples(), `n`] - representing audio data (for some number of channels `n`). - - Waves will be generated at frequencies ranging from A4 to A5. - - Arguments: - logdir: the top-level directory into which to write summary data - run_name: the name of this run; will be created as a subdirectory - under logdir - wave_name: the name of the wave being generated - wave_constructor: see above - """ - tf.compat.v1.reset_default_graph() - tf.compat.v1.set_random_seed(0) - - # On each step `i`, we'll set this placeholder to `i`. This allows us - # to know "what time it is" at each step. - step_placeholder = tf.compat.v1.placeholder(tf.float32, shape=[]) - - # We want to linearly interpolate a frequency between A4 (440 Hz) and - # A5 (880 Hz). - with tf.name_scope('compute_frequency'): - f_min = 440.0 - f_max = 880.0 - t = step_placeholder / (FLAGS.steps - 1) - frequency = f_min * (1.0 - t) + f_max * t - - # Let's log this frequency, just so that we can make sure that it's as - # expected. - tf.compat.v1.summary.scalar('frequency', frequency) - - # Now, we pass this to the wave constructor to get our waveform. Doing - # so within a name scope means that any summaries that the wave - # constructor produces will be namespaced. - with tf.name_scope(wave_name): - waveform = wave_constructor(frequency) - - # We also have the opportunity to annotate each audio clip with a - # label. This is a good place to include the frequency, because it'll - # be visible immediately next to the audio clip. - with tf.name_scope('compute_labels'): - samples = tf.shape(input=waveform)[0] - wave_types = tf.tile(["*Wave type:* `%s`." % wave_name], [samples]) - frequencies = tf.strings.join([ - "*Frequency:* ", - tf.tile([tf.as_string(frequency, precision=2)], [samples]), - " Hz.", - ]) - samples = tf.strings.join([ - "*Sample:* ", tf.as_string(tf.range(samples) + 1), - " of ", tf.as_string(samples), ".", - ]) - labels = tf.strings.join([wave_types, frequencies, samples], separator=" ") - - # We can place a description next to the summary in TensorBoard. This - # is a good place to explain what the summary represents, methodology - # for creating it, etc. Let's include the source code of the function - # that generated the wave. - source = '\n'.join(' %s' % line.rstrip() - for line in inspect.getsourcelines(wave_constructor)[0]) - description = ("A wave of type `%r`, generated via:\n\n%s" - % (wave_name, source)) - - # Here's the crucial piece: we interpret this result as audio. - summary.op('waveform', waveform, FLAGS.sample_rate, - labels=labels, - display_name=wave_name, - description=description) - - # Now, we can collect up all the summaries and begin the run. - summ = tf.compat.v1.summary.merge_all() - - sess = tf.compat.v1.Session() - writer = tf.summary.FileWriter(os.path.join(logdir, run_name)) - writer.add_graph(sess.graph) - sess.run(tf.compat.v1.global_variables_initializer()) - for step in xrange(FLAGS.steps): - s = sess.run(summ, feed_dict={step_placeholder: float(step)}) - writer.add_summary(s, global_step=step) - writer.close() + """Generate wave data of the given form. + + The provided function `wave_constructor` should accept a scalar tensor + of type float32, representing the frequency (in Hz) at which to + construct a wave, and return a tensor of shape [1, _samples(), `n`] + representing audio data (for some number of channels `n`). + + Waves will be generated at frequencies ranging from A4 to A5. + + Arguments: + logdir: the top-level directory into which to write summary data + run_name: the name of this run; will be created as a subdirectory + under logdir + wave_name: the name of the wave being generated + wave_constructor: see above + """ + tf.compat.v1.reset_default_graph() + tf.compat.v1.set_random_seed(0) + + # On each step `i`, we'll set this placeholder to `i`. This allows us + # to know "what time it is" at each step. + step_placeholder = tf.compat.v1.placeholder(tf.float32, shape=[]) + + # We want to linearly interpolate a frequency between A4 (440 Hz) and + # A5 (880 Hz). + with tf.name_scope("compute_frequency"): + f_min = 440.0 + f_max = 880.0 + t = step_placeholder / (FLAGS.steps - 1) + frequency = f_min * (1.0 - t) + f_max * t + + # Let's log this frequency, just so that we can make sure that it's as + # expected. + tf.compat.v1.summary.scalar("frequency", frequency) + + # Now, we pass this to the wave constructor to get our waveform. Doing + # so within a name scope means that any summaries that the wave + # constructor produces will be namespaced. + with tf.name_scope(wave_name): + waveform = wave_constructor(frequency) + + # We also have the opportunity to annotate each audio clip with a + # label. This is a good place to include the frequency, because it'll + # be visible immediately next to the audio clip. + with tf.name_scope("compute_labels"): + samples = tf.shape(input=waveform)[0] + wave_types = tf.tile(["*Wave type:* `%s`." % wave_name], [samples]) + frequencies = tf.strings.join( + [ + "*Frequency:* ", + tf.tile([tf.as_string(frequency, precision=2)], [samples]), + " Hz.", + ] + ) + samples = tf.strings.join( + [ + "*Sample:* ", + tf.as_string(tf.range(samples) + 1), + " of ", + tf.as_string(samples), + ".", + ] + ) + labels = tf.strings.join( + [wave_types, frequencies, samples], separator=" " + ) + + # We can place a description next to the summary in TensorBoard. This + # is a good place to explain what the summary represents, methodology + # for creating it, etc. Let's include the source code of the function + # that generated the wave. + source = "\n".join( + " %s" % line.rstrip() + for line in inspect.getsourcelines(wave_constructor)[0] + ) + description = "A wave of type `%r`, generated via:\n\n%s" % ( + wave_name, + source, + ) + + # Here's the crucial piece: we interpret this result as audio. + summary.op( + "waveform", + waveform, + FLAGS.sample_rate, + labels=labels, + display_name=wave_name, + description=description, + ) + + # Now, we can collect up all the summaries and begin the run. + summ = tf.compat.v1.summary.merge_all() + + sess = tf.compat.v1.Session() + writer = tf.summary.FileWriter(os.path.join(logdir, run_name)) + writer.add_graph(sess.graph) + sess.run(tf.compat.v1.global_variables_initializer()) + for step in xrange(FLAGS.steps): + s = sess.run(summ, feed_dict={step_placeholder: float(step)}) + writer.add_summary(s, global_step=step) + writer.close() # Now, let's take a look at the kinds of waves that we can generate. def sine_wave(frequency): - """Emit a sine wave at the given frequency.""" - xs = tf.reshape(tf.range(_samples(), dtype=tf.float32), [1, _samples(), 1]) - ts = xs / FLAGS.sample_rate - return tf.sin(2 * math.pi * frequency * ts) + """Emit a sine wave at the given frequency.""" + xs = tf.reshape(tf.range(_samples(), dtype=tf.float32), [1, _samples(), 1]) + ts = xs / FLAGS.sample_rate + return tf.sin(2 * math.pi * frequency * ts) def square_wave(frequency): - """Emit a square wave at the given frequency.""" - # The square is just the sign of the sine! - return tf.sign(sine_wave(frequency)) + """Emit a square wave at the given frequency.""" + # The square is just the sign of the sine! + return tf.sign(sine_wave(frequency)) def triangle_wave(frequency): - """Emit a triangle wave at the given frequency.""" - xs = tf.reshape(tf.range(_samples(), dtype=tf.float32), [1, _samples(), 1]) - ts = xs / FLAGS.sample_rate - # - # A triangle wave looks like this: - # - # /\ /\ - # / \ / \ - # \ / \ / - # \/ \/ - # - # If we look at just half a period (the first four slashes in the - # diagram above), we can see that it looks like a transformed absolute - # value function. - # - # Let's start by computing the times relative to the start of each - # half-wave pulse (each individual "mountain" or "valley", of which - # there are four in the above diagram). - half_pulse_index = ts * (frequency * 2) - half_pulse_angle = half_pulse_index % 1.0 # in [0, 1] - # - # Now, we can see that each positive half-pulse ("mountain") has - # amplitude given by A(z) = 0.5 - abs(z - 0.5), and then normalized: - absolute_amplitude = (0.5 - tf.abs(half_pulse_angle - 0.5)) / 0.5 - # - # But every other half-pulse is negative, so we should invert these. - half_pulse_parity = tf.sign(1 - (half_pulse_index % 2.0)) - amplitude = half_pulse_parity * absolute_amplitude - # - # This is precisely the desired result, so we're done! - return amplitude + """Emit a triangle wave at the given frequency.""" + xs = tf.reshape(tf.range(_samples(), dtype=tf.float32), [1, _samples(), 1]) + ts = xs / FLAGS.sample_rate + # + # A triangle wave looks like this: + # + # /\ /\ + # / \ / \ + # \ / \ / + # \/ \/ + # + # If we look at just half a period (the first four slashes in the + # diagram above), we can see that it looks like a transformed absolute + # value function. + # + # Let's start by computing the times relative to the start of each + # half-wave pulse (each individual "mountain" or "valley", of which + # there are four in the above diagram). + half_pulse_index = ts * (frequency * 2) + half_pulse_angle = half_pulse_index % 1.0 # in [0, 1] + # + # Now, we can see that each positive half-pulse ("mountain") has + # amplitude given by A(z) = 0.5 - abs(z - 0.5), and then normalized: + absolute_amplitude = (0.5 - tf.abs(half_pulse_angle - 0.5)) / 0.5 + # + # But every other half-pulse is negative, so we should invert these. + half_pulse_parity = tf.sign(1 - (half_pulse_index % 2.0)) + amplitude = half_pulse_parity * absolute_amplitude + # + # This is precisely the desired result, so we're done! + return amplitude # If we want to get fancy, we can use our above waves as primitives to @@ -188,74 +209,79 @@ def triangle_wave(frequency): def bisine_wave(frequency): - """Emit two sine waves, in stereo at different octaves.""" - # - # We can first our existing sine generator to generate two different - # waves. - f_hi = frequency - f_lo = frequency / 2.0 - with tf.name_scope('hi'): - sine_hi = sine_wave(f_hi) - with tf.name_scope('lo'): - sine_lo = sine_wave(f_lo) - # - # Now, we have two tensors of shape [1, _samples(), 1]. By concatenating - # them along axis 2, we get a tensor of shape [1, _samples(), 2]---a - # stereo waveform. - return tf.concat([sine_lo, sine_hi], axis=2) + """Emit two sine waves, in stereo at different octaves.""" + # + # We can first our existing sine generator to generate two different + # waves. + f_hi = frequency + f_lo = frequency / 2.0 + with tf.name_scope("hi"): + sine_hi = sine_wave(f_hi) + with tf.name_scope("lo"): + sine_lo = sine_wave(f_lo) + # + # Now, we have two tensors of shape [1, _samples(), 1]. By concatenating + # them along axis 2, we get a tensor of shape [1, _samples(), 2]---a + # stereo waveform. + return tf.concat([sine_lo, sine_hi], axis=2) def bisine_wahwah_wave(frequency): - """Emit two sine waves with balance oscillating left and right.""" - # - # This is clearly intended to build on the bisine wave defined above, - # so we can start by generating that. - waves_a = bisine_wave(frequency) - # - # Then, by reversing axis 2, we swap the stereo channels. By mixing - # this with `waves_a`, we'll be able to create the desired effect. - waves_b = tf.reverse(waves_a, axis=[2]) - # - # Let's have the balance oscillate from left to right four times. - iterations = 4 - # - # Now, we compute the balance for each sample: `ts` has values - # in [0, 1] that indicate how much we should use `waves_a`. - xs = tf.reshape(tf.range(_samples(), dtype=tf.float32), [1, _samples(), 1]) - thetas = xs / _samples() * iterations - ts = (tf.sin(math.pi * 2 * thetas) + 1) / 2 - # - # Finally, we can mix the two together, and we're done. - wave = ts * waves_a + (1.0 - ts) * waves_b - # - # Alternately, we can make the effect more pronounced by exaggerating - # the sample data. Let's emit both variations. - exaggerated_wave = wave ** 3.0 - return tf.concat([wave, exaggerated_wave], axis=0) + """Emit two sine waves with balance oscillating left and right.""" + # + # This is clearly intended to build on the bisine wave defined above, + # so we can start by generating that. + waves_a = bisine_wave(frequency) + # + # Then, by reversing axis 2, we swap the stereo channels. By mixing + # this with `waves_a`, we'll be able to create the desired effect. + waves_b = tf.reverse(waves_a, axis=[2]) + # + # Let's have the balance oscillate from left to right four times. + iterations = 4 + # + # Now, we compute the balance for each sample: `ts` has values + # in [0, 1] that indicate how much we should use `waves_a`. + xs = tf.reshape(tf.range(_samples(), dtype=tf.float32), [1, _samples(), 1]) + thetas = xs / _samples() * iterations + ts = (tf.sin(math.pi * 2 * thetas) + 1) / 2 + # + # Finally, we can mix the two together, and we're done. + wave = ts * waves_a + (1.0 - ts) * waves_b + # + # Alternately, we can make the effect more pronounced by exaggerating + # the sample data. Let's emit both variations. + exaggerated_wave = wave ** 3.0 + return tf.concat([wave, exaggerated_wave], axis=0) def run_all(logdir, verbose=False): - """Generate waves of the shapes defined above. - - Arguments: - logdir: the directory into which to store all the runs' data - verbose: if true, print out each run's name as it begins - """ - waves = [sine_wave, square_wave, triangle_wave, - bisine_wave, bisine_wahwah_wave] - for (i, wave_constructor) in enumerate(waves): - wave_name = wave_constructor.__name__ - run_name = 'wave:%02d,%s' % (i + 1, wave_name) - if verbose: - print('--- Running: %s' % run_name) - run(logdir, run_name, wave_name, wave_constructor) + """Generate waves of the shapes defined above. + + Arguments: + logdir: the directory into which to store all the runs' data + verbose: if true, print out each run's name as it begins + """ + waves = [ + sine_wave, + square_wave, + triangle_wave, + bisine_wave, + bisine_wahwah_wave, + ] + for (i, wave_constructor) in enumerate(waves): + wave_name = wave_constructor.__name__ + run_name = "wave:%02d,%s" % (i + 1, wave_name) + if verbose: + print("--- Running: %s" % run_name) + run(logdir, run_name, wave_name, wave_constructor) def main(unused_argv): - print('Saving output to %s.' % FLAGS.logdir) - run_all(FLAGS.logdir, verbose=True) - print('Done. Output saved to %s.' % FLAGS.logdir) + print("Saving output to %s." % FLAGS.logdir) + run_all(FLAGS.logdir, verbose=True) + print("Done. Output saved to %s." % FLAGS.logdir) -if __name__ == '__main__': - app.run(main) +if __name__ == "__main__": + app.run(main) diff --git a/tensorboard/plugins/audio/audio_plugin.py b/tensorboard/plugins/audio/audio_plugin.py index 542ab6d312..906e12aeb0 100644 --- a/tensorboard/plugins/audio/audio_plugin.py +++ b/tensorboard/plugins/audio/audio_plugin.py @@ -30,204 +30,230 @@ from tensorboard.util import tensor_util -_DEFAULT_MIME_TYPE = 'application/octet-stream' +_DEFAULT_MIME_TYPE = "application/octet-stream" _MIME_TYPES = { - metadata.Encoding.Value('WAV'): 'audio/wav', + metadata.Encoding.Value("WAV"): "audio/wav", } class AudioPlugin(base_plugin.TBPlugin): - """Audio Plugin for TensorBoard.""" - - plugin_name = metadata.PLUGIN_NAME - - def __init__(self, context): - """Instantiates AudioPlugin via TensorBoard core. - - Args: - context: A base_plugin.TBContext instance. - """ - self._multiplexer = context.multiplexer - - def get_plugin_apps(self): - return { - '/audio': self._serve_audio_metadata, - '/individualAudio': self._serve_individual_audio, - '/tags': self._serve_tags, - } - - def is_active(self): - """The audio plugin is active iff any run has at least one relevant tag.""" - if not self._multiplexer: - return False - return bool(self._multiplexer.PluginRunToTagToContent(metadata.PLUGIN_NAME)) - - def frontend_metadata(self): - return base_plugin.FrontendMetadata(element_name='tf-audio-dashboard') - - def _index_impl(self): - """Return information about the tags in each run. - - Result is a dictionary of the form - - { - "runName1": { - "tagName1": { - "displayName": "The first tag", - "description": "Long ago there was just one tag...
", - "samples": 3 - }, - "tagName2": ..., - ... - }, - "runName2": ..., - ... + """Audio Plugin for TensorBoard.""" + + plugin_name = metadata.PLUGIN_NAME + + def __init__(self, context): + """Instantiates AudioPlugin via TensorBoard core. + + Args: + context: A base_plugin.TBContext instance. + """ + self._multiplexer = context.multiplexer + + def get_plugin_apps(self): + return { + "/audio": self._serve_audio_metadata, + "/individualAudio": self._serve_individual_audio, + "/tags": self._serve_tags, } - For each tag, `samples` is the greatest number of audio clips that - appear at any particular step. (It's not related to "samples of a - waveform.") For example, if for tag `minibatch_input` there are - five audio clips at step 0 and ten audio clips at step 1, then the - dictionary for `"minibatch_input"` will contain `"samples": 10`. - """ - runs = self._multiplexer.Runs() - result = {run: {} for run in runs} - - mapping = self._multiplexer.PluginRunToTagToContent(metadata.PLUGIN_NAME) - for (run, tag_to_content) in six.iteritems(mapping): - for tag in tag_to_content: - summary_metadata = self._multiplexer.SummaryMetadata(run, tag) - tensor_events = self._multiplexer.Tensors(run, tag) - samples = max([self._number_of_samples(event.tensor_proto) - for event in tensor_events] + [0]) - result[run][tag] = {'displayName': summary_metadata.display_name, - 'description': plugin_util.markdown_to_safe_html( - summary_metadata.summary_description), - 'samples': samples} - - return result - - def _number_of_samples(self, tensor_proto): - """Count the number of samples of an audio TensorProto.""" - # We directly inspect the `tensor_shape` of the proto instead of - # using the preferred `tensor_util.make_ndarray(...).shape`, because - # these protos can contain a large amount of encoded audio data, - # and we don't want to have to convert them all to numpy arrays - # just to look at their shape. - return tensor_proto.tensor_shape.dim[0].size - - def _filter_by_sample(self, tensor_events, sample): - return [tensor_event for tensor_event in tensor_events - if self._number_of_samples(tensor_event.tensor_proto) > sample] - - @wrappers.Request.application - def _serve_audio_metadata(self, request): - """Given a tag and list of runs, serve a list of metadata for audio. - - Note that the actual audio data are not sent; instead, we respond - with URLs to the audio. The frontend should treat these URLs as - opaque and should not try to parse information about them or - generate them itself, as the format may change. - - Args: - request: A werkzeug.wrappers.Request object. - - Returns: - A werkzeug.Response application. - """ - tag = request.args.get('tag') - run = request.args.get('run') - sample = int(request.args.get('sample', 0)) - - events = self._multiplexer.Tensors(run, tag) - try: - response = self._audio_response_for_run(events, run, tag, sample) - except KeyError: - return http_util.Respond( - request, 'Invalid run or tag', 'text/plain', code=400 - ) - return http_util.Respond(request, response, 'application/json') - - def _audio_response_for_run(self, tensor_events, run, tag, sample): - """Builds a JSON-serializable object with information about audio. - - Args: - tensor_events: A list of image event_accumulator.TensorEvent objects. - run: The name of the run. - tag: The name of the tag the audio entries all belong to. - sample: The zero-indexed sample of the audio sample for which to - retrieve information. For instance, setting `sample` to `2` will - fetch information about only the third audio clip of each batch, - and steps with fewer than three audio clips will be omitted from - the results. - - Returns: - A list of dictionaries containing the wall time, step, URL, width, and - height for each audio entry. - """ - response = [] - index = 0 - filtered_events = self._filter_by_sample(tensor_events, sample) - content_type = self._get_mime_type(run, tag) - for (index, tensor_event) in enumerate(filtered_events): - data = tensor_util.make_ndarray(tensor_event.tensor_proto) - label = data[sample, 1] - response.append({ - 'wall_time': tensor_event.wall_time, - 'step': tensor_event.step, - 'label': plugin_util.markdown_to_safe_html(label), - 'contentType': content_type, - 'query': self._query_for_individual_audio(run, tag, sample, index) - }) - return response - - def _query_for_individual_audio(self, run, tag, sample, index): - """Builds a URL for accessing the specified audio. - - This should be kept in sync with _serve_audio_metadata. Note that the URL is - *not* guaranteed to always return the same audio, since audio may be - unloaded from the reservoir as new audio entries come in. - - Args: - run: The name of the run. - tag: The tag. - index: The index of the audio entry. Negative values are OK. - - Returns: - A string representation of a URL that will load the index-th sampled audio - in the given run with the given tag. - """ - query_string = urllib.parse.urlencode({ - 'run': run, - 'tag': tag, - 'sample': sample, - 'index': index, - }) - return query_string - - def _get_mime_type(self, run, tag): - content = self._multiplexer.SummaryMetadata(run, tag).plugin_data.content - parsed = metadata.parse_plugin_metadata(content) - return _MIME_TYPES.get(parsed.encoding, _DEFAULT_MIME_TYPE) - - @wrappers.Request.application - def _serve_individual_audio(self, request): - """Serve encoded audio data.""" - tag = request.args.get('tag') - run = request.args.get('run') - index = int(request.args.get('index', '0')) - sample = int(request.args.get('sample', '0')) - try: - events = self._filter_by_sample(self._multiplexer.Tensors(run, tag), sample) - data = tensor_util.make_ndarray(events[index].tensor_proto)[sample, 0] - except (KeyError, IndexError): - return http_util.Respond( - request, 'Invalid run, tag, index, or sample', 'text/plain', code=400 - ) - mime_type = self._get_mime_type(run, tag) - return http_util.Respond(request, data, mime_type) - - @wrappers.Request.application - def _serve_tags(self, request): - index = self._index_impl() - return http_util.Respond(request, index, 'application/json') + def is_active(self): + """The audio plugin is active iff any run has at least one relevant + tag.""" + if not self._multiplexer: + return False + return bool( + self._multiplexer.PluginRunToTagToContent(metadata.PLUGIN_NAME) + ) + + def frontend_metadata(self): + return base_plugin.FrontendMetadata(element_name="tf-audio-dashboard") + + def _index_impl(self): + """Return information about the tags in each run. + + Result is a dictionary of the form + + { + "runName1": { + "tagName1": { + "displayName": "The first tag", + "description": "Long ago there was just one tag...
", + "samples": 3 + }, + "tagName2": ..., + ... + }, + "runName2": ..., + ... + } + + For each tag, `samples` is the greatest number of audio clips that + appear at any particular step. (It's not related to "samples of a + waveform.") For example, if for tag `minibatch_input` there are + five audio clips at step 0 and ten audio clips at step 1, then the + dictionary for `"minibatch_input"` will contain `"samples": 10`. + """ + runs = self._multiplexer.Runs() + result = {run: {} for run in runs} + + mapping = self._multiplexer.PluginRunToTagToContent( + metadata.PLUGIN_NAME + ) + for (run, tag_to_content) in six.iteritems(mapping): + for tag in tag_to_content: + summary_metadata = self._multiplexer.SummaryMetadata(run, tag) + tensor_events = self._multiplexer.Tensors(run, tag) + samples = max( + [ + self._number_of_samples(event.tensor_proto) + for event in tensor_events + ] + + [0] + ) + result[run][tag] = { + "displayName": summary_metadata.display_name, + "description": plugin_util.markdown_to_safe_html( + summary_metadata.summary_description + ), + "samples": samples, + } + + return result + + def _number_of_samples(self, tensor_proto): + """Count the number of samples of an audio TensorProto.""" + # We directly inspect the `tensor_shape` of the proto instead of + # using the preferred `tensor_util.make_ndarray(...).shape`, because + # these protos can contain a large amount of encoded audio data, + # and we don't want to have to convert them all to numpy arrays + # just to look at their shape. + return tensor_proto.tensor_shape.dim[0].size + + def _filter_by_sample(self, tensor_events, sample): + return [ + tensor_event + for tensor_event in tensor_events + if self._number_of_samples(tensor_event.tensor_proto) > sample + ] + + @wrappers.Request.application + def _serve_audio_metadata(self, request): + """Given a tag and list of runs, serve a list of metadata for audio. + + Note that the actual audio data are not sent; instead, we respond + with URLs to the audio. The frontend should treat these URLs as + opaque and should not try to parse information about them or + generate them itself, as the format may change. + + Args: + request: A werkzeug.wrappers.Request object. + + Returns: + A werkzeug.Response application. + """ + tag = request.args.get("tag") + run = request.args.get("run") + sample = int(request.args.get("sample", 0)) + + events = self._multiplexer.Tensors(run, tag) + try: + response = self._audio_response_for_run(events, run, tag, sample) + except KeyError: + return http_util.Respond( + request, "Invalid run or tag", "text/plain", code=400 + ) + return http_util.Respond(request, response, "application/json") + + def _audio_response_for_run(self, tensor_events, run, tag, sample): + """Builds a JSON-serializable object with information about audio. + + Args: + tensor_events: A list of image event_accumulator.TensorEvent objects. + run: The name of the run. + tag: The name of the tag the audio entries all belong to. + sample: The zero-indexed sample of the audio sample for which to + retrieve information. For instance, setting `sample` to `2` will + fetch information about only the third audio clip of each batch, + and steps with fewer than three audio clips will be omitted from + the results. + + Returns: + A list of dictionaries containing the wall time, step, URL, width, and + height for each audio entry. + """ + response = [] + index = 0 + filtered_events = self._filter_by_sample(tensor_events, sample) + content_type = self._get_mime_type(run, tag) + for (index, tensor_event) in enumerate(filtered_events): + data = tensor_util.make_ndarray(tensor_event.tensor_proto) + label = data[sample, 1] + response.append( + { + "wall_time": tensor_event.wall_time, + "step": tensor_event.step, + "label": plugin_util.markdown_to_safe_html(label), + "contentType": content_type, + "query": self._query_for_individual_audio( + run, tag, sample, index + ), + } + ) + return response + + def _query_for_individual_audio(self, run, tag, sample, index): + """Builds a URL for accessing the specified audio. + + This should be kept in sync with _serve_audio_metadata. Note that the URL is + *not* guaranteed to always return the same audio, since audio may be + unloaded from the reservoir as new audio entries come in. + + Args: + run: The name of the run. + tag: The tag. + index: The index of the audio entry. Negative values are OK. + + Returns: + A string representation of a URL that will load the index-th sampled audio + in the given run with the given tag. + """ + query_string = urllib.parse.urlencode( + {"run": run, "tag": tag, "sample": sample, "index": index,} + ) + return query_string + + def _get_mime_type(self, run, tag): + content = self._multiplexer.SummaryMetadata( + run, tag + ).plugin_data.content + parsed = metadata.parse_plugin_metadata(content) + return _MIME_TYPES.get(parsed.encoding, _DEFAULT_MIME_TYPE) + + @wrappers.Request.application + def _serve_individual_audio(self, request): + """Serve encoded audio data.""" + tag = request.args.get("tag") + run = request.args.get("run") + index = int(request.args.get("index", "0")) + sample = int(request.args.get("sample", "0")) + try: + events = self._filter_by_sample( + self._multiplexer.Tensors(run, tag), sample + ) + data = tensor_util.make_ndarray(events[index].tensor_proto)[ + sample, 0 + ] + except (KeyError, IndexError): + return http_util.Respond( + request, + "Invalid run, tag, index, or sample", + "text/plain", + code=400, + ) + mime_type = self._get_mime_type(run, tag) + return http_util.Respond(request, data, mime_type) + + @wrappers.Request.application + def _serve_tags(self, request): + index = self._index_impl() + return http_util.Respond(request, index, "application/json") diff --git a/tensorboard/plugins/audio/audio_plugin_test.py b/tensorboard/plugins/audio/audio_plugin_test.py index 0672c7bffd..088ef0acca 100644 --- a/tensorboard/plugins/audio/audio_plugin_test.py +++ b/tensorboard/plugins/audio/audio_plugin_test.py @@ -32,7 +32,9 @@ from werkzeug import wrappers from tensorboard.backend import application -from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer +from tensorboard.backend.event_processing import ( + plugin_event_multiplexer as event_multiplexer, +) from tensorboard.plugins import base_plugin from tensorboard.plugins.audio import audio_plugin from tensorboard.plugins.audio import summary @@ -40,201 +42,237 @@ class AudioPluginTest(tf.test.TestCase): + def setUp(self): + self.log_dir = tempfile.mkdtemp() - def setUp(self): - self.log_dir = tempfile.mkdtemp() - - # We use numpy.random to generate audio. We seed to avoid non-determinism - # in this test. - numpy.random.seed(42) - - # Create old-style audio summaries for run "foo". - tf.compat.v1.reset_default_graph() - with tf.compat.v1.Graph().as_default(): - sess = tf.compat.v1.Session() - placeholder = tf.compat.v1.placeholder(tf.float32) - tf.compat.v1.summary.audio(name="baz", tensor=placeholder, sample_rate=44100) - merged_summary_op = tf.compat.v1.summary.merge_all() - foo_directory = os.path.join(self.log_dir, "foo") - with test_util.FileWriterCache.get(foo_directory) as writer: - writer.add_graph(sess.graph) - for step in xrange(2): - # The floats (sample data) range from -1 to 1. - writer.add_summary(sess.run(merged_summary_op, feed_dict={ - placeholder: numpy.random.rand(42, 22050) * 2 - 1 - }), global_step=step) - - # Create new-style audio summaries for run "bar". - tf.compat.v1.reset_default_graph() - with tf.compat.v1.Graph().as_default(): - sess = tf.compat.v1.Session() - audio_placeholder = tf.compat.v1.placeholder(tf.float32) - labels_placeholder = tf.compat.v1.placeholder(tf.string) - summary.op("quux", audio_placeholder, sample_rate=44100, - labels=labels_placeholder, - description="how do you pronounce that, anyway?") - merged_summary_op = tf.compat.v1.summary.merge_all() - bar_directory = os.path.join(self.log_dir, "bar") - with test_util.FileWriterCache.get(bar_directory) as writer: - writer.add_graph(sess.graph) - for step in xrange(2): - # The floats (sample data) range from -1 to 1. - writer.add_summary(sess.run(merged_summary_op, feed_dict={ - audio_placeholder: numpy.random.rand(42, 11025, 1) * 2 - 1, - labels_placeholder: [ - tf.compat.as_bytes('step **%s**, sample %s' % (step, sample)) - for sample in xrange(42) - ], - }), global_step=step) - - # Start a server with the plugin. - multiplexer = event_multiplexer.EventMultiplexer({ - "foo": foo_directory, - "bar": bar_directory, - }) - multiplexer.Reload() - context = base_plugin.TBContext( - logdir=self.log_dir, multiplexer=multiplexer) - self.plugin = audio_plugin.AudioPlugin(context) - wsgi_app = application.TensorBoardWSGI([self.plugin]) - self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) - - def tearDown(self): - shutil.rmtree(self.log_dir, ignore_errors=True) - - def _DeserializeResponse(self, byte_content): - """Deserializes byte content that is a JSON encoding. - - Args: - byte_content: The byte content of a response. - - Returns: - The deserialized python object decoded from JSON. - """ - return json.loads(byte_content.decode("utf-8")) - - def testRoutesProvided(self): - """Tests that the plugin offers the correct routes.""" - routes = self.plugin.get_plugin_apps() - self.assertIsInstance(routes["/audio"], collections.Callable) - self.assertIsInstance(routes["/individualAudio"], collections.Callable) - self.assertIsInstance(routes["/tags"], collections.Callable) - - def testOldStyleAudioRoute(self): - """Tests that the /audio routes returns correct old-style data.""" - response = self.server.get( - "/data/plugin/audio/audio?run=foo&tag=baz/audio/0&sample=0") - self.assertEqual(200, response.status_code) - - # Verify that the correct entries are returned. - entries = self._DeserializeResponse(response.get_data()) - self.assertEqual(2, len(entries)) - - # Verify that the 1st entry is correct. - entry = entries[0] - self.assertEqual("audio/wav", entry["contentType"]) - self.assertEqual("", entry["label"]) - self.assertEqual(0, entry["step"]) - parsed_query = urllib.parse.parse_qs(entry["query"]) - self.assertListEqual(["foo"], parsed_query["run"]) - self.assertListEqual(["baz/audio/0"], parsed_query["tag"]) - self.assertListEqual(["0"], parsed_query["sample"]) - self.assertListEqual(["0"], parsed_query["index"]) - - # Verify that the 2nd entry is correct. - entry = entries[1] - self.assertEqual("audio/wav", entry["contentType"]) - self.assertEqual("", entry["label"]) - self.assertEqual(1, entry["step"]) - parsed_query = urllib.parse.parse_qs(entry["query"]) - self.assertListEqual(["foo"], parsed_query["run"]) - self.assertListEqual(["baz/audio/0"], parsed_query["tag"]) - self.assertListEqual(["0"], parsed_query["sample"]) - self.assertListEqual(["1"], parsed_query["index"]) - - def testNewStyleAudioRoute(self): - """Tests that the /audio routes returns correct new-style data.""" - response = self.server.get( - "/data/plugin/audio/audio?run=bar&tag=quux/audio_summary&sample=0") - self.assertEqual(200, response.status_code) - - # Verify that the correct entries are returned. - entries = self._DeserializeResponse(response.get_data()) - self.assertEqual(2, len(entries)) - - # Verify that the 1st entry is correct. - entry = entries[0] - self.assertEqual("audio/wav", entry["contentType"]) - self.assertEqual( - "step %s, sample 0
" % entry["step"], - entry["label"]) - self.assertEqual(0, entry["step"]) - parsed_query = urllib.parse.parse_qs(entry["query"]) - self.assertListEqual(["bar"], parsed_query["run"]) - self.assertListEqual(["quux/audio_summary"], parsed_query["tag"]) - self.assertListEqual(["0"], parsed_query["sample"]) - self.assertListEqual(["0"], parsed_query["index"]) - - # Verify that the 2nd entry is correct. - entry = entries[1] - self.assertEqual("audio/wav", entry["contentType"]) - self.assertEqual( - "step %s, sample 0
" % entry["step"], - entry["label"]) - self.assertEqual(1, entry["step"]) - parsed_query = urllib.parse.parse_qs(entry["query"]) - self.assertListEqual(["bar"], parsed_query["run"]) - self.assertListEqual(["quux/audio_summary"], parsed_query["tag"]) - self.assertListEqual(["0"], parsed_query["sample"]) - self.assertListEqual(["1"], parsed_query["index"]) - - def testOldStyleIndividualAudioRoute(self): - """Tests fetching an individual audio clip from an old-style summary.""" - response = self.server.get( - "/data/plugin/audio/individualAudio" - "?run=foo&tag=baz/audio/0&sample=0&index=0") - self.assertEqual(200, response.status_code) - self.assertEqual("audio/wav", response.headers.get("content-type")) - - def testNewStyleIndividualAudioRoute(self): - """Tests fetching an individual audio clip from an old-style summary.""" - response = self.server.get( - "/data/plugin/audio/individualAudio" - "?run=bar&tag=quux/audio_summary&sample=0&index=0") - self.assertEqual(200, response.status_code) - self.assertEqual("audio/wav", response.headers.get("content-type")) - - def testTagsRoute(self): - """Tests that the /tags route offers the correct run to tag mapping.""" - response = self.server.get("/data/plugin/audio/tags") - self.assertEqual(200, response.status_code) - self.assertDictEqual({ - "foo": { - "baz/audio/0": { - "displayName": "baz/audio/0", - "description": "", - "samples": 1, - }, - "baz/audio/1": { - "displayName": "baz/audio/1", - "description": "", - "samples": 1, - }, - "baz/audio/2": { - "displayName": "baz/audio/2", - "description": "", - "samples": 1, - }, - }, - "bar": { - "quux/audio_summary": { - "displayName": "quux", - "description": "how do you pronounce that, anyway?
", - "samples": 3, # 42 inputs, but max_outputs=3 + # We use numpy.random to generate audio. We seed to avoid non-determinism + # in this test. + numpy.random.seed(42) + + # Create old-style audio summaries for run "foo". + tf.compat.v1.reset_default_graph() + with tf.compat.v1.Graph().as_default(): + sess = tf.compat.v1.Session() + placeholder = tf.compat.v1.placeholder(tf.float32) + tf.compat.v1.summary.audio( + name="baz", tensor=placeholder, sample_rate=44100 + ) + merged_summary_op = tf.compat.v1.summary.merge_all() + foo_directory = os.path.join(self.log_dir, "foo") + with test_util.FileWriterCache.get(foo_directory) as writer: + writer.add_graph(sess.graph) + for step in xrange(2): + # The floats (sample data) range from -1 to 1. + writer.add_summary( + sess.run( + merged_summary_op, + feed_dict={ + placeholder: numpy.random.rand(42, 22050) * 2 + - 1 + }, + ), + global_step=step, + ) + + # Create new-style audio summaries for run "bar". + tf.compat.v1.reset_default_graph() + with tf.compat.v1.Graph().as_default(): + sess = tf.compat.v1.Session() + audio_placeholder = tf.compat.v1.placeholder(tf.float32) + labels_placeholder = tf.compat.v1.placeholder(tf.string) + summary.op( + "quux", + audio_placeholder, + sample_rate=44100, + labels=labels_placeholder, + description="how do you pronounce that, anyway?", + ) + merged_summary_op = tf.compat.v1.summary.merge_all() + bar_directory = os.path.join(self.log_dir, "bar") + with test_util.FileWriterCache.get(bar_directory) as writer: + writer.add_graph(sess.graph) + for step in xrange(2): + # The floats (sample data) range from -1 to 1. + writer.add_summary( + sess.run( + merged_summary_op, + feed_dict={ + audio_placeholder: numpy.random.rand( + 42, 11025, 1 + ) + * 2 + - 1, + labels_placeholder: [ + tf.compat.as_bytes( + "step **%s**, sample %s" + % (step, sample) + ) + for sample in xrange(42) + ], + }, + ), + global_step=step, + ) + + # Start a server with the plugin. + multiplexer = event_multiplexer.EventMultiplexer( + {"foo": foo_directory, "bar": bar_directory,} + ) + multiplexer.Reload() + context = base_plugin.TBContext( + logdir=self.log_dir, multiplexer=multiplexer + ) + self.plugin = audio_plugin.AudioPlugin(context) + wsgi_app = application.TensorBoardWSGI([self.plugin]) + self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) + + def tearDown(self): + shutil.rmtree(self.log_dir, ignore_errors=True) + + def _DeserializeResponse(self, byte_content): + """Deserializes byte content that is a JSON encoding. + + Args: + byte_content: The byte content of a response. + + Returns: + The deserialized python object decoded from JSON. + """ + return json.loads(byte_content.decode("utf-8")) + + def testRoutesProvided(self): + """Tests that the plugin offers the correct routes.""" + routes = self.plugin.get_plugin_apps() + self.assertIsInstance(routes["/audio"], collections.Callable) + self.assertIsInstance(routes["/individualAudio"], collections.Callable) + self.assertIsInstance(routes["/tags"], collections.Callable) + + def testOldStyleAudioRoute(self): + """Tests that the /audio routes returns correct old-style data.""" + response = self.server.get( + "/data/plugin/audio/audio?run=foo&tag=baz/audio/0&sample=0" + ) + self.assertEqual(200, response.status_code) + + # Verify that the correct entries are returned. + entries = self._DeserializeResponse(response.get_data()) + self.assertEqual(2, len(entries)) + + # Verify that the 1st entry is correct. + entry = entries[0] + self.assertEqual("audio/wav", entry["contentType"]) + self.assertEqual("", entry["label"]) + self.assertEqual(0, entry["step"]) + parsed_query = urllib.parse.parse_qs(entry["query"]) + self.assertListEqual(["foo"], parsed_query["run"]) + self.assertListEqual(["baz/audio/0"], parsed_query["tag"]) + self.assertListEqual(["0"], parsed_query["sample"]) + self.assertListEqual(["0"], parsed_query["index"]) + + # Verify that the 2nd entry is correct. + entry = entries[1] + self.assertEqual("audio/wav", entry["contentType"]) + self.assertEqual("", entry["label"]) + self.assertEqual(1, entry["step"]) + parsed_query = urllib.parse.parse_qs(entry["query"]) + self.assertListEqual(["foo"], parsed_query["run"]) + self.assertListEqual(["baz/audio/0"], parsed_query["tag"]) + self.assertListEqual(["0"], parsed_query["sample"]) + self.assertListEqual(["1"], parsed_query["index"]) + + def testNewStyleAudioRoute(self): + """Tests that the /audio routes returns correct new-style data.""" + response = self.server.get( + "/data/plugin/audio/audio?run=bar&tag=quux/audio_summary&sample=0" + ) + self.assertEqual(200, response.status_code) + + # Verify that the correct entries are returned. + entries = self._DeserializeResponse(response.get_data()) + self.assertEqual(2, len(entries)) + + # Verify that the 1st entry is correct. + entry = entries[0] + self.assertEqual("audio/wav", entry["contentType"]) + self.assertEqual( + "step %s, sample 0
" % entry["step"], + entry["label"], + ) + self.assertEqual(0, entry["step"]) + parsed_query = urllib.parse.parse_qs(entry["query"]) + self.assertListEqual(["bar"], parsed_query["run"]) + self.assertListEqual(["quux/audio_summary"], parsed_query["tag"]) + self.assertListEqual(["0"], parsed_query["sample"]) + self.assertListEqual(["0"], parsed_query["index"]) + + # Verify that the 2nd entry is correct. + entry = entries[1] + self.assertEqual("audio/wav", entry["contentType"]) + self.assertEqual( + "step %s, sample 0
" % entry["step"], + entry["label"], + ) + self.assertEqual(1, entry["step"]) + parsed_query = urllib.parse.parse_qs(entry["query"]) + self.assertListEqual(["bar"], parsed_query["run"]) + self.assertListEqual(["quux/audio_summary"], parsed_query["tag"]) + self.assertListEqual(["0"], parsed_query["sample"]) + self.assertListEqual(["1"], parsed_query["index"]) + + def testOldStyleIndividualAudioRoute(self): + """Tests fetching an individual audio clip from an old-style + summary.""" + response = self.server.get( + "/data/plugin/audio/individualAudio" + "?run=foo&tag=baz/audio/0&sample=0&index=0" + ) + self.assertEqual(200, response.status_code) + self.assertEqual("audio/wav", response.headers.get("content-type")) + + def testNewStyleIndividualAudioRoute(self): + """Tests fetching an individual audio clip from an old-style + summary.""" + response = self.server.get( + "/data/plugin/audio/individualAudio" + "?run=bar&tag=quux/audio_summary&sample=0&index=0" + ) + self.assertEqual(200, response.status_code) + self.assertEqual("audio/wav", response.headers.get("content-type")) + + def testTagsRoute(self): + """Tests that the /tags route offers the correct run to tag mapping.""" + response = self.server.get("/data/plugin/audio/tags") + self.assertEqual(200, response.status_code) + self.assertDictEqual( + { + "foo": { + "baz/audio/0": { + "displayName": "baz/audio/0", + "description": "", + "samples": 1, + }, + "baz/audio/1": { + "displayName": "baz/audio/1", + "description": "", + "samples": 1, + }, + "baz/audio/2": { + "displayName": "baz/audio/2", + "description": "", + "samples": 1, + }, + }, + "bar": { + "quux/audio_summary": { + "displayName": "quux", + "description": "how do you pronounce that, anyway?
", + "samples": 3, # 42 inputs, but max_outputs=3 + }, + }, }, - }, - }, self._DeserializeResponse(response.get_data())) + self._DeserializeResponse(response.get_data()), + ) if __name__ == "__main__": - tf.test.main() + tf.test.main() diff --git a/tensorboard/plugins/audio/metadata.py b/tensorboard/plugins/audio/metadata.py index 2b4baaedef..d6d5cdc6db 100644 --- a/tensorboard/plugins/audio/metadata.py +++ b/tensorboard/plugins/audio/metadata.py @@ -24,7 +24,7 @@ logger = tb_logging.get_logger() -PLUGIN_NAME = 'audio' +PLUGIN_NAME = "audio" # The most recent value for the `version` field of the `AudioPluginData` # proto. @@ -35,40 +35,45 @@ def create_summary_metadata(display_name, description, encoding): - """Create a `SummaryMetadata` proto for audio plugin data. + """Create a `SummaryMetadata` proto for audio plugin data. - Returns: - A `SummaryMetadata` protobuf object. - """ - content = plugin_data_pb2.AudioPluginData( - version=PROTO_VERSION, encoding=encoding) - metadata = summary_pb2.SummaryMetadata( - display_name=display_name, - summary_description=description, - plugin_data=summary_pb2.SummaryMetadata.PluginData( - plugin_name=PLUGIN_NAME, - content=content.SerializeToString())) - return metadata + Returns: + A `SummaryMetadata` protobuf object. + """ + content = plugin_data_pb2.AudioPluginData( + version=PROTO_VERSION, encoding=encoding + ) + metadata = summary_pb2.SummaryMetadata( + display_name=display_name, + summary_description=description, + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME, content=content.SerializeToString() + ), + ) + return metadata def parse_plugin_metadata(content): - """Parse summary metadata to a Python object. + """Parse summary metadata to a Python object. - Arguments: - content: The `content` field of a `SummaryMetadata` proto - corresponding to the audio plugin. + Arguments: + content: The `content` field of a `SummaryMetadata` proto + corresponding to the audio plugin. - Returns: - An `AudioPluginData` protobuf object. - """ - if not isinstance(content, bytes): - raise TypeError('Content type must be bytes') - result = plugin_data_pb2.AudioPluginData.FromString(content) - if result.version == 0: - return result - else: - logger.warn( - 'Unknown metadata version: %s. The latest version known to ' - 'this build of TensorBoard is %s; perhaps a newer build is ' - 'available?', result.version, PROTO_VERSION) - return result + Returns: + An `AudioPluginData` protobuf object. + """ + if not isinstance(content, bytes): + raise TypeError("Content type must be bytes") + result = plugin_data_pb2.AudioPluginData.FromString(content) + if result.version == 0: + return result + else: + logger.warn( + "Unknown metadata version: %s. The latest version known to " + "this build of TensorBoard is %s; perhaps a newer build is " + "available?", + result.version, + PROTO_VERSION, + ) + return result diff --git a/tensorboard/plugins/audio/summary.py b/tensorboard/plugins/audio/summary.py index 6498623dde..506a07e6c5 100644 --- a/tensorboard/plugins/audio/summary.py +++ b/tensorboard/plugins/audio/summary.py @@ -41,176 +41,190 @@ audio = summary_v2.audio -def op(name, - audio, - sample_rate, - labels=None, - max_outputs=3, - encoding=None, - display_name=None, - description=None, - collections=None): - """Create a legacy audio summary op for use in a TensorFlow graph. - - Arguments: - name: A unique name for the generated summary node. - audio: A `Tensor` representing audio data with shape `[k, t, c]`, - where `k` is the number of audio clips, `t` is the number of - frames, and `c` is the number of channels. Elements should be - floating-point values in `[-1.0, 1.0]`. Any of the dimensions may - be statically unknown (i.e., `None`). - sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the - sample rate, in Hz. Must be positive. - labels: Optional `string` `Tensor`, a vector whose length is the - first dimension of `audio`, where `labels[i]` contains arbitrary - textual information about `audio[i]`. (For instance, this could be - some text that a TTS system was supposed to produce.) Markdown is - supported. Contents should be UTF-8. - max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this - many audio clips will be emitted at each step. When more than - `max_outputs` many clips are provided, the first `max_outputs` - many clips will be used and the rest silently discarded. - encoding: A constant `str` (not string tensor) indicating the - desired encoding. You can choose any format you like, as long as - it's "wav". Please see the "API compatibility note" below. - display_name: Optional name for this summary in TensorBoard, as a - constant `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - collections: Optional list of graph collections keys. The new - summary op is added to these collections. Defaults to - `[Graph Keys.SUMMARIES]`. - - Returns: - A TensorFlow summary op. - - API compatibility note: The default value of the `encoding` - argument is _not_ guaranteed to remain unchanged across TensorBoard - versions. In the future, we will by default encode as FLAC instead of - as WAV. If the specific format is important to you, please provide a - file format explicitly. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - if display_name is None: - display_name = name - if encoding is None: - encoding = 'wav' - - if encoding == 'wav': - encoding = metadata.Encoding.Value('WAV') - encoder = functools.partial(tf.audio.encode_wav, sample_rate=sample_rate) - else: - raise ValueError('Unknown encoding: %r' % encoding) - - with tf.name_scope(name), \ - tf.control_dependencies([tf.assert_rank(audio, 3)]): +def op( + name, + audio, + sample_rate, + labels=None, + max_outputs=3, + encoding=None, + display_name=None, + description=None, + collections=None, +): + """Create a legacy audio summary op for use in a TensorFlow graph. + + Arguments: + name: A unique name for the generated summary node. + audio: A `Tensor` representing audio data with shape `[k, t, c]`, + where `k` is the number of audio clips, `t` is the number of + frames, and `c` is the number of channels. Elements should be + floating-point values in `[-1.0, 1.0]`. Any of the dimensions may + be statically unknown (i.e., `None`). + sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the + sample rate, in Hz. Must be positive. + labels: Optional `string` `Tensor`, a vector whose length is the + first dimension of `audio`, where `labels[i]` contains arbitrary + textual information about `audio[i]`. (For instance, this could be + some text that a TTS system was supposed to produce.) Markdown is + supported. Contents should be UTF-8. + max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this + many audio clips will be emitted at each step. When more than + `max_outputs` many clips are provided, the first `max_outputs` + many clips will be used and the rest silently discarded. + encoding: A constant `str` (not string tensor) indicating the + desired encoding. You can choose any format you like, as long as + it's "wav". Please see the "API compatibility note" below. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A TensorFlow summary op. + + API compatibility note: The default value of the `encoding` + argument is _not_ guaranteed to remain unchanged across TensorBoard + versions. In the future, we will by default encode as FLAC instead of + as WAV. If the specific format is important to you, please provide a + file format explicitly. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if display_name is None: + display_name = name + if encoding is None: + encoding = "wav" + + if encoding == "wav": + encoding = metadata.Encoding.Value("WAV") + encoder = functools.partial( + tf.audio.encode_wav, sample_rate=sample_rate + ) + else: + raise ValueError("Unknown encoding: %r" % encoding) + + with tf.name_scope(name), tf.control_dependencies( + [tf.assert_rank(audio, 3)] + ): + limited_audio = audio[:max_outputs] + encoded_audio = tf.map_fn( + encoder, limited_audio, dtype=tf.string, name="encode_each_audio" + ) + if labels is None: + limited_labels = tf.tile([""], tf.shape(input=limited_audio)[:1]) + else: + limited_labels = labels[:max_outputs] + tensor = tf.transpose(a=tf.stack([encoded_audio, limited_labels])) + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, + description=description, + encoding=encoding, + ) + return tf.summary.tensor_summary( + name="audio_summary", + tensor=tensor, + collections=collections, + summary_metadata=summary_metadata, + ) + + +def pb( + name, + audio, + sample_rate, + labels=None, + max_outputs=3, + encoding=None, + display_name=None, + description=None, +): + """Create a legacy audio summary protobuf. + + This behaves as if you were to create an `op` with the same arguments + (wrapped with constant tensors where appropriate) and then execute + that summary op in a TensorFlow session. + + Arguments: + name: A unique name for the generated summary node. + audio: An `np.array` representing audio data with shape `[k, t, c]`, + where `k` is the number of audio clips, `t` is the number of + frames, and `c` is the number of channels. Elements should be + floating-point values in `[-1.0, 1.0]`. + sample_rate: An `int` that represents the sample rate, in Hz. + Must be positive. + labels: Optional list (or rank-1 `np.array`) of textstrings or UTF-8 + bytestrings whose length is the first dimension of `audio`, where + `labels[i]` contains arbitrary textual information about + `audio[i]`. (For instance, this could be some text that a TTS + system was supposed to produce.) Markdown is supported. + max_outputs: Optional `int`. At most this many audio clips will be + emitted. When more than `max_outputs` many clips are provided, the + first `max_outputs` many clips will be used and the rest silently + discarded. + encoding: A constant `str` indicating the desired encoding. You + can choose any format you like, as long as it's "wav". Please see + the "API compatibility note" below. + display_name: Optional name for this summary in TensorBoard, as a + `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + `str`. Markdown is supported. Defaults to empty. + + Returns: + A `tf.Summary` protobuf object. + + API compatibility note: The default value of the `encoding` + argument is _not_ guaranteed to remain unchanged across TensorBoard + versions. In the future, we will by default encode as FLAC instead of + as WAV. If the specific format is important to you, please provide a + file format explicitly. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + audio = np.array(audio) + if audio.ndim != 3: + raise ValueError("Shape %r must have rank 3" % (audio.shape,)) + if encoding is None: + encoding = "wav" + + if encoding == "wav": + encoding = metadata.Encoding.Value("WAV") + encoder = functools.partial( + encoder_util.encode_wav, samples_per_second=sample_rate + ) + else: + raise ValueError("Unknown encoding: %r" % encoding) + limited_audio = audio[:max_outputs] - encoded_audio = tf.map_fn(encoder, limited_audio, - dtype=tf.string, - name='encode_each_audio') if labels is None: - limited_labels = tf.tile([''], tf.shape(input=limited_audio)[:1]) + limited_labels = [b""] * len(limited_audio) else: - limited_labels = labels[:max_outputs] - tensor = tf.transpose(a=tf.stack([encoded_audio, limited_labels])) + limited_labels = [ + tf.compat.as_bytes(label) for label in labels[:max_outputs] + ] + + encoded_audio = [encoder(a) for a in limited_audio] + content = np.array([encoded_audio, limited_labels]).transpose() + tensor = tf.make_tensor_proto(content, dtype=tf.string) + + if display_name is None: + display_name = name summary_metadata = metadata.create_summary_metadata( - display_name=display_name, - description=description, - encoding=encoding) - return tf.summary.tensor_summary(name='audio_summary', - tensor=tensor, - collections=collections, - summary_metadata=summary_metadata) - - -def pb(name, - audio, - sample_rate, - labels=None, - max_outputs=3, - encoding=None, - display_name=None, - description=None): - """Create a legacy audio summary protobuf. - - This behaves as if you were to create an `op` with the same arguments - (wrapped with constant tensors where appropriate) and then execute - that summary op in a TensorFlow session. - - Arguments: - name: A unique name for the generated summary node. - audio: An `np.array` representing audio data with shape `[k, t, c]`, - where `k` is the number of audio clips, `t` is the number of - frames, and `c` is the number of channels. Elements should be - floating-point values in `[-1.0, 1.0]`. - sample_rate: An `int` that represents the sample rate, in Hz. - Must be positive. - labels: Optional list (or rank-1 `np.array`) of textstrings or UTF-8 - bytestrings whose length is the first dimension of `audio`, where - `labels[i]` contains arbitrary textual information about - `audio[i]`. (For instance, this could be some text that a TTS - system was supposed to produce.) Markdown is supported. - max_outputs: Optional `int`. At most this many audio clips will be - emitted. When more than `max_outputs` many clips are provided, the - first `max_outputs` many clips will be used and the rest silently - discarded. - encoding: A constant `str` indicating the desired encoding. You - can choose any format you like, as long as it's "wav". Please see - the "API compatibility note" below. - display_name: Optional name for this summary in TensorBoard, as a - `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - `str`. Markdown is supported. Defaults to empty. - - Returns: - A `tf.Summary` protobuf object. - - API compatibility note: The default value of the `encoding` - argument is _not_ guaranteed to remain unchanged across TensorBoard - versions. In the future, we will by default encode as FLAC instead of - as WAV. If the specific format is important to you, please provide a - file format explicitly. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - audio = np.array(audio) - if audio.ndim != 3: - raise ValueError('Shape %r must have rank 3' % (audio.shape,)) - if encoding is None: - encoding = 'wav' - - if encoding == 'wav': - encoding = metadata.Encoding.Value('WAV') - encoder = functools.partial(encoder_util.encode_wav, - samples_per_second=sample_rate) - else: - raise ValueError('Unknown encoding: %r' % encoding) - - limited_audio = audio[:max_outputs] - if labels is None: - limited_labels = [b''] * len(limited_audio) - else: - limited_labels = [tf.compat.as_bytes(label) - for label in labels[:max_outputs]] - - encoded_audio = [encoder(a) for a in limited_audio] - content = np.array([encoded_audio, limited_labels]).transpose() - tensor = tf.make_tensor_proto(content, dtype=tf.string) - - if display_name is None: - display_name = name - summary_metadata = metadata.create_summary_metadata( - display_name=display_name, - description=description, - encoding=encoding) - tf_summary_metadata = tf.SummaryMetadata.FromString( - summary_metadata.SerializeToString()) - - summary = tf.Summary() - summary.value.add(tag='%s/audio_summary' % name, - metadata=tf_summary_metadata, - tensor=tensor) - return summary + display_name=display_name, description=description, encoding=encoding + ) + tf_summary_metadata = tf.SummaryMetadata.FromString( + summary_metadata.SerializeToString() + ) + + summary = tf.Summary() + summary.value.add( + tag="%s/audio_summary" % name, + metadata=tf_summary_metadata, + tensor=tensor, + ) + return summary diff --git a/tensorboard/plugins/audio/summary_test.py b/tensorboard/plugins/audio/summary_test.py index 25fabdd54e..e4ba03bea1 100644 --- a/tensorboard/plugins/audio/summary_test.py +++ b/tensorboard/plugins/audio/summary_test.py @@ -33,212 +33,223 @@ try: - tf2.__version__ # Force lazy import to resolve + tf2.__version__ # Force lazy import to resolve except ImportError: - tf2 = None + tf2 = None try: - tf.compat.v1.enable_eager_execution() + tf.compat.v1.enable_eager_execution() except AttributeError: - # TF 2.0 doesn't have this symbol because eager is the default. - pass + # TF 2.0 doesn't have this symbol because eager is the default. + pass -audio_ops = getattr(tf, 'audio', None) +audio_ops = getattr(tf, "audio", None) if audio_ops is None: - # Fallback for older versions of TF without tf.audio. - from tensorflow.python.ops import gen_audio_ops as audio_ops + # Fallback for older versions of TF without tf.audio. + from tensorflow.python.ops import gen_audio_ops as audio_ops class SummaryBaseTest(object): - - def setUp(self): - super(SummaryBaseTest, self).setUp() - self.samples_rate = 44100 - self.audio_count = 1 - self.num_samples = 20 - self.num_channels = 2 - - def _generate_audio(self, **kwargs): - size = [ - kwargs.get('k', self.audio_count), - kwargs.get('n', self.num_samples), - kwargs.get('c', self.num_channels), - ] - return np.sin(np.linspace(0.0, 100.0, np.prod(size), - dtype=np.float32)).reshape(size) - - def audio(self, *args, **kwargs): - raise NotImplementedError() - - def test_metadata(self): - data = np.array(1, np.float32, ndmin=3) - description = 'Piano Concerto No. 23 (K488), in **A major.**' - pb = self.audio('k488', data, 44100, description=description) - self.assertEqual(len(pb.value), 1) - summary_metadata = pb.value[0].metadata - self.assertEqual(summary_metadata.summary_description, description) - plugin_data = summary_metadata.plugin_data - self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) - content = summary_metadata.plugin_data.content - parsed = metadata.parse_plugin_metadata(content) - self.assertEqual(parsed.encoding, metadata.Encoding.Value('WAV')) - - def test_wav_format_roundtrip(self): - audio = self._generate_audio(c=1) - pb = self.audio('k888', audio, 44100) - encoded = tensor_util.make_ndarray(pb.value[0].tensor) - decoded, sample_rate = audio_ops.decode_wav(encoded.flat[0]) - # WAV roundtrip goes from float32 to int16 and back, so expect some - # precision loss, but not more than 2 applications of rounding error from - # mapping the range [-1.0, 1.0] to 2^16. - epsilon = 2 * 2.0 / (2**16) - self.assertAllClose(audio[0], decoded, atol=epsilon) - self.assertEqual(44100, sample_rate.numpy()) - - def _test_dimensions(self, audio): - pb = self.audio('k888', audio, 44100) - self.assertEqual(1, len(pb.value)) - results = tensor_util.make_ndarray(pb.value[0].tensor) - for i, (encoded, _) in enumerate(results): - decoded, _ = audio_ops.decode_wav(encoded) - self.assertEqual(audio[i].shape, decoded.shape) - - def test_dimensions(self): - # Check mono and stereo. - self._test_dimensions(self._generate_audio(c=1)) - self._test_dimensions(self._generate_audio(c=2)) - - def test_audio_count_zero(self): - shape = (0, self.num_samples, 2) - audio = np.array([]).reshape(shape).astype(np.float32) - pb = self.audio('k488', audio, 44100, max_outputs=3) - self.assertEqual(1, len(pb.value)) - results = tensor_util.make_ndarray(pb.value[0].tensor) - self.assertEqual(results.shape, (0, 2)) - - def test_audio_count_less_than_max_outputs(self): - max_outputs = 3 - data = self._generate_audio(k=(max_outputs - 1)) - pb = self.audio('k488', data, 44100, max_outputs=max_outputs) - self.assertEqual(1, len(pb.value)) - results = tensor_util.make_ndarray(pb.value[0].tensor) - self.assertEqual(results.shape, (len(data), 2)) - - def test_audio_count_when_more_than_max(self): - max_outputs = 3 - data = self._generate_audio(k=(max_outputs + 1)) - pb = self.audio('k488', data, 44100, max_outputs=max_outputs) - self.assertEqual(1, len(pb.value)) - results = tensor_util.make_ndarray(pb.value[0].tensor) - self.assertEqual(results.shape, (max_outputs, 2)) - - def test_requires_nonnegative_max_outputs(self): - data = np.array(1, np.float32, ndmin=3) - with six.assertRaisesRegex( - self, (ValueError, tf.errors.InvalidArgumentError), '>= 0'): - self.audio('k488', data, 44100, max_outputs=-1) - - def test_requires_rank_3(self): - with six.assertRaisesRegex(self, ValueError, 'must have rank 3'): - self.audio('k488', np.array([[1]]), 44100) - - def test_requires_wav(self): - data = np.array(1, np.float32, ndmin=3) - with six.assertRaisesRegex(self, ValueError, 'Unknown encoding'): - self.audio('k488', data, 44100, encoding='pptx') + def setUp(self): + super(SummaryBaseTest, self).setUp() + self.samples_rate = 44100 + self.audio_count = 1 + self.num_samples = 20 + self.num_channels = 2 + + def _generate_audio(self, **kwargs): + size = [ + kwargs.get("k", self.audio_count), + kwargs.get("n", self.num_samples), + kwargs.get("c", self.num_channels), + ] + return np.sin( + np.linspace(0.0, 100.0, np.prod(size), dtype=np.float32) + ).reshape(size) + + def audio(self, *args, **kwargs): + raise NotImplementedError() + + def test_metadata(self): + data = np.array(1, np.float32, ndmin=3) + description = "Piano Concerto No. 23 (K488), in **A major.**" + pb = self.audio("k488", data, 44100, description=description) + self.assertEqual(len(pb.value), 1) + summary_metadata = pb.value[0].metadata + self.assertEqual(summary_metadata.summary_description, description) + plugin_data = summary_metadata.plugin_data + self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) + content = summary_metadata.plugin_data.content + parsed = metadata.parse_plugin_metadata(content) + self.assertEqual(parsed.encoding, metadata.Encoding.Value("WAV")) + + def test_wav_format_roundtrip(self): + audio = self._generate_audio(c=1) + pb = self.audio("k888", audio, 44100) + encoded = tensor_util.make_ndarray(pb.value[0].tensor) + decoded, sample_rate = audio_ops.decode_wav(encoded.flat[0]) + # WAV roundtrip goes from float32 to int16 and back, so expect some + # precision loss, but not more than 2 applications of rounding error from + # mapping the range [-1.0, 1.0] to 2^16. + epsilon = 2 * 2.0 / (2 ** 16) + self.assertAllClose(audio[0], decoded, atol=epsilon) + self.assertEqual(44100, sample_rate.numpy()) + + def _test_dimensions(self, audio): + pb = self.audio("k888", audio, 44100) + self.assertEqual(1, len(pb.value)) + results = tensor_util.make_ndarray(pb.value[0].tensor) + for i, (encoded, _) in enumerate(results): + decoded, _ = audio_ops.decode_wav(encoded) + self.assertEqual(audio[i].shape, decoded.shape) + + def test_dimensions(self): + # Check mono and stereo. + self._test_dimensions(self._generate_audio(c=1)) + self._test_dimensions(self._generate_audio(c=2)) + + def test_audio_count_zero(self): + shape = (0, self.num_samples, 2) + audio = np.array([]).reshape(shape).astype(np.float32) + pb = self.audio("k488", audio, 44100, max_outputs=3) + self.assertEqual(1, len(pb.value)) + results = tensor_util.make_ndarray(pb.value[0].tensor) + self.assertEqual(results.shape, (0, 2)) + + def test_audio_count_less_than_max_outputs(self): + max_outputs = 3 + data = self._generate_audio(k=(max_outputs - 1)) + pb = self.audio("k488", data, 44100, max_outputs=max_outputs) + self.assertEqual(1, len(pb.value)) + results = tensor_util.make_ndarray(pb.value[0].tensor) + self.assertEqual(results.shape, (len(data), 2)) + + def test_audio_count_when_more_than_max(self): + max_outputs = 3 + data = self._generate_audio(k=(max_outputs + 1)) + pb = self.audio("k488", data, 44100, max_outputs=max_outputs) + self.assertEqual(1, len(pb.value)) + results = tensor_util.make_ndarray(pb.value[0].tensor) + self.assertEqual(results.shape, (max_outputs, 2)) + + def test_requires_nonnegative_max_outputs(self): + data = np.array(1, np.float32, ndmin=3) + with six.assertRaisesRegex( + self, (ValueError, tf.errors.InvalidArgumentError), ">= 0" + ): + self.audio("k488", data, 44100, max_outputs=-1) + + def test_requires_rank_3(self): + with six.assertRaisesRegex(self, ValueError, "must have rank 3"): + self.audio("k488", np.array([[1]]), 44100) + + def test_requires_wav(self): + data = np.array(1, np.float32, ndmin=3) + with six.assertRaisesRegex(self, ValueError, "Unknown encoding"): + self.audio("k488", data, 44100, encoding="pptx") class SummaryV1PbTest(SummaryBaseTest, tf.test.TestCase): - def setUp(self): - super(SummaryV1PbTest, self).setUp() + def setUp(self): + super(SummaryV1PbTest, self).setUp() - def audio(self, *args, **kwargs): - return summary.pb(*args, **kwargs) + def audio(self, *args, **kwargs): + return summary.pb(*args, **kwargs) - def test_tag(self): - data = np.array(1, np.float32, ndmin=3) - self.assertEqual('a/audio_summary', - self.audio('a', data, 44100).value[0].tag) - self.assertEqual('a/b/audio_summary', - self.audio('a/b', data, 44100).value[0].tag) + def test_tag(self): + data = np.array(1, np.float32, ndmin=3) + self.assertEqual( + "a/audio_summary", self.audio("a", data, 44100).value[0].tag + ) + self.assertEqual( + "a/b/audio_summary", self.audio("a/b", data, 44100).value[0].tag + ) - def test_requires_nonnegative_max_outputs(self): - self.skipTest('summary V1 pb does not actually enforce this') + def test_requires_nonnegative_max_outputs(self): + self.skipTest("summary V1 pb does not actually enforce this") class SummaryV1OpTest(SummaryBaseTest, tf.test.TestCase): - def setUp(self): - super(SummaryV1OpTest, self).setUp() - - def audio(self, *args, **kwargs): - return tf.compat.v1.Summary.FromString(summary.op(*args, **kwargs).numpy()) - - def test_tag(self): - data = np.array(1, np.float32, ndmin=3) - self.assertEqual('a/audio_summary', - self.audio('a', data, 44100).value[0].tag) - self.assertEqual('a/b/audio_summary', - self.audio('a/b', data, 44100).value[0].tag) - - def test_scoped_tag(self): - data = np.array(1, np.float32, ndmin=3) - with tf.name_scope('scope'): - self.assertEqual('scope/a/audio_summary', - self.audio('a', data, 44100).value[0].tag) - - def test_audio_count_zero(self): - self.skipTest('fails under eager because map_fn() returns float dtype') - - def test_requires_nonnegative_max_outputs(self): - self.skipTest('summary V1 op does not actually enforce this') + def setUp(self): + super(SummaryV1OpTest, self).setUp() + + def audio(self, *args, **kwargs): + return tf.compat.v1.Summary.FromString( + summary.op(*args, **kwargs).numpy() + ) + + def test_tag(self): + data = np.array(1, np.float32, ndmin=3) + self.assertEqual( + "a/audio_summary", self.audio("a", data, 44100).value[0].tag + ) + self.assertEqual( + "a/b/audio_summary", self.audio("a/b", data, 44100).value[0].tag + ) + + def test_scoped_tag(self): + data = np.array(1, np.float32, ndmin=3) + with tf.name_scope("scope"): + self.assertEqual( + "scope/a/audio_summary", + self.audio("a", data, 44100).value[0].tag, + ) + + def test_audio_count_zero(self): + self.skipTest("fails under eager because map_fn() returns float dtype") + + def test_requires_nonnegative_max_outputs(self): + self.skipTest("summary V1 op does not actually enforce this") class SummaryV2OpTest(SummaryBaseTest, tf.test.TestCase): - def setUp(self): - super(SummaryV2OpTest, self).setUp() - if tf2 is None: - self.skipTest('TF v2 summary API not available') - - def audio(self, *args, **kwargs): - return self.audio_event(*args, **kwargs).summary - - def audio_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - summary.audio(*args, **kwargs) - writer.close() - event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), '*'))) - self.assertEqual(len(event_files), 1) - events = list(tf.compat.v1.train.summary_iterator(event_files[0])) - # Expect a boilerplate event for the file_version, then the summary one. - self.assertEqual(len(events), 2) - # Delete the event file to reset to an empty directory for later calls. - # TODO(nickfelt): use a unique subdirectory per writer instead. - os.remove(event_files[0]) - return events[1] - - def test_scoped_tag(self): - data = np.array(1, np.float32, ndmin=3) - with tf.name_scope('scope'): - self.assertEqual('scope/a', self.audio('a', data, 44100).value[0].tag) - - def test_step(self): - data = np.array(1, np.float32, ndmin=3) - event = self.audio_event('a', data, 44100, step=333) - self.assertEqual(333, event.step) - - def test_default_step(self): - data = np.array(1, np.float32, ndmin=3) - try: - tf2.summary.experimental.set_step(333) - # TODO(nickfelt): change test logic so we can just omit `step` entirely. - event = self.audio_event('a', data, 44100, step=None) - self.assertEqual(333, event.step) - finally: - # Reset to default state for other tests. - tf2.summary.experimental.set_step(None) - - -if __name__ == '__main__': - tf.test.main() + def setUp(self): + super(SummaryV2OpTest, self).setUp() + if tf2 is None: + self.skipTest("TF v2 summary API not available") + + def audio(self, *args, **kwargs): + return self.audio_event(*args, **kwargs).summary + + def audio_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + summary.audio(*args, **kwargs) + writer.close() + event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), "*"))) + self.assertEqual(len(event_files), 1) + events = list(tf.compat.v1.train.summary_iterator(event_files[0])) + # Expect a boilerplate event for the file_version, then the summary one. + self.assertEqual(len(events), 2) + # Delete the event file to reset to an empty directory for later calls. + # TODO(nickfelt): use a unique subdirectory per writer instead. + os.remove(event_files[0]) + return events[1] + + def test_scoped_tag(self): + data = np.array(1, np.float32, ndmin=3) + with tf.name_scope("scope"): + self.assertEqual( + "scope/a", self.audio("a", data, 44100).value[0].tag + ) + + def test_step(self): + data = np.array(1, np.float32, ndmin=3) + event = self.audio_event("a", data, 44100, step=333) + self.assertEqual(333, event.step) + + def test_default_step(self): + data = np.array(1, np.float32, ndmin=3) + try: + tf2.summary.experimental.set_step(333) + # TODO(nickfelt): change test logic so we can just omit `step` entirely. + event = self.audio_event("a", data, 44100, step=None) + self.assertEqual(333, event.step) + finally: + # Reset to default state for other tests. + tf2.summary.experimental.set_step(None) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/audio/summary_v2.py b/tensorboard/plugins/audio/summary_v2.py index 157c7db537..a4d078a82f 100644 --- a/tensorboard/plugins/audio/summary_v2.py +++ b/tensorboard/plugins/audio/summary_v2.py @@ -32,87 +32,97 @@ from tensorboard.util import lazy_tensor_creator -def audio(name, - data, - sample_rate, - step=None, - max_outputs=3, - encoding=None, - description=None): - """Write an audio summary. +def audio( + name, + data, + sample_rate, + step=None, + max_outputs=3, + encoding=None, + description=None, +): + """Write an audio summary. - Arguments: - name: A name for this summary. The summary tag used for TensorBoard will - be this name prefixed by any active name scopes. - data: A `Tensor` representing audio data with shape `[k, t, c]`, - where `k` is the number of audio clips, `t` is the number of - frames, and `c` is the number of channels. Elements should be - floating-point values in `[-1.0, 1.0]`. Any of the dimensions may - be statically unknown (i.e., `None`). - sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the - sample rate, in Hz. Must be positive. - step: Explicit `int64`-castable monotonic step value for this summary. If - omitted, this defaults to `tf.summary.experimental.get_step()`, which must - not be None. - max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this - many audio clips will be emitted at each step. When more than - `max_outputs` many clips are provided, the first `max_outputs` - many clips will be used and the rest silently discarded. - encoding: Optional constant `str` for the desired encoding. Only "wav" - is currently supported, but this is not guaranteed to remain the - default, so if you want "wav" in particular, set this explicitly. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. + Arguments: + name: A name for this summary. The summary tag used for TensorBoard will + be this name prefixed by any active name scopes. + data: A `Tensor` representing audio data with shape `[k, t, c]`, + where `k` is the number of audio clips, `t` is the number of + frames, and `c` is the number of channels. Elements should be + floating-point values in `[-1.0, 1.0]`. Any of the dimensions may + be statically unknown (i.e., `None`). + sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the + sample rate, in Hz. Must be positive. + step: Explicit `int64`-castable monotonic step value for this summary. If + omitted, this defaults to `tf.summary.experimental.get_step()`, which must + not be None. + max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this + many audio clips will be emitted at each step. When more than + `max_outputs` many clips are provided, the first `max_outputs` + many clips will be used and the rest silently discarded. + encoding: Optional constant `str` for the desired encoding. Only "wav" + is currently supported, but this is not guaranteed to remain the + default, so if you want "wav" in particular, set this explicitly. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. - Returns: - True on success, or false if no summary was emitted because no default - summary writer was available. + Returns: + True on success, or false if no summary was emitted because no default + summary writer was available. - Raises: - ValueError: if a default writer exists, but no step was provided and - `tf.summary.experimental.get_step()` is None. - """ - audio_ops = getattr(tf, 'audio', None) - if audio_ops is None: - # Fallback for older versions of TF without tf.audio. - from tensorflow.python.ops import gen_audio_ops as audio_ops + Raises: + ValueError: if a default writer exists, but no step was provided and + `tf.summary.experimental.get_step()` is None. + """ + audio_ops = getattr(tf, "audio", None) + if audio_ops is None: + # Fallback for older versions of TF without tf.audio. + from tensorflow.python.ops import gen_audio_ops as audio_ops - if encoding is None: - encoding = 'wav' - if encoding != 'wav': - raise ValueError('Unknown encoding: %r' % encoding) - summary_metadata = metadata.create_summary_metadata( - display_name=None, - description=description, - encoding=metadata.Encoding.Value('WAV')) - inputs = [data, sample_rate, max_outputs, step] - # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback - summary_scope = ( - getattr(tf.summary.experimental, 'summary_scope', None) or - tf.summary.summary_scope) - with summary_scope( - name, 'audio_summary', values=inputs) as (tag, _): - # Defer audio encoding preprocessing by passing it as a callable to write(), - # wrapped in a LazyTensorCreator for backwards compatibility, so that we - # only do this work when summaries are actually written. - @lazy_tensor_creator.LazyTensorCreator - def lazy_tensor(): - tf.debugging.assert_rank(data, 3) - tf.debugging.assert_non_negative(max_outputs) - limited_audio = data[:max_outputs] - encode_fn = functools.partial(audio_ops.encode_wav, - sample_rate=sample_rate) - encoded_audio = tf.map_fn(encode_fn, limited_audio, - dtype=tf.string, - name='encode_each_audio') - # Workaround for map_fn returning float dtype for an empty elems input. - encoded_audio = tf.cond( - tf.shape(input=encoded_audio)[0] > 0, - lambda: encoded_audio, lambda: tf.constant([], tf.string)) - limited_labels = tf.tile([''], tf.shape(input=limited_audio)[:1]) - return tf.transpose(a=tf.stack([encoded_audio, limited_labels])) + if encoding is None: + encoding = "wav" + if encoding != "wav": + raise ValueError("Unknown encoding: %r" % encoding) + summary_metadata = metadata.create_summary_metadata( + display_name=None, + description=description, + encoding=metadata.Encoding.Value("WAV"), + ) + inputs = [data, sample_rate, max_outputs, step] + # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback + summary_scope = ( + getattr(tf.summary.experimental, "summary_scope", None) + or tf.summary.summary_scope + ) + with summary_scope(name, "audio_summary", values=inputs) as (tag, _): + # Defer audio encoding preprocessing by passing it as a callable to write(), + # wrapped in a LazyTensorCreator for backwards compatibility, so that we + # only do this work when summaries are actually written. + @lazy_tensor_creator.LazyTensorCreator + def lazy_tensor(): + tf.debugging.assert_rank(data, 3) + tf.debugging.assert_non_negative(max_outputs) + limited_audio = data[:max_outputs] + encode_fn = functools.partial( + audio_ops.encode_wav, sample_rate=sample_rate + ) + encoded_audio = tf.map_fn( + encode_fn, + limited_audio, + dtype=tf.string, + name="encode_each_audio", + ) + # Workaround for map_fn returning float dtype for an empty elems input. + encoded_audio = tf.cond( + tf.shape(input=encoded_audio)[0] > 0, + lambda: encoded_audio, + lambda: tf.constant([], tf.string), + ) + limited_labels = tf.tile([""], tf.shape(input=limited_audio)[:1]) + return tf.transpose(a=tf.stack([encoded_audio, limited_labels])) - # To ensure that audio encoding logic is only executed when summaries - # are written, we pass callable to `tensor` parameter. - return tf.summary.write( - tag=tag, tensor=lazy_tensor, step=step, metadata=summary_metadata) + # To ensure that audio encoding logic is only executed when summaries + # are written, we pass callable to `tensor` parameter. + return tf.summary.write( + tag=tag, tensor=lazy_tensor, step=step, metadata=summary_metadata + ) diff --git a/tensorboard/plugins/base_plugin.py b/tensorboard/plugins/base_plugin.py index 2b87567e80..04f93dd00e 100644 --- a/tensorboard/plugins/base_plugin.py +++ b/tensorboard/plugins/base_plugin.py @@ -14,8 +14,8 @@ # ============================================================================== """TensorBoard Plugin abstract base class. -Every plugin in TensorBoard must extend and implement the abstract methods of -this base class. +Every plugin in TensorBoard must extend and implement the abstract +methods of this base class. """ from __future__ import absolute_import @@ -31,331 +31,339 @@ @six.add_metaclass(ABCMeta) class TBPlugin(object): - """TensorBoard plugin interface. + """TensorBoard plugin interface. + + Every plugin must extend from this class. + + Subclasses should have a trivial constructor that takes a TBContext + argument. Any operation that might throw an exception should either be + done lazily or made safe with a TBLoader subclass, so the plugin won't + negatively impact the rest of TensorBoard. + + Fields: + plugin_name: The plugin_name will also be a prefix in the http + handlers, e.g. `data/plugins/$PLUGIN_NAME/$HANDLER` The plugin + name must be unique for each registered plugin, or a ValueError + will be thrown when the application is constructed. The plugin + name must only contain characters among [A-Za-z0-9_.-], and must + be nonempty, or a ValueError will similarly be thrown. + """ - Every plugin must extend from this class. + plugin_name = None - Subclasses should have a trivial constructor that takes a TBContext - argument. Any operation that might throw an exception should either be - done lazily or made safe with a TBLoader subclass, so the plugin won't - negatively impact the rest of TensorBoard. + def __init__(self, context): + """Initializes this plugin. - Fields: - plugin_name: The plugin_name will also be a prefix in the http - handlers, e.g. `data/plugins/$PLUGIN_NAME/$HANDLER` The plugin - name must be unique for each registered plugin, or a ValueError - will be thrown when the application is constructed. The plugin - name must only contain characters among [A-Za-z0-9_.-], and must - be nonempty, or a ValueError will similarly be thrown. - """ + The default implementation does nothing. Subclasses are encouraged + to override this and save any necessary fields from the `context`. - plugin_name = None + Args: + context: A `base_plugin.TBContext` object. + """ + pass - def __init__(self, context): - """Initializes this plugin. + @abstractmethod + def get_plugin_apps(self): + """Returns a set of WSGI applications that the plugin implements. - The default implementation does nothing. Subclasses are encouraged - to override this and save any necessary fields from the `context`. + Each application gets registered with the tensorboard app and is served + under a prefix path that includes the name of the plugin. - Args: - context: A `base_plugin.TBContext` object. - """ - pass + Returns: + A dict mapping route paths to WSGI applications. Each route path + should include a leading slash. + """ + raise NotImplementedError() - @abstractmethod - def get_plugin_apps(self): - """Returns a set of WSGI applications that the plugin implements. + @abstractmethod + def is_active(self): + """Determines whether this plugin is active. - Each application gets registered with the tensorboard app and is served - under a prefix path that includes the name of the plugin. + A plugin may not be active for instance if it lacks relevant data. If a + plugin is inactive, the frontend may avoid issuing requests to its routes. - Returns: - A dict mapping route paths to WSGI applications. Each route path - should include a leading slash. - """ - raise NotImplementedError() + Returns: + A boolean value. Whether this plugin is active. + """ + raise NotImplementedError() - @abstractmethod - def is_active(self): - """Determines whether this plugin is active. + def frontend_metadata(self): + """Defines how the plugin will be displayed on the frontend. - A plugin may not be active for instance if it lacks relevant data. If a - plugin is inactive, the frontend may avoid issuing requests to its routes. + The base implementation returns a default value. Subclasses + should override this and specify either an `es_module_path` or + (for legacy plugins) an `element_name`, and are encouraged to + set any other relevant attributes. + """ + return FrontendMetadata() - Returns: - A boolean value. Whether this plugin is active. - """ - raise NotImplementedError() - def frontend_metadata(self): - """Defines how the plugin will be displayed on the frontend. +class FrontendMetadata(object): + """Metadata required to render a plugin on the frontend. - The base implementation returns a default value. Subclasses should - override this and specify either an `es_module_path` or (for legacy - plugins) an `element_name`, and are encouraged to set any other - relevant attributes. + Each argument to the constructor is publicly accessible under a + field of the same name. See constructor docs for further details. """ - return FrontendMetadata() - -class FrontendMetadata(object): - """Metadata required to render a plugin on the frontend. - - Each argument to the constructor is publicly accessible under a field - of the same name. See constructor docs for further details. - """ - - def __init__( - self, - disable_reload=None, - element_name=None, - es_module_path=None, - remove_dom=None, - tab_name=None, - is_ng_component=False, - ): - """Creates a `FrontendMetadata` value. - - The argument list is sorted and may be extended in the future; - therefore, callers must pass only named arguments to this - constructor. - - Args: - disable_reload: Whether to disable the reload button and - auto-reload timer. A `bool`; defaults to `False`. - element_name: For legacy plugins, name of the custom element - defining the plugin frontend: e.g., `"tf-scalar-dashboard"`. - A `str` or `None` (for iframed plugins). Mutually exclusive - with `es_module_path`. - es_module_path: ES module to use as an entry point to this plugin. - A `str` that is a key in the result of `get_plugin_apps()`, or - `None` for legacy plugins bundled with TensorBoard as part of - `webfiles.zip`. Mutually exclusive with legacy `element_name` - remove_dom: Whether to remove the plugin DOM when switching to a - different plugin, to trigger the Polymer 'detached' event. - A `bool`; defaults to `False`. - tab_name: Name to show in the menu item for this dashboard within - the navigation bar. May differ from the plugin name: for - instance, the tab name should not use underscores to separate - words. Should be a `str` or `None` (the default; indicates to - use the plugin name as the tab name). - is_ng_component: Set to `True` only for built-in Agnular plugins. - In this case, the `plugin_name` property of the Plugin, which is - mapped to the `id` property in JavaScript's `UiPluginMetadata` type, - is used to select the Angular component. A `True` value is mutually - exclusive with `element_name` and `es_module_path`. - """ - self._disable_reload = False if disable_reload is None else disable_reload - self._element_name = element_name - self._es_module_path = es_module_path - self._remove_dom = False if remove_dom is None else remove_dom - self._tab_name = tab_name - self._is_ng_component = is_ng_component - - @property - def disable_reload(self): - return self._disable_reload - - @property - def element_name(self): - return self._element_name - - @property - def is_ng_component(self): - return self._is_ng_component - - @property - def es_module_path(self): - return self._es_module_path - - @property - def remove_dom(self): - return self._remove_dom - - @property - def tab_name(self): - return self._tab_name - - def __eq__(self, other): - if not isinstance(other, FrontendMetadata): - return False - if self._disable_reload != other._disable_reload: - return False - if self._disable_reload != other._disable_reload: - return False - if self._element_name != other._element_name: - return False - if self._es_module_path != other._es_module_path: - return False - if self._remove_dom != other._remove_dom: - return False - if self._tab_name != other._tab_name: - return False - return True - - def __hash__(self): - return hash(( - self._disable_reload, - self._element_name, - self._es_module_path, - self._remove_dom, - self._tab_name, - self._is_ng_component, - )) - - def __repr__(self): - return "FrontendMetadata(%s)" % ", ".join(( - "disable_reload=%r" % self._disable_reload, - "element_name=%r" % self._element_name, - "es_module_path=%r" % self._es_module_path, - "remove_dom=%r" % self._remove_dom, - "tab_name=%r" % self._tab_name, - "is_ng_component=%r" % self._is_ng_component, - )) + def __init__( + self, + disable_reload=None, + element_name=None, + es_module_path=None, + remove_dom=None, + tab_name=None, + is_ng_component=False, + ): + """Creates a `FrontendMetadata` value. + + The argument list is sorted and may be extended in the future; + therefore, callers must pass only named arguments to this + constructor. + + Args: + disable_reload: Whether to disable the reload button and + auto-reload timer. A `bool`; defaults to `False`. + element_name: For legacy plugins, name of the custom element + defining the plugin frontend: e.g., `"tf-scalar-dashboard"`. + A `str` or `None` (for iframed plugins). Mutually exclusive + with `es_module_path`. + es_module_path: ES module to use as an entry point to this plugin. + A `str` that is a key in the result of `get_plugin_apps()`, or + `None` for legacy plugins bundled with TensorBoard as part of + `webfiles.zip`. Mutually exclusive with legacy `element_name` + remove_dom: Whether to remove the plugin DOM when switching to a + different plugin, to trigger the Polymer 'detached' event. + A `bool`; defaults to `False`. + tab_name: Name to show in the menu item for this dashboard within + the navigation bar. May differ from the plugin name: for + instance, the tab name should not use underscores to separate + words. Should be a `str` or `None` (the default; indicates to + use the plugin name as the tab name). + is_ng_component: Set to `True` only for built-in Agnular plugins. + In this case, the `plugin_name` property of the Plugin, which is + mapped to the `id` property in JavaScript's `UiPluginMetadata` type, + is used to select the Angular component. A `True` value is mutually + exclusive with `element_name` and `es_module_path`. + """ + self._disable_reload = ( + False if disable_reload is None else disable_reload + ) + self._element_name = element_name + self._es_module_path = es_module_path + self._remove_dom = False if remove_dom is None else remove_dom + self._tab_name = tab_name + self._is_ng_component = is_ng_component + + @property + def disable_reload(self): + return self._disable_reload + + @property + def element_name(self): + return self._element_name + + @property + def is_ng_component(self): + return self._is_ng_component + + @property + def es_module_path(self): + return self._es_module_path + + @property + def remove_dom(self): + return self._remove_dom + + @property + def tab_name(self): + return self._tab_name + + def __eq__(self, other): + if not isinstance(other, FrontendMetadata): + return False + if self._disable_reload != other._disable_reload: + return False + if self._disable_reload != other._disable_reload: + return False + if self._element_name != other._element_name: + return False + if self._es_module_path != other._es_module_path: + return False + if self._remove_dom != other._remove_dom: + return False + if self._tab_name != other._tab_name: + return False + return True + + def __hash__(self): + return hash( + ( + self._disable_reload, + self._element_name, + self._es_module_path, + self._remove_dom, + self._tab_name, + self._is_ng_component, + ) + ) + + def __repr__(self): + return "FrontendMetadata(%s)" % ", ".join( + ( + "disable_reload=%r" % self._disable_reload, + "element_name=%r" % self._element_name, + "es_module_path=%r" % self._es_module_path, + "remove_dom=%r" % self._remove_dom, + "tab_name=%r" % self._tab_name, + "is_ng_component=%r" % self._is_ng_component, + ) + ) class TBContext(object): - """Magic container of information passed from TensorBoard core to plugins. - - A TBContext instance is passed to the constructor of a TBPlugin class. Plugins - are strongly encouraged to assume that any of these fields can be None. In - cases when a field is considered mandatory by a plugin, it can either crash - with ValueError, or silently choose to disable itself by returning False from - its is_active method. - - All fields in this object are thread safe. - """ - - def __init__( - self, - assets_zip_provider=None, - data_provider=None, - db_connection_provider=None, - db_uri=None, - flags=None, - logdir=None, - multiplexer=None, - plugin_name_to_instance=None, - window_title=None): - """Instantiates magic container. - - The argument list is sorted and may be extended in the future; therefore, - callers must pass only named arguments to this constructor. - - Args: - assets_zip_provider: A function that returns a newly opened file handle - for a zip file containing all static assets. The file names inside the - zip file are considered absolute paths on the web server. The file - handle this function returns must be closed. It is assumed that you - will pass this file handle to zipfile.ZipFile. This zip file should - also have been created by the tensorboard_zip_file build rule. - data_provider: Instance of `tensorboard.data.provider.DataProvider`. May - be `None` if `flags.generic_data` is set to `"false"`. - db_connection_provider: Function taking no arguments that returns a - PEP-249 database Connection object, or None if multiplexer should be - used instead. The returned value must be closed, and is safe to use in - a `with` statement. It is also safe to assume that calling this - function is cheap. The returned connection must only be used by a - single thread. Things like connection pooling are considered - implementation details of the provider. - db_uri: The string db URI TensorBoard was started with. If this is set, - the logdir should be None. - flags: An object of the runtime flags provided to TensorBoard to their - values. - logdir: The string logging directory TensorBoard was started with. If this - is set, the db_uri should be None. - multiplexer: An EventMultiplexer with underlying TB data. Plugins should - copy this data over to the database when the db fields are set. - plugin_name_to_instance: A mapping between plugin name to instance. - Plugins may use this property to access other plugins. The context - object is passed to plugins during their construction, so a given - plugin may be absent from this mapping until it is registered. Plugin - logic should handle cases in which a plugin is absent from this - mapping, lest a KeyError is raised. - window_title: A string specifying the window title. + """Magic container of information passed from TensorBoard core to plugins. + + A TBContext instance is passed to the constructor of a TBPlugin class. Plugins + are strongly encouraged to assume that any of these fields can be None. In + cases when a field is considered mandatory by a plugin, it can either crash + with ValueError, or silently choose to disable itself by returning False from + its is_active method. + + All fields in this object are thread safe. """ - self.assets_zip_provider = assets_zip_provider - self.data_provider = data_provider - self.db_connection_provider = db_connection_provider - self.db_uri = db_uri - self.flags = flags - self.logdir = logdir - self.multiplexer = multiplexer - self.plugin_name_to_instance = plugin_name_to_instance - self.window_title = window_title + + def __init__( + self, + assets_zip_provider=None, + data_provider=None, + db_connection_provider=None, + db_uri=None, + flags=None, + logdir=None, + multiplexer=None, + plugin_name_to_instance=None, + window_title=None, + ): + """Instantiates magic container. + + The argument list is sorted and may be extended in the future; therefore, + callers must pass only named arguments to this constructor. + + Args: + assets_zip_provider: A function that returns a newly opened file handle + for a zip file containing all static assets. The file names inside the + zip file are considered absolute paths on the web server. The file + handle this function returns must be closed. It is assumed that you + will pass this file handle to zipfile.ZipFile. This zip file should + also have been created by the tensorboard_zip_file build rule. + data_provider: Instance of `tensorboard.data.provider.DataProvider`. May + be `None` if `flags.generic_data` is set to `"false"`. + db_connection_provider: Function taking no arguments that returns a + PEP-249 database Connection object, or None if multiplexer should be + used instead. The returned value must be closed, and is safe to use in + a `with` statement. It is also safe to assume that calling this + function is cheap. The returned connection must only be used by a + single thread. Things like connection pooling are considered + implementation details of the provider. + db_uri: The string db URI TensorBoard was started with. If this is set, + the logdir should be None. + flags: An object of the runtime flags provided to TensorBoard to their + values. + logdir: The string logging directory TensorBoard was started with. If this + is set, the db_uri should be None. + multiplexer: An EventMultiplexer with underlying TB data. Plugins should + copy this data over to the database when the db fields are set. + plugin_name_to_instance: A mapping between plugin name to instance. + Plugins may use this property to access other plugins. The context + object is passed to plugins during their construction, so a given + plugin may be absent from this mapping until it is registered. Plugin + logic should handle cases in which a plugin is absent from this + mapping, lest a KeyError is raised. + window_title: A string specifying the window title. + """ + self.assets_zip_provider = assets_zip_provider + self.data_provider = data_provider + self.db_connection_provider = db_connection_provider + self.db_uri = db_uri + self.flags = flags + self.logdir = logdir + self.multiplexer = multiplexer + self.plugin_name_to_instance = plugin_name_to_instance + self.window_title = window_title class TBLoader(object): - """TBPlugin factory base class. + """TBPlugin factory base class. - Plugins can override this class to customize how a plugin is loaded at - startup. This might entail adding command-line arguments, checking if - optional dependencies are installed, and potentially also specializing - the plugin class at runtime. + Plugins can override this class to customize how a plugin is loaded at + startup. This might entail adding command-line arguments, checking if + optional dependencies are installed, and potentially also specializing + the plugin class at runtime. - When plugins use optional dependencies, the loader needs to be - specified in its own module. That way it's guaranteed to be - importable, even if the `TBPlugin` itself can't be imported. + When plugins use optional dependencies, the loader needs to be + specified in its own module. That way it's guaranteed to be + importable, even if the `TBPlugin` itself can't be imported. - Subclasses must have trivial constructors. - """ + Subclasses must have trivial constructors. + """ - def define_flags(self, parser): - """Adds plugin-specific CLI flags to parser. + def define_flags(self, parser): + """Adds plugin-specific CLI flags to parser. - The default behavior is to do nothing. + The default behavior is to do nothing. - When overriding this method, it's recommended that plugins call the - `parser.add_argument_group(plugin_name)` method for readability. No - flags should be specified that would cause `parse_args([])` to fail. + When overriding this method, it's recommended that plugins call the + `parser.add_argument_group(plugin_name)` method for readability. No + flags should be specified that would cause `parse_args([])` to fail. - Args: - parser: The argument parsing object, which may be mutated. - """ - pass + Args: + parser: The argument parsing object, which may be mutated. + """ + pass - def fix_flags(self, flags): - """Allows flag values to be corrected or validated after parsing. + def fix_flags(self, flags): + """Allows flag values to be corrected or validated after parsing. - Args: - flags: The parsed argparse.Namespace object. + Args: + flags: The parsed argparse.Namespace object. - Raises: - base_plugin.FlagsError: If a flag is invalid or a required - flag is not passed. - """ - pass + Raises: + base_plugin.FlagsError: If a flag is invalid or a required + flag is not passed. + """ + pass - def load(self, context): - """Loads a TBPlugin instance during the setup phase. + def load(self, context): + """Loads a TBPlugin instance during the setup phase. - Args: - context: The TBContext instance. + Args: + context: The TBContext instance. - Returns: - A plugin instance or None if it could not be loaded. Loaders that return - None are skipped. + Returns: + A plugin instance or None if it could not be loaded. Loaders that return + None are skipped. - :type context: TBContext - :rtype: TBPlugin | None - """ - return None + :type context: TBContext + :rtype: TBPlugin | None + """ + return None class BasicLoader(TBLoader): - """Simple TBLoader that's sufficient for most plugins.""" + """Simple TBLoader that's sufficient for most plugins.""" - def __init__(self, plugin_class): - """Creates simple plugin instance maker. + def __init__(self, plugin_class): + """Creates simple plugin instance maker. - :param plugin_class: :class:`TBPlugin` - """ - self.plugin_class = plugin_class + :param plugin_class: :class:`TBPlugin` + """ + self.plugin_class = plugin_class - def load(self, context): - return self.plugin_class(context) + def load(self, context): + return self.plugin_class(context) class FlagsError(ValueError): - """Raised when a command line flag is not specified or is invalid.""" - pass + """Raised when a command line flag is not specified or is invalid.""" + + pass diff --git a/tensorboard/plugins/base_plugin_test.py b/tensorboard/plugins/base_plugin_test.py index 772fff2e05..dbb67fa037 100644 --- a/tensorboard/plugins/base_plugin_test.py +++ b/tensorboard/plugins/base_plugin_test.py @@ -23,50 +23,49 @@ class FrontendMetadataTest(tb_test.TestCase): + def _create_metadata(self): + return base_plugin.FrontendMetadata( + disable_reload="my disable_reload", + element_name="my element_name", + es_module_path="my es_module_path", + remove_dom="my remove_dom", + tab_name="my tab_name", + ) - def _create_metadata(self): - return base_plugin.FrontendMetadata( - disable_reload="my disable_reload", - element_name="my element_name", - es_module_path="my es_module_path", - remove_dom="my remove_dom", - tab_name="my tab_name", - ) + def test_basics(self): + md = self._create_metadata() + self.assertEqual(md.disable_reload, "my disable_reload") + self.assertEqual(md.element_name, "my element_name") + self.assertEqual(md.es_module_path, "my es_module_path") + self.assertEqual(md.remove_dom, "my remove_dom") + self.assertEqual(md.tab_name, "my tab_name") - def test_basics(self): - md = self._create_metadata() - self.assertEqual(md.disable_reload, "my disable_reload") - self.assertEqual(md.element_name, "my element_name") - self.assertEqual(md.es_module_path, "my es_module_path") - self.assertEqual(md.remove_dom, "my remove_dom") - self.assertEqual(md.tab_name, "my tab_name") + def test_repr(self): + repr_ = repr(self._create_metadata()) + self.assertIn(repr("my disable_reload"), repr_) + self.assertIn(repr("my element_name"), repr_) + self.assertIn(repr("my es_module_path"), repr_) + self.assertIn(repr("my remove_dom"), repr_) + self.assertIn(repr("my tab_name"), repr_) - def test_repr(self): - repr_ = repr(self._create_metadata()) - self.assertIn(repr("my disable_reload"), repr_) - self.assertIn(repr("my element_name"), repr_) - self.assertIn(repr("my es_module_path"), repr_) - self.assertIn(repr("my remove_dom"), repr_) - self.assertIn(repr("my tab_name"), repr_) + def test_eq(self): + md1 = base_plugin.FrontendMetadata(element_name="foo") + md2 = base_plugin.FrontendMetadata(element_name="foo") + md3 = base_plugin.FrontendMetadata(element_name="bar") + self.assertEqual(md1, md2) + self.assertNotEqual(md1, md3) + self.assertNotEqual(md1, "hmm") - def test_eq(self): - md1 = base_plugin.FrontendMetadata(element_name="foo") - md2 = base_plugin.FrontendMetadata(element_name="foo") - md3 = base_plugin.FrontendMetadata(element_name="bar") - self.assertEqual(md1, md2) - self.assertNotEqual(md1, md3) - self.assertNotEqual(md1, "hmm") + def test_hash(self): + md1 = base_plugin.FrontendMetadata(element_name="foo") + md2 = base_plugin.FrontendMetadata(element_name="foo") + md3 = base_plugin.FrontendMetadata(element_name="bar") + self.assertEqual(hash(md1), hash(md2)) + # The next check is technically not required by the `__hash__` + # contract, but _should_ pass; failure on this assertion would at + # least warrant some scrutiny. + self.assertNotEqual(hash(md1), hash(md3)) - def test_hash(self): - md1 = base_plugin.FrontendMetadata(element_name="foo") - md2 = base_plugin.FrontendMetadata(element_name="foo") - md3 = base_plugin.FrontendMetadata(element_name="bar") - self.assertEqual(hash(md1), hash(md2)) - # The next check is technically not required by the `__hash__` - # contract, but _should_ pass; failure on this assertion would at - # least warrant some scrutiny. - self.assertNotEqual(hash(md1), hash(md3)) - -if __name__ == '__main__': - tb_test.main() +if __name__ == "__main__": + tb_test.main() diff --git a/tensorboard/plugins/beholder/__init__.py b/tensorboard/plugins/beholder/__init__.py index d5d71a2d9d..2c6234da13 100644 --- a/tensorboard/plugins/beholder/__init__.py +++ b/tensorboard/plugins/beholder/__init__.py @@ -14,10 +14,10 @@ # Only import Beholder API when tensorflow is available. try: - # pylint: disable=unused-import - import tensorflow + # pylint: disable=unused-import + import tensorflow except ImportError: - pass + pass else: - from tensorboard.plugins.beholder.beholder import Beholder - from tensorboard.plugins.beholder.beholder import BeholderHook + from tensorboard.plugins.beholder.beholder import Beholder + from tensorboard.plugins.beholder.beholder import BeholderHook diff --git a/tensorboard/plugins/beholder/beholder.py b/tensorboard/plugins/beholder/beholder.py index 10ea9ddc8f..30f445f4d7 100644 --- a/tensorboard/plugins/beholder/beholder.py +++ b/tensorboard/plugins/beholder/beholder.py @@ -23,10 +23,19 @@ import tensorflow as tf from tensorboard.plugins.beholder import im_util -from tensorboard.plugins.beholder.file_system_tools import read_pickle,\ - write_pickle, write_file -from tensorboard.plugins.beholder.shared_config import PLUGIN_NAME, TAG_NAME,\ - SUMMARY_FILENAME, DEFAULT_CONFIG, CONFIG_FILENAME, SUMMARY_COLLECTION_KEY_NAME +from tensorboard.plugins.beholder.file_system_tools import ( + read_pickle, + write_pickle, + write_file, +) +from tensorboard.plugins.beholder.shared_config import ( + PLUGIN_NAME, + TAG_NAME, + SUMMARY_FILENAME, + DEFAULT_CONFIG, + CONFIG_FILENAME, + SUMMARY_COLLECTION_KEY_NAME, +) from tensorboard.plugins.beholder import video_writing from tensorboard.plugins.beholder.visualizer import Visualizer from tensorboard.util import tb_logging @@ -36,187 +45,192 @@ class Beholder(object): - - def __init__(self, logdir): - self.PLUGIN_LOGDIR = logdir + '/plugins/' + PLUGIN_NAME - - self.is_recording = False - self.video_writer = video_writing.VideoWriter( - self.PLUGIN_LOGDIR, - outputs=[ - video_writing.FFmpegVideoOutput, - video_writing.PNGVideoOutput]) - - self.frame_placeholder = tf.compat.v1.placeholder(tf.uint8, [None, None, None]) - self.summary_op = tf.compat.v1.summary.tensor_summary(TAG_NAME, - self.frame_placeholder, - collections=[ - SUMMARY_COLLECTION_KEY_NAME - ]) - - self.last_image_shape = [] - self.last_update_time = time.time() - self.config_last_modified_time = -1 - self.previous_config = dict(DEFAULT_CONFIG) - - if not tf.io.gfile.exists(self.PLUGIN_LOGDIR + '/config.pkl'): - tf.io.gfile.makedirs(self.PLUGIN_LOGDIR) - write_pickle(DEFAULT_CONFIG, '{}/{}'.format(self.PLUGIN_LOGDIR, - CONFIG_FILENAME)) - - self.visualizer = Visualizer(self.PLUGIN_LOGDIR) - - - def _get_config(self): - '''Reads the config file from disk or creates a new one.''' - filename = '{}/{}'.format(self.PLUGIN_LOGDIR, CONFIG_FILENAME) - modified_time = os.path.getmtime(filename) - - if modified_time != self.config_last_modified_time: - config = read_pickle(filename, default=self.previous_config) - self.previous_config = config - else: - config = self.previous_config - - self.config_last_modified_time = modified_time - return config - - - def _write_summary(self, session, frame): - '''Writes the frame to disk as a tensor summary.''' - summary = session.run(self.summary_op, feed_dict={ - self.frame_placeholder: frame - }) - path = '{}/{}'.format(self.PLUGIN_LOGDIR, SUMMARY_FILENAME) - write_file(summary, path) - - - def _get_final_image(self, session, config, arrays=None, frame=None): - if config['values'] == 'frames': - if frame is None: - final_image = im_util.get_image_relative_to_script('frame-missing.png') - else: - frame = frame() if callable(frame) else frame - final_image = im_util.scale_image_for_display(frame) - - elif config['values'] == 'arrays': - if arrays is None: - final_image = im_util.get_image_relative_to_script('arrays-missing.png') - # TODO: hack to clear the info. Should be cleaner. - self.visualizer._save_section_info([], []) - else: - final_image = self.visualizer.build_frame(arrays) - - elif config['values'] == 'trainable_variables': - arrays = [session.run(x) for x in tf.compat.v1.trainable_variables()] - final_image = self.visualizer.build_frame(arrays) - - if len(final_image.shape) == 2: - # Map grayscale images to 3D tensors. - final_image = np.expand_dims(final_image, -1) - - return final_image - - - def _enough_time_has_passed(self, FPS): - '''For limiting how often frames are computed.''' - if FPS == 0: - return False - else: - earliest_time = self.last_update_time + (1.0 / FPS) - return time.time() >= earliest_time - - - def _update_frame(self, session, arrays, frame, config): - final_image = self._get_final_image(session, config, arrays, frame) - self._write_summary(session, final_image) - self.last_image_shape = final_image.shape - - return final_image - - - def _update_recording(self, frame, config): - '''Adds a frame to the current video output.''' - # pylint: disable=redefined-variable-type - should_record = config['is_recording'] - - if should_record: - if not self.is_recording: - self.is_recording = True - logger.info( - 'Starting recording using %s', - self.video_writer.current_output().name()) - self.video_writer.write_frame(frame) - elif self.is_recording: - self.is_recording = False - self.video_writer.finish() - logger.info('Finished recording') - - - # TODO: blanket try and except for production? I don't someone's script to die - # after weeks of running because of a visualization. - def update(self, session, arrays=None, frame=None): - '''Creates a frame and writes it to disk. - - Args: - arrays: a list of np arrays. Use the "custom" option in the client. - frame: a 2D np array. This way the plugin can be used for video of any - kind, not just the visualization that comes with the plugin. - - frame can also be a function, which only is evaluated when the - "frame" option is selected by the client. - ''' - new_config = self._get_config() - - if self._enough_time_has_passed(self.previous_config['FPS']): - self.visualizer.update(new_config) - self.last_update_time = time.time() - final_image = self._update_frame(session, arrays, frame, new_config) - self._update_recording(final_image, new_config) - - - ############################################################################## - - @staticmethod - def gradient_helper(optimizer, loss, var_list=None): - '''A helper to get the gradients out at each step. - - Args: - optimizer: the optimizer op. - loss: the op that computes your loss value. - - Returns: the gradient tensors and the train_step op. - ''' - if var_list is None: - var_list = tf.compat.v1.trainable_variables() - - grads_and_vars = optimizer.compute_gradients(loss, var_list=var_list) - grads = [pair[0] for pair in grads_and_vars] - - return grads, optimizer.apply_gradients(grads_and_vars) + def __init__(self, logdir): + self.PLUGIN_LOGDIR = logdir + "/plugins/" + PLUGIN_NAME + + self.is_recording = False + self.video_writer = video_writing.VideoWriter( + self.PLUGIN_LOGDIR, + outputs=[ + video_writing.FFmpegVideoOutput, + video_writing.PNGVideoOutput, + ], + ) + + self.frame_placeholder = tf.compat.v1.placeholder( + tf.uint8, [None, None, None] + ) + self.summary_op = tf.compat.v1.summary.tensor_summary( + TAG_NAME, + self.frame_placeholder, + collections=[SUMMARY_COLLECTION_KEY_NAME], + ) + + self.last_image_shape = [] + self.last_update_time = time.time() + self.config_last_modified_time = -1 + self.previous_config = dict(DEFAULT_CONFIG) + + if not tf.io.gfile.exists(self.PLUGIN_LOGDIR + "/config.pkl"): + tf.io.gfile.makedirs(self.PLUGIN_LOGDIR) + write_pickle( + DEFAULT_CONFIG, + "{}/{}".format(self.PLUGIN_LOGDIR, CONFIG_FILENAME), + ) + + self.visualizer = Visualizer(self.PLUGIN_LOGDIR) + + def _get_config(self): + """Reads the config file from disk or creates a new one.""" + filename = "{}/{}".format(self.PLUGIN_LOGDIR, CONFIG_FILENAME) + modified_time = os.path.getmtime(filename) + + if modified_time != self.config_last_modified_time: + config = read_pickle(filename, default=self.previous_config) + self.previous_config = config + else: + config = self.previous_config + + self.config_last_modified_time = modified_time + return config + + def _write_summary(self, session, frame): + """Writes the frame to disk as a tensor summary.""" + summary = session.run( + self.summary_op, feed_dict={self.frame_placeholder: frame} + ) + path = "{}/{}".format(self.PLUGIN_LOGDIR, SUMMARY_FILENAME) + write_file(summary, path) + + def _get_final_image(self, session, config, arrays=None, frame=None): + if config["values"] == "frames": + if frame is None: + final_image = im_util.get_image_relative_to_script( + "frame-missing.png" + ) + else: + frame = frame() if callable(frame) else frame + final_image = im_util.scale_image_for_display(frame) + + elif config["values"] == "arrays": + if arrays is None: + final_image = im_util.get_image_relative_to_script( + "arrays-missing.png" + ) + # TODO: hack to clear the info. Should be cleaner. + self.visualizer._save_section_info([], []) + else: + final_image = self.visualizer.build_frame(arrays) + + elif config["values"] == "trainable_variables": + arrays = [ + session.run(x) for x in tf.compat.v1.trainable_variables() + ] + final_image = self.visualizer.build_frame(arrays) + + if len(final_image.shape) == 2: + # Map grayscale images to 3D tensors. + final_image = np.expand_dims(final_image, -1) + + return final_image + + def _enough_time_has_passed(self, FPS): + """For limiting how often frames are computed.""" + if FPS == 0: + return False + else: + earliest_time = self.last_update_time + (1.0 / FPS) + return time.time() >= earliest_time + + def _update_frame(self, session, arrays, frame, config): + final_image = self._get_final_image(session, config, arrays, frame) + self._write_summary(session, final_image) + self.last_image_shape = final_image.shape + + return final_image + + def _update_recording(self, frame, config): + """Adds a frame to the current video output.""" + # pylint: disable=redefined-variable-type + should_record = config["is_recording"] + + if should_record: + if not self.is_recording: + self.is_recording = True + logger.info( + "Starting recording using %s", + self.video_writer.current_output().name(), + ) + self.video_writer.write_frame(frame) + elif self.is_recording: + self.is_recording = False + self.video_writer.finish() + logger.info("Finished recording") + + # TODO: blanket try and except for production? I don't someone's script to die + # after weeks of running because of a visualization. + def update(self, session, arrays=None, frame=None): + """Creates a frame and writes it to disk. + + Args: + arrays: a list of np arrays. Use the "custom" option in the client. + frame: a 2D np array. This way the plugin can be used for video of any + kind, not just the visualization that comes with the plugin. + + frame can also be a function, which only is evaluated when the + "frame" option is selected by the client. + """ + new_config = self._get_config() + + if self._enough_time_has_passed(self.previous_config["FPS"]): + self.visualizer.update(new_config) + self.last_update_time = time.time() + final_image = self._update_frame(session, arrays, frame, new_config) + self._update_recording(final_image, new_config) + + ############################################################################## + + @staticmethod + def gradient_helper(optimizer, loss, var_list=None): + """A helper to get the gradients out at each step. + + Args: + optimizer: the optimizer op. + loss: the op that computes your loss value. + + Returns: the gradient tensors and the train_step op. + """ + if var_list is None: + var_list = tf.compat.v1.trainable_variables() + + grads_and_vars = optimizer.compute_gradients(loss, var_list=var_list) + grads = [pair[0] for pair in grads_and_vars] + + return grads, optimizer.apply_gradients(grads_and_vars) class BeholderHook(tf.estimator.SessionRunHook): - """SessionRunHook implementation that runs Beholder every step. - - Convenient when using tf.train.MonitoredSession: - ```python - beholder_hook = BeholderHook(LOG_DIRECTORY) - with MonitoredSession(..., hooks=[beholder_hook]) as sess: - sess.run(train_op) - ``` - """ - def __init__(self, logdir): - """Creates new Hook instance - - Args: - logdir: Directory where Beholder should write data. + """SessionRunHook implementation that runs Beholder every step. + + Convenient when using tf.train.MonitoredSession: + ```python + beholder_hook = BeholderHook(LOG_DIRECTORY) + with MonitoredSession(..., hooks=[beholder_hook]) as sess: + sess.run(train_op) + ``` """ - self._logdir = logdir - self.beholder = None - def begin(self): - self.beholder = Beholder(self._logdir) + def __init__(self, logdir): + """Creates new Hook instance. + + Args: + logdir: Directory where Beholder should write data. + """ + self._logdir = logdir + self.beholder = None + + def begin(self): + self.beholder = Beholder(self._logdir) - def after_run(self, run_context, unused_run_values): - self.beholder.update(run_context.session) + def after_run(self, run_context, unused_run_values): + self.beholder.update(run_context.session) diff --git a/tensorboard/plugins/beholder/beholder_demo.py b/tensorboard/plugins/beholder/beholder_demo.py index 583e991415..ccec17bca8 100644 --- a/tensorboard/plugins/beholder/beholder_demo.py +++ b/tensorboard/plugins/beholder/beholder_demo.py @@ -32,186 +32,229 @@ FLAGS = None -LOG_DIRECTORY = '/tmp/beholder-demo' +LOG_DIRECTORY = "/tmp/beholder-demo" + def train(): - mnist_data = mnist.input_data.read_data_sets( - FLAGS.data_dir, one_hot=True, fake_data=FLAGS.fake_data) - - sess = tf.compat.v1.InteractiveSession() - - with tf.name_scope('input'): - x = tf.compat.v1.placeholder(tf.float32, [None, 784], name='x-input') - y_ = tf.compat.v1.placeholder(tf.float32, [None, 10], name='y-input') - - with tf.name_scope('input_reshape'): - image_shaped_input = tf.reshape(x, [-1, 28, 28, 1]) - tf.compat.v1.summary.image('input', image_shaped_input, 10) - - def weight_variable(shape): - """Create a weight variable with appropriate initialization.""" - initial = tf.random.truncated_normal(shape, stddev=0.01) - return tf.Variable(initial) - - def bias_variable(shape): - """Create a bias variable with appropriate initialization.""" - initial = tf.constant(0.1, shape=shape) - return tf.Variable(initial) - - def variable_summaries(var): - """Attach a lot of summaries to a Tensor (for TensorBoard visualization).""" - with tf.name_scope('summaries'): - mean = tf.reduce_mean(input_tensor=var) - tf.compat.v1.summary.scalar('mean', mean) - with tf.name_scope('stddev'): - stddev = tf.sqrt(tf.reduce_mean(input_tensor=tf.square(var - mean))) - tf.compat.v1.summary.scalar('stddev', stddev) - tf.compat.v1.summary.scalar('max', tf.reduce_max(input_tensor=var)) - tf.compat.v1.summary.scalar('min', tf.reduce_min(input_tensor=var)) - tf.compat.v1.summary.histogram('histogram', var) - - def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu): - """Reusable code for making a simple neural net layer. - - It does a matrix multiply, bias add, and then uses ReLU to nonlinearize. - It also sets up name scoping so that the resultant graph is easy to read, - and adds a number of summary ops. - """ - # Adding a name scope ensures logical grouping of the layers in the graph. - with tf.name_scope(layer_name): - # This Variable will hold the state of the weights for the layer - with tf.name_scope('weights'): - weights = weight_variable([input_dim, output_dim]) - variable_summaries(weights) - with tf.name_scope('biases'): - biases = bias_variable([output_dim]) - variable_summaries(biases) - with tf.name_scope('Wx_plus_b'): - preactivate = tf.matmul(input_tensor, weights) + biases - tf.compat.v1.summary.histogram('pre_activations', preactivate) - activations = act(preactivate, name='activation') - tf.compat.v1.summary.histogram('activations', activations) - return activations - - #conv1 - kernel = tf.Variable(tf.random.truncated_normal([5, 5, 1, 10], - dtype=tf.float32, - stddev=1e-1), - name='conv-weights') - conv = tf.nn.conv2d(image_shaped_input, kernel, [1, 1, 1, 1], padding='VALID') - biases_init = tf.constant( - 0.0, shape=[kernel.get_shape().as_list()[-1]], dtype=tf.float32) - biases = tf.Variable(biases_init, trainable=True, name='biases') - out = tf.nn.bias_add(conv, biases) - conv1 = tf.nn.relu(out, name='relu') - - #conv2 - kernel2_init = tf.random.truncated_normal( - [3, 3, 10, 20], dtype=tf.float32, stddev=1e-1) - kernel2 = tf.Variable(kernel2_init, name='conv-weights2') - conv2 = tf.nn.conv2d(conv1, kernel2, [1, 1, 1, 1], padding='VALID') - biases2_init = tf.constant( - 0.0, shape=[kernel2.get_shape().as_list()[-1]], dtype=tf.float32) - biases2 = tf.Variable(biases2_init, trainable=True, name='biases') - out2 = tf.nn.bias_add(conv2, biases2) - conv2 = tf.nn.relu(out2, name='relu') - - flattened = tf.contrib.layers.flatten(conv2) - hidden1 = nn_layer( - flattened, flattened.get_shape().as_list()[1], 10, 'layer1') - - with tf.name_scope('dropout'): - keep_prob = tf.compat.v1.placeholder(tf.float32) - tf.compat.v1.summary.scalar('dropout_keep_probability', keep_prob) - dropped = tf.nn.dropout(hidden1, 1 - keep_prob) - - y = nn_layer(dropped, 10, 10, 'layer2', act=tf.identity) - - with tf.name_scope('cross_entropy'): - diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y) - with tf.name_scope('total'): - cross_entropy = tf.reduce_mean(input_tensor=diff) - tf.compat.v1.summary.scalar('cross_entropy', cross_entropy) - - with tf.name_scope('train'): - optimizer = tf.compat.v1.train.AdamOptimizer(FLAGS.learning_rate) - gradients, train_step = beholder_lib.Beholder.gradient_helper( - optimizer, cross_entropy) - - with tf.name_scope('accuracy'): - with tf.name_scope('correct_prediction'): - correct_prediction = tf.equal(tf.argmax(input=y, axis=1), tf.argmax(input=y_, axis=1)) - with tf.name_scope('accuracy'): - accuracy = tf.reduce_mean(input_tensor=tf.cast(correct_prediction, tf.float32)) - tf.compat.v1.summary.scalar('accuracy', accuracy) - - merged = tf.compat.v1.summary.merge_all() - train_writer = tf.summary.FileWriter(LOG_DIRECTORY + '/train', sess.graph) - test_writer = tf.summary.FileWriter(LOG_DIRECTORY + '/test') - tf.compat.v1.global_variables_initializer().run() - - beholder = beholder_lib.Beholder(logdir=LOG_DIRECTORY) - - - def feed_dict(is_train): - if is_train or FLAGS.fake_data: - xs, ys = mnist_data.train.next_batch(100, fake_data=FLAGS.fake_data) - k = FLAGS.dropout - else: - xs, ys = mnist_data.test.images, mnist_data.test.labels - k = 1.0 - return {x: xs, y_: ys, keep_prob: k} - - for i in range(FLAGS.max_steps): - summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False)) - test_writer.add_summary(summary, i) - print('Accuracy at step %s: %s' % (i, acc)) - print('i', i) - feed_dictionary = feed_dict(True) - summary, gradient_arrays, activations, _ = sess.run( - [ - merged, - gradients, - [image_shaped_input, conv1, conv2, hidden1, y], - train_step - ], - feed_dict=feed_dictionary) - first_of_batch = sess.run(x, feed_dict=feed_dictionary)[0].reshape(28, 28) - beholder.update( - session=sess, - arrays=activations + [first_of_batch] + gradient_arrays, - frame=first_of_batch, + mnist_data = mnist.input_data.read_data_sets( + FLAGS.data_dir, one_hot=True, fake_data=FLAGS.fake_data + ) + + sess = tf.compat.v1.InteractiveSession() + + with tf.name_scope("input"): + x = tf.compat.v1.placeholder(tf.float32, [None, 784], name="x-input") + y_ = tf.compat.v1.placeholder(tf.float32, [None, 10], name="y-input") + + with tf.name_scope("input_reshape"): + image_shaped_input = tf.reshape(x, [-1, 28, 28, 1]) + tf.compat.v1.summary.image("input", image_shaped_input, 10) + + def weight_variable(shape): + """Create a weight variable with appropriate initialization.""" + initial = tf.random.truncated_normal(shape, stddev=0.01) + return tf.Variable(initial) + + def bias_variable(shape): + """Create a bias variable with appropriate initialization.""" + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + + def variable_summaries(var): + """Attach a lot of summaries to a Tensor (for TensorBoard + visualization).""" + with tf.name_scope("summaries"): + mean = tf.reduce_mean(input_tensor=var) + tf.compat.v1.summary.scalar("mean", mean) + with tf.name_scope("stddev"): + stddev = tf.sqrt( + tf.reduce_mean(input_tensor=tf.square(var - mean)) + ) + tf.compat.v1.summary.scalar("stddev", stddev) + tf.compat.v1.summary.scalar("max", tf.reduce_max(input_tensor=var)) + tf.compat.v1.summary.scalar("min", tf.reduce_min(input_tensor=var)) + tf.compat.v1.summary.histogram("histogram", var) + + def nn_layer( + input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu + ): + """Reusable code for making a simple neural net layer. + + It does a matrix multiply, bias add, and then uses ReLU to + nonlinearize. It also sets up name scoping so that the resultant + graph is easy to read, and adds a number of summary ops. + """ + # Adding a name scope ensures logical grouping of the layers in the graph. + with tf.name_scope(layer_name): + # This Variable will hold the state of the weights for the layer + with tf.name_scope("weights"): + weights = weight_variable([input_dim, output_dim]) + variable_summaries(weights) + with tf.name_scope("biases"): + biases = bias_variable([output_dim]) + variable_summaries(biases) + with tf.name_scope("Wx_plus_b"): + preactivate = tf.matmul(input_tensor, weights) + biases + tf.compat.v1.summary.histogram("pre_activations", preactivate) + activations = act(preactivate, name="activation") + tf.compat.v1.summary.histogram("activations", activations) + return activations + + # conv1 + kernel = tf.Variable( + tf.random.truncated_normal( + [5, 5, 1, 10], dtype=tf.float32, stddev=1e-1 + ), + name="conv-weights", + ) + conv = tf.nn.conv2d( + image_shaped_input, kernel, [1, 1, 1, 1], padding="VALID" + ) + biases_init = tf.constant( + 0.0, shape=[kernel.get_shape().as_list()[-1]], dtype=tf.float32 ) - train_writer.add_summary(summary, i) + biases = tf.Variable(biases_init, trainable=True, name="biases") + out = tf.nn.bias_add(conv, biases) + conv1 = tf.nn.relu(out, name="relu") + + # conv2 + kernel2_init = tf.random.truncated_normal( + [3, 3, 10, 20], dtype=tf.float32, stddev=1e-1 + ) + kernel2 = tf.Variable(kernel2_init, name="conv-weights2") + conv2 = tf.nn.conv2d(conv1, kernel2, [1, 1, 1, 1], padding="VALID") + biases2_init = tf.constant( + 0.0, shape=[kernel2.get_shape().as_list()[-1]], dtype=tf.float32 + ) + biases2 = tf.Variable(biases2_init, trainable=True, name="biases") + out2 = tf.nn.bias_add(conv2, biases2) + conv2 = tf.nn.relu(out2, name="relu") + + flattened = tf.contrib.layers.flatten(conv2) + hidden1 = nn_layer( + flattened, flattened.get_shape().as_list()[1], 10, "layer1" + ) + + with tf.name_scope("dropout"): + keep_prob = tf.compat.v1.placeholder(tf.float32) + tf.compat.v1.summary.scalar("dropout_keep_probability", keep_prob) + dropped = tf.nn.dropout(hidden1, 1 - keep_prob) + + y = nn_layer(dropped, 10, 10, "layer2", act=tf.identity) + + with tf.name_scope("cross_entropy"): + diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y) + with tf.name_scope("total"): + cross_entropy = tf.reduce_mean(input_tensor=diff) + tf.compat.v1.summary.scalar("cross_entropy", cross_entropy) + + with tf.name_scope("train"): + optimizer = tf.compat.v1.train.AdamOptimizer(FLAGS.learning_rate) + gradients, train_step = beholder_lib.Beholder.gradient_helper( + optimizer, cross_entropy + ) + + with tf.name_scope("accuracy"): + with tf.name_scope("correct_prediction"): + correct_prediction = tf.equal( + tf.argmax(input=y, axis=1), tf.argmax(input=y_, axis=1) + ) + with tf.name_scope("accuracy"): + accuracy = tf.reduce_mean( + input_tensor=tf.cast(correct_prediction, tf.float32) + ) + tf.compat.v1.summary.scalar("accuracy", accuracy) + + merged = tf.compat.v1.summary.merge_all() + train_writer = tf.summary.FileWriter(LOG_DIRECTORY + "/train", sess.graph) + test_writer = tf.summary.FileWriter(LOG_DIRECTORY + "/test") + tf.compat.v1.global_variables_initializer().run() + + beholder = beholder_lib.Beholder(logdir=LOG_DIRECTORY) + + def feed_dict(is_train): + if is_train or FLAGS.fake_data: + xs, ys = mnist_data.train.next_batch(100, fake_data=FLAGS.fake_data) + k = FLAGS.dropout + else: + xs, ys = mnist_data.test.images, mnist_data.test.labels + k = 1.0 + return {x: xs, y_: ys, keep_prob: k} + + for i in range(FLAGS.max_steps): + summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False)) + test_writer.add_summary(summary, i) + print("Accuracy at step %s: %s" % (i, acc)) + print("i", i) + feed_dictionary = feed_dict(True) + summary, gradient_arrays, activations, _ = sess.run( + [ + merged, + gradients, + [image_shaped_input, conv1, conv2, hidden1, y], + train_step, + ], + feed_dict=feed_dictionary, + ) + first_of_batch = sess.run(x, feed_dict=feed_dictionary)[0].reshape( + 28, 28 + ) + beholder.update( + session=sess, + arrays=activations + [first_of_batch] + gradient_arrays, + frame=first_of_batch, + ) + train_writer.add_summary(summary, i) + + train_writer.close() + test_writer.close() - train_writer.close() - test_writer.close() def main(_): - if not tf.io.gfile.exists(LOG_DIRECTORY): - tf.io.gfile.makedirs(LOG_DIRECTORY) - train() - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--fake_data', nargs='?', const=True, type=bool, - default=False, - help='If true, uses fake data for unit testing.') - parser.add_argument('--max_steps', type=int, default=1000000, - help='Number of steps to run trainer.') - parser.add_argument('--learning_rate', type=float, default=0.001, - help='Initial learning rate') - parser.add_argument('--dropout', type=float, default=0.9, - help='Keep probability for training dropout.') - parser.add_argument( - '--data_dir', - type=str, - default='/tmp/tensorflow/mnist/input_data', - help='Directory for storing input data') - parser.add_argument( - '--log_dir', - type=str, - default='/tmp/tensorflow/mnist/logs/mnist_with_summaries', - help='Summaries log directory') - FLAGS, unparsed = parser.parse_known_args() - app.run(main=main, argv=[sys.argv[0]] + unparsed) + if not tf.io.gfile.exists(LOG_DIRECTORY): + tf.io.gfile.makedirs(LOG_DIRECTORY) + train() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--fake_data", + nargs="?", + const=True, + type=bool, + default=False, + help="If true, uses fake data for unit testing.", + ) + parser.add_argument( + "--max_steps", + type=int, + default=1000000, + help="Number of steps to run trainer.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=0.001, + help="Initial learning rate", + ) + parser.add_argument( + "--dropout", + type=float, + default=0.9, + help="Keep probability for training dropout.", + ) + parser.add_argument( + "--data_dir", + type=str, + default="/tmp/tensorflow/mnist/input_data", + help="Directory for storing input data", + ) + parser.add_argument( + "--log_dir", + type=str, + default="/tmp/tensorflow/mnist/logs/mnist_with_summaries", + help="Summaries log directory", + ) + FLAGS, unparsed = parser.parse_known_args() + app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorboard/plugins/beholder/beholder_plugin.py b/tensorboard/plugins/beholder/beholder_plugin.py index b01a192de8..87a382688b 100644 --- a/tensorboard/plugins/beholder/beholder_plugin.py +++ b/tensorboard/plugins/beholder/beholder_plugin.py @@ -35,162 +35,180 @@ logger = tb_logging.get_logger() -DEFAULT_INFO = [{ - 'name': 'Waiting for data...', -}] +DEFAULT_INFO = [{"name": "Waiting for data...",}] class BeholderPlugin(base_plugin.TBPlugin): - """ - TensorBoard plugin for viewing model data as a live video during training. - """ - - plugin_name = shared_config.PLUGIN_NAME - - def __init__(self, context): - self._lock = threading.Lock() - self._MULTIPLEXER = context.multiplexer - self.PLUGIN_LOGDIR = pau.PluginDirectory( - context.logdir, shared_config.PLUGIN_NAME) - self.FPS = 10 - self._config_file_lock = threading.Lock() - self.most_recent_frame = None - self.most_recent_info = DEFAULT_INFO - - def get_plugin_apps(self): - return { - '/change-config': self._serve_change_config, - '/beholder-frame': self._serve_beholder_frame, - '/section-info': self._serve_section_info, - '/ping': self._serve_ping, - '/is-active': self._serve_is_active, - } - - def is_active(self): - summary_filename = '{}/{}'.format( - self.PLUGIN_LOGDIR, shared_config.SUMMARY_FILENAME) - info_filename = '{}/{}'.format( - self.PLUGIN_LOGDIR, shared_config.SECTION_INFO_FILENAME) - return tf.io.gfile.exists(summary_filename) and\ - tf.io.gfile.exists(info_filename) - - def frontend_metadata(self): - # TODO(#2338): Keep this in sync with the `registerDashboard` call - # on the frontend until that call is removed. - return base_plugin.FrontendMetadata( - element_name='tf-beholder-dashboard', - remove_dom=True, - ) - - def is_config_writable(self): - try: - if not tf.io.gfile.exists(self.PLUGIN_LOGDIR): - tf.io.gfile.makedirs(self.PLUGIN_LOGDIR) - config_filename = '{}/{}'.format( - self.PLUGIN_LOGDIR, shared_config.CONFIG_FILENAME) - with self._config_file_lock: - file_system_tools.write_pickle( - file_system_tools.read_pickle( - config_filename, shared_config.DEFAULT_CONFIG), - config_filename) - return True - except tf.errors.PermissionDeniedError as e: - logger.warn( - 'Unable to write Beholder config, controls will be disabled: %s', e) - return False - - @wrappers.Request.application - def _serve_is_active(self, request): - is_active = self.is_active() - # If the plugin isn't active, don't check if the configuration is writable - # since that will leave traces on disk; instead return True (the default). - is_config_writable = self.is_config_writable() if is_active else True - response = { - 'is_active': is_active, - 'is_config_writable': is_config_writable, - } - return http_util.Respond(request, response, 'application/json') - - def _fetch_current_frame(self): - path = '{}/{}'.format(self.PLUGIN_LOGDIR, shared_config.SUMMARY_FILENAME) - with self._lock: - try: - frame = file_system_tools.read_tensor_summary(path).astype(np.uint8) - self.most_recent_frame = frame - return frame - except (message.DecodeError, IOError, tf.errors.NotFoundError): - if self.most_recent_frame is None: - self.most_recent_frame = im_util.get_image_relative_to_script( - 'no-data.png') - return self.most_recent_frame - - @wrappers.Request.application - def _serve_change_config(self, request): - config = {} - - for key, value in request.form.items(): - try: - config[key] = int(value) - except ValueError: - if value == 'false': - config[key] = False - elif value == 'true': - config[key] = True - else: - config[key] = value - - self.FPS = config['FPS'] - - with self._config_file_lock: - file_system_tools.write_pickle( - config, - '{}/{}'.format(self.PLUGIN_LOGDIR, shared_config.CONFIG_FILENAME)) - return http_util.Respond(request, {'config': config}, 'application/json') - - @wrappers.Request.application - def _serve_section_info(self, request): - path = '{}/{}'.format( - self.PLUGIN_LOGDIR, shared_config.SECTION_INFO_FILENAME) - with self._lock: - default = self.most_recent_info - info = file_system_tools.read_pickle(path, default=default) - if info is not default: - with self._lock: - self.most_recent_info = info - return http_util.Respond(request, info, 'application/json') - - def _frame_generator(self): - while True: - last_duration = 0 - - if self.FPS == 0: - continue - else: - time.sleep(max(0, 1/(self.FPS) - last_duration)) - - start_time = time.time() - array = self._fetch_current_frame() - image_bytes = encoder.encode_png(array) - - frame_text = b'--frame\r\n' - content_type = b'Content-Type: image/png\r\n\r\n' - - response_content = frame_text + content_type + image_bytes + b'\r\n\r\n' - - last_duration = time.time() - start_time - yield response_content - - - @wrappers.Request.application - def _serve_beholder_frame(self, request): # pylint: disable=unused-argument - # Thanks to Miguel Grinberg for this technique: - # https://blog.miguelgrinberg.com/post/video-streaming-with-flask - mimetype = 'multipart/x-mixed-replace; boundary=frame' - return http_util.Respond(request, - self._frame_generator(), - mimetype, - code=200) - - @wrappers.Request.application - def _serve_ping(self, request): # pylint: disable=unused-argument - return http_util.Respond(request, {'status': 'alive'}, 'application/json') + """TensorBoard plugin for viewing model data as a live video during + training.""" + + plugin_name = shared_config.PLUGIN_NAME + + def __init__(self, context): + self._lock = threading.Lock() + self._MULTIPLEXER = context.multiplexer + self.PLUGIN_LOGDIR = pau.PluginDirectory( + context.logdir, shared_config.PLUGIN_NAME + ) + self.FPS = 10 + self._config_file_lock = threading.Lock() + self.most_recent_frame = None + self.most_recent_info = DEFAULT_INFO + + def get_plugin_apps(self): + return { + "/change-config": self._serve_change_config, + "/beholder-frame": self._serve_beholder_frame, + "/section-info": self._serve_section_info, + "/ping": self._serve_ping, + "/is-active": self._serve_is_active, + } + + def is_active(self): + summary_filename = "{}/{}".format( + self.PLUGIN_LOGDIR, shared_config.SUMMARY_FILENAME + ) + info_filename = "{}/{}".format( + self.PLUGIN_LOGDIR, shared_config.SECTION_INFO_FILENAME + ) + return tf.io.gfile.exists(summary_filename) and tf.io.gfile.exists( + info_filename + ) + + def frontend_metadata(self): + # TODO(#2338): Keep this in sync with the `registerDashboard` call + # on the frontend until that call is removed. + return base_plugin.FrontendMetadata( + element_name="tf-beholder-dashboard", remove_dom=True, + ) + + def is_config_writable(self): + try: + if not tf.io.gfile.exists(self.PLUGIN_LOGDIR): + tf.io.gfile.makedirs(self.PLUGIN_LOGDIR) + config_filename = "{}/{}".format( + self.PLUGIN_LOGDIR, shared_config.CONFIG_FILENAME + ) + with self._config_file_lock: + file_system_tools.write_pickle( + file_system_tools.read_pickle( + config_filename, shared_config.DEFAULT_CONFIG + ), + config_filename, + ) + return True + except tf.errors.PermissionDeniedError as e: + logger.warn( + "Unable to write Beholder config, controls will be disabled: %s", + e, + ) + return False + + @wrappers.Request.application + def _serve_is_active(self, request): + is_active = self.is_active() + # If the plugin isn't active, don't check if the configuration is writable + # since that will leave traces on disk; instead return True (the default). + is_config_writable = self.is_config_writable() if is_active else True + response = { + "is_active": is_active, + "is_config_writable": is_config_writable, + } + return http_util.Respond(request, response, "application/json") + + def _fetch_current_frame(self): + path = "{}/{}".format( + self.PLUGIN_LOGDIR, shared_config.SUMMARY_FILENAME + ) + with self._lock: + try: + frame = file_system_tools.read_tensor_summary(path).astype( + np.uint8 + ) + self.most_recent_frame = frame + return frame + except (message.DecodeError, IOError, tf.errors.NotFoundError): + if self.most_recent_frame is None: + self.most_recent_frame = im_util.get_image_relative_to_script( + "no-data.png" + ) + return self.most_recent_frame + + @wrappers.Request.application + def _serve_change_config(self, request): + config = {} + + for key, value in request.form.items(): + try: + config[key] = int(value) + except ValueError: + if value == "false": + config[key] = False + elif value == "true": + config[key] = True + else: + config[key] = value + + self.FPS = config["FPS"] + + with self._config_file_lock: + file_system_tools.write_pickle( + config, + "{}/{}".format( + self.PLUGIN_LOGDIR, shared_config.CONFIG_FILENAME + ), + ) + return http_util.Respond( + request, {"config": config}, "application/json" + ) + + @wrappers.Request.application + def _serve_section_info(self, request): + path = "{}/{}".format( + self.PLUGIN_LOGDIR, shared_config.SECTION_INFO_FILENAME + ) + with self._lock: + default = self.most_recent_info + info = file_system_tools.read_pickle(path, default=default) + if info is not default: + with self._lock: + self.most_recent_info = info + return http_util.Respond(request, info, "application/json") + + def _frame_generator(self): + while True: + last_duration = 0 + + if self.FPS == 0: + continue + else: + time.sleep(max(0, 1 / (self.FPS) - last_duration)) + + start_time = time.time() + array = self._fetch_current_frame() + image_bytes = encoder.encode_png(array) + + frame_text = b"--frame\r\n" + content_type = b"Content-Type: image/png\r\n\r\n" + + response_content = ( + frame_text + content_type + image_bytes + b"\r\n\r\n" + ) + + last_duration = time.time() - start_time + yield response_content + + @wrappers.Request.application + def _serve_beholder_frame(self, request): # pylint: disable=unused-argument + # Thanks to Miguel Grinberg for this technique: + # https://blog.miguelgrinberg.com/post/video-streaming-with-flask + mimetype = "multipart/x-mixed-replace; boundary=frame" + return http_util.Respond( + request, self._frame_generator(), mimetype, code=200 + ) + + @wrappers.Request.application + def _serve_ping(self, request): # pylint: disable=unused-argument + return http_util.Respond( + request, {"status": "alive"}, "application/json" + ) diff --git a/tensorboard/plugins/beholder/beholder_plugin_loader.py b/tensorboard/plugins/beholder/beholder_plugin_loader.py index 66dd00fca4..c7762a34a4 100644 --- a/tensorboard/plugins/beholder/beholder_plugin_loader.py +++ b/tensorboard/plugins/beholder/beholder_plugin_loader.py @@ -22,25 +22,26 @@ class BeholderPluginLoader(base_plugin.TBLoader): - """BeholderPlugin factory. + """BeholderPlugin factory. - This class checks for `tensorflow` install and dependency. - """ + This class checks for `tensorflow` install and dependency. + """ - def load(self, context): - """Returns the plugin, if possible. + def load(self, context): + """Returns the plugin, if possible. - Args: - context: The TBContext flags. + Args: + context: The TBContext flags. - Returns: - A BeholderPlugin instance or None if it couldn't be loaded. - """ - try: - # pylint: disable=unused-import - import tensorflow - except ImportError: - return - - from tensorboard.plugins.beholder.beholder_plugin import BeholderPlugin - return BeholderPlugin(context) + Returns: + A BeholderPlugin instance or None if it couldn't be loaded. + """ + try: + # pylint: disable=unused-import + import tensorflow + except ImportError: + return + + from tensorboard.plugins.beholder.beholder_plugin import BeholderPlugin + + return BeholderPlugin(context) diff --git a/tensorboard/plugins/beholder/beholder_test.py b/tensorboard/plugins/beholder/beholder_test.py index efd0195526..ba3608ac30 100644 --- a/tensorboard/plugins/beholder/beholder_test.py +++ b/tensorboard/plugins/beholder/beholder_test.py @@ -24,30 +24,29 @@ class BeholderTest(tf.test.TestCase): - - def setUp(self): - self._current_time_seconds = 1554232353 - - def advance_time(self, delta_seconds): - self._current_time_seconds += delta_seconds - - def get_time(self): - return self._current_time_seconds - - @test_util.run_v1_only("Requires sessions") - def test_update(self): - with tf.test.mock.patch("time.time", self.get_time): - b = beholder.Beholder(self.get_temp_dir()) - array = np.array([[0, 1], [1, 0]]) - with tf.Session() as sess: - v = tf.Variable([0, 0], trainable=True) - sess.run(tf.global_variables_initializer()) - # Beholder only updates if at least one frame has passed. The - # default FPS value is 10, but in any case 100 seconds ought to - # do it. - self.advance_time(delta_seconds=100) - b.update(session=sess, arrays=[array]) + def setUp(self): + self._current_time_seconds = 1554232353 + + def advance_time(self, delta_seconds): + self._current_time_seconds += delta_seconds + + def get_time(self): + return self._current_time_seconds + + @test_util.run_v1_only("Requires sessions") + def test_update(self): + with tf.test.mock.patch("time.time", self.get_time): + b = beholder.Beholder(self.get_temp_dir()) + array = np.array([[0, 1], [1, 0]]) + with tf.Session() as sess: + v = tf.Variable([0, 0], trainable=True) + sess.run(tf.global_variables_initializer()) + # Beholder only updates if at least one frame has passed. The + # default FPS value is 10, but in any case 100 seconds ought to + # do it. + self.advance_time(delta_seconds=100) + b.update(session=sess, arrays=[array]) if __name__ == "__main__": - tf.test.main() + tf.test.main() diff --git a/tensorboard/plugins/beholder/colormaps.py b/tensorboard/plugins/beholder/colormaps.py index 9dca3b97ed..099c308d77 100644 --- a/tensorboard/plugins/beholder/colormaps.py +++ b/tensorboard/plugins/beholder/colormaps.py @@ -47,1045 +47,1057 @@ def _convert(colormap_data): - colormap = (np.array(colormap_data) * 255).astype(np.uint8) - colormap.setflags(write=False) - return colormap + colormap = (np.array(colormap_data) * 255).astype(np.uint8) + colormap.setflags(write=False) + return colormap magma = _convert( - ((0.001462, 0.000466, 0.013866), - (0.002258, 0.001295, 0.018331), - (0.003279, 0.002305, 0.023708), - (0.004512, 0.003490, 0.029965), - (0.005950, 0.004843, 0.037130), - (0.007588, 0.006356, 0.044973), - (0.009426, 0.008022, 0.052844), - (0.011465, 0.009828, 0.060750), - (0.013708, 0.011771, 0.068667), - (0.016156, 0.013840, 0.076603), - (0.018815, 0.016026, 0.084584), - (0.021692, 0.018320, 0.092610), - (0.024792, 0.020715, 0.100676), - (0.028123, 0.023201, 0.108787), - (0.031696, 0.025765, 0.116965), - (0.035520, 0.028397, 0.125209), - (0.039608, 0.031090, 0.133515), - (0.043830, 0.033830, 0.141886), - (0.048062, 0.036607, 0.150327), - (0.052320, 0.039407, 0.158841), - (0.056615, 0.042160, 0.167446), - (0.060949, 0.044794, 0.176129), - (0.065330, 0.047318, 0.184892), - (0.069764, 0.049726, 0.193735), - (0.074257, 0.052017, 0.202660), - (0.078815, 0.054184, 0.211667), - (0.083446, 0.056225, 0.220755), - (0.088155, 0.058133, 0.229922), - (0.092949, 0.059904, 0.239164), - (0.097833, 0.061531, 0.248477), - (0.102815, 0.063010, 0.257854), - (0.107899, 0.064335, 0.267289), - (0.113094, 0.065492, 0.276784), - (0.118405, 0.066479, 0.286321), - (0.123833, 0.067295, 0.295879), - (0.129380, 0.067935, 0.305443), - (0.135053, 0.068391, 0.315000), - (0.140858, 0.068654, 0.324538), - (0.146785, 0.068738, 0.334011), - (0.152839, 0.068637, 0.343404), - (0.159018, 0.068354, 0.352688), - (0.165308, 0.067911, 0.361816), - (0.171713, 0.067305, 0.370771), - (0.178212, 0.066576, 0.379497), - (0.184801, 0.065732, 0.387973), - (0.191460, 0.064818, 0.396152), - (0.198177, 0.063862, 0.404009), - (0.204935, 0.062907, 0.411514), - (0.211718, 0.061992, 0.418647), - (0.218512, 0.061158, 0.425392), - (0.225302, 0.060445, 0.431742), - (0.232077, 0.059889, 0.437695), - (0.238826, 0.059517, 0.443256), - (0.245543, 0.059352, 0.448436), - (0.252220, 0.059415, 0.453248), - (0.258857, 0.059706, 0.457710), - (0.265447, 0.060237, 0.461840), - (0.271994, 0.060994, 0.465660), - (0.278493, 0.061978, 0.469190), - (0.284951, 0.063168, 0.472451), - (0.291366, 0.064553, 0.475462), - (0.297740, 0.066117, 0.478243), - (0.304081, 0.067835, 0.480812), - (0.310382, 0.069702, 0.483186), - (0.316654, 0.071690, 0.485380), - (0.322899, 0.073782, 0.487408), - (0.329114, 0.075972, 0.489287), - (0.335308, 0.078236, 0.491024), - (0.341482, 0.080564, 0.492631), - (0.347636, 0.082946, 0.494121), - (0.353773, 0.085373, 0.495501), - (0.359898, 0.087831, 0.496778), - (0.366012, 0.090314, 0.497960), - (0.372116, 0.092816, 0.499053), - (0.378211, 0.095332, 0.500067), - (0.384299, 0.097855, 0.501002), - (0.390384, 0.100379, 0.501864), - (0.396467, 0.102902, 0.502658), - (0.402548, 0.105420, 0.503386), - (0.408629, 0.107930, 0.504052), - (0.414709, 0.110431, 0.504662), - (0.420791, 0.112920, 0.505215), - (0.426877, 0.115395, 0.505714), - (0.432967, 0.117855, 0.506160), - (0.439062, 0.120298, 0.506555), - (0.445163, 0.122724, 0.506901), - (0.451271, 0.125132, 0.507198), - (0.457386, 0.127522, 0.507448), - (0.463508, 0.129893, 0.507652), - (0.469640, 0.132245, 0.507809), - (0.475780, 0.134577, 0.507921), - (0.481929, 0.136891, 0.507989), - (0.488088, 0.139186, 0.508011), - (0.494258, 0.141462, 0.507988), - (0.500438, 0.143719, 0.507920), - (0.506629, 0.145958, 0.507806), - (0.512831, 0.148179, 0.507648), - (0.519045, 0.150383, 0.507443), - (0.525270, 0.152569, 0.507192), - (0.531507, 0.154739, 0.506895), - (0.537755, 0.156894, 0.506551), - (0.544015, 0.159033, 0.506159), - (0.550287, 0.161158, 0.505719), - (0.556571, 0.163269, 0.505230), - (0.562866, 0.165368, 0.504692), - (0.569172, 0.167454, 0.504105), - (0.575490, 0.169530, 0.503466), - (0.581819, 0.171596, 0.502777), - (0.588158, 0.173652, 0.502035), - (0.594508, 0.175701, 0.501241), - (0.600868, 0.177743, 0.500394), - (0.607238, 0.179779, 0.499492), - (0.613617, 0.181811, 0.498536), - (0.620005, 0.183840, 0.497524), - (0.626401, 0.185867, 0.496456), - (0.632805, 0.187893, 0.495332), - (0.639216, 0.189921, 0.494150), - (0.645633, 0.191952, 0.492910), - (0.652056, 0.193986, 0.491611), - (0.658483, 0.196027, 0.490253), - (0.664915, 0.198075, 0.488836), - (0.671349, 0.200133, 0.487358), - (0.677786, 0.202203, 0.485819), - (0.684224, 0.204286, 0.484219), - (0.690661, 0.206384, 0.482558), - (0.697098, 0.208501, 0.480835), - (0.703532, 0.210638, 0.479049), - (0.709962, 0.212797, 0.477201), - (0.716387, 0.214982, 0.475290), - (0.722805, 0.217194, 0.473316), - (0.729216, 0.219437, 0.471279), - (0.735616, 0.221713, 0.469180), - (0.742004, 0.224025, 0.467018), - (0.748378, 0.226377, 0.464794), - (0.754737, 0.228772, 0.462509), - (0.761077, 0.231214, 0.460162), - (0.767398, 0.233705, 0.457755), - (0.773695, 0.236249, 0.455289), - (0.779968, 0.238851, 0.452765), - (0.786212, 0.241514, 0.450184), - (0.792427, 0.244242, 0.447543), - (0.798608, 0.247040, 0.444848), - (0.804752, 0.249911, 0.442102), - (0.810855, 0.252861, 0.439305), - (0.816914, 0.255895, 0.436461), - (0.822926, 0.259016, 0.433573), - (0.828886, 0.262229, 0.430644), - (0.834791, 0.265540, 0.427671), - (0.840636, 0.268953, 0.424666), - (0.846416, 0.272473, 0.421631), - (0.852126, 0.276106, 0.418573), - (0.857763, 0.279857, 0.415496), - (0.863320, 0.283729, 0.412403), - (0.868793, 0.287728, 0.409303), - (0.874176, 0.291859, 0.406205), - (0.879464, 0.296125, 0.403118), - (0.884651, 0.300530, 0.400047), - (0.889731, 0.305079, 0.397002), - (0.894700, 0.309773, 0.393995), - (0.899552, 0.314616, 0.391037), - (0.904281, 0.319610, 0.388137), - (0.908884, 0.324755, 0.385308), - (0.913354, 0.330052, 0.382563), - (0.917689, 0.335500, 0.379915), - (0.921884, 0.341098, 0.377376), - (0.925937, 0.346844, 0.374959), - (0.929845, 0.352734, 0.372677), - (0.933606, 0.358764, 0.370541), - (0.937221, 0.364929, 0.368567), - (0.940687, 0.371224, 0.366762), - (0.944006, 0.377643, 0.365136), - (0.947180, 0.384178, 0.363701), - (0.950210, 0.390820, 0.362468), - (0.953099, 0.397563, 0.361438), - (0.955849, 0.404400, 0.360619), - (0.958464, 0.411324, 0.360014), - (0.960949, 0.418323, 0.359630), - (0.963310, 0.425390, 0.359469), - (0.965549, 0.432519, 0.359529), - (0.967671, 0.439703, 0.359810), - (0.969680, 0.446936, 0.360311), - (0.971582, 0.454210, 0.361030), - (0.973381, 0.461520, 0.361965), - (0.975082, 0.468861, 0.363111), - (0.976690, 0.476226, 0.364466), - (0.978210, 0.483612, 0.366025), - (0.979645, 0.491014, 0.367783), - (0.981000, 0.498428, 0.369734), - (0.982279, 0.505851, 0.371874), - (0.983485, 0.513280, 0.374198), - (0.984622, 0.520713, 0.376698), - (0.985693, 0.528148, 0.379371), - (0.986700, 0.535582, 0.382210), - (0.987646, 0.543015, 0.385210), - (0.988533, 0.550446, 0.388365), - (0.989363, 0.557873, 0.391671), - (0.990138, 0.565296, 0.395122), - (0.990871, 0.572706, 0.398714), - (0.991558, 0.580107, 0.402441), - (0.992196, 0.587502, 0.406299), - (0.992785, 0.594891, 0.410283), - (0.993326, 0.602275, 0.414390), - (0.993834, 0.609644, 0.418613), - (0.994309, 0.616999, 0.422950), - (0.994738, 0.624350, 0.427397), - (0.995122, 0.631696, 0.431951), - (0.995480, 0.639027, 0.436607), - (0.995810, 0.646344, 0.441361), - (0.996096, 0.653659, 0.446213), - (0.996341, 0.660969, 0.451160), - (0.996580, 0.668256, 0.456192), - (0.996775, 0.675541, 0.461314), - (0.996925, 0.682828, 0.466526), - (0.997077, 0.690088, 0.471811), - (0.997186, 0.697349, 0.477182), - (0.997254, 0.704611, 0.482635), - (0.997325, 0.711848, 0.488154), - (0.997351, 0.719089, 0.493755), - (0.997351, 0.726324, 0.499428), - (0.997341, 0.733545, 0.505167), - (0.997285, 0.740772, 0.510983), - (0.997228, 0.747981, 0.516859), - (0.997138, 0.755190, 0.522806), - (0.997019, 0.762398, 0.528821), - (0.996898, 0.769591, 0.534892), - (0.996727, 0.776795, 0.541039), - (0.996571, 0.783977, 0.547233), - (0.996369, 0.791167, 0.553499), - (0.996162, 0.798348, 0.559820), - (0.995932, 0.805527, 0.566202), - (0.995680, 0.812706, 0.572645), - (0.995424, 0.819875, 0.579140), - (0.995131, 0.827052, 0.585701), - (0.994851, 0.834213, 0.592307), - (0.994524, 0.841387, 0.598983), - (0.994222, 0.848540, 0.605696), - (0.993866, 0.855711, 0.612482), - (0.993545, 0.862859, 0.619299), - (0.993170, 0.870024, 0.626189), - (0.992831, 0.877168, 0.633109), - (0.992440, 0.884330, 0.640099), - (0.992089, 0.891470, 0.647116), - (0.991688, 0.898627, 0.654202), - (0.991332, 0.905763, 0.661309), - (0.990930, 0.912915, 0.668481), - (0.990570, 0.920049, 0.675675), - (0.990175, 0.927196, 0.682926), - (0.989815, 0.934329, 0.690198), - (0.989434, 0.941470, 0.697519), - (0.989077, 0.948604, 0.704863), - (0.988717, 0.955742, 0.712242), - (0.988367, 0.962878, 0.719649), - (0.988033, 0.970012, 0.727077), - (0.987691, 0.977154, 0.734536), - (0.987387, 0.984288, 0.742002), - (0.987053, 0.991438, 0.749504))) + ( + (0.001462, 0.000466, 0.013866), + (0.002258, 0.001295, 0.018331), + (0.003279, 0.002305, 0.023708), + (0.004512, 0.003490, 0.029965), + (0.005950, 0.004843, 0.037130), + (0.007588, 0.006356, 0.044973), + (0.009426, 0.008022, 0.052844), + (0.011465, 0.009828, 0.060750), + (0.013708, 0.011771, 0.068667), + (0.016156, 0.013840, 0.076603), + (0.018815, 0.016026, 0.084584), + (0.021692, 0.018320, 0.092610), + (0.024792, 0.020715, 0.100676), + (0.028123, 0.023201, 0.108787), + (0.031696, 0.025765, 0.116965), + (0.035520, 0.028397, 0.125209), + (0.039608, 0.031090, 0.133515), + (0.043830, 0.033830, 0.141886), + (0.048062, 0.036607, 0.150327), + (0.052320, 0.039407, 0.158841), + (0.056615, 0.042160, 0.167446), + (0.060949, 0.044794, 0.176129), + (0.065330, 0.047318, 0.184892), + (0.069764, 0.049726, 0.193735), + (0.074257, 0.052017, 0.202660), + (0.078815, 0.054184, 0.211667), + (0.083446, 0.056225, 0.220755), + (0.088155, 0.058133, 0.229922), + (0.092949, 0.059904, 0.239164), + (0.097833, 0.061531, 0.248477), + (0.102815, 0.063010, 0.257854), + (0.107899, 0.064335, 0.267289), + (0.113094, 0.065492, 0.276784), + (0.118405, 0.066479, 0.286321), + (0.123833, 0.067295, 0.295879), + (0.129380, 0.067935, 0.305443), + (0.135053, 0.068391, 0.315000), + (0.140858, 0.068654, 0.324538), + (0.146785, 0.068738, 0.334011), + (0.152839, 0.068637, 0.343404), + (0.159018, 0.068354, 0.352688), + (0.165308, 0.067911, 0.361816), + (0.171713, 0.067305, 0.370771), + (0.178212, 0.066576, 0.379497), + (0.184801, 0.065732, 0.387973), + (0.191460, 0.064818, 0.396152), + (0.198177, 0.063862, 0.404009), + (0.204935, 0.062907, 0.411514), + (0.211718, 0.061992, 0.418647), + (0.218512, 0.061158, 0.425392), + (0.225302, 0.060445, 0.431742), + (0.232077, 0.059889, 0.437695), + (0.238826, 0.059517, 0.443256), + (0.245543, 0.059352, 0.448436), + (0.252220, 0.059415, 0.453248), + (0.258857, 0.059706, 0.457710), + (0.265447, 0.060237, 0.461840), + (0.271994, 0.060994, 0.465660), + (0.278493, 0.061978, 0.469190), + (0.284951, 0.063168, 0.472451), + (0.291366, 0.064553, 0.475462), + (0.297740, 0.066117, 0.478243), + (0.304081, 0.067835, 0.480812), + (0.310382, 0.069702, 0.483186), + (0.316654, 0.071690, 0.485380), + (0.322899, 0.073782, 0.487408), + (0.329114, 0.075972, 0.489287), + (0.335308, 0.078236, 0.491024), + (0.341482, 0.080564, 0.492631), + (0.347636, 0.082946, 0.494121), + (0.353773, 0.085373, 0.495501), + (0.359898, 0.087831, 0.496778), + (0.366012, 0.090314, 0.497960), + (0.372116, 0.092816, 0.499053), + (0.378211, 0.095332, 0.500067), + (0.384299, 0.097855, 0.501002), + (0.390384, 0.100379, 0.501864), + (0.396467, 0.102902, 0.502658), + (0.402548, 0.105420, 0.503386), + (0.408629, 0.107930, 0.504052), + (0.414709, 0.110431, 0.504662), + (0.420791, 0.112920, 0.505215), + (0.426877, 0.115395, 0.505714), + (0.432967, 0.117855, 0.506160), + (0.439062, 0.120298, 0.506555), + (0.445163, 0.122724, 0.506901), + (0.451271, 0.125132, 0.507198), + (0.457386, 0.127522, 0.507448), + (0.463508, 0.129893, 0.507652), + (0.469640, 0.132245, 0.507809), + (0.475780, 0.134577, 0.507921), + (0.481929, 0.136891, 0.507989), + (0.488088, 0.139186, 0.508011), + (0.494258, 0.141462, 0.507988), + (0.500438, 0.143719, 0.507920), + (0.506629, 0.145958, 0.507806), + (0.512831, 0.148179, 0.507648), + (0.519045, 0.150383, 0.507443), + (0.525270, 0.152569, 0.507192), + (0.531507, 0.154739, 0.506895), + (0.537755, 0.156894, 0.506551), + (0.544015, 0.159033, 0.506159), + (0.550287, 0.161158, 0.505719), + (0.556571, 0.163269, 0.505230), + (0.562866, 0.165368, 0.504692), + (0.569172, 0.167454, 0.504105), + (0.575490, 0.169530, 0.503466), + (0.581819, 0.171596, 0.502777), + (0.588158, 0.173652, 0.502035), + (0.594508, 0.175701, 0.501241), + (0.600868, 0.177743, 0.500394), + (0.607238, 0.179779, 0.499492), + (0.613617, 0.181811, 0.498536), + (0.620005, 0.183840, 0.497524), + (0.626401, 0.185867, 0.496456), + (0.632805, 0.187893, 0.495332), + (0.639216, 0.189921, 0.494150), + (0.645633, 0.191952, 0.492910), + (0.652056, 0.193986, 0.491611), + (0.658483, 0.196027, 0.490253), + (0.664915, 0.198075, 0.488836), + (0.671349, 0.200133, 0.487358), + (0.677786, 0.202203, 0.485819), + (0.684224, 0.204286, 0.484219), + (0.690661, 0.206384, 0.482558), + (0.697098, 0.208501, 0.480835), + (0.703532, 0.210638, 0.479049), + (0.709962, 0.212797, 0.477201), + (0.716387, 0.214982, 0.475290), + (0.722805, 0.217194, 0.473316), + (0.729216, 0.219437, 0.471279), + (0.735616, 0.221713, 0.469180), + (0.742004, 0.224025, 0.467018), + (0.748378, 0.226377, 0.464794), + (0.754737, 0.228772, 0.462509), + (0.761077, 0.231214, 0.460162), + (0.767398, 0.233705, 0.457755), + (0.773695, 0.236249, 0.455289), + (0.779968, 0.238851, 0.452765), + (0.786212, 0.241514, 0.450184), + (0.792427, 0.244242, 0.447543), + (0.798608, 0.247040, 0.444848), + (0.804752, 0.249911, 0.442102), + (0.810855, 0.252861, 0.439305), + (0.816914, 0.255895, 0.436461), + (0.822926, 0.259016, 0.433573), + (0.828886, 0.262229, 0.430644), + (0.834791, 0.265540, 0.427671), + (0.840636, 0.268953, 0.424666), + (0.846416, 0.272473, 0.421631), + (0.852126, 0.276106, 0.418573), + (0.857763, 0.279857, 0.415496), + (0.863320, 0.283729, 0.412403), + (0.868793, 0.287728, 0.409303), + (0.874176, 0.291859, 0.406205), + (0.879464, 0.296125, 0.403118), + (0.884651, 0.300530, 0.400047), + (0.889731, 0.305079, 0.397002), + (0.894700, 0.309773, 0.393995), + (0.899552, 0.314616, 0.391037), + (0.904281, 0.319610, 0.388137), + (0.908884, 0.324755, 0.385308), + (0.913354, 0.330052, 0.382563), + (0.917689, 0.335500, 0.379915), + (0.921884, 0.341098, 0.377376), + (0.925937, 0.346844, 0.374959), + (0.929845, 0.352734, 0.372677), + (0.933606, 0.358764, 0.370541), + (0.937221, 0.364929, 0.368567), + (0.940687, 0.371224, 0.366762), + (0.944006, 0.377643, 0.365136), + (0.947180, 0.384178, 0.363701), + (0.950210, 0.390820, 0.362468), + (0.953099, 0.397563, 0.361438), + (0.955849, 0.404400, 0.360619), + (0.958464, 0.411324, 0.360014), + (0.960949, 0.418323, 0.359630), + (0.963310, 0.425390, 0.359469), + (0.965549, 0.432519, 0.359529), + (0.967671, 0.439703, 0.359810), + (0.969680, 0.446936, 0.360311), + (0.971582, 0.454210, 0.361030), + (0.973381, 0.461520, 0.361965), + (0.975082, 0.468861, 0.363111), + (0.976690, 0.476226, 0.364466), + (0.978210, 0.483612, 0.366025), + (0.979645, 0.491014, 0.367783), + (0.981000, 0.498428, 0.369734), + (0.982279, 0.505851, 0.371874), + (0.983485, 0.513280, 0.374198), + (0.984622, 0.520713, 0.376698), + (0.985693, 0.528148, 0.379371), + (0.986700, 0.535582, 0.382210), + (0.987646, 0.543015, 0.385210), + (0.988533, 0.550446, 0.388365), + (0.989363, 0.557873, 0.391671), + (0.990138, 0.565296, 0.395122), + (0.990871, 0.572706, 0.398714), + (0.991558, 0.580107, 0.402441), + (0.992196, 0.587502, 0.406299), + (0.992785, 0.594891, 0.410283), + (0.993326, 0.602275, 0.414390), + (0.993834, 0.609644, 0.418613), + (0.994309, 0.616999, 0.422950), + (0.994738, 0.624350, 0.427397), + (0.995122, 0.631696, 0.431951), + (0.995480, 0.639027, 0.436607), + (0.995810, 0.646344, 0.441361), + (0.996096, 0.653659, 0.446213), + (0.996341, 0.660969, 0.451160), + (0.996580, 0.668256, 0.456192), + (0.996775, 0.675541, 0.461314), + (0.996925, 0.682828, 0.466526), + (0.997077, 0.690088, 0.471811), + (0.997186, 0.697349, 0.477182), + (0.997254, 0.704611, 0.482635), + (0.997325, 0.711848, 0.488154), + (0.997351, 0.719089, 0.493755), + (0.997351, 0.726324, 0.499428), + (0.997341, 0.733545, 0.505167), + (0.997285, 0.740772, 0.510983), + (0.997228, 0.747981, 0.516859), + (0.997138, 0.755190, 0.522806), + (0.997019, 0.762398, 0.528821), + (0.996898, 0.769591, 0.534892), + (0.996727, 0.776795, 0.541039), + (0.996571, 0.783977, 0.547233), + (0.996369, 0.791167, 0.553499), + (0.996162, 0.798348, 0.559820), + (0.995932, 0.805527, 0.566202), + (0.995680, 0.812706, 0.572645), + (0.995424, 0.819875, 0.579140), + (0.995131, 0.827052, 0.585701), + (0.994851, 0.834213, 0.592307), + (0.994524, 0.841387, 0.598983), + (0.994222, 0.848540, 0.605696), + (0.993866, 0.855711, 0.612482), + (0.993545, 0.862859, 0.619299), + (0.993170, 0.870024, 0.626189), + (0.992831, 0.877168, 0.633109), + (0.992440, 0.884330, 0.640099), + (0.992089, 0.891470, 0.647116), + (0.991688, 0.898627, 0.654202), + (0.991332, 0.905763, 0.661309), + (0.990930, 0.912915, 0.668481), + (0.990570, 0.920049, 0.675675), + (0.990175, 0.927196, 0.682926), + (0.989815, 0.934329, 0.690198), + (0.989434, 0.941470, 0.697519), + (0.989077, 0.948604, 0.704863), + (0.988717, 0.955742, 0.712242), + (0.988367, 0.962878, 0.719649), + (0.988033, 0.970012, 0.727077), + (0.987691, 0.977154, 0.734536), + (0.987387, 0.984288, 0.742002), + (0.987053, 0.991438, 0.749504), + ) +) inferno = _convert( - ((0.001462, 0.000466, 0.013866), - (0.002267, 0.001270, 0.018570), - (0.003299, 0.002249, 0.024239), - (0.004547, 0.003392, 0.030909), - (0.006006, 0.004692, 0.038558), - (0.007676, 0.006136, 0.046836), - (0.009561, 0.007713, 0.055143), - (0.011663, 0.009417, 0.063460), - (0.013995, 0.011225, 0.071862), - (0.016561, 0.013136, 0.080282), - (0.019373, 0.015133, 0.088767), - (0.022447, 0.017199, 0.097327), - (0.025793, 0.019331, 0.105930), - (0.029432, 0.021503, 0.114621), - (0.033385, 0.023702, 0.123397), - (0.037668, 0.025921, 0.132232), - (0.042253, 0.028139, 0.141141), - (0.046915, 0.030324, 0.150164), - (0.051644, 0.032474, 0.159254), - (0.056449, 0.034569, 0.168414), - (0.061340, 0.036590, 0.177642), - (0.066331, 0.038504, 0.186962), - (0.071429, 0.040294, 0.196354), - (0.076637, 0.041905, 0.205799), - (0.081962, 0.043328, 0.215289), - (0.087411, 0.044556, 0.224813), - (0.092990, 0.045583, 0.234358), - (0.098702, 0.046402, 0.243904), - (0.104551, 0.047008, 0.253430), - (0.110536, 0.047399, 0.262912), - (0.116656, 0.047574, 0.272321), - (0.122908, 0.047536, 0.281624), - (0.129285, 0.047293, 0.290788), - (0.135778, 0.046856, 0.299776), - (0.142378, 0.046242, 0.308553), - (0.149073, 0.045468, 0.317085), - (0.155850, 0.044559, 0.325338), - (0.162689, 0.043554, 0.333277), - (0.169575, 0.042489, 0.340874), - (0.176493, 0.041402, 0.348111), - (0.183429, 0.040329, 0.354971), - (0.190367, 0.039309, 0.361447), - (0.197297, 0.038400, 0.367535), - (0.204209, 0.037632, 0.373238), - (0.211095, 0.037030, 0.378563), - (0.217949, 0.036615, 0.383522), - (0.224763, 0.036405, 0.388129), - (0.231538, 0.036405, 0.392400), - (0.238273, 0.036621, 0.396353), - (0.244967, 0.037055, 0.400007), - (0.251620, 0.037705, 0.403378), - (0.258234, 0.038571, 0.406485), - (0.264810, 0.039647, 0.409345), - (0.271347, 0.040922, 0.411976), - (0.277850, 0.042353, 0.414392), - (0.284321, 0.043933, 0.416608), - (0.290763, 0.045644, 0.418637), - (0.297178, 0.047470, 0.420491), - (0.303568, 0.049396, 0.422182), - (0.309935, 0.051407, 0.423721), - (0.316282, 0.053490, 0.425116), - (0.322610, 0.055634, 0.426377), - (0.328921, 0.057827, 0.427511), - (0.335217, 0.060060, 0.428524), - (0.341500, 0.062325, 0.429425), - (0.347771, 0.064616, 0.430217), - (0.354032, 0.066925, 0.430906), - (0.360284, 0.069247, 0.431497), - (0.366529, 0.071579, 0.431994), - (0.372768, 0.073915, 0.432400), - (0.379001, 0.076253, 0.432719), - (0.385228, 0.078591, 0.432955), - (0.391453, 0.080927, 0.433109), - (0.397674, 0.083257, 0.433183), - (0.403894, 0.085580, 0.433179), - (0.410113, 0.087896, 0.433098), - (0.416331, 0.090203, 0.432943), - (0.422549, 0.092501, 0.432714), - (0.428768, 0.094790, 0.432412), - (0.434987, 0.097069, 0.432039), - (0.441207, 0.099338, 0.431594), - (0.447428, 0.101597, 0.431080), - (0.453651, 0.103848, 0.430498), - (0.459875, 0.106089, 0.429846), - (0.466100, 0.108322, 0.429125), - (0.472328, 0.110547, 0.428334), - (0.478558, 0.112764, 0.427475), - (0.484789, 0.114974, 0.426548), - (0.491022, 0.117179, 0.425552), - (0.497257, 0.119379, 0.424488), - (0.503493, 0.121575, 0.423356), - (0.509730, 0.123769, 0.422156), - (0.515967, 0.125960, 0.420887), - (0.522206, 0.128150, 0.419549), - (0.528444, 0.130341, 0.418142), - (0.534683, 0.132534, 0.416667), - (0.540920, 0.134729, 0.415123), - (0.547157, 0.136929, 0.413511), - (0.553392, 0.139134, 0.411829), - (0.559624, 0.141346, 0.410078), - (0.565854, 0.143567, 0.408258), - (0.572081, 0.145797, 0.406369), - (0.578304, 0.148039, 0.404411), - (0.584521, 0.150294, 0.402385), - (0.590734, 0.152563, 0.400290), - (0.596940, 0.154848, 0.398125), - (0.603139, 0.157151, 0.395891), - (0.609330, 0.159474, 0.393589), - (0.615513, 0.161817, 0.391219), - (0.621685, 0.164184, 0.388781), - (0.627847, 0.166575, 0.386276), - (0.633998, 0.168992, 0.383704), - (0.640135, 0.171438, 0.381065), - (0.646260, 0.173914, 0.378359), - (0.652369, 0.176421, 0.375586), - (0.658463, 0.178962, 0.372748), - (0.664540, 0.181539, 0.369846), - (0.670599, 0.184153, 0.366879), - (0.676638, 0.186807, 0.363849), - (0.682656, 0.189501, 0.360757), - (0.688653, 0.192239, 0.357603), - (0.694627, 0.195021, 0.354388), - (0.700576, 0.197851, 0.351113), - (0.706500, 0.200728, 0.347777), - (0.712396, 0.203656, 0.344383), - (0.718264, 0.206636, 0.340931), - (0.724103, 0.209670, 0.337424), - (0.729909, 0.212759, 0.333861), - (0.735683, 0.215906, 0.330245), - (0.741423, 0.219112, 0.326576), - (0.747127, 0.222378, 0.322856), - (0.752794, 0.225706, 0.319085), - (0.758422, 0.229097, 0.315266), - (0.764010, 0.232554, 0.311399), - (0.769556, 0.236077, 0.307485), - (0.775059, 0.239667, 0.303526), - (0.780517, 0.243327, 0.299523), - (0.785929, 0.247056, 0.295477), - (0.791293, 0.250856, 0.291390), - (0.796607, 0.254728, 0.287264), - (0.801871, 0.258674, 0.283099), - (0.807082, 0.262692, 0.278898), - (0.812239, 0.266786, 0.274661), - (0.817341, 0.270954, 0.270390), - (0.822386, 0.275197, 0.266085), - (0.827372, 0.279517, 0.261750), - (0.832299, 0.283913, 0.257383), - (0.837165, 0.288385, 0.252988), - (0.841969, 0.292933, 0.248564), - (0.846709, 0.297559, 0.244113), - (0.851384, 0.302260, 0.239636), - (0.855992, 0.307038, 0.235133), - (0.860533, 0.311892, 0.230606), - (0.865006, 0.316822, 0.226055), - (0.869409, 0.321827, 0.221482), - (0.873741, 0.326906, 0.216886), - (0.878001, 0.332060, 0.212268), - (0.882188, 0.337287, 0.207628), - (0.886302, 0.342586, 0.202968), - (0.890341, 0.347957, 0.198286), - (0.894305, 0.353399, 0.193584), - (0.898192, 0.358911, 0.188860), - (0.902003, 0.364492, 0.184116), - (0.905735, 0.370140, 0.179350), - (0.909390, 0.375856, 0.174563), - (0.912966, 0.381636, 0.169755), - (0.916462, 0.387481, 0.164924), - (0.919879, 0.393389, 0.160070), - (0.923215, 0.399359, 0.155193), - (0.926470, 0.405389, 0.150292), - (0.929644, 0.411479, 0.145367), - (0.932737, 0.417627, 0.140417), - (0.935747, 0.423831, 0.135440), - (0.938675, 0.430091, 0.130438), - (0.941521, 0.436405, 0.125409), - (0.944285, 0.442772, 0.120354), - (0.946965, 0.449191, 0.115272), - (0.949562, 0.455660, 0.110164), - (0.952075, 0.462178, 0.105031), - (0.954506, 0.468744, 0.099874), - (0.956852, 0.475356, 0.094695), - (0.959114, 0.482014, 0.089499), - (0.961293, 0.488716, 0.084289), - (0.963387, 0.495462, 0.079073), - (0.965397, 0.502249, 0.073859), - (0.967322, 0.509078, 0.068659), - (0.969163, 0.515946, 0.063488), - (0.970919, 0.522853, 0.058367), - (0.972590, 0.529798, 0.053324), - (0.974176, 0.536780, 0.048392), - (0.975677, 0.543798, 0.043618), - (0.977092, 0.550850, 0.039050), - (0.978422, 0.557937, 0.034931), - (0.979666, 0.565057, 0.031409), - (0.980824, 0.572209, 0.028508), - (0.981895, 0.579392, 0.026250), - (0.982881, 0.586606, 0.024661), - (0.983779, 0.593849, 0.023770), - (0.984591, 0.601122, 0.023606), - (0.985315, 0.608422, 0.024202), - (0.985952, 0.615750, 0.025592), - (0.986502, 0.623105, 0.027814), - (0.986964, 0.630485, 0.030908), - (0.987337, 0.637890, 0.034916), - (0.987622, 0.645320, 0.039886), - (0.987819, 0.652773, 0.045581), - (0.987926, 0.660250, 0.051750), - (0.987945, 0.667748, 0.058329), - (0.987874, 0.675267, 0.065257), - (0.987714, 0.682807, 0.072489), - (0.987464, 0.690366, 0.079990), - (0.987124, 0.697944, 0.087731), - (0.986694, 0.705540, 0.095694), - (0.986175, 0.713153, 0.103863), - (0.985566, 0.720782, 0.112229), - (0.984865, 0.728427, 0.120785), - (0.984075, 0.736087, 0.129527), - (0.983196, 0.743758, 0.138453), - (0.982228, 0.751442, 0.147565), - (0.981173, 0.759135, 0.156863), - (0.980032, 0.766837, 0.166353), - (0.978806, 0.774545, 0.176037), - (0.977497, 0.782258, 0.185923), - (0.976108, 0.789974, 0.196018), - (0.974638, 0.797692, 0.206332), - (0.973088, 0.805409, 0.216877), - (0.971468, 0.813122, 0.227658), - (0.969783, 0.820825, 0.238686), - (0.968041, 0.828515, 0.249972), - (0.966243, 0.836191, 0.261534), - (0.964394, 0.843848, 0.273391), - (0.962517, 0.851476, 0.285546), - (0.960626, 0.859069, 0.298010), - (0.958720, 0.866624, 0.310820), - (0.956834, 0.874129, 0.323974), - (0.954997, 0.881569, 0.337475), - (0.953215, 0.888942, 0.351369), - (0.951546, 0.896226, 0.365627), - (0.950018, 0.903409, 0.380271), - (0.948683, 0.910473, 0.395289), - (0.947594, 0.917399, 0.410665), - (0.946809, 0.924168, 0.426373), - (0.946392, 0.930761, 0.442367), - (0.946403, 0.937159, 0.458592), - (0.946903, 0.943348, 0.474970), - (0.947937, 0.949318, 0.491426), - (0.949545, 0.955063, 0.507860), - (0.951740, 0.960587, 0.524203), - (0.954529, 0.965896, 0.540361), - (0.957896, 0.971003, 0.556275), - (0.961812, 0.975924, 0.571925), - (0.966249, 0.980678, 0.587206), - (0.971162, 0.985282, 0.602154), - (0.976511, 0.989753, 0.616760), - (0.982257, 0.994109, 0.631017), - (0.988362, 0.998364, 0.644924))) + ( + (0.001462, 0.000466, 0.013866), + (0.002267, 0.001270, 0.018570), + (0.003299, 0.002249, 0.024239), + (0.004547, 0.003392, 0.030909), + (0.006006, 0.004692, 0.038558), + (0.007676, 0.006136, 0.046836), + (0.009561, 0.007713, 0.055143), + (0.011663, 0.009417, 0.063460), + (0.013995, 0.011225, 0.071862), + (0.016561, 0.013136, 0.080282), + (0.019373, 0.015133, 0.088767), + (0.022447, 0.017199, 0.097327), + (0.025793, 0.019331, 0.105930), + (0.029432, 0.021503, 0.114621), + (0.033385, 0.023702, 0.123397), + (0.037668, 0.025921, 0.132232), + (0.042253, 0.028139, 0.141141), + (0.046915, 0.030324, 0.150164), + (0.051644, 0.032474, 0.159254), + (0.056449, 0.034569, 0.168414), + (0.061340, 0.036590, 0.177642), + (0.066331, 0.038504, 0.186962), + (0.071429, 0.040294, 0.196354), + (0.076637, 0.041905, 0.205799), + (0.081962, 0.043328, 0.215289), + (0.087411, 0.044556, 0.224813), + (0.092990, 0.045583, 0.234358), + (0.098702, 0.046402, 0.243904), + (0.104551, 0.047008, 0.253430), + (0.110536, 0.047399, 0.262912), + (0.116656, 0.047574, 0.272321), + (0.122908, 0.047536, 0.281624), + (0.129285, 0.047293, 0.290788), + (0.135778, 0.046856, 0.299776), + (0.142378, 0.046242, 0.308553), + (0.149073, 0.045468, 0.317085), + (0.155850, 0.044559, 0.325338), + (0.162689, 0.043554, 0.333277), + (0.169575, 0.042489, 0.340874), + (0.176493, 0.041402, 0.348111), + (0.183429, 0.040329, 0.354971), + (0.190367, 0.039309, 0.361447), + (0.197297, 0.038400, 0.367535), + (0.204209, 0.037632, 0.373238), + (0.211095, 0.037030, 0.378563), + (0.217949, 0.036615, 0.383522), + (0.224763, 0.036405, 0.388129), + (0.231538, 0.036405, 0.392400), + (0.238273, 0.036621, 0.396353), + (0.244967, 0.037055, 0.400007), + (0.251620, 0.037705, 0.403378), + (0.258234, 0.038571, 0.406485), + (0.264810, 0.039647, 0.409345), + (0.271347, 0.040922, 0.411976), + (0.277850, 0.042353, 0.414392), + (0.284321, 0.043933, 0.416608), + (0.290763, 0.045644, 0.418637), + (0.297178, 0.047470, 0.420491), + (0.303568, 0.049396, 0.422182), + (0.309935, 0.051407, 0.423721), + (0.316282, 0.053490, 0.425116), + (0.322610, 0.055634, 0.426377), + (0.328921, 0.057827, 0.427511), + (0.335217, 0.060060, 0.428524), + (0.341500, 0.062325, 0.429425), + (0.347771, 0.064616, 0.430217), + (0.354032, 0.066925, 0.430906), + (0.360284, 0.069247, 0.431497), + (0.366529, 0.071579, 0.431994), + (0.372768, 0.073915, 0.432400), + (0.379001, 0.076253, 0.432719), + (0.385228, 0.078591, 0.432955), + (0.391453, 0.080927, 0.433109), + (0.397674, 0.083257, 0.433183), + (0.403894, 0.085580, 0.433179), + (0.410113, 0.087896, 0.433098), + (0.416331, 0.090203, 0.432943), + (0.422549, 0.092501, 0.432714), + (0.428768, 0.094790, 0.432412), + (0.434987, 0.097069, 0.432039), + (0.441207, 0.099338, 0.431594), + (0.447428, 0.101597, 0.431080), + (0.453651, 0.103848, 0.430498), + (0.459875, 0.106089, 0.429846), + (0.466100, 0.108322, 0.429125), + (0.472328, 0.110547, 0.428334), + (0.478558, 0.112764, 0.427475), + (0.484789, 0.114974, 0.426548), + (0.491022, 0.117179, 0.425552), + (0.497257, 0.119379, 0.424488), + (0.503493, 0.121575, 0.423356), + (0.509730, 0.123769, 0.422156), + (0.515967, 0.125960, 0.420887), + (0.522206, 0.128150, 0.419549), + (0.528444, 0.130341, 0.418142), + (0.534683, 0.132534, 0.416667), + (0.540920, 0.134729, 0.415123), + (0.547157, 0.136929, 0.413511), + (0.553392, 0.139134, 0.411829), + (0.559624, 0.141346, 0.410078), + (0.565854, 0.143567, 0.408258), + (0.572081, 0.145797, 0.406369), + (0.578304, 0.148039, 0.404411), + (0.584521, 0.150294, 0.402385), + (0.590734, 0.152563, 0.400290), + (0.596940, 0.154848, 0.398125), + (0.603139, 0.157151, 0.395891), + (0.609330, 0.159474, 0.393589), + (0.615513, 0.161817, 0.391219), + (0.621685, 0.164184, 0.388781), + (0.627847, 0.166575, 0.386276), + (0.633998, 0.168992, 0.383704), + (0.640135, 0.171438, 0.381065), + (0.646260, 0.173914, 0.378359), + (0.652369, 0.176421, 0.375586), + (0.658463, 0.178962, 0.372748), + (0.664540, 0.181539, 0.369846), + (0.670599, 0.184153, 0.366879), + (0.676638, 0.186807, 0.363849), + (0.682656, 0.189501, 0.360757), + (0.688653, 0.192239, 0.357603), + (0.694627, 0.195021, 0.354388), + (0.700576, 0.197851, 0.351113), + (0.706500, 0.200728, 0.347777), + (0.712396, 0.203656, 0.344383), + (0.718264, 0.206636, 0.340931), + (0.724103, 0.209670, 0.337424), + (0.729909, 0.212759, 0.333861), + (0.735683, 0.215906, 0.330245), + (0.741423, 0.219112, 0.326576), + (0.747127, 0.222378, 0.322856), + (0.752794, 0.225706, 0.319085), + (0.758422, 0.229097, 0.315266), + (0.764010, 0.232554, 0.311399), + (0.769556, 0.236077, 0.307485), + (0.775059, 0.239667, 0.303526), + (0.780517, 0.243327, 0.299523), + (0.785929, 0.247056, 0.295477), + (0.791293, 0.250856, 0.291390), + (0.796607, 0.254728, 0.287264), + (0.801871, 0.258674, 0.283099), + (0.807082, 0.262692, 0.278898), + (0.812239, 0.266786, 0.274661), + (0.817341, 0.270954, 0.270390), + (0.822386, 0.275197, 0.266085), + (0.827372, 0.279517, 0.261750), + (0.832299, 0.283913, 0.257383), + (0.837165, 0.288385, 0.252988), + (0.841969, 0.292933, 0.248564), + (0.846709, 0.297559, 0.244113), + (0.851384, 0.302260, 0.239636), + (0.855992, 0.307038, 0.235133), + (0.860533, 0.311892, 0.230606), + (0.865006, 0.316822, 0.226055), + (0.869409, 0.321827, 0.221482), + (0.873741, 0.326906, 0.216886), + (0.878001, 0.332060, 0.212268), + (0.882188, 0.337287, 0.207628), + (0.886302, 0.342586, 0.202968), + (0.890341, 0.347957, 0.198286), + (0.894305, 0.353399, 0.193584), + (0.898192, 0.358911, 0.188860), + (0.902003, 0.364492, 0.184116), + (0.905735, 0.370140, 0.179350), + (0.909390, 0.375856, 0.174563), + (0.912966, 0.381636, 0.169755), + (0.916462, 0.387481, 0.164924), + (0.919879, 0.393389, 0.160070), + (0.923215, 0.399359, 0.155193), + (0.926470, 0.405389, 0.150292), + (0.929644, 0.411479, 0.145367), + (0.932737, 0.417627, 0.140417), + (0.935747, 0.423831, 0.135440), + (0.938675, 0.430091, 0.130438), + (0.941521, 0.436405, 0.125409), + (0.944285, 0.442772, 0.120354), + (0.946965, 0.449191, 0.115272), + (0.949562, 0.455660, 0.110164), + (0.952075, 0.462178, 0.105031), + (0.954506, 0.468744, 0.099874), + (0.956852, 0.475356, 0.094695), + (0.959114, 0.482014, 0.089499), + (0.961293, 0.488716, 0.084289), + (0.963387, 0.495462, 0.079073), + (0.965397, 0.502249, 0.073859), + (0.967322, 0.509078, 0.068659), + (0.969163, 0.515946, 0.063488), + (0.970919, 0.522853, 0.058367), + (0.972590, 0.529798, 0.053324), + (0.974176, 0.536780, 0.048392), + (0.975677, 0.543798, 0.043618), + (0.977092, 0.550850, 0.039050), + (0.978422, 0.557937, 0.034931), + (0.979666, 0.565057, 0.031409), + (0.980824, 0.572209, 0.028508), + (0.981895, 0.579392, 0.026250), + (0.982881, 0.586606, 0.024661), + (0.983779, 0.593849, 0.023770), + (0.984591, 0.601122, 0.023606), + (0.985315, 0.608422, 0.024202), + (0.985952, 0.615750, 0.025592), + (0.986502, 0.623105, 0.027814), + (0.986964, 0.630485, 0.030908), + (0.987337, 0.637890, 0.034916), + (0.987622, 0.645320, 0.039886), + (0.987819, 0.652773, 0.045581), + (0.987926, 0.660250, 0.051750), + (0.987945, 0.667748, 0.058329), + (0.987874, 0.675267, 0.065257), + (0.987714, 0.682807, 0.072489), + (0.987464, 0.690366, 0.079990), + (0.987124, 0.697944, 0.087731), + (0.986694, 0.705540, 0.095694), + (0.986175, 0.713153, 0.103863), + (0.985566, 0.720782, 0.112229), + (0.984865, 0.728427, 0.120785), + (0.984075, 0.736087, 0.129527), + (0.983196, 0.743758, 0.138453), + (0.982228, 0.751442, 0.147565), + (0.981173, 0.759135, 0.156863), + (0.980032, 0.766837, 0.166353), + (0.978806, 0.774545, 0.176037), + (0.977497, 0.782258, 0.185923), + (0.976108, 0.789974, 0.196018), + (0.974638, 0.797692, 0.206332), + (0.973088, 0.805409, 0.216877), + (0.971468, 0.813122, 0.227658), + (0.969783, 0.820825, 0.238686), + (0.968041, 0.828515, 0.249972), + (0.966243, 0.836191, 0.261534), + (0.964394, 0.843848, 0.273391), + (0.962517, 0.851476, 0.285546), + (0.960626, 0.859069, 0.298010), + (0.958720, 0.866624, 0.310820), + (0.956834, 0.874129, 0.323974), + (0.954997, 0.881569, 0.337475), + (0.953215, 0.888942, 0.351369), + (0.951546, 0.896226, 0.365627), + (0.950018, 0.903409, 0.380271), + (0.948683, 0.910473, 0.395289), + (0.947594, 0.917399, 0.410665), + (0.946809, 0.924168, 0.426373), + (0.946392, 0.930761, 0.442367), + (0.946403, 0.937159, 0.458592), + (0.946903, 0.943348, 0.474970), + (0.947937, 0.949318, 0.491426), + (0.949545, 0.955063, 0.507860), + (0.951740, 0.960587, 0.524203), + (0.954529, 0.965896, 0.540361), + (0.957896, 0.971003, 0.556275), + (0.961812, 0.975924, 0.571925), + (0.966249, 0.980678, 0.587206), + (0.971162, 0.985282, 0.602154), + (0.976511, 0.989753, 0.616760), + (0.982257, 0.994109, 0.631017), + (0.988362, 0.998364, 0.644924), + ) +) plasma = _convert( - ((0.050383, 0.029803, 0.527975), - (0.063536, 0.028426, 0.533124), - (0.075353, 0.027206, 0.538007), - (0.086222, 0.026125, 0.542658), - (0.096379, 0.025165, 0.547103), - (0.105980, 0.024309, 0.551368), - (0.115124, 0.023556, 0.555468), - (0.123903, 0.022878, 0.559423), - (0.132381, 0.022258, 0.563250), - (0.140603, 0.021687, 0.566959), - (0.148607, 0.021154, 0.570562), - (0.156421, 0.020651, 0.574065), - (0.164070, 0.020171, 0.577478), - (0.171574, 0.019706, 0.580806), - (0.178950, 0.019252, 0.584054), - (0.186213, 0.018803, 0.587228), - (0.193374, 0.018354, 0.590330), - (0.200445, 0.017902, 0.593364), - (0.207435, 0.017442, 0.596333), - (0.214350, 0.016973, 0.599239), - (0.221197, 0.016497, 0.602083), - (0.227983, 0.016007, 0.604867), - (0.234715, 0.015502, 0.607592), - (0.241396, 0.014979, 0.610259), - (0.248032, 0.014439, 0.612868), - (0.254627, 0.013882, 0.615419), - (0.261183, 0.013308, 0.617911), - (0.267703, 0.012716, 0.620346), - (0.274191, 0.012109, 0.622722), - (0.280648, 0.011488, 0.625038), - (0.287076, 0.010855, 0.627295), - (0.293478, 0.010213, 0.629490), - (0.299855, 0.009561, 0.631624), - (0.306210, 0.008902, 0.633694), - (0.312543, 0.008239, 0.635700), - (0.318856, 0.007576, 0.637640), - (0.325150, 0.006915, 0.639512), - (0.331426, 0.006261, 0.641316), - (0.337683, 0.005618, 0.643049), - (0.343925, 0.004991, 0.644710), - (0.350150, 0.004382, 0.646298), - (0.356359, 0.003798, 0.647810), - (0.362553, 0.003243, 0.649245), - (0.368733, 0.002724, 0.650601), - (0.374897, 0.002245, 0.651876), - (0.381047, 0.001814, 0.653068), - (0.387183, 0.001434, 0.654177), - (0.393304, 0.001114, 0.655199), - (0.399411, 0.000859, 0.656133), - (0.405503, 0.000678, 0.656977), - (0.411580, 0.000577, 0.657730), - (0.417642, 0.000564, 0.658390), - (0.423689, 0.000646, 0.658956), - (0.429719, 0.000831, 0.659425), - (0.435734, 0.001127, 0.659797), - (0.441732, 0.001540, 0.660069), - (0.447714, 0.002080, 0.660240), - (0.453677, 0.002755, 0.660310), - (0.459623, 0.003574, 0.660277), - (0.465550, 0.004545, 0.660139), - (0.471457, 0.005678, 0.659897), - (0.477344, 0.006980, 0.659549), - (0.483210, 0.008460, 0.659095), - (0.489055, 0.010127, 0.658534), - (0.494877, 0.011990, 0.657865), - (0.500678, 0.014055, 0.657088), - (0.506454, 0.016333, 0.656202), - (0.512206, 0.018833, 0.655209), - (0.517933, 0.021563, 0.654109), - (0.523633, 0.024532, 0.652901), - (0.529306, 0.027747, 0.651586), - (0.534952, 0.031217, 0.650165), - (0.540570, 0.034950, 0.648640), - (0.546157, 0.038954, 0.647010), - (0.551715, 0.043136, 0.645277), - (0.557243, 0.047331, 0.643443), - (0.562738, 0.051545, 0.641509), - (0.568201, 0.055778, 0.639477), - (0.573632, 0.060028, 0.637349), - (0.579029, 0.064296, 0.635126), - (0.584391, 0.068579, 0.632812), - (0.589719, 0.072878, 0.630408), - (0.595011, 0.077190, 0.627917), - (0.600266, 0.081516, 0.625342), - (0.605485, 0.085854, 0.622686), - (0.610667, 0.090204, 0.619951), - (0.615812, 0.094564, 0.617140), - (0.620919, 0.098934, 0.614257), - (0.625987, 0.103312, 0.611305), - (0.631017, 0.107699, 0.608287), - (0.636008, 0.112092, 0.605205), - (0.640959, 0.116492, 0.602065), - (0.645872, 0.120898, 0.598867), - (0.650746, 0.125309, 0.595617), - (0.655580, 0.129725, 0.592317), - (0.660374, 0.134144, 0.588971), - (0.665129, 0.138566, 0.585582), - (0.669845, 0.142992, 0.582154), - (0.674522, 0.147419, 0.578688), - (0.679160, 0.151848, 0.575189), - (0.683758, 0.156278, 0.571660), - (0.688318, 0.160709, 0.568103), - (0.692840, 0.165141, 0.564522), - (0.697324, 0.169573, 0.560919), - (0.701769, 0.174005, 0.557296), - (0.706178, 0.178437, 0.553657), - (0.710549, 0.182868, 0.550004), - (0.714883, 0.187299, 0.546338), - (0.719181, 0.191729, 0.542663), - (0.723444, 0.196158, 0.538981), - (0.727670, 0.200586, 0.535293), - (0.731862, 0.205013, 0.531601), - (0.736019, 0.209439, 0.527908), - (0.740143, 0.213864, 0.524216), - (0.744232, 0.218288, 0.520524), - (0.748289, 0.222711, 0.516834), - (0.752312, 0.227133, 0.513149), - (0.756304, 0.231555, 0.509468), - (0.760264, 0.235976, 0.505794), - (0.764193, 0.240396, 0.502126), - (0.768090, 0.244817, 0.498465), - (0.771958, 0.249237, 0.494813), - (0.775796, 0.253658, 0.491171), - (0.779604, 0.258078, 0.487539), - (0.783383, 0.262500, 0.483918), - (0.787133, 0.266922, 0.480307), - (0.790855, 0.271345, 0.476706), - (0.794549, 0.275770, 0.473117), - (0.798216, 0.280197, 0.469538), - (0.801855, 0.284626, 0.465971), - (0.805467, 0.289057, 0.462415), - (0.809052, 0.293491, 0.458870), - (0.812612, 0.297928, 0.455338), - (0.816144, 0.302368, 0.451816), - (0.819651, 0.306812, 0.448306), - (0.823132, 0.311261, 0.444806), - (0.826588, 0.315714, 0.441316), - (0.830018, 0.320172, 0.437836), - (0.833422, 0.324635, 0.434366), - (0.836801, 0.329105, 0.430905), - (0.840155, 0.333580, 0.427455), - (0.843484, 0.338062, 0.424013), - (0.846788, 0.342551, 0.420579), - (0.850066, 0.347048, 0.417153), - (0.853319, 0.351553, 0.413734), - (0.856547, 0.356066, 0.410322), - (0.859750, 0.360588, 0.406917), - (0.862927, 0.365119, 0.403519), - (0.866078, 0.369660, 0.400126), - (0.869203, 0.374212, 0.396738), - (0.872303, 0.378774, 0.393355), - (0.875376, 0.383347, 0.389976), - (0.878423, 0.387932, 0.386600), - (0.881443, 0.392529, 0.383229), - (0.884436, 0.397139, 0.379860), - (0.887402, 0.401762, 0.376494), - (0.890340, 0.406398, 0.373130), - (0.893250, 0.411048, 0.369768), - (0.896131, 0.415712, 0.366407), - (0.898984, 0.420392, 0.363047), - (0.901807, 0.425087, 0.359688), - (0.904601, 0.429797, 0.356329), - (0.907365, 0.434524, 0.352970), - (0.910098, 0.439268, 0.349610), - (0.912800, 0.444029, 0.346251), - (0.915471, 0.448807, 0.342890), - (0.918109, 0.453603, 0.339529), - (0.920714, 0.458417, 0.336166), - (0.923287, 0.463251, 0.332801), - (0.925825, 0.468103, 0.329435), - (0.928329, 0.472975, 0.326067), - (0.930798, 0.477867, 0.322697), - (0.933232, 0.482780, 0.319325), - (0.935630, 0.487712, 0.315952), - (0.937990, 0.492667, 0.312575), - (0.940313, 0.497642, 0.309197), - (0.942598, 0.502639, 0.305816), - (0.944844, 0.507658, 0.302433), - (0.947051, 0.512699, 0.299049), - (0.949217, 0.517763, 0.295662), - (0.951344, 0.522850, 0.292275), - (0.953428, 0.527960, 0.288883), - (0.955470, 0.533093, 0.285490), - (0.957469, 0.538250, 0.282096), - (0.959424, 0.543431, 0.278701), - (0.961336, 0.548636, 0.275305), - (0.963203, 0.553865, 0.271909), - (0.965024, 0.559118, 0.268513), - (0.966798, 0.564396, 0.265118), - (0.968526, 0.569700, 0.261721), - (0.970205, 0.575028, 0.258325), - (0.971835, 0.580382, 0.254931), - (0.973416, 0.585761, 0.251540), - (0.974947, 0.591165, 0.248151), - (0.976428, 0.596595, 0.244767), - (0.977856, 0.602051, 0.241387), - (0.979233, 0.607532, 0.238013), - (0.980556, 0.613039, 0.234646), - (0.981826, 0.618572, 0.231287), - (0.983041, 0.624131, 0.227937), - (0.984199, 0.629718, 0.224595), - (0.985301, 0.635330, 0.221265), - (0.986345, 0.640969, 0.217948), - (0.987332, 0.646633, 0.214648), - (0.988260, 0.652325, 0.211364), - (0.989128, 0.658043, 0.208100), - (0.989935, 0.663787, 0.204859), - (0.990681, 0.669558, 0.201642), - (0.991365, 0.675355, 0.198453), - (0.991985, 0.681179, 0.195295), - (0.992541, 0.687030, 0.192170), - (0.993032, 0.692907, 0.189084), - (0.993456, 0.698810, 0.186041), - (0.993814, 0.704741, 0.183043), - (0.994103, 0.710698, 0.180097), - (0.994324, 0.716681, 0.177208), - (0.994474, 0.722691, 0.174381), - (0.994553, 0.728728, 0.171622), - (0.994561, 0.734791, 0.168938), - (0.994495, 0.740880, 0.166335), - (0.994355, 0.746995, 0.163821), - (0.994141, 0.753137, 0.161404), - (0.993851, 0.759304, 0.159092), - (0.993482, 0.765499, 0.156891), - (0.993033, 0.771720, 0.154808), - (0.992505, 0.777967, 0.152855), - (0.991897, 0.784239, 0.151042), - (0.991209, 0.790537, 0.149377), - (0.990439, 0.796859, 0.147870), - (0.989587, 0.803205, 0.146529), - (0.988648, 0.809579, 0.145357), - (0.987621, 0.815978, 0.144363), - (0.986509, 0.822401, 0.143557), - (0.985314, 0.828846, 0.142945), - (0.984031, 0.835315, 0.142528), - (0.982653, 0.841812, 0.142303), - (0.981190, 0.848329, 0.142279), - (0.979644, 0.854866, 0.142453), - (0.977995, 0.861432, 0.142808), - (0.976265, 0.868016, 0.143351), - (0.974443, 0.874622, 0.144061), - (0.972530, 0.881250, 0.144923), - (0.970533, 0.887896, 0.145919), - (0.968443, 0.894564, 0.147014), - (0.966271, 0.901249, 0.148180), - (0.964021, 0.907950, 0.149370), - (0.961681, 0.914672, 0.150520), - (0.959276, 0.921407, 0.151566), - (0.956808, 0.928152, 0.152409), - (0.954287, 0.934908, 0.152921), - (0.951726, 0.941671, 0.152925), - (0.949151, 0.948435, 0.152178), - (0.946602, 0.955190, 0.150328), - (0.944152, 0.961916, 0.146861), - (0.941896, 0.968590, 0.140956), - (0.940015, 0.975158, 0.131326))) + ( + (0.050383, 0.029803, 0.527975), + (0.063536, 0.028426, 0.533124), + (0.075353, 0.027206, 0.538007), + (0.086222, 0.026125, 0.542658), + (0.096379, 0.025165, 0.547103), + (0.105980, 0.024309, 0.551368), + (0.115124, 0.023556, 0.555468), + (0.123903, 0.022878, 0.559423), + (0.132381, 0.022258, 0.563250), + (0.140603, 0.021687, 0.566959), + (0.148607, 0.021154, 0.570562), + (0.156421, 0.020651, 0.574065), + (0.164070, 0.020171, 0.577478), + (0.171574, 0.019706, 0.580806), + (0.178950, 0.019252, 0.584054), + (0.186213, 0.018803, 0.587228), + (0.193374, 0.018354, 0.590330), + (0.200445, 0.017902, 0.593364), + (0.207435, 0.017442, 0.596333), + (0.214350, 0.016973, 0.599239), + (0.221197, 0.016497, 0.602083), + (0.227983, 0.016007, 0.604867), + (0.234715, 0.015502, 0.607592), + (0.241396, 0.014979, 0.610259), + (0.248032, 0.014439, 0.612868), + (0.254627, 0.013882, 0.615419), + (0.261183, 0.013308, 0.617911), + (0.267703, 0.012716, 0.620346), + (0.274191, 0.012109, 0.622722), + (0.280648, 0.011488, 0.625038), + (0.287076, 0.010855, 0.627295), + (0.293478, 0.010213, 0.629490), + (0.299855, 0.009561, 0.631624), + (0.306210, 0.008902, 0.633694), + (0.312543, 0.008239, 0.635700), + (0.318856, 0.007576, 0.637640), + (0.325150, 0.006915, 0.639512), + (0.331426, 0.006261, 0.641316), + (0.337683, 0.005618, 0.643049), + (0.343925, 0.004991, 0.644710), + (0.350150, 0.004382, 0.646298), + (0.356359, 0.003798, 0.647810), + (0.362553, 0.003243, 0.649245), + (0.368733, 0.002724, 0.650601), + (0.374897, 0.002245, 0.651876), + (0.381047, 0.001814, 0.653068), + (0.387183, 0.001434, 0.654177), + (0.393304, 0.001114, 0.655199), + (0.399411, 0.000859, 0.656133), + (0.405503, 0.000678, 0.656977), + (0.411580, 0.000577, 0.657730), + (0.417642, 0.000564, 0.658390), + (0.423689, 0.000646, 0.658956), + (0.429719, 0.000831, 0.659425), + (0.435734, 0.001127, 0.659797), + (0.441732, 0.001540, 0.660069), + (0.447714, 0.002080, 0.660240), + (0.453677, 0.002755, 0.660310), + (0.459623, 0.003574, 0.660277), + (0.465550, 0.004545, 0.660139), + (0.471457, 0.005678, 0.659897), + (0.477344, 0.006980, 0.659549), + (0.483210, 0.008460, 0.659095), + (0.489055, 0.010127, 0.658534), + (0.494877, 0.011990, 0.657865), + (0.500678, 0.014055, 0.657088), + (0.506454, 0.016333, 0.656202), + (0.512206, 0.018833, 0.655209), + (0.517933, 0.021563, 0.654109), + (0.523633, 0.024532, 0.652901), + (0.529306, 0.027747, 0.651586), + (0.534952, 0.031217, 0.650165), + (0.540570, 0.034950, 0.648640), + (0.546157, 0.038954, 0.647010), + (0.551715, 0.043136, 0.645277), + (0.557243, 0.047331, 0.643443), + (0.562738, 0.051545, 0.641509), + (0.568201, 0.055778, 0.639477), + (0.573632, 0.060028, 0.637349), + (0.579029, 0.064296, 0.635126), + (0.584391, 0.068579, 0.632812), + (0.589719, 0.072878, 0.630408), + (0.595011, 0.077190, 0.627917), + (0.600266, 0.081516, 0.625342), + (0.605485, 0.085854, 0.622686), + (0.610667, 0.090204, 0.619951), + (0.615812, 0.094564, 0.617140), + (0.620919, 0.098934, 0.614257), + (0.625987, 0.103312, 0.611305), + (0.631017, 0.107699, 0.608287), + (0.636008, 0.112092, 0.605205), + (0.640959, 0.116492, 0.602065), + (0.645872, 0.120898, 0.598867), + (0.650746, 0.125309, 0.595617), + (0.655580, 0.129725, 0.592317), + (0.660374, 0.134144, 0.588971), + (0.665129, 0.138566, 0.585582), + (0.669845, 0.142992, 0.582154), + (0.674522, 0.147419, 0.578688), + (0.679160, 0.151848, 0.575189), + (0.683758, 0.156278, 0.571660), + (0.688318, 0.160709, 0.568103), + (0.692840, 0.165141, 0.564522), + (0.697324, 0.169573, 0.560919), + (0.701769, 0.174005, 0.557296), + (0.706178, 0.178437, 0.553657), + (0.710549, 0.182868, 0.550004), + (0.714883, 0.187299, 0.546338), + (0.719181, 0.191729, 0.542663), + (0.723444, 0.196158, 0.538981), + (0.727670, 0.200586, 0.535293), + (0.731862, 0.205013, 0.531601), + (0.736019, 0.209439, 0.527908), + (0.740143, 0.213864, 0.524216), + (0.744232, 0.218288, 0.520524), + (0.748289, 0.222711, 0.516834), + (0.752312, 0.227133, 0.513149), + (0.756304, 0.231555, 0.509468), + (0.760264, 0.235976, 0.505794), + (0.764193, 0.240396, 0.502126), + (0.768090, 0.244817, 0.498465), + (0.771958, 0.249237, 0.494813), + (0.775796, 0.253658, 0.491171), + (0.779604, 0.258078, 0.487539), + (0.783383, 0.262500, 0.483918), + (0.787133, 0.266922, 0.480307), + (0.790855, 0.271345, 0.476706), + (0.794549, 0.275770, 0.473117), + (0.798216, 0.280197, 0.469538), + (0.801855, 0.284626, 0.465971), + (0.805467, 0.289057, 0.462415), + (0.809052, 0.293491, 0.458870), + (0.812612, 0.297928, 0.455338), + (0.816144, 0.302368, 0.451816), + (0.819651, 0.306812, 0.448306), + (0.823132, 0.311261, 0.444806), + (0.826588, 0.315714, 0.441316), + (0.830018, 0.320172, 0.437836), + (0.833422, 0.324635, 0.434366), + (0.836801, 0.329105, 0.430905), + (0.840155, 0.333580, 0.427455), + (0.843484, 0.338062, 0.424013), + (0.846788, 0.342551, 0.420579), + (0.850066, 0.347048, 0.417153), + (0.853319, 0.351553, 0.413734), + (0.856547, 0.356066, 0.410322), + (0.859750, 0.360588, 0.406917), + (0.862927, 0.365119, 0.403519), + (0.866078, 0.369660, 0.400126), + (0.869203, 0.374212, 0.396738), + (0.872303, 0.378774, 0.393355), + (0.875376, 0.383347, 0.389976), + (0.878423, 0.387932, 0.386600), + (0.881443, 0.392529, 0.383229), + (0.884436, 0.397139, 0.379860), + (0.887402, 0.401762, 0.376494), + (0.890340, 0.406398, 0.373130), + (0.893250, 0.411048, 0.369768), + (0.896131, 0.415712, 0.366407), + (0.898984, 0.420392, 0.363047), + (0.901807, 0.425087, 0.359688), + (0.904601, 0.429797, 0.356329), + (0.907365, 0.434524, 0.352970), + (0.910098, 0.439268, 0.349610), + (0.912800, 0.444029, 0.346251), + (0.915471, 0.448807, 0.342890), + (0.918109, 0.453603, 0.339529), + (0.920714, 0.458417, 0.336166), + (0.923287, 0.463251, 0.332801), + (0.925825, 0.468103, 0.329435), + (0.928329, 0.472975, 0.326067), + (0.930798, 0.477867, 0.322697), + (0.933232, 0.482780, 0.319325), + (0.935630, 0.487712, 0.315952), + (0.937990, 0.492667, 0.312575), + (0.940313, 0.497642, 0.309197), + (0.942598, 0.502639, 0.305816), + (0.944844, 0.507658, 0.302433), + (0.947051, 0.512699, 0.299049), + (0.949217, 0.517763, 0.295662), + (0.951344, 0.522850, 0.292275), + (0.953428, 0.527960, 0.288883), + (0.955470, 0.533093, 0.285490), + (0.957469, 0.538250, 0.282096), + (0.959424, 0.543431, 0.278701), + (0.961336, 0.548636, 0.275305), + (0.963203, 0.553865, 0.271909), + (0.965024, 0.559118, 0.268513), + (0.966798, 0.564396, 0.265118), + (0.968526, 0.569700, 0.261721), + (0.970205, 0.575028, 0.258325), + (0.971835, 0.580382, 0.254931), + (0.973416, 0.585761, 0.251540), + (0.974947, 0.591165, 0.248151), + (0.976428, 0.596595, 0.244767), + (0.977856, 0.602051, 0.241387), + (0.979233, 0.607532, 0.238013), + (0.980556, 0.613039, 0.234646), + (0.981826, 0.618572, 0.231287), + (0.983041, 0.624131, 0.227937), + (0.984199, 0.629718, 0.224595), + (0.985301, 0.635330, 0.221265), + (0.986345, 0.640969, 0.217948), + (0.987332, 0.646633, 0.214648), + (0.988260, 0.652325, 0.211364), + (0.989128, 0.658043, 0.208100), + (0.989935, 0.663787, 0.204859), + (0.990681, 0.669558, 0.201642), + (0.991365, 0.675355, 0.198453), + (0.991985, 0.681179, 0.195295), + (0.992541, 0.687030, 0.192170), + (0.993032, 0.692907, 0.189084), + (0.993456, 0.698810, 0.186041), + (0.993814, 0.704741, 0.183043), + (0.994103, 0.710698, 0.180097), + (0.994324, 0.716681, 0.177208), + (0.994474, 0.722691, 0.174381), + (0.994553, 0.728728, 0.171622), + (0.994561, 0.734791, 0.168938), + (0.994495, 0.740880, 0.166335), + (0.994355, 0.746995, 0.163821), + (0.994141, 0.753137, 0.161404), + (0.993851, 0.759304, 0.159092), + (0.993482, 0.765499, 0.156891), + (0.993033, 0.771720, 0.154808), + (0.992505, 0.777967, 0.152855), + (0.991897, 0.784239, 0.151042), + (0.991209, 0.790537, 0.149377), + (0.990439, 0.796859, 0.147870), + (0.989587, 0.803205, 0.146529), + (0.988648, 0.809579, 0.145357), + (0.987621, 0.815978, 0.144363), + (0.986509, 0.822401, 0.143557), + (0.985314, 0.828846, 0.142945), + (0.984031, 0.835315, 0.142528), + (0.982653, 0.841812, 0.142303), + (0.981190, 0.848329, 0.142279), + (0.979644, 0.854866, 0.142453), + (0.977995, 0.861432, 0.142808), + (0.976265, 0.868016, 0.143351), + (0.974443, 0.874622, 0.144061), + (0.972530, 0.881250, 0.144923), + (0.970533, 0.887896, 0.145919), + (0.968443, 0.894564, 0.147014), + (0.966271, 0.901249, 0.148180), + (0.964021, 0.907950, 0.149370), + (0.961681, 0.914672, 0.150520), + (0.959276, 0.921407, 0.151566), + (0.956808, 0.928152, 0.152409), + (0.954287, 0.934908, 0.152921), + (0.951726, 0.941671, 0.152925), + (0.949151, 0.948435, 0.152178), + (0.946602, 0.955190, 0.150328), + (0.944152, 0.961916, 0.146861), + (0.941896, 0.968590, 0.140956), + (0.940015, 0.975158, 0.131326), + ) +) viridis = _convert( - ((0.267004, 0.004874, 0.329415), - (0.268510, 0.009605, 0.335427), - (0.269944, 0.014625, 0.341379), - (0.271305, 0.019942, 0.347269), - (0.272594, 0.025563, 0.353093), - (0.273809, 0.031497, 0.358853), - (0.274952, 0.037752, 0.364543), - (0.276022, 0.044167, 0.370164), - (0.277018, 0.050344, 0.375715), - (0.277941, 0.056324, 0.381191), - (0.278791, 0.062145, 0.386592), - (0.279566, 0.067836, 0.391917), - (0.280267, 0.073417, 0.397163), - (0.280894, 0.078907, 0.402329), - (0.281446, 0.084320, 0.407414), - (0.281924, 0.089666, 0.412415), - (0.282327, 0.094955, 0.417331), - (0.282656, 0.100196, 0.422160), - (0.282910, 0.105393, 0.426902), - (0.283091, 0.110553, 0.431554), - (0.283197, 0.115680, 0.436115), - (0.283229, 0.120777, 0.440584), - (0.283187, 0.125848, 0.444960), - (0.283072, 0.130895, 0.449241), - (0.282884, 0.135920, 0.453427), - (0.282623, 0.140926, 0.457517), - (0.282290, 0.145912, 0.461510), - (0.281887, 0.150881, 0.465405), - (0.281412, 0.155834, 0.469201), - (0.280868, 0.160771, 0.472899), - (0.280255, 0.165693, 0.476498), - (0.279574, 0.170599, 0.479997), - (0.278826, 0.175490, 0.483397), - (0.278012, 0.180367, 0.486697), - (0.277134, 0.185228, 0.489898), - (0.276194, 0.190074, 0.493001), - (0.275191, 0.194905, 0.496005), - (0.274128, 0.199721, 0.498911), - (0.273006, 0.204520, 0.501721), - (0.271828, 0.209303, 0.504434), - (0.270595, 0.214069, 0.507052), - (0.269308, 0.218818, 0.509577), - (0.267968, 0.223549, 0.512008), - (0.266580, 0.228262, 0.514349), - (0.265145, 0.232956, 0.516599), - (0.263663, 0.237631, 0.518762), - (0.262138, 0.242286, 0.520837), - (0.260571, 0.246922, 0.522828), - (0.258965, 0.251537, 0.524736), - (0.257322, 0.256130, 0.526563), - (0.255645, 0.260703, 0.528312), - (0.253935, 0.265254, 0.529983), - (0.252194, 0.269783, 0.531579), - (0.250425, 0.274290, 0.533103), - (0.248629, 0.278775, 0.534556), - (0.246811, 0.283237, 0.535941), - (0.244972, 0.287675, 0.537260), - (0.243113, 0.292092, 0.538516), - (0.241237, 0.296485, 0.539709), - (0.239346, 0.300855, 0.540844), - (0.237441, 0.305202, 0.541921), - (0.235526, 0.309527, 0.542944), - (0.233603, 0.313828, 0.543914), - (0.231674, 0.318106, 0.544834), - (0.229739, 0.322361, 0.545706), - (0.227802, 0.326594, 0.546532), - (0.225863, 0.330805, 0.547314), - (0.223925, 0.334994, 0.548053), - (0.221989, 0.339161, 0.548752), - (0.220057, 0.343307, 0.549413), - (0.218130, 0.347432, 0.550038), - (0.216210, 0.351535, 0.550627), - (0.214298, 0.355619, 0.551184), - (0.212395, 0.359683, 0.551710), - (0.210503, 0.363727, 0.552206), - (0.208623, 0.367752, 0.552675), - (0.206756, 0.371758, 0.553117), - (0.204903, 0.375746, 0.553533), - (0.203063, 0.379716, 0.553925), - (0.201239, 0.383670, 0.554294), - (0.199430, 0.387607, 0.554642), - (0.197636, 0.391528, 0.554969), - (0.195860, 0.395433, 0.555276), - (0.194100, 0.399323, 0.555565), - (0.192357, 0.403199, 0.555836), - (0.190631, 0.407061, 0.556089), - (0.188923, 0.410910, 0.556326), - (0.187231, 0.414746, 0.556547), - (0.185556, 0.418570, 0.556753), - (0.183898, 0.422383, 0.556944), - (0.182256, 0.426184, 0.557120), - (0.180629, 0.429975, 0.557282), - (0.179019, 0.433756, 0.557430), - (0.177423, 0.437527, 0.557565), - (0.175841, 0.441290, 0.557685), - (0.174274, 0.445044, 0.557792), - (0.172719, 0.448791, 0.557885), - (0.171176, 0.452530, 0.557965), - (0.169646, 0.456262, 0.558030), - (0.168126, 0.459988, 0.558082), - (0.166617, 0.463708, 0.558119), - (0.165117, 0.467423, 0.558141), - (0.163625, 0.471133, 0.558148), - (0.162142, 0.474838, 0.558140), - (0.160665, 0.478540, 0.558115), - (0.159194, 0.482237, 0.558073), - (0.157729, 0.485932, 0.558013), - (0.156270, 0.489624, 0.557936), - (0.154815, 0.493313, 0.557840), - (0.153364, 0.497000, 0.557724), - (0.151918, 0.500685, 0.557587), - (0.150476, 0.504369, 0.557430), - (0.149039, 0.508051, 0.557250), - (0.147607, 0.511733, 0.557049), - (0.146180, 0.515413, 0.556823), - (0.144759, 0.519093, 0.556572), - (0.143343, 0.522773, 0.556295), - (0.141935, 0.526453, 0.555991), - (0.140536, 0.530132, 0.555659), - (0.139147, 0.533812, 0.555298), - (0.137770, 0.537492, 0.554906), - (0.136408, 0.541173, 0.554483), - (0.135066, 0.544853, 0.554029), - (0.133743, 0.548535, 0.553541), - (0.132444, 0.552216, 0.553018), - (0.131172, 0.555899, 0.552459), - (0.129933, 0.559582, 0.551864), - (0.128729, 0.563265, 0.551229), - (0.127568, 0.566949, 0.550556), - (0.126453, 0.570633, 0.549841), - (0.125394, 0.574318, 0.549086), - (0.124395, 0.578002, 0.548287), - (0.123463, 0.581687, 0.547445), - (0.122606, 0.585371, 0.546557), - (0.121831, 0.589055, 0.545623), - (0.121148, 0.592739, 0.544641), - (0.120565, 0.596422, 0.543611), - (0.120092, 0.600104, 0.542530), - (0.119738, 0.603785, 0.541400), - (0.119512, 0.607464, 0.540218), - (0.119423, 0.611141, 0.538982), - (0.119483, 0.614817, 0.537692), - (0.119699, 0.618490, 0.536347), - (0.120081, 0.622161, 0.534946), - (0.120638, 0.625828, 0.533488), - (0.121380, 0.629492, 0.531973), - (0.122312, 0.633153, 0.530398), - (0.123444, 0.636809, 0.528763), - (0.124780, 0.640461, 0.527068), - (0.126326, 0.644107, 0.525311), - (0.128087, 0.647749, 0.523491), - (0.130067, 0.651384, 0.521608), - (0.132268, 0.655014, 0.519661), - (0.134692, 0.658636, 0.517649), - (0.137339, 0.662252, 0.515571), - (0.140210, 0.665859, 0.513427), - (0.143303, 0.669459, 0.511215), - (0.146616, 0.673050, 0.508936), - (0.150148, 0.676631, 0.506589), - (0.153894, 0.680203, 0.504172), - (0.157851, 0.683765, 0.501686), - (0.162016, 0.687316, 0.499129), - (0.166383, 0.690856, 0.496502), - (0.170948, 0.694384, 0.493803), - (0.175707, 0.697900, 0.491033), - (0.180653, 0.701402, 0.488189), - (0.185783, 0.704891, 0.485273), - (0.191090, 0.708366, 0.482284), - (0.196571, 0.711827, 0.479221), - (0.202219, 0.715272, 0.476084), - (0.208030, 0.718701, 0.472873), - (0.214000, 0.722114, 0.469588), - (0.220124, 0.725509, 0.466226), - (0.226397, 0.728888, 0.462789), - (0.232815, 0.732247, 0.459277), - (0.239374, 0.735588, 0.455688), - (0.246070, 0.738910, 0.452024), - (0.252899, 0.742211, 0.448284), - (0.259857, 0.745492, 0.444467), - (0.266941, 0.748751, 0.440573), - (0.274149, 0.751988, 0.436601), - (0.281477, 0.755203, 0.432552), - (0.288921, 0.758394, 0.428426), - (0.296479, 0.761561, 0.424223), - (0.304148, 0.764704, 0.419943), - (0.311925, 0.767822, 0.415586), - (0.319809, 0.770914, 0.411152), - (0.327796, 0.773980, 0.406640), - (0.335885, 0.777018, 0.402049), - (0.344074, 0.780029, 0.397381), - (0.352360, 0.783011, 0.392636), - (0.360741, 0.785964, 0.387814), - (0.369214, 0.788888, 0.382914), - (0.377779, 0.791781, 0.377939), - (0.386433, 0.794644, 0.372886), - (0.395174, 0.797475, 0.367757), - (0.404001, 0.800275, 0.362552), - (0.412913, 0.803041, 0.357269), - (0.421908, 0.805774, 0.351910), - (0.430983, 0.808473, 0.346476), - (0.440137, 0.811138, 0.340967), - (0.449368, 0.813768, 0.335384), - (0.458674, 0.816363, 0.329727), - (0.468053, 0.818921, 0.323998), - (0.477504, 0.821444, 0.318195), - (0.487026, 0.823929, 0.312321), - (0.496615, 0.826376, 0.306377), - (0.506271, 0.828786, 0.300362), - (0.515992, 0.831158, 0.294279), - (0.525776, 0.833491, 0.288127), - (0.535621, 0.835785, 0.281908), - (0.545524, 0.838039, 0.275626), - (0.555484, 0.840254, 0.269281), - (0.565498, 0.842430, 0.262877), - (0.575563, 0.844566, 0.256415), - (0.585678, 0.846661, 0.249897), - (0.595839, 0.848717, 0.243329), - (0.606045, 0.850733, 0.236712), - (0.616293, 0.852709, 0.230052), - (0.626579, 0.854645, 0.223353), - (0.636902, 0.856542, 0.216620), - (0.647257, 0.858400, 0.209861), - (0.657642, 0.860219, 0.203082), - (0.668054, 0.861999, 0.196293), - (0.678489, 0.863742, 0.189503), - (0.688944, 0.865448, 0.182725), - (0.699415, 0.867117, 0.175971), - (0.709898, 0.868751, 0.169257), - (0.720391, 0.870350, 0.162603), - (0.730889, 0.871916, 0.156029), - (0.741388, 0.873449, 0.149561), - (0.751884, 0.874951, 0.143228), - (0.762373, 0.876424, 0.137064), - (0.772852, 0.877868, 0.131109), - (0.783315, 0.879285, 0.125405), - (0.793760, 0.880678, 0.120005), - (0.804182, 0.882046, 0.114965), - (0.814576, 0.883393, 0.110347), - (0.824940, 0.884720, 0.106217), - (0.835270, 0.886029, 0.102646), - (0.845561, 0.887322, 0.099702), - (0.855810, 0.888601, 0.097452), - (0.866013, 0.889868, 0.095953), - (0.876168, 0.891125, 0.095250), - (0.886271, 0.892374, 0.095374), - (0.896320, 0.893616, 0.096335), - (0.906311, 0.894855, 0.098125), - (0.916242, 0.896091, 0.100717), - (0.926106, 0.897330, 0.104071), - (0.935904, 0.898570, 0.108131), - (0.945636, 0.899815, 0.112838), - (0.955300, 0.901065, 0.118128), - (0.964894, 0.902323, 0.123941), - (0.974417, 0.903590, 0.130215), - (0.983868, 0.904867, 0.136897), - (0.993248, 0.906157, 0.143936))) + ( + (0.267004, 0.004874, 0.329415), + (0.268510, 0.009605, 0.335427), + (0.269944, 0.014625, 0.341379), + (0.271305, 0.019942, 0.347269), + (0.272594, 0.025563, 0.353093), + (0.273809, 0.031497, 0.358853), + (0.274952, 0.037752, 0.364543), + (0.276022, 0.044167, 0.370164), + (0.277018, 0.050344, 0.375715), + (0.277941, 0.056324, 0.381191), + (0.278791, 0.062145, 0.386592), + (0.279566, 0.067836, 0.391917), + (0.280267, 0.073417, 0.397163), + (0.280894, 0.078907, 0.402329), + (0.281446, 0.084320, 0.407414), + (0.281924, 0.089666, 0.412415), + (0.282327, 0.094955, 0.417331), + (0.282656, 0.100196, 0.422160), + (0.282910, 0.105393, 0.426902), + (0.283091, 0.110553, 0.431554), + (0.283197, 0.115680, 0.436115), + (0.283229, 0.120777, 0.440584), + (0.283187, 0.125848, 0.444960), + (0.283072, 0.130895, 0.449241), + (0.282884, 0.135920, 0.453427), + (0.282623, 0.140926, 0.457517), + (0.282290, 0.145912, 0.461510), + (0.281887, 0.150881, 0.465405), + (0.281412, 0.155834, 0.469201), + (0.280868, 0.160771, 0.472899), + (0.280255, 0.165693, 0.476498), + (0.279574, 0.170599, 0.479997), + (0.278826, 0.175490, 0.483397), + (0.278012, 0.180367, 0.486697), + (0.277134, 0.185228, 0.489898), + (0.276194, 0.190074, 0.493001), + (0.275191, 0.194905, 0.496005), + (0.274128, 0.199721, 0.498911), + (0.273006, 0.204520, 0.501721), + (0.271828, 0.209303, 0.504434), + (0.270595, 0.214069, 0.507052), + (0.269308, 0.218818, 0.509577), + (0.267968, 0.223549, 0.512008), + (0.266580, 0.228262, 0.514349), + (0.265145, 0.232956, 0.516599), + (0.263663, 0.237631, 0.518762), + (0.262138, 0.242286, 0.520837), + (0.260571, 0.246922, 0.522828), + (0.258965, 0.251537, 0.524736), + (0.257322, 0.256130, 0.526563), + (0.255645, 0.260703, 0.528312), + (0.253935, 0.265254, 0.529983), + (0.252194, 0.269783, 0.531579), + (0.250425, 0.274290, 0.533103), + (0.248629, 0.278775, 0.534556), + (0.246811, 0.283237, 0.535941), + (0.244972, 0.287675, 0.537260), + (0.243113, 0.292092, 0.538516), + (0.241237, 0.296485, 0.539709), + (0.239346, 0.300855, 0.540844), + (0.237441, 0.305202, 0.541921), + (0.235526, 0.309527, 0.542944), + (0.233603, 0.313828, 0.543914), + (0.231674, 0.318106, 0.544834), + (0.229739, 0.322361, 0.545706), + (0.227802, 0.326594, 0.546532), + (0.225863, 0.330805, 0.547314), + (0.223925, 0.334994, 0.548053), + (0.221989, 0.339161, 0.548752), + (0.220057, 0.343307, 0.549413), + (0.218130, 0.347432, 0.550038), + (0.216210, 0.351535, 0.550627), + (0.214298, 0.355619, 0.551184), + (0.212395, 0.359683, 0.551710), + (0.210503, 0.363727, 0.552206), + (0.208623, 0.367752, 0.552675), + (0.206756, 0.371758, 0.553117), + (0.204903, 0.375746, 0.553533), + (0.203063, 0.379716, 0.553925), + (0.201239, 0.383670, 0.554294), + (0.199430, 0.387607, 0.554642), + (0.197636, 0.391528, 0.554969), + (0.195860, 0.395433, 0.555276), + (0.194100, 0.399323, 0.555565), + (0.192357, 0.403199, 0.555836), + (0.190631, 0.407061, 0.556089), + (0.188923, 0.410910, 0.556326), + (0.187231, 0.414746, 0.556547), + (0.185556, 0.418570, 0.556753), + (0.183898, 0.422383, 0.556944), + (0.182256, 0.426184, 0.557120), + (0.180629, 0.429975, 0.557282), + (0.179019, 0.433756, 0.557430), + (0.177423, 0.437527, 0.557565), + (0.175841, 0.441290, 0.557685), + (0.174274, 0.445044, 0.557792), + (0.172719, 0.448791, 0.557885), + (0.171176, 0.452530, 0.557965), + (0.169646, 0.456262, 0.558030), + (0.168126, 0.459988, 0.558082), + (0.166617, 0.463708, 0.558119), + (0.165117, 0.467423, 0.558141), + (0.163625, 0.471133, 0.558148), + (0.162142, 0.474838, 0.558140), + (0.160665, 0.478540, 0.558115), + (0.159194, 0.482237, 0.558073), + (0.157729, 0.485932, 0.558013), + (0.156270, 0.489624, 0.557936), + (0.154815, 0.493313, 0.557840), + (0.153364, 0.497000, 0.557724), + (0.151918, 0.500685, 0.557587), + (0.150476, 0.504369, 0.557430), + (0.149039, 0.508051, 0.557250), + (0.147607, 0.511733, 0.557049), + (0.146180, 0.515413, 0.556823), + (0.144759, 0.519093, 0.556572), + (0.143343, 0.522773, 0.556295), + (0.141935, 0.526453, 0.555991), + (0.140536, 0.530132, 0.555659), + (0.139147, 0.533812, 0.555298), + (0.137770, 0.537492, 0.554906), + (0.136408, 0.541173, 0.554483), + (0.135066, 0.544853, 0.554029), + (0.133743, 0.548535, 0.553541), + (0.132444, 0.552216, 0.553018), + (0.131172, 0.555899, 0.552459), + (0.129933, 0.559582, 0.551864), + (0.128729, 0.563265, 0.551229), + (0.127568, 0.566949, 0.550556), + (0.126453, 0.570633, 0.549841), + (0.125394, 0.574318, 0.549086), + (0.124395, 0.578002, 0.548287), + (0.123463, 0.581687, 0.547445), + (0.122606, 0.585371, 0.546557), + (0.121831, 0.589055, 0.545623), + (0.121148, 0.592739, 0.544641), + (0.120565, 0.596422, 0.543611), + (0.120092, 0.600104, 0.542530), + (0.119738, 0.603785, 0.541400), + (0.119512, 0.607464, 0.540218), + (0.119423, 0.611141, 0.538982), + (0.119483, 0.614817, 0.537692), + (0.119699, 0.618490, 0.536347), + (0.120081, 0.622161, 0.534946), + (0.120638, 0.625828, 0.533488), + (0.121380, 0.629492, 0.531973), + (0.122312, 0.633153, 0.530398), + (0.123444, 0.636809, 0.528763), + (0.124780, 0.640461, 0.527068), + (0.126326, 0.644107, 0.525311), + (0.128087, 0.647749, 0.523491), + (0.130067, 0.651384, 0.521608), + (0.132268, 0.655014, 0.519661), + (0.134692, 0.658636, 0.517649), + (0.137339, 0.662252, 0.515571), + (0.140210, 0.665859, 0.513427), + (0.143303, 0.669459, 0.511215), + (0.146616, 0.673050, 0.508936), + (0.150148, 0.676631, 0.506589), + (0.153894, 0.680203, 0.504172), + (0.157851, 0.683765, 0.501686), + (0.162016, 0.687316, 0.499129), + (0.166383, 0.690856, 0.496502), + (0.170948, 0.694384, 0.493803), + (0.175707, 0.697900, 0.491033), + (0.180653, 0.701402, 0.488189), + (0.185783, 0.704891, 0.485273), + (0.191090, 0.708366, 0.482284), + (0.196571, 0.711827, 0.479221), + (0.202219, 0.715272, 0.476084), + (0.208030, 0.718701, 0.472873), + (0.214000, 0.722114, 0.469588), + (0.220124, 0.725509, 0.466226), + (0.226397, 0.728888, 0.462789), + (0.232815, 0.732247, 0.459277), + (0.239374, 0.735588, 0.455688), + (0.246070, 0.738910, 0.452024), + (0.252899, 0.742211, 0.448284), + (0.259857, 0.745492, 0.444467), + (0.266941, 0.748751, 0.440573), + (0.274149, 0.751988, 0.436601), + (0.281477, 0.755203, 0.432552), + (0.288921, 0.758394, 0.428426), + (0.296479, 0.761561, 0.424223), + (0.304148, 0.764704, 0.419943), + (0.311925, 0.767822, 0.415586), + (0.319809, 0.770914, 0.411152), + (0.327796, 0.773980, 0.406640), + (0.335885, 0.777018, 0.402049), + (0.344074, 0.780029, 0.397381), + (0.352360, 0.783011, 0.392636), + (0.360741, 0.785964, 0.387814), + (0.369214, 0.788888, 0.382914), + (0.377779, 0.791781, 0.377939), + (0.386433, 0.794644, 0.372886), + (0.395174, 0.797475, 0.367757), + (0.404001, 0.800275, 0.362552), + (0.412913, 0.803041, 0.357269), + (0.421908, 0.805774, 0.351910), + (0.430983, 0.808473, 0.346476), + (0.440137, 0.811138, 0.340967), + (0.449368, 0.813768, 0.335384), + (0.458674, 0.816363, 0.329727), + (0.468053, 0.818921, 0.323998), + (0.477504, 0.821444, 0.318195), + (0.487026, 0.823929, 0.312321), + (0.496615, 0.826376, 0.306377), + (0.506271, 0.828786, 0.300362), + (0.515992, 0.831158, 0.294279), + (0.525776, 0.833491, 0.288127), + (0.535621, 0.835785, 0.281908), + (0.545524, 0.838039, 0.275626), + (0.555484, 0.840254, 0.269281), + (0.565498, 0.842430, 0.262877), + (0.575563, 0.844566, 0.256415), + (0.585678, 0.846661, 0.249897), + (0.595839, 0.848717, 0.243329), + (0.606045, 0.850733, 0.236712), + (0.616293, 0.852709, 0.230052), + (0.626579, 0.854645, 0.223353), + (0.636902, 0.856542, 0.216620), + (0.647257, 0.858400, 0.209861), + (0.657642, 0.860219, 0.203082), + (0.668054, 0.861999, 0.196293), + (0.678489, 0.863742, 0.189503), + (0.688944, 0.865448, 0.182725), + (0.699415, 0.867117, 0.175971), + (0.709898, 0.868751, 0.169257), + (0.720391, 0.870350, 0.162603), + (0.730889, 0.871916, 0.156029), + (0.741388, 0.873449, 0.149561), + (0.751884, 0.874951, 0.143228), + (0.762373, 0.876424, 0.137064), + (0.772852, 0.877868, 0.131109), + (0.783315, 0.879285, 0.125405), + (0.793760, 0.880678, 0.120005), + (0.804182, 0.882046, 0.114965), + (0.814576, 0.883393, 0.110347), + (0.824940, 0.884720, 0.106217), + (0.835270, 0.886029, 0.102646), + (0.845561, 0.887322, 0.099702), + (0.855810, 0.888601, 0.097452), + (0.866013, 0.889868, 0.095953), + (0.876168, 0.891125, 0.095250), + (0.886271, 0.892374, 0.095374), + (0.896320, 0.893616, 0.096335), + (0.906311, 0.894855, 0.098125), + (0.916242, 0.896091, 0.100717), + (0.926106, 0.897330, 0.104071), + (0.935904, 0.898570, 0.108131), + (0.945636, 0.899815, 0.112838), + (0.955300, 0.901065, 0.118128), + (0.964894, 0.902323, 0.123941), + (0.974417, 0.903590, 0.130215), + (0.983868, 0.904867, 0.136897), + (0.993248, 0.906157, 0.143936), + ) +) -__all__ = ['magma', 'inferno', 'plasma', 'viridis'] +__all__ = ["magma", "inferno", "plasma", "viridis"] diff --git a/tensorboard/plugins/beholder/file_system_tools.py b/tensorboard/plugins/beholder/file_system_tools.py index f533182e10..296edcf219 100644 --- a/tensorboard/plugins/beholder/file_system_tools.py +++ b/tensorboard/plugins/beholder/file_system_tools.py @@ -27,42 +27,42 @@ logger = tb_logging.get_logger() -def write_file(contents, path, mode='wb'): - with tf.io.gfile.GFile(path, mode) as new_file: - new_file.write(contents) +def write_file(contents, path, mode="wb"): + with tf.io.gfile.GFile(path, mode) as new_file: + new_file.write(contents) def read_tensor_summary(path): - with tf.io.gfile.GFile(path, 'rb') as summary_file: - summary_string = summary_file.read() + with tf.io.gfile.GFile(path, "rb") as summary_file: + summary_string = summary_file.read() - if not summary_string: - raise message.DecodeError('Empty summary.') + if not summary_string: + raise message.DecodeError("Empty summary.") - summary_proto = summary_pb2.Summary() - summary_proto.ParseFromString(summary_string) - tensor_proto = summary_proto.value[0].tensor - array = tensor_util.make_ndarray(tensor_proto) + summary_proto = summary_pb2.Summary() + summary_proto.ParseFromString(summary_string) + tensor_proto = summary_proto.value[0].tensor + array = tensor_util.make_ndarray(tensor_proto) - return array + return array def write_pickle(obj, path): - with tf.io.gfile.GFile(path, 'wb') as new_file: - pickle.dump(obj, new_file) + with tf.io.gfile.GFile(path, "wb") as new_file: + pickle.dump(obj, new_file) def read_pickle(path, default=None): - try: - with tf.io.gfile.GFile(path, 'rb') as pickle_file: - result = pickle.load(pickle_file) - - except (IOError, EOFError, ValueError, tf.errors.NotFoundError) as e: - if not isinstance(e, tf.errors.NotFoundError): - logger.error('Error reading pickle value: %s', e) - if default is not None: - result = default - else: - raise - - return result + try: + with tf.io.gfile.GFile(path, "rb") as pickle_file: + result = pickle.load(pickle_file) + + except (IOError, EOFError, ValueError, tf.errors.NotFoundError) as e: + if not isinstance(e, tf.errors.NotFoundError): + logger.error("Error reading pickle value: %s", e) + if default is not None: + result = default + else: + raise + + return result diff --git a/tensorboard/plugins/beholder/im_util.py b/tensorboard/plugins/beholder/im_util.py index fd80ff9b30..db3e6e9ef6 100644 --- a/tensorboard/plugins/beholder/im_util.py +++ b/tensorboard/plugins/beholder/im_util.py @@ -28,130 +28,128 @@ # pylint: disable=not-context-manager + def global_extrema(arrays): - return min([x.min() for x in arrays]), max([x.max() for x in arrays]) + return min([x.min() for x in arrays]), max([x.max() for x in arrays]) def scale_sections(sections, scaling_scope): - ''' + """ input: unscaled sections. returns: sections scaled to [0, 255] - ''' - new_sections = [] + """ + new_sections = [] - if scaling_scope == 'layer': - for section in sections: - new_sections.append(scale_image_for_display(section)) + if scaling_scope == "layer": + for section in sections: + new_sections.append(scale_image_for_display(section)) - elif scaling_scope == 'network': - global_min, global_max = global_extrema(sections) + elif scaling_scope == "network": + global_min, global_max = global_extrema(sections) - for section in sections: - new_sections.append(scale_image_for_display(section, - global_min, - global_max)) - return new_sections + for section in sections: + new_sections.append( + scale_image_for_display(section, global_min, global_max) + ) + return new_sections def scale_image_for_display(image, minimum=None, maximum=None): - image = image.astype(float) + image = image.astype(float) - minimum = image.min() if minimum is None else minimum - image -= minimum + minimum = image.min() if minimum is None else minimum + image -= minimum - maximum = image.max() if maximum is None else maximum + maximum = image.max() if maximum is None else maximum - if maximum == 0: - return image - else: - image *= 255 / maximum - return image.astype(np.uint8) + if maximum == 0: + return image + else: + image *= 255 / maximum + return image.astype(np.uint8) def pad_to_shape(array, shape, constant=245): - padding = [] + padding = [] - for actual_dim, target_dim in zip(array.shape, shape): - start_padding = 0 - end_padding = target_dim - actual_dim + for actual_dim, target_dim in zip(array.shape, shape): + start_padding = 0 + end_padding = target_dim - actual_dim - padding.append((start_padding, end_padding)) + padding.append((start_padding, end_padding)) - return np.pad(array, padding, mode='constant', constant_values=constant) + return np.pad(array, padding, mode="constant", constant_values=constant) -def apply_colormap(image, colormap='magma'): - if colormap == 'grayscale': - return image - cm = getattr(colormaps, colormap) - return image if cm is None else cm[image] +def apply_colormap(image, colormap="magma"): + if colormap == "grayscale": + return image + cm = getattr(colormaps, colormap) + return image if cm is None else cm[image] class PNGDecoder(op_evaluator.PersistentOpEvaluator): + def __init__(self): + super(PNGDecoder, self).__init__() + self._image_placeholder = None + self._decode_op = None - def __init__(self): - super(PNGDecoder, self).__init__() - self._image_placeholder = None - self._decode_op = None - - - def initialize_graph(self): - self._image_placeholder = tf.compat.v1.placeholder(dtype=tf.string) - self._decode_op = tf.image.decode_png(self._image_placeholder) + def initialize_graph(self): + self._image_placeholder = tf.compat.v1.placeholder(dtype=tf.string) + self._decode_op = tf.image.decode_png(self._image_placeholder) - - # pylint: disable=arguments-differ - def run(self, image): - return self._decode_op.eval(feed_dict={ - self._image_placeholder: image, - }) + # pylint: disable=arguments-differ + def run(self, image): + return self._decode_op.eval(feed_dict={self._image_placeholder: image,}) class Resizer(op_evaluator.PersistentOpEvaluator): - - def __init__(self): - super(Resizer, self).__init__() - self._image_placeholder = None - self._size_placeholder = None - self._resize_op = None - - - def initialize_graph(self): - self._image_placeholder = tf.compat.v1.placeholder(dtype=tf.float32) - self._size_placeholder = tf.compat.v1.placeholder(dtype=tf.int32) - self._resize_op = tf.compat.v1.image.resize_nearest_neighbor( - self._image_placeholder, - self._size_placeholder, - ) - - # pylint: disable=arguments-differ - def run(self, image, height, width): - if len(image.shape) == 2: - image = image.reshape([image.shape[0], image.shape[1], 1]) - - resized = np.squeeze(self._resize_op.eval(feed_dict={ - self._image_placeholder: [image], - self._size_placeholder: [height, width] - })) - - return resized + def __init__(self): + super(Resizer, self).__init__() + self._image_placeholder = None + self._size_placeholder = None + self._resize_op = None + + def initialize_graph(self): + self._image_placeholder = tf.compat.v1.placeholder(dtype=tf.float32) + self._size_placeholder = tf.compat.v1.placeholder(dtype=tf.int32) + self._resize_op = tf.compat.v1.image.resize_nearest_neighbor( + self._image_placeholder, self._size_placeholder, + ) + + # pylint: disable=arguments-differ + def run(self, image, height, width): + if len(image.shape) == 2: + image = image.reshape([image.shape[0], image.shape[1], 1]) + + resized = np.squeeze( + self._resize_op.eval( + feed_dict={ + self._image_placeholder: [image], + self._size_placeholder: [height, width], + } + ) + ) + + return resized decode_png = PNGDecoder() resize = Resizer() + def read_image(filename): - with tf.io.gfile.GFile(filename, 'rb') as image_file: - return np.array(decode_png(image_file.read())) + with tf.io.gfile.GFile(filename, "rb") as image_file: + return np.array(decode_png(image_file.read())) def write_image(array, filename): - with tf.io.gfile.GFile(filename, 'w') as image_file: - image_file.write(encoder.encode_png(array)) + with tf.io.gfile.GFile(filename, "w") as image_file: + image_file.write(encoder.encode_png(array)) def get_image_relative_to_script(filename): - script_directory = os.path.dirname(__file__) - filename = os.path.join(script_directory, 'resources', filename) + script_directory = os.path.dirname(__file__) + filename = os.path.join(script_directory, "resources", filename) - return read_image(filename) + return read_image(filename) diff --git a/tensorboard/plugins/beholder/shared_config.py b/tensorboard/plugins/beholder/shared_config.py index b987a50cfb..0660ddfa5d 100644 --- a/tensorboard/plugins/beholder/shared_config.py +++ b/tensorboard/plugins/beholder/shared_config.py @@ -16,22 +16,22 @@ from __future__ import division from __future__ import print_function -PLUGIN_NAME = 'beholder' -TAG_NAME = 'beholder-frame' -SUMMARY_FILENAME = 'frame.summary' -CONFIG_FILENAME = 'config.pkl' -SECTION_INFO_FILENAME = 'section-info.pkl' -SUMMARY_COLLECTION_KEY_NAME = 'summaries_beholder' +PLUGIN_NAME = "beholder" +TAG_NAME = "beholder-frame" +SUMMARY_FILENAME = "frame.summary" +CONFIG_FILENAME = "config.pkl" +SECTION_INFO_FILENAME = "section-info.pkl" +SUMMARY_COLLECTION_KEY_NAME = "summaries_beholder" DEFAULT_CONFIG = { - 'values': 'trainable_variables', - 'mode': 'variance', - 'scaling': 'layer', - 'window_size': 15, - 'FPS': 10, - 'is_recording': False, - 'show_all': False, - 'colormap': 'magma' + "values": "trainable_variables", + "mode": "variance", + "scaling": "layer", + "window_size": 15, + "FPS": 10, + "is_recording": False, + "show_all": False, + "colormap": "magma", } SECTION_HEIGHT = 128 diff --git a/tensorboard/plugins/beholder/video_writing.py b/tensorboard/plugins/beholder/video_writing.py index b2b2d2598a..d02af7c338 100644 --- a/tensorboard/plugins/beholder/video_writing.py +++ b/tensorboard/plugins/beholder/video_writing.py @@ -31,172 +31,194 @@ class VideoWriter(object): - """Video file writer that can use different output types. - - Each VideoWriter instance writes video files to a specified directory, using - the first available VideoOutput from the provided list. - """ - - def __init__(self, directory, outputs): - self.directory = directory - # Filter to the available outputs - self.outputs = [out for out in outputs if out.available()] - if not self.outputs: - raise IOError('No available video outputs') - self.output_index = 0 - self.output = None - self.frame_shape = None - - def current_output(self): - return self.outputs[self.output_index] - - def write_frame(self, np_array): - # Reset whenever we encounter a new frame shape. - if self.frame_shape != np_array.shape: - if self.output: - self.output.close() - self.output = None - self.frame_shape = np_array.shape - logger.info('Starting video with frame shape: %s', self.frame_shape) - # Write the frame, advancing across output types as necessary. - original_output_index = self.output_index - for self.output_index in range(original_output_index, len(self.outputs)): - try: - if not self.output: - new_output = self.outputs[self.output_index] - if self.output_index > original_output_index: - logger.warn( - 'Falling back to video output %s', new_output.name()) - self.output = new_output(self.directory, self.frame_shape) - self.output.emit_frame(np_array) - return - except (IOError, OSError) as e: - logger.warn( - 'Video output type %s not available: %s', - self.current_output().name(), str(e)) + """Video file writer that can use different output types. + + Each VideoWriter instance writes video files to a specified + directory, using the first available VideoOutput from the provided + list. + """ + + def __init__(self, directory, outputs): + self.directory = directory + # Filter to the available outputs + self.outputs = [out for out in outputs if out.available()] + if not self.outputs: + raise IOError("No available video outputs") + self.output_index = 0 + self.output = None + self.frame_shape = None + + def current_output(self): + return self.outputs[self.output_index] + + def write_frame(self, np_array): + # Reset whenever we encounter a new frame shape. + if self.frame_shape != np_array.shape: + if self.output: + self.output.close() + self.output = None + self.frame_shape = np_array.shape + logger.info("Starting video with frame shape: %s", self.frame_shape) + # Write the frame, advancing across output types as necessary. + original_output_index = self.output_index + for self.output_index in range( + original_output_index, len(self.outputs) + ): + try: + if not self.output: + new_output = self.outputs[self.output_index] + if self.output_index > original_output_index: + logger.warn( + "Falling back to video output %s", new_output.name() + ) + self.output = new_output(self.directory, self.frame_shape) + self.output.emit_frame(np_array) + return + except (IOError, OSError) as e: + logger.warn( + "Video output type %s not available: %s", + self.current_output().name(), + str(e), + ) + if self.output: + self.output.close() + self.output = None + raise IOError("Exhausted available video outputs") + + def finish(self): if self.output: - self.output.close() + self.output.close() self.output = None - raise IOError('Exhausted available video outputs') - - def finish(self): - if self.output: - self.output.close() - self.output = None - self.frame_shape = None - # Reconsider failed outputs when video is manually restarted. - self.output_index = 0 + self.frame_shape = None + # Reconsider failed outputs when video is manually restarted. + self.output_index = 0 @six.add_metaclass(abc.ABCMeta) class VideoOutput(object): - """Base class for video outputs supported by VideoWriter.""" + """Base class for video outputs supported by VideoWriter.""" - # Would add @abc.abstractmethod in python 3.3+ - @classmethod - def available(cls): - raise NotImplementedError() + # Would add @abc.abstractmethod in python 3.3+ + @classmethod + def available(cls): + raise NotImplementedError() - @classmethod - def name(cls): - return cls.__name__ + @classmethod + def name(cls): + return cls.__name__ - @abc.abstractmethod - def emit_frame(self, np_array): - raise NotImplementedError() + @abc.abstractmethod + def emit_frame(self, np_array): + raise NotImplementedError() - @abc.abstractmethod - def close(self): - raise NotImplementedError() + @abc.abstractmethod + def close(self): + raise NotImplementedError() class PNGVideoOutput(VideoOutput): - """Video output implemented by writing individual PNGs to disk.""" + """Video output implemented by writing individual PNGs to disk.""" - @classmethod - def available(cls): - return True + @classmethod + def available(cls): + return True - def __init__(self, directory, frame_shape): - del frame_shape # unused - self.directory = directory + '/video-frames-{}'.format(time.time()) - self.frame_num = 0 - tf.io.gfile.makedirs(self.directory) + def __init__(self, directory, frame_shape): + del frame_shape # unused + self.directory = directory + "/video-frames-{}".format(time.time()) + self.frame_num = 0 + tf.io.gfile.makedirs(self.directory) - def emit_frame(self, np_array): - filename = self.directory + '/{:05}.png'.format(self.frame_num) - im_util.write_image(np_array.astype(np.uint8), filename) - self.frame_num += 1 + def emit_frame(self, np_array): + filename = self.directory + "/{:05}.png".format(self.frame_num) + im_util.write_image(np_array.astype(np.uint8), filename) + self.frame_num += 1 - def close(self): - pass + def close(self): + pass class FFmpegVideoOutput(VideoOutput): - """Video output implemented by streaming to FFmpeg with .mp4 output.""" - - @classmethod - def available(cls): - # Silently check if ffmpeg is available. - try: - with open(os.devnull, 'wb') as devnull: - subprocess.check_call( - ['ffmpeg', '-version'], stdout=devnull, stderr=devnull) - return True - except (OSError, subprocess.CalledProcessError): - return False - - def __init__(self, directory, frame_shape): - self.filename = directory + '/video-{}.webm'.format(time.time()) - if len(frame_shape) != 3: - raise ValueError( - 'Expected rank-3 array for frame, got %s' % str(frame_shape)) - # Set input pixel format based on channel count. - if frame_shape[2] == 1: - pix_fmt = 'gray' - elif frame_shape[2] == 3: - pix_fmt = 'rgb24' - else: - raise ValueError('Unsupported channel count %d' % frame_shape[2]) - - command = [ - 'ffmpeg', - '-y', # Overwite output - # Input options - raw video file format and codec. - '-f', 'rawvideo', - '-vcodec', 'rawvideo', - '-s', '%dx%d' % (frame_shape[1], frame_shape[0]), # Width x height. - '-pix_fmt', pix_fmt, - '-r', '15', # Frame rate: arbitrarily use 15 frames per second. - '-i', '-', # Use stdin. - '-an', # No audio. - # Output options - use lossless VP9 codec inside .webm. - '-vcodec', 'libvpx-vp9', - '-lossless', '1', - # Using YUV is most compatible, though conversion from RGB skews colors. - '-pix_fmt', 'yuv420p', - self.filename - ] - PIPE = subprocess.PIPE - self.ffmpeg = subprocess.Popen( - command, stdin=PIPE, stdout=PIPE, stderr=PIPE) - - def _handle_error(self): - _, stderr = self.ffmpeg.communicate() - bar = '=' * 40 - logger.error( - 'Error writing to FFmpeg:\n%s\n%s\n%s', bar, stderr.rstrip('\n'), bar) - - def emit_frame(self, np_array): - try: - self.ffmpeg.stdin.write(np_array.tobytes()) - self.ffmpeg.stdin.flush() - except IOError: - self._handle_error() - raise IOError('Failure invoking FFmpeg') - - def close(self): - if self.ffmpeg.poll() is None: - # Close stdin and consume and discard stderr/stdout. - self.ffmpeg.communicate() - self.ffmpeg = None + """Video output implemented by streaming to FFmpeg with .mp4 output.""" + + @classmethod + def available(cls): + # Silently check if ffmpeg is available. + try: + with open(os.devnull, "wb") as devnull: + subprocess.check_call( + ["ffmpeg", "-version"], stdout=devnull, stderr=devnull + ) + return True + except (OSError, subprocess.CalledProcessError): + return False + + def __init__(self, directory, frame_shape): + self.filename = directory + "/video-{}.webm".format(time.time()) + if len(frame_shape) != 3: + raise ValueError( + "Expected rank-3 array for frame, got %s" % str(frame_shape) + ) + # Set input pixel format based on channel count. + if frame_shape[2] == 1: + pix_fmt = "gray" + elif frame_shape[2] == 3: + pix_fmt = "rgb24" + else: + raise ValueError("Unsupported channel count %d" % frame_shape[2]) + + command = [ + "ffmpeg", + "-y", # Overwite output + # Input options - raw video file format and codec. + "-f", + "rawvideo", + "-vcodec", + "rawvideo", + "-s", + "%dx%d" % (frame_shape[1], frame_shape[0]), # Width x height. + "-pix_fmt", + pix_fmt, + "-r", + "15", # Frame rate: arbitrarily use 15 frames per second. + "-i", + "-", # Use stdin. + "-an", # No audio. + # Output options - use lossless VP9 codec inside .webm. + "-vcodec", + "libvpx-vp9", + "-lossless", + "1", + # Using YUV is most compatible, though conversion from RGB skews colors. + "-pix_fmt", + "yuv420p", + self.filename, + ] + PIPE = subprocess.PIPE + self.ffmpeg = subprocess.Popen( + command, stdin=PIPE, stdout=PIPE, stderr=PIPE + ) + + def _handle_error(self): + _, stderr = self.ffmpeg.communicate() + bar = "=" * 40 + logger.error( + "Error writing to FFmpeg:\n%s\n%s\n%s", + bar, + stderr.rstrip("\n"), + bar, + ) + + def emit_frame(self, np_array): + try: + self.ffmpeg.stdin.write(np_array.tobytes()) + self.ffmpeg.stdin.flush() + except IOError: + self._handle_error() + raise IOError("Failure invoking FFmpeg") + + def close(self): + if self.ffmpeg.poll() is None: + # Close stdin and consume and discard stderr/stdout. + self.ffmpeg.communicate() + self.ffmpeg = None diff --git a/tensorboard/plugins/beholder/visualizer.py b/tensorboard/plugins/beholder/visualizer.py index b8ea95c6ec..46fbca7a19 100644 --- a/tensorboard/plugins/beholder/visualizer.py +++ b/tensorboard/plugins/beholder/visualizer.py @@ -23,286 +23,300 @@ import tensorflow as tf from tensorboard.plugins.beholder import im_util -from tensorboard.plugins.beholder.shared_config import SECTION_HEIGHT,\ - IMAGE_WIDTH, DEFAULT_CONFIG, SECTION_INFO_FILENAME +from tensorboard.plugins.beholder.shared_config import ( + SECTION_HEIGHT, + IMAGE_WIDTH, + DEFAULT_CONFIG, + SECTION_INFO_FILENAME, +) from tensorboard.plugins.beholder.file_system_tools import write_pickle MIN_SQUARE_SIZE = 3 class Visualizer(object): - - def __init__(self, logdir): - self.logdir = logdir - self.sections_over_time = deque([], DEFAULT_CONFIG['window_size']) - self.config = dict(DEFAULT_CONFIG) - self.old_config = dict(DEFAULT_CONFIG) - - - def _reshape_conv_array(self, array, section_height, image_width): - '''Reshape a rank 4 array to be rank 2, where each column of block_width is - a filter, and each row of block height is an input channel. For example: - - [[[[ 11, 21, 31, 41], - [ 51, 61, 71, 81], - [ 91, 101, 111, 121]], - [[ 12, 22, 32, 42], - [ 52, 62, 72, 82], - [ 92, 102, 112, 122]], - [[ 13, 23, 33, 43], - [ 53, 63, 73, 83], - [ 93, 103, 113, 123]]], - [[[ 14, 24, 34, 44], - [ 54, 64, 74, 84], - [ 94, 104, 114, 124]], - [[ 15, 25, 35, 45], - [ 55, 65, 75, 85], - [ 95, 105, 115, 125]], - [[ 16, 26, 36, 46], - [ 56, 66, 76, 86], - [ 96, 106, 116, 126]]], - [[[ 17, 27, 37, 47], - [ 57, 67, 77, 87], - [ 97, 107, 117, 127]], - [[ 18, 28, 38, 48], - [ 58, 68, 78, 88], - [ 98, 108, 118, 128]], - [[ 19, 29, 39, 49], - [ 59, 69, 79, 89], - [ 99, 109, 119, 129]]]] - - should be reshaped to: - - [[ 11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43], - [ 14, 15, 16, 24, 25, 26, 34, 35, 36, 44, 45, 46], - [ 17, 18, 19, 27, 28, 29, 37, 38, 39, 47, 48, 49], - [ 51, 52, 53, 61, 62, 63, 71, 72, 73, 81, 82, 83], - [ 54, 55, 56, 64, 65, 66, 74, 75, 76, 84, 85, 86], - [ 57, 58, 59, 67, 68, 69, 77, 78, 79, 87, 88, 89], - [ 91, 92, 93, 101, 102, 103, 111, 112, 113, 121, 122, 123], - [ 94, 95, 96, 104, 105, 106, 114, 115, 116, 124, 125, 126], - [ 97, 98, 99, 107, 108, 109, 117, 118, 119, 127, 128, 129]] - ''' - - # E.g. [100, 24, 24, 10]: this shouldn't be reshaped like normal. - if array.shape[1] == array.shape[2] and array.shape[0] != array.shape[1]: - array = np.rollaxis(np.rollaxis(array, 2), 2) - - block_height, block_width, in_channels = array.shape[:3] - rows = [] - - max_element_count = section_height * int(image_width / MIN_SQUARE_SIZE) - element_count = 0 - - for i in range(in_channels): - rows.append(array[:, :, i, :].reshape(block_height, -1, order='F')) - - # This line should be left in this position. Gives it one extra row. - if element_count >= max_element_count and not self.config['show_all']: - break - - element_count += block_height * in_channels * block_width - - return np.vstack(rows) - - - def _reshape_irregular_array(self, array, section_height, image_width): - '''Reshapes arrays of ranks not in {1, 2, 4} - ''' - section_area = section_height * image_width - flattened_array = np.ravel(array) - - if not self.config['show_all']: - flattened_array = flattened_array[:int(section_area/MIN_SQUARE_SIZE)] - - cell_count = np.prod(flattened_array.shape) - cell_area = section_area / cell_count - - cell_side_length = max(1, floor(sqrt(cell_area))) - row_count = max(1, int(section_height / cell_side_length)) - col_count = int(cell_count / row_count) - - # Reshape the truncated array so that it has the same aspect ratio as - # the section. - - # Truncate whatever remaining values there are that don't fit. Hopefully - # it doesn't matter that the last few (< section count) aren't there. - section = np.reshape(flattened_array[:row_count * col_count], - (row_count, col_count)) - - return section - - - def _determine_image_width(self, arrays, show_all): - final_width = IMAGE_WIDTH - - if show_all: - for array in arrays: + def __init__(self, logdir): + self.logdir = logdir + self.sections_over_time = deque([], DEFAULT_CONFIG["window_size"]) + self.config = dict(DEFAULT_CONFIG) + self.old_config = dict(DEFAULT_CONFIG) + + def _reshape_conv_array(self, array, section_height, image_width): + """Reshape a rank 4 array to be rank 2, where each column of + block_width is a filter, and each row of block height is an input + channel. For example: + + [[[[ 11, 21, 31, 41], + [ 51, 61, 71, 81], + [ 91, 101, 111, 121]], + [[ 12, 22, 32, 42], + [ 52, 62, 72, 82], + [ 92, 102, 112, 122]], + [[ 13, 23, 33, 43], + [ 53, 63, 73, 83], + [ 93, 103, 113, 123]]], + [[[ 14, 24, 34, 44], + [ 54, 64, 74, 84], + [ 94, 104, 114, 124]], + [[ 15, 25, 35, 45], + [ 55, 65, 75, 85], + [ 95, 105, 115, 125]], + [[ 16, 26, 36, 46], + [ 56, 66, 76, 86], + [ 96, 106, 116, 126]]], + [[[ 17, 27, 37, 47], + [ 57, 67, 77, 87], + [ 97, 107, 117, 127]], + [[ 18, 28, 38, 48], + [ 58, 68, 78, 88], + [ 98, 108, 118, 128]], + [[ 19, 29, 39, 49], + [ 59, 69, 79, 89], + [ 99, 109, 119, 129]]]] + + should be reshaped to: + + [[ 11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43], + [ 14, 15, 16, 24, 25, 26, 34, 35, 36, 44, 45, 46], + [ 17, 18, 19, 27, 28, 29, 37, 38, 39, 47, 48, 49], + [ 51, 52, 53, 61, 62, 63, 71, 72, 73, 81, 82, 83], + [ 54, 55, 56, 64, 65, 66, 74, 75, 76, 84, 85, 86], + [ 57, 58, 59, 67, 68, 69, 77, 78, 79, 87, 88, 89], + [ 91, 92, 93, 101, 102, 103, 111, 112, 113, 121, 122, 123], + [ 94, 95, 96, 104, 105, 106, 114, 115, 116, 124, 125, 126], + [ 97, 98, 99, 107, 108, 109, 117, 118, 119, 127, 128, 129]] + """ + + # E.g. [100, 24, 24, 10]: this shouldn't be reshaped like normal. + if ( + array.shape[1] == array.shape[2] + and array.shape[0] != array.shape[1] + ): + array = np.rollaxis(np.rollaxis(array, 2), 2) + + block_height, block_width, in_channels = array.shape[:3] + rows = [] + + max_element_count = section_height * int(image_width / MIN_SQUARE_SIZE) + element_count = 0 + + for i in range(in_channels): + rows.append(array[:, :, i, :].reshape(block_height, -1, order="F")) + + # This line should be left in this position. Gives it one extra row. + if ( + element_count >= max_element_count + and not self.config["show_all"] + ): + break + + element_count += block_height * in_channels * block_width + + return np.vstack(rows) + + def _reshape_irregular_array(self, array, section_height, image_width): + """Reshapes arrays of ranks not in {1, 2, 4}""" + section_area = section_height * image_width + flattened_array = np.ravel(array) + + if not self.config["show_all"]: + flattened_array = flattened_array[ + : int(section_area / MIN_SQUARE_SIZE) + ] + + cell_count = np.prod(flattened_array.shape) + cell_area = section_area / cell_count + + cell_side_length = max(1, floor(sqrt(cell_area))) + row_count = max(1, int(section_height / cell_side_length)) + col_count = int(cell_count / row_count) + + # Reshape the truncated array so that it has the same aspect ratio as + # the section. + + # Truncate whatever remaining values there are that don't fit. Hopefully + # it doesn't matter that the last few (< section count) aren't there. + section = np.reshape( + flattened_array[: row_count * col_count], (row_count, col_count) + ) + + return section + + def _determine_image_width(self, arrays, show_all): + final_width = IMAGE_WIDTH + + if show_all: + for array in arrays: + rank = len(array.shape) + + if rank == 1: + width = len(array) + elif rank == 2: + width = array.shape[1] + elif rank == 4: + width = array.shape[1] * array.shape[3] + else: + width = IMAGE_WIDTH + + if width > final_width: + final_width = width + + return final_width + + def _determine_section_height(self, array, show_all): rank = len(array.shape) - - if rank == 1: - width = len(array) - elif rank == 2: - width = array.shape[1] - elif rank == 4: - width = array.shape[1] * array.shape[3] - else: - width = IMAGE_WIDTH - - if width > final_width: - final_width = width - - return final_width - - - def _determine_section_height(self, array, show_all): - rank = len(array.shape) - height = SECTION_HEIGHT - - if show_all: - if rank == 1: height = SECTION_HEIGHT - if rank == 2: - height = max(SECTION_HEIGHT, array.shape[0]) - elif rank == 4: - height = max(SECTION_HEIGHT, array.shape[0] * array.shape[2]) - else: - height = max(SECTION_HEIGHT, np.prod(array.shape) // IMAGE_WIDTH) - - return height - - def _arrays_to_sections(self, arrays): - ''' + if show_all: + if rank == 1: + height = SECTION_HEIGHT + if rank == 2: + height = max(SECTION_HEIGHT, array.shape[0]) + elif rank == 4: + height = max(SECTION_HEIGHT, array.shape[0] * array.shape[2]) + else: + height = max( + SECTION_HEIGHT, np.prod(array.shape) // IMAGE_WIDTH + ) + + return height + + def _arrays_to_sections(self, arrays): + """ input: unprocessed numpy arrays. returns: columns of the size that they will appear in the image, not scaled for display. That needs to wait until after variance is computed. - ''' - sections = [] - sections_to_resize_later = {} - show_all = self.config['show_all'] - image_width = self._determine_image_width(arrays, show_all) - - for array_number, array in enumerate(arrays): - rank = len(array.shape) - section_height = self._determine_section_height(array, show_all) - - if rank == 1: - section = np.atleast_2d(array) - elif rank == 2: - section = array - elif rank == 4: - section = self._reshape_conv_array(array, section_height, image_width) - else: - section = self._reshape_irregular_array(array, - section_height, - image_width) - # Only calculate variance for what we have to. In some cases (biases), - # the section is larger than the array, so we don't want to calculate - # variance for the same value over and over - better to resize later. - # About a 6-7x speedup for a big network with a big variance window. - section_size = section_height * image_width - array_size = np.prod(array.shape) - - if section_size > array_size: - sections.append(section) - sections_to_resize_later[array_number] = section_height - else: - sections.append(im_util.resize(section, section_height, image_width)) - - self.sections_over_time.append(sections) - - if self.config['mode'] == 'variance': - sections = self._sections_to_variance_sections(self.sections_over_time) - - for array_number, height in sections_to_resize_later.items(): - sections[array_number] = im_util.resize(sections[array_number], - height, - image_width) - return sections - - - def _sections_to_variance_sections(self, sections_over_time): - '''Computes the variance of corresponding sections over time. - - Returns: - a list of np arrays. - ''' - variance_sections = [] - - for i in range(len(sections_over_time[0])): - time_sections = [sections[i] for sections in sections_over_time] - variance = np.var(time_sections, axis=0) - variance_sections.append(variance) - - return variance_sections - - - def _sections_to_image(self, sections): - padding_size = 5 - - sections = im_util.scale_sections(sections, self.config['scaling']) - - final_stack = [sections[0]] - padding = np.zeros((padding_size, sections[0].shape[1])) - - for section in sections[1:]: - final_stack.append(padding) - final_stack.append(section) - - return np.vstack(final_stack).astype(np.uint8) - - - def _maybe_clear_deque(self): - '''Clears the deque if certain parts of the config have changed.''' - - for config_item in ['values', 'mode', 'show_all']: - if self.config[config_item] != self.old_config[config_item]: - self.sections_over_time.clear() - break - - self.old_config = self.config - - window_size = self.config['window_size'] - if window_size != self.sections_over_time.maxlen: - self.sections_over_time = deque(self.sections_over_time, window_size) - - - def _save_section_info(self, arrays, sections): - infos = [] - - if self.config['values'] == 'trainable_variables': - names = [x.name for x in tf.compat.v1.trainable_variables()] - else: - names = range(len(arrays)) - - for array, section, name in zip(arrays, sections, names): - info = {} + """ + sections = [] + sections_to_resize_later = {} + show_all = self.config["show_all"] + image_width = self._determine_image_width(arrays, show_all) + + for array_number, array in enumerate(arrays): + rank = len(array.shape) + section_height = self._determine_section_height(array, show_all) + + if rank == 1: + section = np.atleast_2d(array) + elif rank == 2: + section = array + elif rank == 4: + section = self._reshape_conv_array( + array, section_height, image_width + ) + else: + section = self._reshape_irregular_array( + array, section_height, image_width + ) + # Only calculate variance for what we have to. In some cases (biases), + # the section is larger than the array, so we don't want to calculate + # variance for the same value over and over - better to resize later. + # About a 6-7x speedup for a big network with a big variance window. + section_size = section_height * image_width + array_size = np.prod(array.shape) + + if section_size > array_size: + sections.append(section) + sections_to_resize_later[array_number] = section_height + else: + sections.append( + im_util.resize(section, section_height, image_width) + ) + + self.sections_over_time.append(sections) + + if self.config["mode"] == "variance": + sections = self._sections_to_variance_sections( + self.sections_over_time + ) + + for array_number, height in sections_to_resize_later.items(): + sections[array_number] = im_util.resize( + sections[array_number], height, image_width + ) + return sections + + def _sections_to_variance_sections(self, sections_over_time): + """Computes the variance of corresponding sections over time. + + Returns: + a list of np arrays. + """ + variance_sections = [] + + for i in range(len(sections_over_time[0])): + time_sections = [sections[i] for sections in sections_over_time] + variance = np.var(time_sections, axis=0) + variance_sections.append(variance) + + return variance_sections + + def _sections_to_image(self, sections): + padding_size = 5 + + sections = im_util.scale_sections(sections, self.config["scaling"]) + + final_stack = [sections[0]] + padding = np.zeros((padding_size, sections[0].shape[1])) + + for section in sections[1:]: + final_stack.append(padding) + final_stack.append(section) + + return np.vstack(final_stack).astype(np.uint8) + + def _maybe_clear_deque(self): + """Clears the deque if certain parts of the config have changed.""" + + for config_item in ["values", "mode", "show_all"]: + if self.config[config_item] != self.old_config[config_item]: + self.sections_over_time.clear() + break + + self.old_config = self.config + + window_size = self.config["window_size"] + if window_size != self.sections_over_time.maxlen: + self.sections_over_time = deque( + self.sections_over_time, window_size + ) + + def _save_section_info(self, arrays, sections): + infos = [] + + if self.config["values"] == "trainable_variables": + names = [x.name for x in tf.compat.v1.trainable_variables()] + else: + names = range(len(arrays)) - info['name'] = name - info['shape'] = str(array.shape) - info['min'] = '{:.3e}'.format(section.min()) - info['mean'] = '{:.3e}'.format(section.mean()) - info['max'] = '{:.3e}'.format(section.max()) - info['range'] = '{:.3e}'.format(section.max() - section.min()) - info['height'] = section.shape[0] + for array, section, name in zip(arrays, sections, names): + info = {} - infos.append(info) + info["name"] = name + info["shape"] = str(array.shape) + info["min"] = "{:.3e}".format(section.min()) + info["mean"] = "{:.3e}".format(section.mean()) + info["max"] = "{:.3e}".format(section.max()) + info["range"] = "{:.3e}".format(section.max() - section.min()) + info["height"] = section.shape[0] - write_pickle(infos, '{}/{}'.format(self.logdir, SECTION_INFO_FILENAME)) + infos.append(info) + write_pickle(infos, "{}/{}".format(self.logdir, SECTION_INFO_FILENAME)) - def build_frame(self, arrays): - self._maybe_clear_deque() + def build_frame(self, arrays): + self._maybe_clear_deque() - arrays = arrays if isinstance(arrays, list) else [arrays] + arrays = arrays if isinstance(arrays, list) else [arrays] - sections = self._arrays_to_sections(arrays) - self._save_section_info(arrays, sections) - final_image = self._sections_to_image(sections) - final_image = im_util.apply_colormap(final_image, self.config['colormap']) + sections = self._arrays_to_sections(arrays) + self._save_section_info(arrays, sections) + final_image = self._sections_to_image(sections) + final_image = im_util.apply_colormap( + final_image, self.config["colormap"] + ) - return final_image + return final_image - def update(self, config): - self.config = config + def update(self, config): + self.config = config diff --git a/tensorboard/plugins/core/core_plugin.py b/tensorboard/plugins/core/core_plugin.py index d09c2e5e23..4dfe678f1e 100644 --- a/tensorboard/plugins/core/core_plugin.py +++ b/tensorboard/plugins/core/core_plugin.py @@ -44,179 +44,199 @@ class CorePlugin(base_plugin.TBPlugin): - """Core plugin for TensorBoard. + """Core plugin for TensorBoard. - This plugin serves runs, configuration data, and static assets. This plugin - should always be present in a TensorBoard WSGI application. - """ - - plugin_name = 'core' - - def __init__(self, context): - """Instantiates CorePlugin. - - Args: - context: A base_plugin.TBContext instance. - """ - logdir_spec = context.flags.logdir_spec if context.flags else '' - self._logdir = context.logdir or logdir_spec - self._db_uri = context.db_uri - self._window_title = context.window_title - self._multiplexer = context.multiplexer - self._db_connection_provider = context.db_connection_provider - self._assets_zip_provider = context.assets_zip_provider - if context.flags and context.flags.generic_data == 'true': - self._data_provider = context.data_provider - else: - self._data_provider = None - - def is_active(self): - return True - - def get_plugin_apps(self): - apps = { - '/___rPc_sWiTcH___': self._send_404_without_logging, - '/audio': self._redirect_to_index, - '/data/environment': self._serve_environment, - '/data/logdir': self._serve_logdir, - '/data/runs': self._serve_runs, - '/data/experiments': self._serve_experiments, - '/data/experiment_runs': self._serve_experiment_runs, - '/data/window_properties': self._serve_window_properties, - '/events': self._redirect_to_index, - '/favicon.ico': self._send_404_without_logging, - '/graphs': self._redirect_to_index, - '/histograms': self._redirect_to_index, - '/images': self._redirect_to_index, - } - apps.update(self.get_resource_apps()) - return apps - - def get_resource_apps(self): - apps = {} - if not self._assets_zip_provider: - return apps - - with self._assets_zip_provider() as fp: - with zipfile.ZipFile(fp) as zip_: - for path in zip_.namelist(): - gzipped_asset_bytes = _gzip(zip_.read(path)) - wsgi_app = functools.partial( - self._serve_asset, path, gzipped_asset_bytes) - apps['/' + path] = wsgi_app - apps['/'] = apps['/index.html'] - return apps - - @wrappers.Request.application - def _send_404_without_logging(self, request): - return http_util.Respond(request, 'Not found', 'text/plain', code=404) - - @wrappers.Request.application - def _redirect_to_index(self, unused_request): - return utils.redirect('/') - - @wrappers.Request.application - def _serve_asset(self, path, gzipped_asset_bytes, request): - """Serves a pre-gzipped static asset from the zip file.""" - mimetype = mimetypes.guess_type(path)[0] or 'application/octet-stream' - return http_util.Respond( - request, gzipped_asset_bytes, mimetype, content_encoding='gzip') - - @wrappers.Request.application - def _serve_environment(self, request): - """Serve a JSON object containing some base properties used by the frontend. - - * data_location is either a path to a directory or an address to a - database (depending on which mode TensorBoard is running in). - * window_title is the title of the TensorBoard web page. + This plugin serves runs, configuration data, and static assets. This + plugin should always be present in a TensorBoard WSGI application. """ - if self._data_provider: - experiment = plugin_util.experiment_id(request.environ) - data_location = self._data_provider.data_location(experiment) - else: - data_location = self._logdir or self._db_uri - return http_util.Respond( - request, - { - 'data_location': data_location, - 'window_title': self._window_title, - }, - 'application/json') - - @wrappers.Request.application - def _serve_logdir(self, request): - """Respond with a JSON object containing this TensorBoard's logdir.""" - # TODO(chihuahua): Remove this method once the frontend instead uses the - # /data/environment route (and no deps throughout Google use the - # /data/logdir route). - return http_util.Respond( - request, {'logdir': self._logdir}, 'application/json') - - @wrappers.Request.application - def _serve_window_properties(self, request): - """Serve a JSON object containing this TensorBoard's window properties.""" - # TODO(chihuahua): Remove this method once the frontend instead uses the - # /data/environment route. - return http_util.Respond( - request, {'window_title': self._window_title}, 'application/json') - - @wrappers.Request.application - def _serve_runs(self, request): - """Serve a JSON array of run names, ordered by run started time. - - Sort order is by started time (aka first event time) with empty times sorted - last, and then ties are broken by sorting on the run name. - """ - if self._data_provider: - experiment = plugin_util.experiment_id(request.environ) - runs = sorted( - self._data_provider.list_runs(experiment_id=experiment), - key=lambda run: ( - run.start_time if run.start_time is not None else float('inf'), - run.run_name, - ) - ) - run_names = [run.run_name for run in runs] - elif self._db_connection_provider: - db = self._db_connection_provider() - cursor = db.execute(''' + + plugin_name = "core" + + def __init__(self, context): + """Instantiates CorePlugin. + + Args: + context: A base_plugin.TBContext instance. + """ + logdir_spec = context.flags.logdir_spec if context.flags else "" + self._logdir = context.logdir or logdir_spec + self._db_uri = context.db_uri + self._window_title = context.window_title + self._multiplexer = context.multiplexer + self._db_connection_provider = context.db_connection_provider + self._assets_zip_provider = context.assets_zip_provider + if context.flags and context.flags.generic_data == "true": + self._data_provider = context.data_provider + else: + self._data_provider = None + + def is_active(self): + return True + + def get_plugin_apps(self): + apps = { + "/___rPc_sWiTcH___": self._send_404_without_logging, + "/audio": self._redirect_to_index, + "/data/environment": self._serve_environment, + "/data/logdir": self._serve_logdir, + "/data/runs": self._serve_runs, + "/data/experiments": self._serve_experiments, + "/data/experiment_runs": self._serve_experiment_runs, + "/data/window_properties": self._serve_window_properties, + "/events": self._redirect_to_index, + "/favicon.ico": self._send_404_without_logging, + "/graphs": self._redirect_to_index, + "/histograms": self._redirect_to_index, + "/images": self._redirect_to_index, + } + apps.update(self.get_resource_apps()) + return apps + + def get_resource_apps(self): + apps = {} + if not self._assets_zip_provider: + return apps + + with self._assets_zip_provider() as fp: + with zipfile.ZipFile(fp) as zip_: + for path in zip_.namelist(): + gzipped_asset_bytes = _gzip(zip_.read(path)) + wsgi_app = functools.partial( + self._serve_asset, path, gzipped_asset_bytes + ) + apps["/" + path] = wsgi_app + apps["/"] = apps["/index.html"] + return apps + + @wrappers.Request.application + def _send_404_without_logging(self, request): + return http_util.Respond(request, "Not found", "text/plain", code=404) + + @wrappers.Request.application + def _redirect_to_index(self, unused_request): + return utils.redirect("/") + + @wrappers.Request.application + def _serve_asset(self, path, gzipped_asset_bytes, request): + """Serves a pre-gzipped static asset from the zip file.""" + mimetype = mimetypes.guess_type(path)[0] or "application/octet-stream" + return http_util.Respond( + request, gzipped_asset_bytes, mimetype, content_encoding="gzip" + ) + + @wrappers.Request.application + def _serve_environment(self, request): + """Serve a JSON object containing some base properties used by the + frontend. + + * data_location is either a path to a directory or an address to a + database (depending on which mode TensorBoard is running in). + * window_title is the title of the TensorBoard web page. + """ + if self._data_provider: + experiment = plugin_util.experiment_id(request.environ) + data_location = self._data_provider.data_location(experiment) + else: + data_location = self._logdir or self._db_uri + return http_util.Respond( + request, + { + "data_location": data_location, + "window_title": self._window_title, + }, + "application/json", + ) + + @wrappers.Request.application + def _serve_logdir(self, request): + """Respond with a JSON object containing this TensorBoard's logdir.""" + # TODO(chihuahua): Remove this method once the frontend instead uses the + # /data/environment route (and no deps throughout Google use the + # /data/logdir route). + return http_util.Respond( + request, {"logdir": self._logdir}, "application/json" + ) + + @wrappers.Request.application + def _serve_window_properties(self, request): + """Serve a JSON object containing this TensorBoard's window + properties.""" + # TODO(chihuahua): Remove this method once the frontend instead uses the + # /data/environment route. + return http_util.Respond( + request, {"window_title": self._window_title}, "application/json" + ) + + @wrappers.Request.application + def _serve_runs(self, request): + """Serve a JSON array of run names, ordered by run started time. + + Sort order is by started time (aka first event time) with empty + times sorted last, and then ties are broken by sorting on the + run name. + """ + if self._data_provider: + experiment = plugin_util.experiment_id(request.environ) + runs = sorted( + self._data_provider.list_runs(experiment_id=experiment), + key=lambda run: ( + run.start_time + if run.start_time is not None + else float("inf"), + run.run_name, + ), + ) + run_names = [run.run_name for run in runs] + elif self._db_connection_provider: + db = self._db_connection_provider() + cursor = db.execute( + """ SELECT run_name, started_time IS NULL as started_time_nulls_last, started_time FROM Runs ORDER BY started_time_nulls_last, started_time, run_name - ''') - run_names = [row[0] for row in cursor] - else: - # Python's list.sort is stable, so to order by started time and - # then by name, we can just do the sorts in the reverse order. - run_names = sorted(self._multiplexer.Runs()) - def get_first_event_timestamp(run_name): - try: - return self._multiplexer.FirstEventTimestamp(run_name) - except ValueError as e: - logger.warn( - 'Unable to get first event timestamp for run %s: %s', run_name, e) - # Put runs without a timestamp at the end. - return float('inf') - run_names.sort(key=get_first_event_timestamp) - return http_util.Respond(request, run_names, 'application/json') - - @wrappers.Request.application - def _serve_experiments(self, request): - """Serve a JSON array of experiments. Experiments are ordered by experiment - started time (aka first event time) with empty times sorted last, and then - ties are broken by sorting on the experiment name. - """ - results = self.list_experiments_impl() - return http_util.Respond(request, results, 'application/json') - - def list_experiments_impl(self): - results = [] - if self._db_connection_provider: - db = self._db_connection_provider() - cursor = db.execute(''' + """ + ) + run_names = [row[0] for row in cursor] + else: + # Python's list.sort is stable, so to order by started time and + # then by name, we can just do the sorts in the reverse order. + run_names = sorted(self._multiplexer.Runs()) + + def get_first_event_timestamp(run_name): + try: + return self._multiplexer.FirstEventTimestamp(run_name) + except ValueError as e: + logger.warn( + "Unable to get first event timestamp for run %s: %s", + run_name, + e, + ) + # Put runs without a timestamp at the end. + return float("inf") + + run_names.sort(key=get_first_event_timestamp) + return http_util.Respond(request, run_names, "application/json") + + @wrappers.Request.application + def _serve_experiments(self, request): + """Serve a JSON array of experiments. + + Experiments are ordered by experiment started time (aka first + event time) with empty times sorted last, and then ties are + broken by sorting on the experiment name. + """ + results = self.list_experiments_impl() + return http_util.Respond(request, results, "application/json") + + def list_experiments_impl(self): + results = [] + if self._db_connection_provider: + db = self._db_connection_provider() + cursor = db.execute( + """ SELECT experiment_id, experiment_name, @@ -225,30 +245,33 @@ def list_experiments_impl(self): FROM Experiments ORDER BY started_time_nulls_last, started_time, experiment_name, experiment_id - ''') - results = [{ - "id": row[0], - "name": row[1], - "startTime": row[2], - } for row in cursor] - - return results - - @wrappers.Request.application - def _serve_experiment_runs(self, request): - """Serve a JSON runs of an experiment, specified with query param - `experiment`, with their nested data, tag, populated. Runs returned are - ordered by started time (aka first event time) with empty times sorted last, - and then ties are broken by sorting on the run name. Tags are sorted by - its name, displayName, and lastly, inserted time. - """ - results = [] - if self._db_connection_provider: - exp_id = plugin_util.experiment_id(request.environ) - runs_dict = collections.OrderedDict() - - db = self._db_connection_provider() - cursor = db.execute(''' + """ + ) + results = [ + {"id": row[0], "name": row[1], "startTime": row[2],} + for row in cursor + ] + + return results + + @wrappers.Request.application + def _serve_experiment_runs(self, request): + """Serve a JSON runs of an experiment, specified with query param + `experiment`, with their nested data, tag, populated. + + Runs returned are ordered by started time (aka first event time) + with empty times sorted last, and then ties are broken by + sorting on the run name. Tags are sorted by its name, + displayName, and lastly, inserted time. + """ + results = [] + if self._db_connection_provider: + exp_id = plugin_util.experiment_id(request.environ) + runs_dict = collections.OrderedDict() + + db = self._db_connection_provider() + cursor = db.execute( + """ SELECT Runs.run_id, Runs.run_name, @@ -270,52 +293,58 @@ def _serve_experiment_runs(self, request): Tags.tag_name, Tags.display_name, Tags.inserted_time; - ''', (exp_id,)) - for row in cursor: - run_id = row[0] - if not run_id in runs_dict: - runs_dict[run_id] = { - "id": run_id, - "name": row[1], - "startTime": math.floor(row[2]), - "tags": [], - } - # tag can be missing. - if row[4]: - runs_dict[run_id].get("tags").append({ - "id": row[4], - "displayName": row[6], - "name": row[5], - "pluginName": row[7], - }) - results = list(runs_dict.values()) - return http_util.Respond(request, results, 'application/json') + """, + (exp_id,), + ) + for row in cursor: + run_id = row[0] + if not run_id in runs_dict: + runs_dict[run_id] = { + "id": run_id, + "name": row[1], + "startTime": math.floor(row[2]), + "tags": [], + } + # tag can be missing. + if row[4]: + runs_dict[run_id].get("tags").append( + { + "id": row[4], + "displayName": row[6], + "name": row[5], + "pluginName": row[7], + } + ) + results = list(runs_dict.values()) + return http_util.Respond(request, results, "application/json") + class CorePluginLoader(base_plugin.TBLoader): - """CorePlugin factory.""" - - def define_flags(self, parser): - """Adds standard TensorBoard CLI flags to parser.""" - parser.add_argument( - '--logdir', - metavar='PATH', - type=str, - default='', - help='''\ + """CorePlugin factory.""" + + def define_flags(self, parser): + """Adds standard TensorBoard CLI flags to parser.""" + parser.add_argument( + "--logdir", + metavar="PATH", + type=str, + default="", + help="""\ Directory where TensorBoard will look to find TensorFlow event files that it can display. TensorBoard will recursively walk the directory structure rooted at logdir, looking for .*tfevents.* files. A leading tilde will be expanded with the semantics of Python's os.expanduser function. -''') - - parser.add_argument( - '--logdir_spec', - metavar='PATH_SPEC', - type=str, - default='', - help='''\ +""", + ) + + parser.add_argument( + "--logdir_spec", + metavar="PATH_SPEC", + type=str, + default="", + help="""\ Like `--logdir`, but with special interpretation for commas and colons: commas separate multiple runs, where a colon specifies a new name for a run. For example: @@ -325,77 +354,84 @@ def define_flags(self, parser): log directories recursively; for finer-grained control, prefer using a symlink tree. Some features may not work when using `--logdir_spec` instead of `--logdir`. -''') - - parser.add_argument( - '--host', - metavar='ADDR', - type=str, - default=None, # like localhost, but prints a note about `--bind_all` - help='''\ +""", + ) + + parser.add_argument( + "--host", + metavar="ADDR", + type=str, + default=None, # like localhost, but prints a note about `--bind_all` + help="""\ What host to listen to (default: localhost). To serve to the entire local network on both IPv4 and IPv6, see `--bind_all`, with which this option is mutually exclusive. -''') +""", + ) - parser.add_argument( - '--bind_all', - action='store_true', - help='''\ + parser.add_argument( + "--bind_all", + action="store_true", + help="""\ Serve on all public interfaces. This will expose your TensorBoard instance to the network on both IPv4 and IPv6 (where available). Mutually exclusive with `--host`. -''') - - - parser.add_argument( - '--port', - metavar='PORT', - type=lambda s: (None if s == "default" else int(s)), - default="default", - help='''\ +""", + ) + + parser.add_argument( + "--port", + metavar="PORT", + type=lambda s: (None if s == "default" else int(s)), + default="default", + help="""\ Port to serve TensorBoard on. Pass 0 to request an unused port selected by the operating system, or pass "default" to try to bind to the default port (%s) but search for a nearby free port if the default port is unavailable. (default: "default").\ -''' % DEFAULT_PORT) - - parser.add_argument( - '--purge_orphaned_data', - metavar='BOOL', - # Custom str-to-bool converter since regular bool() doesn't work. - type=lambda v: {'true': True, 'false': False}.get(v.lower(), v), - choices=[True, False], - default=True, - help='''\ +""" + % DEFAULT_PORT, + ) + + parser.add_argument( + "--purge_orphaned_data", + metavar="BOOL", + # Custom str-to-bool converter since regular bool() doesn't work. + type=lambda v: {"true": True, "false": False}.get(v.lower(), v), + choices=[True, False], + default=True, + help="""\ Whether to purge data that may have been orphaned due to TensorBoard restarts. Setting --purge_orphaned_data=False can be used to debug data disappearance. (default: %(default)s)\ -''') - - parser.add_argument( - '--db', - metavar='URI', - type=str, - default='', - help='''\ +""", + ) + + parser.add_argument( + "--db", + metavar="URI", + type=str, + default="", + help="""\ [experimental] sets SQL database URI and enables DB backend mode, which is read-only unless --db_import is also passed.\ -''') +""", + ) - parser.add_argument( - '--db_import', - action='store_true', - help='''\ + parser.add_argument( + "--db_import", + action="store_true", + help="""\ [experimental] enables DB read-and-import mode, which in combination with --logdir imports event files into a DB backend on the fly. The backing DB is temporary unless --db is also passed to specify a DB path to use.\ -''') +""", + ) - parser.add_argument( - '--inspect', - action='store_true', - help='''\ + parser.add_argument( + "--inspect", + action="store_true", + help="""\ Prints digests of event files to command line. This is useful when no data is shown on TensorBoard, or the data shown @@ -407,39 +443,43 @@ def define_flags(self, parser): `tensorboard --inspect --logdir mylogdir --tag loss` See tensorboard/backend/event_processing/event_file_inspector.py for more info.\ -''') - - # This flag has a "_tb" suffix to avoid conflicting with an internal flag - # named --version. Note that due to argparse auto-expansion of unambiguous - # flag prefixes, you can still invoke this as `tensorboard --version`. - parser.add_argument( - '--version_tb', - action='store_true', - help='Prints the version of Tensorboard') - - parser.add_argument( - '--tag', - metavar='TAG', - type=str, - default='', - help='tag to query for; used with --inspect') - - parser.add_argument( - '--event_file', - metavar='PATH', - type=str, - default='', - help='''\ +""", + ) + + # This flag has a "_tb" suffix to avoid conflicting with an internal flag + # named --version. Note that due to argparse auto-expansion of unambiguous + # flag prefixes, you can still invoke this as `tensorboard --version`. + parser.add_argument( + "--version_tb", + action="store_true", + help="Prints the version of Tensorboard", + ) + + parser.add_argument( + "--tag", + metavar="TAG", + type=str, + default="", + help="tag to query for; used with --inspect", + ) + + parser.add_argument( + "--event_file", + metavar="PATH", + type=str, + default="", + help="""\ The particular event file to query for. Only used if --inspect is present and --logdir is not specified.\ -''') - - parser.add_argument( - '--path_prefix', - metavar='PATH', - type=str, - default='', - help='''\ +""", + ) + + parser.add_argument( + "--path_prefix", + metavar="PATH", + type=str, + default="", + help="""\ An optional, relative prefix to the path, e.g. "/path/to/tensorboard". resulting in the new base url being located at localhost:6006/path/to/tensorboard under default settings. A leading @@ -447,59 +487,64 @@ def define_flags(self, parser): optional and has no effect. The path_prefix can be leveraged for path based routing of an ELB when the website base_url is not available e.g. "example.site.com/path/to/tensorboard/".\ -''') - - parser.add_argument( - '--window_title', - metavar='TEXT', - type=str, - default='', - help='changes title of browser window') - - parser.add_argument( - '--max_reload_threads', - metavar='COUNT', - type=int, - default=1, - help='''\ +""", + ) + + parser.add_argument( + "--window_title", + metavar="TEXT", + type=str, + default="", + help="changes title of browser window", + ) + + parser.add_argument( + "--max_reload_threads", + metavar="COUNT", + type=int, + default=1, + help="""\ The max number of threads that TensorBoard can use to reload runs. Not relevant for db read-only mode. Each thread reloads one run at a time. (default: %(default)s)\ -''') - - parser.add_argument( - '--reload_interval', - metavar='SECONDS', - type=float, - default=5.0, - help='''\ +""", + ) + + parser.add_argument( + "--reload_interval", + metavar="SECONDS", + type=float, + default=5.0, + help="""\ How often the backend should load more data, in seconds. Set to 0 to load just once at startup and a negative number to never reload at all. Not relevant for DB read-only mode. (default: %(default)s)\ -''') - - parser.add_argument( - '--reload_task', - metavar='TYPE', - type=str, - default='auto', - choices=['auto', 'thread', 'process', 'blocking'], - help='''\ +""", + ) + + parser.add_argument( + "--reload_task", + metavar="TYPE", + type=str, + default="auto", + choices=["auto", "thread", "process", "blocking"], + help="""\ [experimental] The mechanism to use for the background data reload task. The default "auto" option will conditionally use threads for legacy reloading and a child process for DB import reloading. The "process" option is only useful with DB import mode. The "blocking" option will block startup until reload finishes, and requires --load_interval=0. (default: %(default)s)\ -''') - - parser.add_argument( - '--reload_multifile', - metavar='BOOL', - # Custom str-to-bool converter since regular bool() doesn't work. - type=lambda v: {'true': True, 'false': False}.get(v.lower(), v), - choices=[True, False], - default=None, - help='''\ +""", + ) + + parser.add_argument( + "--reload_multifile", + metavar="BOOL", + # Custom str-to-bool converter since regular bool() doesn't work. + type=lambda v: {"true": True, "false": False}.get(v.lower(), v), + choices=[True, False], + default=None, + help="""\ [experimental] If true, this enables experimental support for continuously polling multiple event files in each run directory for newly appended data (rather than only polling the last event file). Event files will only be @@ -507,14 +552,15 @@ def define_flags(self, parser): defined by --reload_multifile_inactive_secs, to limit resource usage. Beware of running out of memory if the logdir contains many active event files. (default: false)\ -''') - - parser.add_argument( - '--reload_multifile_inactive_secs', - metavar='SECONDS', - type=int, - default=4000, - help='''\ +""", + ) + + parser.add_argument( + "--reload_multifile_inactive_secs", + metavar="SECONDS", + type=int, + default=4000, + help="""\ [experimental] Configures the age threshold in seconds at which an event file that has no event wall time more recent than that will be considered an inactive file and no longer polled (to limit resource usage). If set to -1, @@ -523,25 +569,27 @@ def define_flags(self, parser): last-file-only polling strategy (akin to --reload_multifile=false). (default: %(default)s - intended to ensure an event file remains active if it receives new data at least once per hour)\ -''') - - parser.add_argument( - '--generic_data', - metavar='TYPE', - type=str, - default='auto', - choices=['false', 'auto', 'true'], - help='''\ +""", + ) + + parser.add_argument( + "--generic_data", + metavar="TYPE", + type=str, + default="auto", + choices=["false", "auto", "true"], + help="""\ [experimental] Whether to use generic data provider infrastructure. The "auto" option enables this only for dashboards that are considered stable under the new codepaths. (default: %(default)s)\ -''') - - parser.add_argument( - '--samples_per_plugin', - type=str, - default='', - help='''\ +""", + ) + + parser.add_argument( + "--samples_per_plugin", + type=str, + default="", + help="""\ An optional comma separated list of plugin_name=num_samples pairs to explicitly specify how many samples to keep per tag for that plugin. For unspecified plugins, TensorBoard randomly downsamples logged summaries @@ -550,45 +598,54 @@ def define_flags(self, parser): means keep all samples of that type. For instance "scalars=500,images=0" keeps 500 scalars and all images. Most users should not need to set this flag.\ -''') - - def fix_flags(self, flags): - """Fixes standard TensorBoard CLI flags to parser.""" - FlagsError = base_plugin.FlagsError - if flags.version_tb: - pass - elif flags.inspect: - if flags.logdir_spec: - raise FlagsError('--logdir_spec is not supported with --inspect.') - if flags.logdir and flags.event_file: - raise FlagsError( - 'Must specify either --logdir or --event_file, but not both.') - if not (flags.logdir or flags.event_file): - raise FlagsError('Must specify either --logdir or --event_file.') - elif flags.logdir and flags.logdir_spec: - raise FlagsError( - 'May not specify both --logdir and --logdir_spec') - elif not flags.db and not flags.logdir and not flags.logdir_spec: - raise FlagsError('A logdir or db must be specified. ' - 'For example `tensorboard --logdir mylogdir` ' - 'or `tensorboard --db sqlite:~/.tensorboard.db`. ' - 'Run `tensorboard --helpfull` for details and examples.') - elif flags.host is not None and flags.bind_all: - raise FlagsError('Must not specify both --host and --bind_all.') - - flags.path_prefix = flags.path_prefix.rstrip('/') - if flags.path_prefix and not flags.path_prefix.startswith('/'): - raise FlagsError( - 'Path prefix must start with slash, but got: %r.' % flags.path_prefix) - - def load(self, context): - """Creates CorePlugin instance.""" - return CorePlugin(context) +""", + ) + + def fix_flags(self, flags): + """Fixes standard TensorBoard CLI flags to parser.""" + FlagsError = base_plugin.FlagsError + if flags.version_tb: + pass + elif flags.inspect: + if flags.logdir_spec: + raise FlagsError( + "--logdir_spec is not supported with --inspect." + ) + if flags.logdir and flags.event_file: + raise FlagsError( + "Must specify either --logdir or --event_file, but not both." + ) + if not (flags.logdir or flags.event_file): + raise FlagsError( + "Must specify either --logdir or --event_file." + ) + elif flags.logdir and flags.logdir_spec: + raise FlagsError("May not specify both --logdir and --logdir_spec") + elif not flags.db and not flags.logdir and not flags.logdir_spec: + raise FlagsError( + "A logdir or db must be specified. " + "For example `tensorboard --logdir mylogdir` " + "or `tensorboard --db sqlite:~/.tensorboard.db`. " + "Run `tensorboard --helpfull` for details and examples." + ) + elif flags.host is not None and flags.bind_all: + raise FlagsError("Must not specify both --host and --bind_all.") + + flags.path_prefix = flags.path_prefix.rstrip("/") + if flags.path_prefix and not flags.path_prefix.startswith("/"): + raise FlagsError( + "Path prefix must start with slash, but got: %r." + % flags.path_prefix + ) + + def load(self, context): + """Creates CorePlugin instance.""" + return CorePlugin(context) def _gzip(bytestring): - out = six.BytesIO() - # Set mtime to zero for deterministic results across TensorBoard launches. - with gzip.GzipFile(fileobj=out, mode='wb', compresslevel=3, mtime=0) as f: - f.write(bytestring) - return out.getvalue() + out = six.BytesIO() + # Set mtime to zero for deterministic results across TensorBoard launches. + with gzip.GzipFile(fileobj=out, mode="wb", compresslevel=3, mtime=0) as f: + f.write(bytestring) + return out.getvalue() diff --git a/tensorboard/plugins/core/core_plugin_test.py b/tensorboard/plugins/core/core_plugin_test.py index 8b928cd5dc..f7877b78cc 100644 --- a/tensorboard/plugins/core/core_plugin_test.py +++ b/tensorboard/plugins/core/core_plugin_test.py @@ -33,372 +33,436 @@ from werkzeug import wrappers from tensorboard.backend import application -from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer +from tensorboard.backend.event_processing import ( + plugin_event_multiplexer as event_multiplexer, +) from tensorboard.compat.proto import graph_pb2 from tensorboard.compat.proto import meta_graph_pb2 from tensorboard.plugins import base_plugin from tensorboard.plugins.core import core_plugin from tensorboard.util import test_util -FAKE_INDEX_HTML = b'quod erat dispertiendum
' - - _RUN_WITH_LEGACY_DISTRIBUTION = '_RUN_WITH_LEGACY_DISTRIBUTION' - _RUN_WITH_DISTRIBUTION = '_RUN_WITH_DISTRIBUTION' - _RUN_WITH_SCALARS = '_RUN_WITH_SCALARS' - - def __init__(self, *args, **kwargs): - super(DistributionsPluginTest, self).__init__(*args, **kwargs) - self.logdir = None - self.plugin = None - - def set_up_with_runs(self, run_names): - self.logdir = self.get_temp_dir() - for run_name in run_names: - self.generate_run(run_name) - multiplexer = event_multiplexer.EventMultiplexer(size_guidance={ - # don't truncate my test data, please - event_accumulator.TENSORS: self._STEPS, - }) - multiplexer.AddRunsFromDirectory(self.logdir) - multiplexer.Reload() - context = base_plugin.TBContext(logdir=self.logdir, multiplexer=multiplexer) - self.plugin = distributions_plugin.DistributionsPlugin(context) - - def generate_run(self, run_name): - tf.compat.v1.reset_default_graph() - sess = tf.compat.v1.Session() - placeholder = tf.compat.v1.placeholder(tf.float32, shape=[3]) - - if run_name == self._RUN_WITH_LEGACY_DISTRIBUTION: - tf.compat.v1.summary.histogram(self._LEGACY_DISTRIBUTION_TAG, placeholder) - elif run_name == self._RUN_WITH_DISTRIBUTION: - summary.op(self._DISTRIBUTION_TAG, placeholder, - display_name=self._DISPLAY_NAME, - description=self._DESCRIPTION) - elif run_name == self._RUN_WITH_SCALARS: - tf.compat.v1.summary.scalar(self._SCALAR_TAG, tf.reduce_mean(input_tensor=placeholder)) - else: - assert False, 'Invalid run name: %r' % run_name - summ = tf.compat.v1.summary.merge_all() - - subdir = os.path.join(self.logdir, run_name) - with test_util.FileWriterCache.get(subdir) as writer: - writer.add_graph(sess.graph) - for step in xrange(self._STEPS): - feed_dict = {placeholder: [1 + step, 2 + step, 3 + step]} - s = sess.run(summ, feed_dict=feed_dict) - writer.add_summary(s, global_step=step) - - def test_routes_provided(self): - """Tests that the plugin offers the correct routes.""" - self.set_up_with_runs([self._RUN_WITH_SCALARS]) - routes = self.plugin.get_plugin_apps() - self.assertIsInstance(routes['/distributions'], collections.Callable) - self.assertIsInstance(routes['/tags'], collections.Callable) - - def test_index(self): - self.set_up_with_runs([self._RUN_WITH_SCALARS, - self._RUN_WITH_LEGACY_DISTRIBUTION, - self._RUN_WITH_DISTRIBUTION]) - self.assertEqual({ - # _RUN_WITH_SCALARS omitted: No distribution data. - self._RUN_WITH_LEGACY_DISTRIBUTION: { - self._LEGACY_DISTRIBUTION_TAG: { - 'displayName': self._LEGACY_DISTRIBUTION_TAG, - 'description': '', - }, - }, - self._RUN_WITH_DISTRIBUTION: { - '%s/histogram_summary' % self._DISTRIBUTION_TAG: { - 'displayName': self._DISPLAY_NAME, - 'description': self._HTML_DESCRIPTION, + _STEPS = 99 + + _LEGACY_DISTRIBUTION_TAG = "my-ancient-distribution" + _DISTRIBUTION_TAG = "my-favorite-distribution" + _SCALAR_TAG = "my-boring-scalars" + + _DISPLAY_NAME = "Very important production statistics" + _DESCRIPTION = "quod *erat* dispertiendum" + _HTML_DESCRIPTION = "quod erat dispertiendum
" + + _RUN_WITH_LEGACY_DISTRIBUTION = "_RUN_WITH_LEGACY_DISTRIBUTION" + _RUN_WITH_DISTRIBUTION = "_RUN_WITH_DISTRIBUTION" + _RUN_WITH_SCALARS = "_RUN_WITH_SCALARS" + + def __init__(self, *args, **kwargs): + super(DistributionsPluginTest, self).__init__(*args, **kwargs) + self.logdir = None + self.plugin = None + + def set_up_with_runs(self, run_names): + self.logdir = self.get_temp_dir() + for run_name in run_names: + self.generate_run(run_name) + multiplexer = event_multiplexer.EventMultiplexer( + size_guidance={ + # don't truncate my test data, please + event_accumulator.TENSORS: self._STEPS, + } + ) + multiplexer.AddRunsFromDirectory(self.logdir) + multiplexer.Reload() + context = base_plugin.TBContext( + logdir=self.logdir, multiplexer=multiplexer + ) + self.plugin = distributions_plugin.DistributionsPlugin(context) + + def generate_run(self, run_name): + tf.compat.v1.reset_default_graph() + sess = tf.compat.v1.Session() + placeholder = tf.compat.v1.placeholder(tf.float32, shape=[3]) + + if run_name == self._RUN_WITH_LEGACY_DISTRIBUTION: + tf.compat.v1.summary.histogram( + self._LEGACY_DISTRIBUTION_TAG, placeholder + ) + elif run_name == self._RUN_WITH_DISTRIBUTION: + summary.op( + self._DISTRIBUTION_TAG, + placeholder, + display_name=self._DISPLAY_NAME, + description=self._DESCRIPTION, + ) + elif run_name == self._RUN_WITH_SCALARS: + tf.compat.v1.summary.scalar( + self._SCALAR_TAG, tf.reduce_mean(input_tensor=placeholder) + ) + else: + assert False, "Invalid run name: %r" % run_name + summ = tf.compat.v1.summary.merge_all() + + subdir = os.path.join(self.logdir, run_name) + with test_util.FileWriterCache.get(subdir) as writer: + writer.add_graph(sess.graph) + for step in xrange(self._STEPS): + feed_dict = {placeholder: [1 + step, 2 + step, 3 + step]} + s = sess.run(summ, feed_dict=feed_dict) + writer.add_summary(s, global_step=step) + + def test_routes_provided(self): + """Tests that the plugin offers the correct routes.""" + self.set_up_with_runs([self._RUN_WITH_SCALARS]) + routes = self.plugin.get_plugin_apps() + self.assertIsInstance(routes["/distributions"], collections.Callable) + self.assertIsInstance(routes["/tags"], collections.Callable) + + def test_index(self): + self.set_up_with_runs( + [ + self._RUN_WITH_SCALARS, + self._RUN_WITH_LEGACY_DISTRIBUTION, + self._RUN_WITH_DISTRIBUTION, + ] + ) + self.assertEqual( + { + # _RUN_WITH_SCALARS omitted: No distribution data. + self._RUN_WITH_LEGACY_DISTRIBUTION: { + self._LEGACY_DISTRIBUTION_TAG: { + "displayName": self._LEGACY_DISTRIBUTION_TAG, + "description": "", + }, + }, + self._RUN_WITH_DISTRIBUTION: { + "%s/histogram_summary" + % self._DISTRIBUTION_TAG: { + "displayName": self._DISPLAY_NAME, + "description": self._HTML_DESCRIPTION, + }, + }, }, - }, - }, self.plugin.index_impl(experiment='exp')) - - def _test_distributions(self, run_name, tag_name, should_work=True): - self.set_up_with_runs([self._RUN_WITH_SCALARS, - self._RUN_WITH_LEGACY_DISTRIBUTION, - self._RUN_WITH_DISTRIBUTION]) - if should_work: - (data, mime_type) = self.plugin.distributions_impl( - tag_name, run_name, experiment='exp' - ) - self.assertEqual('application/json', mime_type) - self.assertEqual(len(data), self._STEPS) - for i in xrange(self._STEPS): - [_unused_wall_time, step, bps_and_icdfs] = data[i] - self.assertEqual(i, step) - (bps, _unused_icdfs) = zip(*bps_and_icdfs) - self.assertEqual(bps, compressor.NORMAL_HISTOGRAM_BPS) - else: - with self.assertRaises(errors.NotFoundError): - self.plugin.distributions_impl( - self._DISTRIBUTION_TAG, run_name, experiment='exp' + self.plugin.index_impl(experiment="exp"), ) - def test_distributions_with_scalars(self): - self._test_distributions(self._RUN_WITH_SCALARS, self._DISTRIBUTION_TAG, - should_work=False) + def _test_distributions(self, run_name, tag_name, should_work=True): + self.set_up_with_runs( + [ + self._RUN_WITH_SCALARS, + self._RUN_WITH_LEGACY_DISTRIBUTION, + self._RUN_WITH_DISTRIBUTION, + ] + ) + if should_work: + (data, mime_type) = self.plugin.distributions_impl( + tag_name, run_name, experiment="exp" + ) + self.assertEqual("application/json", mime_type) + self.assertEqual(len(data), self._STEPS) + for i in xrange(self._STEPS): + [_unused_wall_time, step, bps_and_icdfs] = data[i] + self.assertEqual(i, step) + (bps, _unused_icdfs) = zip(*bps_and_icdfs) + self.assertEqual(bps, compressor.NORMAL_HISTOGRAM_BPS) + else: + with self.assertRaises(errors.NotFoundError): + self.plugin.distributions_impl( + self._DISTRIBUTION_TAG, run_name, experiment="exp" + ) + + def test_distributions_with_scalars(self): + self._test_distributions( + self._RUN_WITH_SCALARS, self._DISTRIBUTION_TAG, should_work=False + ) - def test_distributions_with_legacy_distribution(self): - self._test_distributions(self._RUN_WITH_LEGACY_DISTRIBUTION, - self._LEGACY_DISTRIBUTION_TAG) + def test_distributions_with_legacy_distribution(self): + self._test_distributions( + self._RUN_WITH_LEGACY_DISTRIBUTION, self._LEGACY_DISTRIBUTION_TAG + ) - def test_distributions_with_distribution(self): - self._test_distributions(self._RUN_WITH_DISTRIBUTION, - '%s/histogram_summary' % self._DISTRIBUTION_TAG) + def test_distributions_with_distribution(self): + self._test_distributions( + self._RUN_WITH_DISTRIBUTION, + "%s/histogram_summary" % self._DISTRIBUTION_TAG, + ) - def test_active_with_distribution(self): - self.set_up_with_runs([self._RUN_WITH_DISTRIBUTION]) - self.assertTrue(self.plugin.is_active()) + def test_active_with_distribution(self): + self.set_up_with_runs([self._RUN_WITH_DISTRIBUTION]) + self.assertTrue(self.plugin.is_active()) - def test_active_with_scalars(self): - self.set_up_with_runs([self._RUN_WITH_SCALARS]) - self.assertFalse(self.plugin.is_active()) + def test_active_with_scalars(self): + self.set_up_with_runs([self._RUN_WITH_SCALARS]) + self.assertFalse(self.plugin.is_active()) - def test_active_with_both(self): - self.set_up_with_runs([self._RUN_WITH_DISTRIBUTION, - self._RUN_WITH_SCALARS]) - self.assertTrue(self.plugin.is_active()) + def test_active_with_both(self): + self.set_up_with_runs( + [self._RUN_WITH_DISTRIBUTION, self._RUN_WITH_SCALARS] + ) + self.assertTrue(self.plugin.is_active()) -if __name__ == '__main__': - tf.test.main() +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/graph/graph_util.py b/tensorboard/plugins/graph/graph_util.py index db477674b8..c820752808 100644 --- a/tensorboard/plugins/graph/graph_util.py +++ b/tensorboard/plugins/graph/graph_util.py @@ -17,134 +17,150 @@ class _ProtoListDuplicateKeyError(Exception): - pass + pass class _SameKeyDiffContentError(Exception): - pass + pass def _safe_copy_proto_list_values(dst_proto_list, src_proto_list, get_key): - """Safely merge values from `src_proto_list` into `dst_proto_list`. - - Each element in `dst_proto_list` must be mapped by `get_key` to a key - value that is unique within that list; likewise for `src_proto_list`. - If an element of `src_proto_list` has the same key as an existing - element in `dst_proto_list`, then the elements must also be equal. - - Args: - dst_proto_list: A `RepeatedCompositeContainer` or - `RepeatedScalarContainer` into which values should be copied. - src_proto_list: A container holding the same kind of values as in - `dst_proto_list` from which values should be copied. - get_key: A function that takes an element of `dst_proto_list` or - `src_proto_list` and returns a key, such that if two elements have - the same key then it is required that they be deep-equal. For - instance, if `dst_proto_list` is a list of nodes, then `get_key` - might be `lambda node: node.name` to indicate that if two nodes - have the same name then they must be the same node. All keys must - be hashable. - - Raises: - _ProtoListDuplicateKeyError: A proto_list contains items with duplicate - keys. - _SameKeyDiffContentError: An item with the same key has different contents. - """ - - def _assert_proto_container_unique_keys(proto_list, get_key): - """Asserts proto_list to only contains unique keys. + """Safely merge values from `src_proto_list` into `dst_proto_list`. + + Each element in `dst_proto_list` must be mapped by `get_key` to a key + value that is unique within that list; likewise for `src_proto_list`. + If an element of `src_proto_list` has the same key as an existing + element in `dst_proto_list`, then the elements must also be equal. Args: - proto_list: A `RepeatedCompositeContainer` or `RepeatedScalarContainer`. - get_key: A function that takes an element of `proto_list` and returns a - hashable key. + dst_proto_list: A `RepeatedCompositeContainer` or + `RepeatedScalarContainer` into which values should be copied. + src_proto_list: A container holding the same kind of values as in + `dst_proto_list` from which values should be copied. + get_key: A function that takes an element of `dst_proto_list` or + `src_proto_list` and returns a key, such that if two elements have + the same key then it is required that they be deep-equal. For + instance, if `dst_proto_list` is a list of nodes, then `get_key` + might be `lambda node: node.name` to indicate that if two nodes + have the same name then they must be the same node. All keys must + be hashable. Raises: _ProtoListDuplicateKeyError: A proto_list contains items with duplicate keys. + _SameKeyDiffContentError: An item with the same key has different contents. """ - keys = set() - for item in proto_list: - key = get_key(item) - if key in keys: - raise _ProtoListDuplicateKeyError(key) - keys.add(key) - - _assert_proto_container_unique_keys(dst_proto_list, get_key) - _assert_proto_container_unique_keys(src_proto_list, get_key) - - key_to_proto = {} - for proto in dst_proto_list: - key = get_key(proto) - key_to_proto[key] = proto - - for proto in src_proto_list: - key = get_key(proto) - if key in key_to_proto: - if proto != key_to_proto.get(key): - raise _SameKeyDiffContentError(key) - else: - dst_proto_list.add().CopyFrom(proto) + + def _assert_proto_container_unique_keys(proto_list, get_key): + """Asserts proto_list to only contains unique keys. + + Args: + proto_list: A `RepeatedCompositeContainer` or `RepeatedScalarContainer`. + get_key: A function that takes an element of `proto_list` and returns a + hashable key. + + Raises: + _ProtoListDuplicateKeyError: A proto_list contains items with duplicate + keys. + """ + keys = set() + for item in proto_list: + key = get_key(item) + if key in keys: + raise _ProtoListDuplicateKeyError(key) + keys.add(key) + + _assert_proto_container_unique_keys(dst_proto_list, get_key) + _assert_proto_container_unique_keys(src_proto_list, get_key) + + key_to_proto = {} + for proto in dst_proto_list: + key = get_key(proto) + key_to_proto[key] = proto + + for proto in src_proto_list: + key = get_key(proto) + if key in key_to_proto: + if proto != key_to_proto.get(key): + raise _SameKeyDiffContentError(key) + else: + dst_proto_list.add().CopyFrom(proto) def combine_graph_defs(to_proto, from_proto): - """Combines two GraphDefs by adding nodes from from_proto into to_proto. - - All GraphDefs are expected to be of TensorBoard's. - It assumes node names are unique across GraphDefs if contents differ. The - names can be the same if the NodeDef content are exactly the same. - - Args: - to_proto: A destination TensorBoard GraphDef. - from_proto: A TensorBoard GraphDef to copy contents from. - - Returns: - to_proto - - Raises: - ValueError in case any assumption about GraphDef is violated: A - GraphDef should have unique node, function, and gradient function - names. Also, when merging GraphDefs, they should have not have nodes, - functions, or gradient function mappings that share the name but details - do not match. - """ - if from_proto.version != to_proto.version: - raise ValueError('Cannot combine GraphDefs of different versions.') - - try: - _safe_copy_proto_list_values( - to_proto.node, - from_proto.node, - lambda n: n.name) - except _ProtoListDuplicateKeyError as exc: - raise ValueError('A GraphDef contains non-unique node names: %s' % exc) - except _SameKeyDiffContentError as exc: - raise ValueError( - ('Cannot combine GraphDefs because nodes share a name ' - 'but contents are different: %s') % exc) - try: - _safe_copy_proto_list_values( - to_proto.library.function, - from_proto.library.function, - lambda n: n.signature.name) - except _ProtoListDuplicateKeyError as exc: - raise ValueError('A GraphDef contains non-unique function names: %s' % exc) - except _SameKeyDiffContentError as exc: - raise ValueError( - ('Cannot combine GraphDefs because functions share a name ' - 'but are different: %s') % exc) - - try: - _safe_copy_proto_list_values( - to_proto.library.gradient, - from_proto.library.gradient, - lambda g: g.gradient_func) - except _ProtoListDuplicateKeyError as exc: - raise ValueError( - 'A GraphDef contains non-unique gradient function names: %s' % exc) - except _SameKeyDiffContentError as exc: - raise ValueError( - ('Cannot combine GraphDefs because gradients share a gradient_func name ' - 'but map to different functions: %s') % exc) - - return to_proto + """Combines two GraphDefs by adding nodes from from_proto into to_proto. + + All GraphDefs are expected to be of TensorBoard's. + It assumes node names are unique across GraphDefs if contents differ. The + names can be the same if the NodeDef content are exactly the same. + + Args: + to_proto: A destination TensorBoard GraphDef. + from_proto: A TensorBoard GraphDef to copy contents from. + + Returns: + to_proto + + Raises: + ValueError in case any assumption about GraphDef is violated: A + GraphDef should have unique node, function, and gradient function + names. Also, when merging GraphDefs, they should have not have nodes, + functions, or gradient function mappings that share the name but details + do not match. + """ + if from_proto.version != to_proto.version: + raise ValueError("Cannot combine GraphDefs of different versions.") + + try: + _safe_copy_proto_list_values( + to_proto.node, from_proto.node, lambda n: n.name + ) + except _ProtoListDuplicateKeyError as exc: + raise ValueError("A GraphDef contains non-unique node names: %s" % exc) + except _SameKeyDiffContentError as exc: + raise ValueError( + ( + "Cannot combine GraphDefs because nodes share a name " + "but contents are different: %s" + ) + % exc + ) + try: + _safe_copy_proto_list_values( + to_proto.library.function, + from_proto.library.function, + lambda n: n.signature.name, + ) + except _ProtoListDuplicateKeyError as exc: + raise ValueError( + "A GraphDef contains non-unique function names: %s" % exc + ) + except _SameKeyDiffContentError as exc: + raise ValueError( + ( + "Cannot combine GraphDefs because functions share a name " + "but are different: %s" + ) + % exc + ) + + try: + _safe_copy_proto_list_values( + to_proto.library.gradient, + from_proto.library.gradient, + lambda g: g.gradient_func, + ) + except _ProtoListDuplicateKeyError as exc: + raise ValueError( + "A GraphDef contains non-unique gradient function names: %s" % exc + ) + except _SameKeyDiffContentError as exc: + raise ValueError( + ( + "Cannot combine GraphDefs because gradients share a gradient_func name " + "but map to different functions: %s" + ) + % exc + ) + + return to_proto diff --git a/tensorboard/plugins/graph/graph_util_test.py b/tensorboard/plugins/graph/graph_util_test.py index 124f629bdc..c456727b63 100644 --- a/tensorboard/plugins/graph/graph_util_test.py +++ b/tensorboard/plugins/graph/graph_util_test.py @@ -21,9 +21,8 @@ class GraphUtilTest(tf.test.TestCase): - - def test_combine_graph_defs(self): - expected_proto = ''' + def test_combine_graph_defs(self): + expected_proto = """ node { name: "X" op: "Input" @@ -55,10 +54,11 @@ def test_combine_graph_defs(self): versions { producer: 21 } - ''' + """ - graph_def_a = GraphDef() - text_format.Merge(''' + graph_def_a = GraphDef() + text_format.Merge( + """ node { name: "X" op: "Input" @@ -76,10 +76,13 @@ def test_combine_graph_defs(self): versions { producer: 21 } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ node { name: "A" op: "Input" @@ -97,14 +100,17 @@ def test_combine_graph_defs(self): versions { producer: 21 } - ''', graph_def_b) + """, + graph_def_b, + ) - self.assertProtoEquals( - expected_proto, - graph_util.combine_graph_defs(graph_def_a, graph_def_b)) + self.assertProtoEquals( + expected_proto, + graph_util.combine_graph_defs(graph_def_a, graph_def_b), + ) - def test_combine_graph_defs_name_collided_but_same_content(self): - expected_proto = ''' + def test_combine_graph_defs_name_collided_but_same_content(self): + expected_proto = """ node { name: "X" op: "Input" @@ -126,10 +132,11 @@ def test_combine_graph_defs_name_collided_but_same_content(self): versions { producer: 21 } - ''' + """ - graph_def_a = GraphDef() - text_format.Merge(''' + graph_def_a = GraphDef() + text_format.Merge( + """ node { name: "X" op: "Input" @@ -147,10 +154,13 @@ def test_combine_graph_defs_name_collided_but_same_content(self): versions { producer: 21 } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ node { name: "X" op: "Input" @@ -162,15 +172,19 @@ def test_combine_graph_defs_name_collided_but_same_content(self): versions { producer: 21 } - ''', graph_def_b) + """, + graph_def_b, + ) - self.assertProtoEquals( - expected_proto, - graph_util.combine_graph_defs(graph_def_a, graph_def_b)) + self.assertProtoEquals( + expected_proto, + graph_util.combine_graph_defs(graph_def_a, graph_def_b), + ) - def test_combine_graph_defs_name_collided_different_content(self): - graph_def_a = GraphDef() - text_format.Merge(''' + def test_combine_graph_defs_name_collided_different_content(self): + graph_def_a = GraphDef() + text_format.Merge( + """ node { name: "X" op: "Input" @@ -188,10 +202,13 @@ def test_combine_graph_defs_name_collided_different_content(self): versions { producer: 21 } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ node { name: "X" op: "Input" @@ -210,18 +227,24 @@ def test_combine_graph_defs_name_collided_different_content(self): versions { producer: 21 } - ''', graph_def_b) - - with six.assertRaisesRegex( - self, - ValueError, - ('Cannot combine GraphDefs because nodes share a name but ' - 'contents are different: X')): - graph_util.combine_graph_defs(graph_def_a, graph_def_b) - - def test_combine_graph_defs_dst_nodes_duplicate_keys(self): - graph_def_a = GraphDef() - text_format.Merge(''' + """, + graph_def_b, + ) + + with six.assertRaisesRegex( + self, + ValueError, + ( + "Cannot combine GraphDefs because nodes share a name but " + "contents are different: X" + ), + ): + graph_util.combine_graph_defs(graph_def_a, graph_def_b) + + def test_combine_graph_defs_dst_nodes_duplicate_keys(self): + graph_def_a = GraphDef() + text_format.Merge( + """ node { name: "X" op: "Input" @@ -233,10 +256,13 @@ def test_combine_graph_defs_dst_nodes_duplicate_keys(self): versions { producer: 21 } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ node { name: "X" op: "Input" @@ -248,17 +274,19 @@ def test_combine_graph_defs_dst_nodes_duplicate_keys(self): versions { producer: 21 } - ''', graph_def_b) + """, + graph_def_b, + ) - with six.assertRaisesRegex( - self, - ValueError, - 'A GraphDef contains non-unique node names: X'): - graph_util.combine_graph_defs(graph_def_a, graph_def_b) + with six.assertRaisesRegex( + self, ValueError, "A GraphDef contains non-unique node names: X" + ): + graph_util.combine_graph_defs(graph_def_a, graph_def_b) - def test_combine_graph_defs_src_nodes_duplicate_keys(self): - graph_def_a = GraphDef() - text_format.Merge(''' + def test_combine_graph_defs_src_nodes_duplicate_keys(self): + graph_def_a = GraphDef() + text_format.Merge( + """ node { name: "X" op: "Input" @@ -270,10 +298,13 @@ def test_combine_graph_defs_src_nodes_duplicate_keys(self): versions { producer: 21 } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ node { name: "W" op: "Input" @@ -286,16 +317,17 @@ def test_combine_graph_defs_src_nodes_duplicate_keys(self): versions { producer: 21 } - ''', graph_def_b) + """, + graph_def_b, + ) - with six.assertRaisesRegex( - self, - ValueError, - 'A GraphDef contains non-unique node names: W'): - graph_util.combine_graph_defs(graph_def_a, graph_def_b) + with six.assertRaisesRegex( + self, ValueError, "A GraphDef contains non-unique node names: W" + ): + graph_util.combine_graph_defs(graph_def_a, graph_def_b) - def test_combine_graph_defs_function(self): - expected_proto = ''' + def test_combine_graph_defs_function(self): + expected_proto = """ library { function { signature { @@ -336,10 +368,11 @@ def test_combine_graph_defs_function(self): } } } - ''' + """ - graph_def_a = GraphDef() - text_format.Merge(''' + graph_def_a = GraphDef() + text_format.Merge( + """ library { function { signature { @@ -361,10 +394,13 @@ def test_combine_graph_defs_function(self): } } } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ library { function { signature { @@ -405,15 +441,19 @@ def test_combine_graph_defs_function(self): } } } - ''', graph_def_b) + """, + graph_def_b, + ) - self.assertProtoEquals( - expected_proto, - graph_util.combine_graph_defs(graph_def_a, graph_def_b)) + self.assertProtoEquals( + expected_proto, + graph_util.combine_graph_defs(graph_def_a, graph_def_b), + ) - def test_combine_graph_defs_function_collison(self): - graph_def_a = GraphDef() - text_format.Merge(''' + def test_combine_graph_defs_function_collison(self): + graph_def_a = GraphDef() + text_format.Merge( + """ library { function { signature { @@ -435,10 +475,13 @@ def test_combine_graph_defs_function_collison(self): } } } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ library { function { signature { @@ -479,18 +522,24 @@ def test_combine_graph_defs_function_collison(self): } } } - ''', graph_def_b) - - with six.assertRaisesRegex( - self, - ValueError, - ('Cannot combine GraphDefs because functions share a name but ' - 'are different: foo')): - graph_util.combine_graph_defs(graph_def_a, graph_def_b) - - def test_combine_graph_defs_dst_function_duplicate_keys(self): - graph_def_a = GraphDef() - text_format.Merge(''' + """, + graph_def_b, + ) + + with six.assertRaisesRegex( + self, + ValueError, + ( + "Cannot combine GraphDefs because functions share a name but " + "are different: foo" + ), + ): + graph_util.combine_graph_defs(graph_def_a, graph_def_b) + + def test_combine_graph_defs_dst_function_duplicate_keys(self): + graph_def_a = GraphDef() + text_format.Merge( + """ library { function { signature { @@ -525,10 +574,13 @@ def test_combine_graph_defs_dst_function_duplicate_keys(self): } } } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ library { function { signature { @@ -550,17 +602,21 @@ def test_combine_graph_defs_dst_function_duplicate_keys(self): } } } - ''', graph_def_b) - - with six.assertRaisesRegex( - self, - ValueError, - ('A GraphDef contains non-unique function names: foo')): - graph_util.combine_graph_defs(graph_def_a, graph_def_b) - - def test_combine_graph_defs_src_function_duplicate_keys(self): - graph_def_a = GraphDef() - text_format.Merge(''' + """, + graph_def_b, + ) + + with six.assertRaisesRegex( + self, + ValueError, + ("A GraphDef contains non-unique function names: foo"), + ): + graph_util.combine_graph_defs(graph_def_a, graph_def_b) + + def test_combine_graph_defs_src_function_duplicate_keys(self): + graph_def_a = GraphDef() + text_format.Merge( + """ library { function { signature { @@ -582,10 +638,13 @@ def test_combine_graph_defs_src_function_duplicate_keys(self): } } } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ library { function { signature { @@ -614,16 +673,19 @@ def test_combine_graph_defs_src_function_duplicate_keys(self): } } } - ''', graph_def_b) + """, + graph_def_b, + ) - with six.assertRaisesRegex( - self, - ValueError, - 'A GraphDef contains non-unique function names: bar'): - graph_util.combine_graph_defs(graph_def_a, graph_def_b) + with six.assertRaisesRegex( + self, + ValueError, + "A GraphDef contains non-unique function names: bar", + ): + graph_util.combine_graph_defs(graph_def_a, graph_def_b) - def test_combine_graph_defs_gradient(self): - expected_proto = ''' + def test_combine_graph_defs_gradient(self): + expected_proto = """ library { gradient { function_name: "foo" @@ -634,20 +696,24 @@ def test_combine_graph_defs_gradient(self): gradient_func: "bar_grad" } } - ''' + """ - graph_def_a = GraphDef() - text_format.Merge(''' + graph_def_a = GraphDef() + text_format.Merge( + """ library { gradient { function_name: "foo" gradient_func: "foo_grad" } } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ library { gradient { function_name: "foo" @@ -658,25 +724,32 @@ def test_combine_graph_defs_gradient(self): gradient_func: "bar_grad" } } - ''', graph_def_b) + """, + graph_def_b, + ) - self.assertProtoEquals( - expected_proto, - graph_util.combine_graph_defs(graph_def_a, graph_def_b)) + self.assertProtoEquals( + expected_proto, + graph_util.combine_graph_defs(graph_def_a, graph_def_b), + ) - def test_combine_graph_defs_gradient_collison(self): - graph_def_a = GraphDef() - text_format.Merge(''' + def test_combine_graph_defs_gradient_collison(self): + graph_def_a = GraphDef() + text_format.Merge( + """ library { gradient { function_name: "foo" gradient_func: "foo_grad" } } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ library { gradient { function_name: "bar" @@ -687,18 +760,24 @@ def test_combine_graph_defs_gradient_collison(self): gradient_func: "foo_grad" } } - ''', graph_def_b) - - with six.assertRaisesRegex( - self, - ValueError, - ('share a gradient_func name but map to different functions: ' - 'foo_grad')): - graph_util.combine_graph_defs(graph_def_a, graph_def_b) - - def test_combine_graph_defs_dst_gradient_func_non_unique(self): - graph_def_a = GraphDef() - text_format.Merge(''' + """, + graph_def_b, + ) + + with six.assertRaisesRegex( + self, + ValueError, + ( + "share a gradient_func name but map to different functions: " + "foo_grad" + ), + ): + graph_util.combine_graph_defs(graph_def_a, graph_def_b) + + def test_combine_graph_defs_dst_gradient_func_non_unique(self): + graph_def_a = GraphDef() + text_format.Merge( + """ library { gradient { function_name: "foo" @@ -709,37 +788,47 @@ def test_combine_graph_defs_dst_gradient_func_non_unique(self): gradient_func: "foo_grad" } } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ library { gradient { function_name: "bar" gradient_func: "bar_grad" } } - ''', graph_def_b) - - with six.assertRaisesRegex( - self, - ValueError, - 'A GraphDef contains non-unique gradient function names: foo_grad'): - graph_util.combine_graph_defs(graph_def_a, graph_def_b) - - def test_combine_graph_defs_src_gradient_func_non_unique(self): - graph_def_a = GraphDef() - text_format.Merge(''' + """, + graph_def_b, + ) + + with six.assertRaisesRegex( + self, + ValueError, + "A GraphDef contains non-unique gradient function names: foo_grad", + ): + graph_util.combine_graph_defs(graph_def_a, graph_def_b) + + def test_combine_graph_defs_src_gradient_func_non_unique(self): + graph_def_a = GraphDef() + text_format.Merge( + """ library { gradient { function_name: "foo" gradient_func: "foo_grad" } } - ''', graph_def_a) + """, + graph_def_a, + ) - graph_def_b = GraphDef() - text_format.Merge(''' + graph_def_b = GraphDef() + text_format.Merge( + """ library { gradient { function_name: "bar" @@ -750,14 +839,17 @@ def test_combine_graph_defs_src_gradient_func_non_unique(self): gradient_func: "bar_grad" } } - ''', graph_def_b) + """, + graph_def_b, + ) - with six.assertRaisesRegex( - self, - ValueError, - 'A GraphDef contains non-unique gradient function names: bar_grad'): - graph_util.combine_graph_defs(graph_def_a, graph_def_b) + with six.assertRaisesRegex( + self, + ValueError, + "A GraphDef contains non-unique gradient function names: bar_grad", + ): + graph_util.combine_graph_defs(graph_def_a, graph_def_b) -if __name__ == '__main__': - tf.test.main() +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/graph/graphs_plugin.py b/tensorboard/plugins/graph/graphs_plugin.py index ccbbf6b3b5..427c7d614c 100644 --- a/tensorboard/plugins/graph/graphs_plugin.py +++ b/tensorboard/plugins/graph/graphs_plugin.py @@ -25,7 +25,9 @@ from tensorboard import plugin_util from tensorboard.backend import http_util from tensorboard.backend import process_graph -from tensorboard.backend.event_processing import plugin_event_accumulator as event_accumulator +from tensorboard.backend.event_processing import ( + plugin_event_accumulator as event_accumulator, +) from tensorboard.compat.proto import config_pb2 from tensorboard.compat.proto import graph_pb2 from tensorboard.data import provider @@ -41,260 +43,314 @@ # As a result, this SummaryMetadata is a bit unconventional and uses non-public # hardcoded name as the plugin name. Please refer to link below for the summary ops. # https://github.com/tensorflow/tensorflow/blob/11f4ecb54708865ec757ca64e4805957b05d7570/tensorflow/python/ops/summary_ops_v2.py#L757 -_PLUGIN_NAME_RUN_METADATA = 'graph_run_metadata' +_PLUGIN_NAME_RUN_METADATA = "graph_run_metadata" # https://github.com/tensorflow/tensorflow/blob/11f4ecb54708865ec757ca64e4805957b05d7570/tensorflow/python/ops/summary_ops_v2.py#L788 -_PLUGIN_NAME_RUN_METADATA_WITH_GRAPH = 'graph_run_metadata_graph' +_PLUGIN_NAME_RUN_METADATA_WITH_GRAPH = "graph_run_metadata_graph" # https://github.com/tensorflow/tensorflow/blob/565952cc2f17fdfd995e25171cf07be0f6f06180/tensorflow/python/ops/summary_ops_v2.py#L825 -_PLUGIN_NAME_KERAS_MODEL = 'graph_keras_model' +_PLUGIN_NAME_KERAS_MODEL = "graph_keras_model" class GraphsPlugin(base_plugin.TBPlugin): - """Graphs Plugin for TensorBoard.""" - - plugin_name = metadata.PLUGIN_NAME - - def __init__(self, context): - """Instantiates GraphsPlugin via TensorBoard core. - - Args: - context: A base_plugin.TBContext instance. - """ - self._multiplexer = context.multiplexer - if context.flags and context.flags.generic_data == 'true': - self._data_provider = context.data_provider - else: - self._data_provider = None - - def get_plugin_apps(self): - return { - '/graph': self.graph_route, - '/info': self.info_route, - '/run_metadata': self.run_metadata_route, - } - - def is_active(self): - """The graphs plugin is active iff any run has a graph or metadata.""" - if self._data_provider: - # We don't have an experiment ID, and modifying the backend core - # to provide one would break backward compatibility. Hack for now. - return True - - return bool(self.info_impl()) - - def frontend_metadata(self): - return base_plugin.FrontendMetadata( - element_name='tf-graph-dashboard', - # TODO(@chihuahua): Reconcile this setting with Health Pills. - disable_reload=True, - ) - - def info_impl(self, experiment=None): - """Returns a dict of all runs and their data availabilities.""" - result = {} - def add_row_item(run, tag=None): - run_item = result.setdefault(run, { - 'run': run, - 'tags': {}, - # A run-wide GraphDef of ops. - 'run_graph': False}) - - tag_item = None - if tag: - tag_item = run_item.get('tags').setdefault(tag, { - 'tag': tag, - 'conceptual_graph': False, - # A tagged GraphDef of ops. - 'op_graph': False, - 'profile': False}) - return (run_item, tag_item) - - if self._data_provider: - mapping = self._data_provider.list_blob_sequences( - experiment_id=experiment, - plugin_name=metadata.PLUGIN_NAME, - ) - for (run_name, tag_to_time_series) in six.iteritems(mapping): - for tag in tag_to_time_series: - (run_item, tag_item) = add_row_item(run_name, tag) - run_item['run_graph'] = True - if tag_item: - tag_item['op_graph'] = True - return result - - mapping = self._multiplexer.PluginRunToTagToContent( - _PLUGIN_NAME_RUN_METADATA_WITH_GRAPH) - for run_name, tag_to_content in six.iteritems(mapping): - for (tag, content) in six.iteritems(tag_to_content): - # The Summary op is defined in TensorFlow and does not use a stringified proto - # as a content of plugin data. It contains single string that denotes a version. - # https://github.com/tensorflow/tensorflow/blob/11f4ecb54708865ec757ca64e4805957b05d7570/tensorflow/python/ops/summary_ops_v2.py#L789-L790 - if content != b'1': - logger.warn('Ignoring unrecognizable version of RunMetadata.') - continue - (_, tag_item) = add_row_item(run_name, tag) - tag_item['op_graph'] = True - - # Tensors associated with plugin name _PLUGIN_NAME_RUN_METADATA contain - # both op graph and profile information. - mapping = self._multiplexer.PluginRunToTagToContent( - _PLUGIN_NAME_RUN_METADATA) - for run_name, tag_to_content in six.iteritems(mapping): - for (tag, content) in six.iteritems(tag_to_content): - if content != b'1': - logger.warn('Ignoring unrecognizable version of RunMetadata.') - continue - (_, tag_item) = add_row_item(run_name, tag) - tag_item['profile'] = True - tag_item['op_graph'] = True - - # Tensors associated with plugin name _PLUGIN_NAME_KERAS_MODEL contain - # serialized Keras model in JSON format. - mapping = self._multiplexer.PluginRunToTagToContent( - _PLUGIN_NAME_KERAS_MODEL) - for run_name, tag_to_content in six.iteritems(mapping): - for (tag, content) in six.iteritems(tag_to_content): - if content != b'1': - logger.warn('Ignoring unrecognizable version of RunMetadata.') - continue - (_, tag_item) = add_row_item(run_name, tag) - tag_item['conceptual_graph'] = True - - for (run_name, run_data) in six.iteritems(self._multiplexer.Runs()): - if run_data.get(event_accumulator.GRAPH): - (run_item, _) = add_row_item(run_name, None) - run_item['run_graph'] = True - - for (run_name, run_data) in six.iteritems(self._multiplexer.Runs()): - if event_accumulator.RUN_METADATA in run_data: - for tag in run_data[event_accumulator.RUN_METADATA]: - (_, tag_item) = add_row_item(run_name, tag) - tag_item['profile'] = True - - return result - - def graph_impl(self, run, tag, is_conceptual, experiment=None, limit_attr_size=None, large_attrs_key=None): - """Result of the form `(body, mime_type)`, or `None` if no graph exists.""" - if self._data_provider: - graph_blob_sequences = self._data_provider.read_blob_sequences( - experiment_id=experiment, - plugin_name=metadata.PLUGIN_NAME, - run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), - ) - blob_datum_list = graph_blob_sequences.get(run, {}).get(tag, ()) - try: - blob_ref = blob_datum_list[0].values[0] - except IndexError: - return None - # Always use the blob_key approach for now, even if there is a direct url. - graph_raw = self._data_provider.read_blob(blob_ref.blob_key) - # This method ultimately returns pbtxt, but we have to deserialize and - # later reserialize this anyway, because a) this way we accept binary - # protobufs too, and b) below we run `prepare_graph_for_ui` on the graph. - graph = graph_pb2.GraphDef.FromString(graph_raw) - - elif is_conceptual: - tensor_events = self._multiplexer.Tensors(run, tag) - # Take the first event if there are multiple events written from different - # steps. - keras_model_config = json.loads(tensor_events[0].tensor_proto.string_val[0]) - graph = keras_util.keras_model_to_graph_def(keras_model_config) - - elif tag: - tensor_events = self._multiplexer.Tensors(run, tag) - # Take the first event if there are multiple events written from different - # steps. - run_metadata = config_pb2.RunMetadata.FromString( - tensor_events[0].tensor_proto.string_val[0]) - graph = graph_pb2.GraphDef() - - for func_graph in run_metadata.function_graphs: - graph_util.combine_graph_defs(graph, func_graph.pre_optimization_graph) - else: - graph = self._multiplexer.Graph(run) - - # This next line might raise a ValueError if the limit parameters - # are invalid (size is negative, size present but key absent, etc.). - process_graph.prepare_graph_for_ui(graph, limit_attr_size, large_attrs_key) - return (str(graph), 'text/x-protobuf') # pbtxt - - def run_metadata_impl(self, run, tag): - """Result of the form `(body, mime_type)`, or `None` if no data exists.""" - if self._data_provider: - # TODO(davidsoergel, wchargin): Consider plumbing run metadata through data providers. - return None - try: - run_metadata = self._multiplexer.RunMetadata(run, tag) - except ValueError: - # TODO(stephanwlee): Should include whether FE is fetching for v1 or v2 RunMetadata - # so we can remove this try/except. - tensor_events = self._multiplexer.Tensors(run, tag) - if tensor_events is None: - return None - # Take the first event if there are multiple events written from different - # steps. - run_metadata = config_pb2.RunMetadata.FromString( - tensor_events[0].tensor_proto.string_val[0]) - if run_metadata is None: - return None - return (str(run_metadata), 'text/x-protobuf') # pbtxt - - @wrappers.Request.application - def info_route(self, request): - experiment = plugin_util.experiment_id(request.environ) - info = self.info_impl(experiment) - return http_util.Respond(request, info, 'application/json') - - @wrappers.Request.application - def graph_route(self, request): - """Given a single run, return the graph definition in protobuf format.""" - experiment = plugin_util.experiment_id(request.environ) - run = request.args.get('run') - tag = request.args.get('tag') - conceptual_arg = request.args.get('conceptual', False) - is_conceptual = True if conceptual_arg == 'true' else False - - if run is None: - return http_util.Respond( - request, 'query parameter "run" is required', 'text/plain', 400) - - limit_attr_size = request.args.get('limit_attr_size', None) - if limit_attr_size is not None: - try: - limit_attr_size = int(limit_attr_size) - except ValueError: - return http_util.Respond( - request, 'query parameter `limit_attr_size` must be an integer', - 'text/plain', 400) - - large_attrs_key = request.args.get('large_attrs_key', None) - - try: - result = self.graph_impl(run, tag, is_conceptual, experiment, limit_attr_size, large_attrs_key) - except ValueError as e: - return http_util.Respond(request, e.message, 'text/plain', code=400) - else: - if result is not None: - (body, mime_type) = result # pylint: disable=unpacking-non-sequence - return http_util.Respond(request, body, mime_type) - else: - return http_util.Respond(request, '404 Not Found', 'text/plain', - code=404) - - @wrappers.Request.application - def run_metadata_route(self, request): - """Given a tag and a run, return the session.run() metadata.""" - tag = request.args.get('tag') - run = request.args.get('run') - if tag is None: - return http_util.Respond( - request, 'query parameter "tag" is required', 'text/plain', 400) - if run is None: - return http_util.Respond( - request, 'query parameter "run" is required', 'text/plain', 400) - result = self.run_metadata_impl(run, tag) - if result is not None: - (body, mime_type) = result # pylint: disable=unpacking-non-sequence - return http_util.Respond(request, body, mime_type) - else: - return http_util.Respond(request, '404 Not Found', 'text/plain', - code=404) + """Graphs Plugin for TensorBoard.""" + + plugin_name = metadata.PLUGIN_NAME + + def __init__(self, context): + """Instantiates GraphsPlugin via TensorBoard core. + + Args: + context: A base_plugin.TBContext instance. + """ + self._multiplexer = context.multiplexer + if context.flags and context.flags.generic_data == "true": + self._data_provider = context.data_provider + else: + self._data_provider = None + + def get_plugin_apps(self): + return { + "/graph": self.graph_route, + "/info": self.info_route, + "/run_metadata": self.run_metadata_route, + } + + def is_active(self): + """The graphs plugin is active iff any run has a graph or metadata.""" + if self._data_provider: + # We don't have an experiment ID, and modifying the backend core + # to provide one would break backward compatibility. Hack for now. + return True + + return bool(self.info_impl()) + + def frontend_metadata(self): + return base_plugin.FrontendMetadata( + element_name="tf-graph-dashboard", + # TODO(@chihuahua): Reconcile this setting with Health Pills. + disable_reload=True, + ) + + def info_impl(self, experiment=None): + """Returns a dict of all runs and their data availabilities.""" + result = {} + + def add_row_item(run, tag=None): + run_item = result.setdefault( + run, + { + "run": run, + "tags": {}, + # A run-wide GraphDef of ops. + "run_graph": False, + }, + ) + + tag_item = None + if tag: + tag_item = run_item.get("tags").setdefault( + tag, + { + "tag": tag, + "conceptual_graph": False, + # A tagged GraphDef of ops. + "op_graph": False, + "profile": False, + }, + ) + return (run_item, tag_item) + + if self._data_provider: + mapping = self._data_provider.list_blob_sequences( + experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME, + ) + for (run_name, tag_to_time_series) in six.iteritems(mapping): + for tag in tag_to_time_series: + (run_item, tag_item) = add_row_item(run_name, tag) + run_item["run_graph"] = True + if tag_item: + tag_item["op_graph"] = True + return result + + mapping = self._multiplexer.PluginRunToTagToContent( + _PLUGIN_NAME_RUN_METADATA_WITH_GRAPH + ) + for run_name, tag_to_content in six.iteritems(mapping): + for (tag, content) in six.iteritems(tag_to_content): + # The Summary op is defined in TensorFlow and does not use a stringified proto + # as a content of plugin data. It contains single string that denotes a version. + # https://github.com/tensorflow/tensorflow/blob/11f4ecb54708865ec757ca64e4805957b05d7570/tensorflow/python/ops/summary_ops_v2.py#L789-L790 + if content != b"1": + logger.warn( + "Ignoring unrecognizable version of RunMetadata." + ) + continue + (_, tag_item) = add_row_item(run_name, tag) + tag_item["op_graph"] = True + + # Tensors associated with plugin name _PLUGIN_NAME_RUN_METADATA contain + # both op graph and profile information. + mapping = self._multiplexer.PluginRunToTagToContent( + _PLUGIN_NAME_RUN_METADATA + ) + for run_name, tag_to_content in six.iteritems(mapping): + for (tag, content) in six.iteritems(tag_to_content): + if content != b"1": + logger.warn( + "Ignoring unrecognizable version of RunMetadata." + ) + continue + (_, tag_item) = add_row_item(run_name, tag) + tag_item["profile"] = True + tag_item["op_graph"] = True + + # Tensors associated with plugin name _PLUGIN_NAME_KERAS_MODEL contain + # serialized Keras model in JSON format. + mapping = self._multiplexer.PluginRunToTagToContent( + _PLUGIN_NAME_KERAS_MODEL + ) + for run_name, tag_to_content in six.iteritems(mapping): + for (tag, content) in six.iteritems(tag_to_content): + if content != b"1": + logger.warn( + "Ignoring unrecognizable version of RunMetadata." + ) + continue + (_, tag_item) = add_row_item(run_name, tag) + tag_item["conceptual_graph"] = True + + for (run_name, run_data) in six.iteritems(self._multiplexer.Runs()): + if run_data.get(event_accumulator.GRAPH): + (run_item, _) = add_row_item(run_name, None) + run_item["run_graph"] = True + + for (run_name, run_data) in six.iteritems(self._multiplexer.Runs()): + if event_accumulator.RUN_METADATA in run_data: + for tag in run_data[event_accumulator.RUN_METADATA]: + (_, tag_item) = add_row_item(run_name, tag) + tag_item["profile"] = True + + return result + + def graph_impl( + self, + run, + tag, + is_conceptual, + experiment=None, + limit_attr_size=None, + large_attrs_key=None, + ): + """Result of the form `(body, mime_type)`, or `None` if no graph + exists.""" + if self._data_provider: + graph_blob_sequences = self._data_provider.read_blob_sequences( + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), + ) + blob_datum_list = graph_blob_sequences.get(run, {}).get(tag, ()) + try: + blob_ref = blob_datum_list[0].values[0] + except IndexError: + return None + # Always use the blob_key approach for now, even if there is a direct url. + graph_raw = self._data_provider.read_blob(blob_ref.blob_key) + # This method ultimately returns pbtxt, but we have to deserialize and + # later reserialize this anyway, because a) this way we accept binary + # protobufs too, and b) below we run `prepare_graph_for_ui` on the graph. + graph = graph_pb2.GraphDef.FromString(graph_raw) + + elif is_conceptual: + tensor_events = self._multiplexer.Tensors(run, tag) + # Take the first event if there are multiple events written from different + # steps. + keras_model_config = json.loads( + tensor_events[0].tensor_proto.string_val[0] + ) + graph = keras_util.keras_model_to_graph_def(keras_model_config) + + elif tag: + tensor_events = self._multiplexer.Tensors(run, tag) + # Take the first event if there are multiple events written from different + # steps. + run_metadata = config_pb2.RunMetadata.FromString( + tensor_events[0].tensor_proto.string_val[0] + ) + graph = graph_pb2.GraphDef() + + for func_graph in run_metadata.function_graphs: + graph_util.combine_graph_defs( + graph, func_graph.pre_optimization_graph + ) + else: + graph = self._multiplexer.Graph(run) + + # This next line might raise a ValueError if the limit parameters + # are invalid (size is negative, size present but key absent, etc.). + process_graph.prepare_graph_for_ui( + graph, limit_attr_size, large_attrs_key + ) + return (str(graph), "text/x-protobuf") # pbtxt + + def run_metadata_impl(self, run, tag): + """Result of the form `(body, mime_type)`, or `None` if no data + exists.""" + if self._data_provider: + # TODO(davidsoergel, wchargin): Consider plumbing run metadata through data providers. + return None + try: + run_metadata = self._multiplexer.RunMetadata(run, tag) + except ValueError: + # TODO(stephanwlee): Should include whether FE is fetching for v1 or v2 RunMetadata + # so we can remove this try/except. + tensor_events = self._multiplexer.Tensors(run, tag) + if tensor_events is None: + return None + # Take the first event if there are multiple events written from different + # steps. + run_metadata = config_pb2.RunMetadata.FromString( + tensor_events[0].tensor_proto.string_val[0] + ) + if run_metadata is None: + return None + return (str(run_metadata), "text/x-protobuf") # pbtxt + + @wrappers.Request.application + def info_route(self, request): + experiment = plugin_util.experiment_id(request.environ) + info = self.info_impl(experiment) + return http_util.Respond(request, info, "application/json") + + @wrappers.Request.application + def graph_route(self, request): + """Given a single run, return the graph definition in protobuf + format.""" + experiment = plugin_util.experiment_id(request.environ) + run = request.args.get("run") + tag = request.args.get("tag") + conceptual_arg = request.args.get("conceptual", False) + is_conceptual = True if conceptual_arg == "true" else False + + if run is None: + return http_util.Respond( + request, 'query parameter "run" is required', "text/plain", 400 + ) + + limit_attr_size = request.args.get("limit_attr_size", None) + if limit_attr_size is not None: + try: + limit_attr_size = int(limit_attr_size) + except ValueError: + return http_util.Respond( + request, + "query parameter `limit_attr_size` must be an integer", + "text/plain", + 400, + ) + + large_attrs_key = request.args.get("large_attrs_key", None) + + try: + result = self.graph_impl( + run, + tag, + is_conceptual, + experiment, + limit_attr_size, + large_attrs_key, + ) + except ValueError as e: + return http_util.Respond(request, e.message, "text/plain", code=400) + else: + if result is not None: + ( + body, + mime_type, + ) = result # pylint: disable=unpacking-non-sequence + return http_util.Respond(request, body, mime_type) + else: + return http_util.Respond( + request, "404 Not Found", "text/plain", code=404 + ) + + @wrappers.Request.application + def run_metadata_route(self, request): + """Given a tag and a run, return the session.run() metadata.""" + tag = request.args.get("tag") + run = request.args.get("run") + if tag is None: + return http_util.Respond( + request, 'query parameter "tag" is required', "text/plain", 400 + ) + if run is None: + return http_util.Respond( + request, 'query parameter "run" is required', "text/plain", 400 + ) + result = self.run_metadata_impl(run, tag) + if result is not None: + (body, mime_type) = result # pylint: disable=unpacking-non-sequence + return http_util.Respond(request, body, mime_type) + else: + return http_util.Respond( + request, "404 Not Found", "text/plain", code=404 + ) diff --git a/tensorboard/plugins/graph/graphs_plugin_test.py b/tensorboard/plugins/graph/graphs_plugin_test.py index 70dbb51f0b..a573c412f8 100644 --- a/tensorboard/plugins/graph/graphs_plugin_test.py +++ b/tensorboard/plugins/graph/graphs_plugin_test.py @@ -29,7 +29,9 @@ from google.protobuf import text_format from tensorboard.backend.event_processing import data_provider -from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer +from tensorboard.backend.event_processing import ( + plugin_event_multiplexer as event_multiplexer, +) from tensorboard.compat.proto import config_pb2 from tensorboard.plugins import base_plugin from tensorboard.plugins.graph import graphs_plugin @@ -42,227 +44,282 @@ # can write graph and metadata with a TF public API. -_RUN_WITH_GRAPH_WITH_METADATA = ('_RUN_WITH_GRAPH_WITH_METADATA', True, True) -_RUN_WITHOUT_GRAPH_WITH_METADATA = ('_RUN_WITHOUT_GRAPH_WITH_METADATA', False, True) -_RUN_WITH_GRAPH_WITHOUT_METADATA = ('_RUN_WITH_GRAPH_WITHOUT_METADATA', True, False) -_RUN_WITHOUT_GRAPH_WITHOUT_METADATA = ('_RUN_WITHOUT_GRAPH_WITHOUT_METADATA', False, False) +_RUN_WITH_GRAPH_WITH_METADATA = ("_RUN_WITH_GRAPH_WITH_METADATA", True, True) +_RUN_WITHOUT_GRAPH_WITH_METADATA = ( + "_RUN_WITHOUT_GRAPH_WITH_METADATA", + False, + True, +) +_RUN_WITH_GRAPH_WITHOUT_METADATA = ( + "_RUN_WITH_GRAPH_WITHOUT_METADATA", + True, + False, +) +_RUN_WITHOUT_GRAPH_WITHOUT_METADATA = ( + "_RUN_WITHOUT_GRAPH_WITHOUT_METADATA", + False, + False, +) + def with_runs(run_specs): - """Run a test with a bare multiplexer and with a `data_provider`. - - The decorated function will receive an initialized `GraphsPlugin` - object as its first positional argument. - - The receiver argument of the decorated function must be a `TestCase` instance - that also provides `load_runs`.` - """ - def decorator(fn): - @functools.wraps(fn) - def wrapper(self, *args, **kwargs): - (logdir, multiplexer) = self.load_runs(run_specs) - with self.subTest('bare multiplexer'): - ctx = base_plugin.TBContext(logdir=logdir, multiplexer=multiplexer) - fn(self, graphs_plugin.GraphsPlugin(ctx), *args, **kwargs) - with self.subTest('generic data provider'): - flags = argparse.Namespace(generic_data='true') - provider = data_provider.MultiplexerDataProvider(multiplexer, logdir) - ctx = base_plugin.TBContext( - flags=flags, - logdir=logdir, - multiplexer=multiplexer, - data_provider=provider, - ) - fn(self, graphs_plugin.GraphsPlugin(ctx), *args, **kwargs) - return wrapper - return decorator + """Run a test with a bare multiplexer and with a `data_provider`. + + The decorated function will receive an initialized `GraphsPlugin` + object as its first positional argument. + + The receiver argument of the decorated function must be a `TestCase` instance + that also provides `load_runs`.` + """ + + def decorator(fn): + @functools.wraps(fn) + def wrapper(self, *args, **kwargs): + (logdir, multiplexer) = self.load_runs(run_specs) + with self.subTest("bare multiplexer"): + ctx = base_plugin.TBContext( + logdir=logdir, multiplexer=multiplexer + ) + fn(self, graphs_plugin.GraphsPlugin(ctx), *args, **kwargs) + with self.subTest("generic data provider"): + flags = argparse.Namespace(generic_data="true") + provider = data_provider.MultiplexerDataProvider( + multiplexer, logdir + ) + ctx = base_plugin.TBContext( + flags=flags, + logdir=logdir, + multiplexer=multiplexer, + data_provider=provider, + ) + fn(self, graphs_plugin.GraphsPlugin(ctx), *args, **kwargs) + + return wrapper + + return decorator + class GraphsPluginBaseTest(object): - _METADATA_TAG = 'secret-stats' - _MESSAGE_PREFIX_LENGTH_LOWER_BOUND = 1024 + _METADATA_TAG = "secret-stats" + _MESSAGE_PREFIX_LENGTH_LOWER_BOUND = 1024 + + def __init__(self, *args, **kwargs): + super(GraphsPluginBaseTest, self).__init__(*args, **kwargs) + self.plugin = None - def __init__(self, *args, **kwargs): - super(GraphsPluginBaseTest, self).__init__(*args, **kwargs) - self.plugin = None + def setUp(self): + super(GraphsPluginBaseTest, self).setUp() - def setUp(self): - super(GraphsPluginBaseTest, self).setUp() + def generate_run( + self, logdir, run_name, include_graph, include_run_metadata + ): + """Create a run.""" + raise NotImplementedError("Please implement generate_run") - def generate_run(self, logdir, run_name, include_graph, include_run_metadata): - """Create a run""" - raise NotImplementedError('Please implement generate_run') + def load_runs(self, run_specs): + logdir = self.get_temp_dir() + for run_spec in run_specs: + self.generate_run(logdir, *run_spec) + return self.bootstrap_plugin(logdir) - def load_runs(self, run_specs): - logdir = self.get_temp_dir() - for run_spec in run_specs: - self.generate_run(logdir, *run_spec) - return self.bootstrap_plugin(logdir) + def bootstrap_plugin(self, logdir): + multiplexer = event_multiplexer.EventMultiplexer() + multiplexer.AddRunsFromDirectory(logdir) + multiplexer.Reload() + return (logdir, multiplexer) - def bootstrap_plugin(self, logdir): - multiplexer = event_multiplexer.EventMultiplexer() - multiplexer.AddRunsFromDirectory(logdir) - multiplexer.Reload() - return (logdir, multiplexer) + @with_runs( + [_RUN_WITH_GRAPH_WITH_METADATA, _RUN_WITHOUT_GRAPH_WITH_METADATA] + ) + def testRoutesProvided(self, plugin): + """Tests that the plugin offers the correct routes.""" + routes = plugin.get_plugin_apps() + self.assertIsInstance(routes["/graph"], collections.Callable) + self.assertIsInstance(routes["/run_metadata"], collections.Callable) + self.assertIsInstance(routes["/info"], collections.Callable) - @with_runs([_RUN_WITH_GRAPH_WITH_METADATA, _RUN_WITHOUT_GRAPH_WITH_METADATA]) - def testRoutesProvided(self, plugin): - """Tests that the plugin offers the correct routes.""" - routes = plugin.get_plugin_apps() - self.assertIsInstance(routes['/graph'], collections.Callable) - self.assertIsInstance(routes['/run_metadata'], collections.Callable) - self.assertIsInstance(routes['/info'], collections.Callable) class GraphsPluginV1Test(GraphsPluginBaseTest, tf.test.TestCase): + def generate_run( + self, logdir, run_name, include_graph, include_run_metadata + ): + """Create a run with a text summary, metadata, and optionally a + graph.""" + tf.compat.v1.reset_default_graph() + k1 = tf.constant(math.pi, name="k1") + k2 = tf.constant(math.e, name="k2") + result = (k1 ** k2) - k1 + expected = tf.constant(20.0, name="expected") + error = tf.abs(result - expected, name="error") + message_prefix_value = "error " * 1000 + true_length = len(message_prefix_value) + assert ( + true_length > self._MESSAGE_PREFIX_LENGTH_LOWER_BOUND + ), true_length + message_prefix = tf.constant( + message_prefix_value, name="message_prefix" + ) + error_message = tf.strings.join( + [message_prefix, tf.as_string(error, name="error_string")], + name="error_message", + ) + summary_message = tf.compat.v1.summary.text( + "summary_message", error_message + ) - def generate_run(self, logdir, run_name, include_graph, include_run_metadata): - """Create a run with a text summary, metadata, and optionally a graph.""" - tf.compat.v1.reset_default_graph() - k1 = tf.constant(math.pi, name='k1') - k2 = tf.constant(math.e, name='k2') - result = (k1 ** k2) - k1 - expected = tf.constant(20.0, name='expected') - error = tf.abs(result - expected, name='error') - message_prefix_value = 'error ' * 1000 - true_length = len(message_prefix_value) - assert true_length > self._MESSAGE_PREFIX_LENGTH_LOWER_BOUND, true_length - message_prefix = tf.constant(message_prefix_value, name='message_prefix') - error_message = tf.strings.join([message_prefix, - tf.as_string(error, name='error_string')], - name='error_message') - summary_message = tf.compat.v1.summary.text('summary_message', error_message) - - sess = tf.compat.v1.Session() - writer = test_util.FileWriter(os.path.join(logdir, run_name)) - if include_graph: - writer.add_graph(sess.graph) - options = tf.compat.v1.RunOptions(trace_level=tf.compat.v1.RunOptions.FULL_TRACE) - run_metadata = config_pb2.RunMetadata() - s = sess.run(summary_message, options=options, run_metadata=run_metadata) - writer.add_summary(s) - if include_run_metadata: - writer.add_run_metadata(run_metadata, self._METADATA_TAG) - writer.close() - - def _get_graph(self, plugin, *args, **kwargs): - """Set up runs, then fetch and return the graph as a proto.""" - (graph_pbtxt, mime_type) = plugin.graph_impl( - _RUN_WITH_GRAPH_WITH_METADATA[0], *args, **kwargs) - self.assertEqual(mime_type, 'text/x-protobuf') - return text_format.Parse(graph_pbtxt, tf.compat.v1.GraphDef()) - - @with_runs([ - _RUN_WITH_GRAPH_WITH_METADATA, - _RUN_WITH_GRAPH_WITHOUT_METADATA, - _RUN_WITHOUT_GRAPH_WITH_METADATA, - _RUN_WITHOUT_GRAPH_WITHOUT_METADATA]) - def test_info(self, plugin): - expected = { - '_RUN_WITH_GRAPH_WITH_METADATA': { - 'run': '_RUN_WITH_GRAPH_WITH_METADATA', - 'run_graph': True, - 'tags': { - 'secret-stats': { - 'conceptual_graph': False, - 'profile': True, - 'tag': 'secret-stats', - 'op_graph': False, - }, - }, - }, - '_RUN_WITH_GRAPH_WITHOUT_METADATA': { - 'run': '_RUN_WITH_GRAPH_WITHOUT_METADATA', - 'run_graph': True, - 'tags': {}, - }, - '_RUN_WITHOUT_GRAPH_WITH_METADATA': { - 'run': '_RUN_WITHOUT_GRAPH_WITH_METADATA', - 'run_graph': False, - 'tags': { - 'secret-stats': { - 'conceptual_graph': False, - 'profile': True, - 'tag': 'secret-stats', - 'op_graph': False, - }, - }, - }, - } - - if plugin._data_provider: - # Hack, for now. - # Data providers don't yet pass RunMetadata, so this entry excludes it. - expected['_RUN_WITH_GRAPH_WITH_METADATA']['tags'] = {} - # Data providers don't yet pass RunMetadata, so this entry is completely omitted. - del expected['_RUN_WITHOUT_GRAPH_WITH_METADATA'] - - actual = plugin.info_impl('eid') - self.assertEqual(expected, actual) - - @with_runs([_RUN_WITH_GRAPH_WITH_METADATA]) - def test_graph_simple(self, plugin): - graph = self._get_graph( - plugin, - tag=None, - is_conceptual=False, - experiment='eid', + sess = tf.compat.v1.Session() + writer = test_util.FileWriter(os.path.join(logdir, run_name)) + if include_graph: + writer.add_graph(sess.graph) + options = tf.compat.v1.RunOptions( + trace_level=tf.compat.v1.RunOptions.FULL_TRACE + ) + run_metadata = config_pb2.RunMetadata() + s = sess.run( + summary_message, options=options, run_metadata=run_metadata + ) + writer.add_summary(s) + if include_run_metadata: + writer.add_run_metadata(run_metadata, self._METADATA_TAG) + writer.close() + + def _get_graph(self, plugin, *args, **kwargs): + """Set up runs, then fetch and return the graph as a proto.""" + (graph_pbtxt, mime_type) = plugin.graph_impl( + _RUN_WITH_GRAPH_WITH_METADATA[0], *args, **kwargs + ) + self.assertEqual(mime_type, "text/x-protobuf") + return text_format.Parse(graph_pbtxt, tf.compat.v1.GraphDef()) + + @with_runs( + [ + _RUN_WITH_GRAPH_WITH_METADATA, + _RUN_WITH_GRAPH_WITHOUT_METADATA, + _RUN_WITHOUT_GRAPH_WITH_METADATA, + _RUN_WITHOUT_GRAPH_WITHOUT_METADATA, + ] ) - node_names = set(node.name for node in graph.node) - self.assertEqual({ - 'k1', 'k2', 'pow', 'sub', 'expected', 'sub_1', 'error', - 'message_prefix', 'error_string', 'error_message', 'summary_message', - 'summary_message/tag', 'summary_message/serialized_summary_metadata', - }, node_names) - - @with_runs([_RUN_WITH_GRAPH_WITH_METADATA]) - def test_graph_large_attrs(self, plugin): - key = 'o---;;-;' - graph = self._get_graph( - plugin, - tag=None, - is_conceptual=False, - experiment='eid', - limit_attr_size=self._MESSAGE_PREFIX_LENGTH_LOWER_BOUND, - large_attrs_key=key) - large_attrs = { - node.name: list(node.attr[key].list.s) - for node in graph.node - if key in node.attr - } - self.assertEqual({'message_prefix': [b'value']}, - large_attrs) - - @with_runs([_RUN_WITH_GRAPH_WITH_METADATA]) - def test_run_metadata(self, plugin): - result = plugin.run_metadata_impl( - _RUN_WITH_GRAPH_WITH_METADATA[0], self._METADATA_TAG) - if plugin._data_provider: - # Hack, for now - self.assertEqual(result, None) - else: - (metadata_pbtxt, mime_type) = result - self.assertEqual(mime_type, 'text/x-protobuf') - text_format.Parse(metadata_pbtxt, config_pb2.RunMetadata()) - # If it parses, we're happy. - - @with_runs([_RUN_WITH_GRAPH_WITHOUT_METADATA]) - def test_is_active_with_graph_without_run_metadata(self, plugin): - self.assertTrue(plugin.is_active()) - - @with_runs([_RUN_WITHOUT_GRAPH_WITH_METADATA]) - def test_is_active_without_graph_with_run_metadata(self, plugin): - self.assertTrue(plugin.is_active()) - - @with_runs([_RUN_WITH_GRAPH_WITH_METADATA]) - def test_is_active_with_both(self, plugin): - self.assertTrue(plugin.is_active()) - - @with_runs([_RUN_WITHOUT_GRAPH_WITHOUT_METADATA]) - def test_is_inactive_without_both(self, plugin): - if plugin._data_provider: - # Hack, for now. - self.assertTrue(plugin.is_active()) - else: - self.assertFalse(plugin.is_active()) - -if __name__ == '__main__': - tf.test.main() + def test_info(self, plugin): + expected = { + "_RUN_WITH_GRAPH_WITH_METADATA": { + "run": "_RUN_WITH_GRAPH_WITH_METADATA", + "run_graph": True, + "tags": { + "secret-stats": { + "conceptual_graph": False, + "profile": True, + "tag": "secret-stats", + "op_graph": False, + }, + }, + }, + "_RUN_WITH_GRAPH_WITHOUT_METADATA": { + "run": "_RUN_WITH_GRAPH_WITHOUT_METADATA", + "run_graph": True, + "tags": {}, + }, + "_RUN_WITHOUT_GRAPH_WITH_METADATA": { + "run": "_RUN_WITHOUT_GRAPH_WITH_METADATA", + "run_graph": False, + "tags": { + "secret-stats": { + "conceptual_graph": False, + "profile": True, + "tag": "secret-stats", + "op_graph": False, + }, + }, + }, + } + + if plugin._data_provider: + # Hack, for now. + # Data providers don't yet pass RunMetadata, so this entry excludes it. + expected["_RUN_WITH_GRAPH_WITH_METADATA"]["tags"] = {} + # Data providers don't yet pass RunMetadata, so this entry is completely omitted. + del expected["_RUN_WITHOUT_GRAPH_WITH_METADATA"] + + actual = plugin.info_impl("eid") + self.assertEqual(expected, actual) + + @with_runs([_RUN_WITH_GRAPH_WITH_METADATA]) + def test_graph_simple(self, plugin): + graph = self._get_graph( + plugin, tag=None, is_conceptual=False, experiment="eid", + ) + node_names = set(node.name for node in graph.node) + self.assertEqual( + { + "k1", + "k2", + "pow", + "sub", + "expected", + "sub_1", + "error", + "message_prefix", + "error_string", + "error_message", + "summary_message", + "summary_message/tag", + "summary_message/serialized_summary_metadata", + }, + node_names, + ) + + @with_runs([_RUN_WITH_GRAPH_WITH_METADATA]) + def test_graph_large_attrs(self, plugin): + key = "o---;;-;" + graph = self._get_graph( + plugin, + tag=None, + is_conceptual=False, + experiment="eid", + limit_attr_size=self._MESSAGE_PREFIX_LENGTH_LOWER_BOUND, + large_attrs_key=key, + ) + large_attrs = { + node.name: list(node.attr[key].list.s) + for node in graph.node + if key in node.attr + } + self.assertEqual({"message_prefix": [b"value"]}, large_attrs) + + @with_runs([_RUN_WITH_GRAPH_WITH_METADATA]) + def test_run_metadata(self, plugin): + result = plugin.run_metadata_impl( + _RUN_WITH_GRAPH_WITH_METADATA[0], self._METADATA_TAG + ) + if plugin._data_provider: + # Hack, for now + self.assertEqual(result, None) + else: + (metadata_pbtxt, mime_type) = result + self.assertEqual(mime_type, "text/x-protobuf") + text_format.Parse(metadata_pbtxt, config_pb2.RunMetadata()) + # If it parses, we're happy. + + @with_runs([_RUN_WITH_GRAPH_WITHOUT_METADATA]) + def test_is_active_with_graph_without_run_metadata(self, plugin): + self.assertTrue(plugin.is_active()) + + @with_runs([_RUN_WITHOUT_GRAPH_WITH_METADATA]) + def test_is_active_without_graph_with_run_metadata(self, plugin): + self.assertTrue(plugin.is_active()) + + @with_runs([_RUN_WITH_GRAPH_WITH_METADATA]) + def test_is_active_with_both(self, plugin): + self.assertTrue(plugin.is_active()) + + @with_runs([_RUN_WITHOUT_GRAPH_WITHOUT_METADATA]) + def test_is_inactive_without_both(self, plugin): + if plugin._data_provider: + # Hack, for now. + self.assertTrue(plugin.is_active()) + else: + self.assertFalse(plugin.is_active()) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/graph/graphs_plugin_v2_test.py b/tensorboard/plugins/graph/graphs_plugin_v2_test.py index ca08955820..782a738c62 100644 --- a/tensorboard/plugins/graph/graphs_plugin_v2_test.py +++ b/tensorboard/plugins/graph/graphs_plugin_v2_test.py @@ -28,76 +28,95 @@ from tensorboard.plugins.graph import graphs_plugin_test -class GraphsPluginV2Test(graphs_plugin_test.GraphsPluginBaseTest, tf.test.TestCase): - - def generate_run(self, logdir, run_name, include_graph, include_run_metadata): - x, y = np.ones((10, 10)), np.ones((10, 1)) - val_x, val_y = np.ones((4, 10)), np.ones((4, 1)) - - model = tf.keras.Sequential([ - tf.keras.layers.Dense(10, activation='relu'), - tf.keras.layers.Dense(1, activation='sigmoid')]) - model.compile('rmsprop', 'binary_crossentropy') - - model.fit( - x, - y, - validation_data=(val_x, val_y), - batch_size=2, - epochs=1, - callbacks=[tf.compat.v2.keras.callbacks.TensorBoard( - log_dir=os.path.join(logdir, run_name), - write_graph=include_graph)]) - - def _get_graph(self, plugin, *args, **kwargs): - """Fetch and return the graph as a proto.""" - (graph_pbtxt, mime_type) = plugin.graph_impl(*args, **kwargs) - self.assertEqual(mime_type, 'text/x-protobuf') - return text_format.Parse(graph_pbtxt, graph_pb2.GraphDef()) - - @graphs_plugin_test.with_runs([ - graphs_plugin_test._RUN_WITH_GRAPH_WITH_METADATA, - graphs_plugin_test._RUN_WITHOUT_GRAPH_WITH_METADATA]) - def test_info(self, plugin): - raise self.skipTest('TODO: enable this after tf-nightly writes a conceptual graph.') - - expected = { - 'w_graph_wo_meta': { - 'run': 'w_graph_wo_meta', - 'run_graph': True, - 'tags': { - 'keras': { - 'conceptual_graph': True, - 'profile': False, - 'tag': 'keras', - 'op_graph': False, - }, - }, - }, - } - - self.generate_run('w_graph_wo_meta', - include_graph=True, - include_run_metadata=False) - self.generate_run('wo_graph_wo_meta', - include_graph=False, - include_run_metadata=False) - self.bootstrap_plugin() - - self.assertEqual(expected, plugin.info_impl()) - - def test_graph_conceptual_graph(self): - raise self.skipTest('TODO: enable this after tf-nightly writes a conceptual graph.') - - self.generate_run(self._RUN_WITH_GRAPH, - include_graph=True, - include_run_metadata=False) - self.bootstrap_plugin() - - graph = self._get_graph(self._RUN_WITH_GRAPH, tag='keras', is_conceptual=True) - node_names = set(node.name for node in graph.node) - self.assertEqual({'sequential/dense', 'sequential/dense_1'}, node_names) - - -if __name__ == '__main__': - tf.test.main() +class GraphsPluginV2Test( + graphs_plugin_test.GraphsPluginBaseTest, tf.test.TestCase +): + def generate_run( + self, logdir, run_name, include_graph, include_run_metadata + ): + x, y = np.ones((10, 10)), np.ones((10, 1)) + val_x, val_y = np.ones((4, 10)), np.ones((4, 1)) + + model = tf.keras.Sequential( + [ + tf.keras.layers.Dense(10, activation="relu"), + tf.keras.layers.Dense(1, activation="sigmoid"), + ] + ) + model.compile("rmsprop", "binary_crossentropy") + + model.fit( + x, + y, + validation_data=(val_x, val_y), + batch_size=2, + epochs=1, + callbacks=[ + tf.compat.v2.keras.callbacks.TensorBoard( + log_dir=os.path.join(logdir, run_name), + write_graph=include_graph, + ) + ], + ) + + def _get_graph(self, plugin, *args, **kwargs): + """Fetch and return the graph as a proto.""" + (graph_pbtxt, mime_type) = plugin.graph_impl(*args, **kwargs) + self.assertEqual(mime_type, "text/x-protobuf") + return text_format.Parse(graph_pbtxt, graph_pb2.GraphDef()) + + @graphs_plugin_test.with_runs( + [ + graphs_plugin_test._RUN_WITH_GRAPH_WITH_METADATA, + graphs_plugin_test._RUN_WITHOUT_GRAPH_WITH_METADATA, + ] + ) + def test_info(self, plugin): + raise self.skipTest( + "TODO: enable this after tf-nightly writes a conceptual graph." + ) + + expected = { + "w_graph_wo_meta": { + "run": "w_graph_wo_meta", + "run_graph": True, + "tags": { + "keras": { + "conceptual_graph": True, + "profile": False, + "tag": "keras", + "op_graph": False, + }, + }, + }, + } + + self.generate_run( + "w_graph_wo_meta", include_graph=True, include_run_metadata=False + ) + self.generate_run( + "wo_graph_wo_meta", include_graph=False, include_run_metadata=False + ) + self.bootstrap_plugin() + + self.assertEqual(expected, plugin.info_impl()) + + def test_graph_conceptual_graph(self): + raise self.skipTest( + "TODO: enable this after tf-nightly writes a conceptual graph." + ) + + self.generate_run( + self._RUN_WITH_GRAPH, include_graph=True, include_run_metadata=False + ) + self.bootstrap_plugin() + + graph = self._get_graph( + self._RUN_WITH_GRAPH, tag="keras", is_conceptual=True + ) + node_names = set(node.name for node in graph.node) + self.assertEqual({"sequential/dense", "sequential/dense_1"}, node_names) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/graph/keras_util.py b/tensorboard/plugins/graph/keras_util.py index dc720c2831..646569361e 100644 --- a/tensorboard/plugins/graph/keras_util.py +++ b/tensorboard/plugins/graph/keras_util.py @@ -46,7 +46,7 @@ def _walk_layers(keras_layer): - """Walks the nested keras layer configuration in preorder. + """Walks the nested keras layer configuration in preorder. Args: keras_layer: Keras configuration from model.to_json. @@ -55,185 +55,207 @@ def _walk_layers(keras_layer): name_scope: a string representing a scope name, similar to that of tf.name_scope. layer_config: a dict representing a Keras layer configuration. """ - yield ('', keras_layer) - if keras_layer.get('config').get('layers'): - name_scope = keras_layer.get('config').get('name') - for layer in keras_layer.get('config').get('layers'): - for (sub_name_scope, sublayer) in _walk_layers(layer): - sub_name_scope = '%s/%s' % ( - name_scope, sub_name_scope) if sub_name_scope else name_scope - yield (sub_name_scope, sublayer) + yield ("", keras_layer) + if keras_layer.get("config").get("layers"): + name_scope = keras_layer.get("config").get("name") + for layer in keras_layer.get("config").get("layers"): + for (sub_name_scope, sublayer) in _walk_layers(layer): + sub_name_scope = ( + "%s/%s" % (name_scope, sub_name_scope) + if sub_name_scope + else name_scope + ) + yield (sub_name_scope, sublayer) def _scoped_name(name_scope, node_name): - """Returns scoped name for a node as a string in the form 'quod erat scribendum
' - - _RUN_WITH_LEGACY_HISTOGRAM = '_RUN_WITH_LEGACY_HISTOGRAM' - _RUN_WITH_HISTOGRAM = '_RUN_WITH_HISTOGRAM' - _RUN_WITH_SCALARS = '_RUN_WITH_SCALARS' - - def __init__(self, *args, **kwargs): - super(HistogramsPluginTest, self).__init__(*args, **kwargs) - self.logdir = None - - def load_runs(self, run_names): - logdir = self.get_temp_dir() - for run_name in run_names: - self.generate_run(logdir, run_name) - multiplexer = event_multiplexer.EventMultiplexer(size_guidance={ - # don't truncate my test data, please - event_accumulator.TENSORS: self._STEPS, - }) - multiplexer.AddRunsFromDirectory(logdir) - multiplexer.Reload() - return (logdir, multiplexer) - - def with_runs(run_names): - """Run a test with a bare multiplexer and with a `data_provider`. - - The decorated function will receive an initialized `HistogramsPlugin` - object as its first positional argument. - """ - def decorator(fn): - @functools.wraps(fn) - def wrapper(self, *args, **kwargs): - (logdir, multiplexer) = self.load_runs(run_names) - with self.subTest('bare multiplexer'): - ctx = base_plugin.TBContext(logdir=logdir, multiplexer=multiplexer) - fn(self, histograms_plugin.HistogramsPlugin(ctx), *args, **kwargs) - with self.subTest('generic data provider'): - flags = argparse.Namespace(generic_data='true') - provider = data_provider.MultiplexerDataProvider(multiplexer, logdir) - ctx = base_plugin.TBContext( - flags=flags, - logdir=logdir, - multiplexer=multiplexer, - data_provider=provider, - ) - fn(self, histograms_plugin.HistogramsPlugin(ctx), *args, **kwargs) - return wrapper - return decorator - - def generate_run(self, logdir, run_name): - tf.compat.v1.reset_default_graph() - sess = tf.compat.v1.Session() - placeholder = tf.compat.v1.placeholder(tf.float32, shape=[3]) - - if run_name == self._RUN_WITH_LEGACY_HISTOGRAM: - tf.compat.v1.summary.histogram(self._LEGACY_HISTOGRAM_TAG, placeholder) - elif run_name == self._RUN_WITH_HISTOGRAM: - summary.op(self._HISTOGRAM_TAG, placeholder, - display_name=self._DISPLAY_NAME, - description=self._DESCRIPTION) - elif run_name == self._RUN_WITH_SCALARS: - tf.compat.v1.summary.scalar(self._SCALAR_TAG, tf.reduce_mean(input_tensor=placeholder)) - else: - assert False, 'Invalid run name: %r' % run_name - summ = tf.compat.v1.summary.merge_all() - - subdir = os.path.join(logdir, run_name) - with test_util.FileWriterCache.get(subdir) as writer: - writer.add_graph(sess.graph) - for step in xrange(self._STEPS): - feed_dict = {placeholder: [1 + step, 2 + step, 3 + step]} - s = sess.run(summ, feed_dict=feed_dict) - writer.add_summary(s, global_step=step) - - @with_runs([_RUN_WITH_SCALARS]) - def test_routes_provided(self, plugin): - """Tests that the plugin offers the correct routes.""" - routes = plugin.get_plugin_apps() - self.assertIsInstance(routes['/histograms'], collections.Callable) - self.assertIsInstance(routes['/tags'], collections.Callable) - - @with_runs([ - _RUN_WITH_SCALARS, - _RUN_WITH_LEGACY_HISTOGRAM, - _RUN_WITH_HISTOGRAM, - ]) - def test_index(self, plugin): - self.assertEqual({ - # _RUN_WITH_SCALARS omitted: No histogram data. - self._RUN_WITH_LEGACY_HISTOGRAM: { - self._LEGACY_HISTOGRAM_TAG: { - 'displayName': self._LEGACY_HISTOGRAM_TAG, - 'description': '', - }, - }, - self._RUN_WITH_HISTOGRAM: { - '%s/histogram_summary' % self._HISTOGRAM_TAG: { - 'displayName': self._DISPLAY_NAME, - 'description': self._HTML_DESCRIPTION, + _STEPS = 99 + + _LEGACY_HISTOGRAM_TAG = "my-ancient-histogram" + _HISTOGRAM_TAG = "my-favorite-histogram" + _SCALAR_TAG = "my-boring-scalars" + + _DISPLAY_NAME = "Important production statistics" + _DESCRIPTION = "quod *erat* scribendum" + _HTML_DESCRIPTION = "quod erat scribendum
" + + _RUN_WITH_LEGACY_HISTOGRAM = "_RUN_WITH_LEGACY_HISTOGRAM" + _RUN_WITH_HISTOGRAM = "_RUN_WITH_HISTOGRAM" + _RUN_WITH_SCALARS = "_RUN_WITH_SCALARS" + + def __init__(self, *args, **kwargs): + super(HistogramsPluginTest, self).__init__(*args, **kwargs) + self.logdir = None + + def load_runs(self, run_names): + logdir = self.get_temp_dir() + for run_name in run_names: + self.generate_run(logdir, run_name) + multiplexer = event_multiplexer.EventMultiplexer( + size_guidance={ + # don't truncate my test data, please + event_accumulator.TENSORS: self._STEPS, + } + ) + multiplexer.AddRunsFromDirectory(logdir) + multiplexer.Reload() + return (logdir, multiplexer) + + def with_runs(run_names): + """Run a test with a bare multiplexer and with a `data_provider`. + + The decorated function will receive an initialized + `HistogramsPlugin` object as its first positional argument. + """ + + def decorator(fn): + @functools.wraps(fn) + def wrapper(self, *args, **kwargs): + (logdir, multiplexer) = self.load_runs(run_names) + with self.subTest("bare multiplexer"): + ctx = base_plugin.TBContext( + logdir=logdir, multiplexer=multiplexer + ) + fn( + self, + histograms_plugin.HistogramsPlugin(ctx), + *args, + **kwargs + ) + with self.subTest("generic data provider"): + flags = argparse.Namespace(generic_data="true") + provider = data_provider.MultiplexerDataProvider( + multiplexer, logdir + ) + ctx = base_plugin.TBContext( + flags=flags, + logdir=logdir, + multiplexer=multiplexer, + data_provider=provider, + ) + fn( + self, + histograms_plugin.HistogramsPlugin(ctx), + *args, + **kwargs + ) + + return wrapper + + return decorator + + def generate_run(self, logdir, run_name): + tf.compat.v1.reset_default_graph() + sess = tf.compat.v1.Session() + placeholder = tf.compat.v1.placeholder(tf.float32, shape=[3]) + + if run_name == self._RUN_WITH_LEGACY_HISTOGRAM: + tf.compat.v1.summary.histogram( + self._LEGACY_HISTOGRAM_TAG, placeholder + ) + elif run_name == self._RUN_WITH_HISTOGRAM: + summary.op( + self._HISTOGRAM_TAG, + placeholder, + display_name=self._DISPLAY_NAME, + description=self._DESCRIPTION, + ) + elif run_name == self._RUN_WITH_SCALARS: + tf.compat.v1.summary.scalar( + self._SCALAR_TAG, tf.reduce_mean(input_tensor=placeholder) + ) + else: + assert False, "Invalid run name: %r" % run_name + summ = tf.compat.v1.summary.merge_all() + + subdir = os.path.join(logdir, run_name) + with test_util.FileWriterCache.get(subdir) as writer: + writer.add_graph(sess.graph) + for step in xrange(self._STEPS): + feed_dict = {placeholder: [1 + step, 2 + step, 3 + step]} + s = sess.run(summ, feed_dict=feed_dict) + writer.add_summary(s, global_step=step) + + @with_runs([_RUN_WITH_SCALARS]) + def test_routes_provided(self, plugin): + """Tests that the plugin offers the correct routes.""" + routes = plugin.get_plugin_apps() + self.assertIsInstance(routes["/histograms"], collections.Callable) + self.assertIsInstance(routes["/tags"], collections.Callable) + + @with_runs( + [_RUN_WITH_SCALARS, _RUN_WITH_LEGACY_HISTOGRAM, _RUN_WITH_HISTOGRAM,] + ) + def test_index(self, plugin): + self.assertEqual( + { + # _RUN_WITH_SCALARS omitted: No histogram data. + self._RUN_WITH_LEGACY_HISTOGRAM: { + self._LEGACY_HISTOGRAM_TAG: { + "displayName": self._LEGACY_HISTOGRAM_TAG, + "description": "", + }, + }, + self._RUN_WITH_HISTOGRAM: { + "%s/histogram_summary" + % self._HISTOGRAM_TAG: { + "displayName": self._DISPLAY_NAME, + "description": self._HTML_DESCRIPTION, + }, + }, }, - }, - }, plugin.index_impl(experiment='exp')) - - @with_runs([ - _RUN_WITH_SCALARS, - _RUN_WITH_LEGACY_HISTOGRAM, - _RUN_WITH_HISTOGRAM, - ]) - def _test_histograms(self, plugin, run_name, tag_name, should_work=True): - if should_work: - self._check_histograms_result(plugin, tag_name, run_name, downsample=False) - self._check_histograms_result(plugin, tag_name, run_name, downsample=True) - else: - with self.assertRaises(errors.NotFoundError): - plugin.histograms_impl(self._HISTOGRAM_TAG, run_name, experiment='exp') - - def _check_histograms_result(self, plugin, tag_name, run_name, downsample): - if downsample: - downsample_to = 50 - expected_length = 50 - else: - downsample_to = None - expected_length = self._STEPS - - (data, mime_type) = plugin.histograms_impl( - tag_name, run_name, experiment='exp', downsample_to=downsample_to + plugin.index_impl(experiment="exp"), + ) + + @with_runs( + [_RUN_WITH_SCALARS, _RUN_WITH_LEGACY_HISTOGRAM, _RUN_WITH_HISTOGRAM,] ) - self.assertEqual('application/json', mime_type) - self.assertEqual(expected_length, len(data), - 'expected %r, got %r (downsample=%r)' - % (expected_length, len(data), downsample)) - last_step_seen = None - for (i, datum) in enumerate(data): - [_unused_wall_time, step, buckets] = datum - if last_step_seen is not None: - self.assertGreater(step, last_step_seen) - last_step_seen = step - if not downsample: - self.assertEqual(i, step) - self.assertEqual(1 + step, buckets[0][0]) # first left-edge - self.assertEqual(3 + step, buckets[-1][1]) # last right-edge - self.assertAlmostEqual( - 3, # three items across all buckets - sum(bucket[2] for bucket in buckets)) - - def test_histograms_with_scalars(self): - self._test_histograms(self._RUN_WITH_SCALARS, self._HISTOGRAM_TAG, - should_work=False) - - def test_histograms_with_legacy_histogram(self): - self._test_histograms(self._RUN_WITH_LEGACY_HISTOGRAM, - self._LEGACY_HISTOGRAM_TAG) - - def test_histograms_with_histogram(self): - self._test_histograms(self._RUN_WITH_HISTOGRAM, - '%s/histogram_summary' % self._HISTOGRAM_TAG) - - @with_runs([_RUN_WITH_LEGACY_HISTOGRAM]) - def test_active_with_legacy_histogram(self, plugin): - self.assertTrue(plugin.is_active()) - - @with_runs([_RUN_WITH_HISTOGRAM]) - def test_active_with_histogram(self, plugin): - self.assertTrue(plugin.is_active()) - - @with_runs([_RUN_WITH_SCALARS]) - def test_active_with_scalars(self, plugin): - if plugin._data_provider: - # Hack, for now. - self.assertTrue(plugin.is_active()) - else: - self.assertFalse(plugin.is_active()) - - @with_runs([ - _RUN_WITH_SCALARS, - _RUN_WITH_LEGACY_HISTOGRAM, - _RUN_WITH_HISTOGRAM, - ]) - def test_active_with_all(self, plugin): - self.assertTrue(plugin.is_active()) - - -if __name__ == '__main__': - tf.test.main() + def _test_histograms(self, plugin, run_name, tag_name, should_work=True): + if should_work: + self._check_histograms_result( + plugin, tag_name, run_name, downsample=False + ) + self._check_histograms_result( + plugin, tag_name, run_name, downsample=True + ) + else: + with self.assertRaises(errors.NotFoundError): + plugin.histograms_impl( + self._HISTOGRAM_TAG, run_name, experiment="exp" + ) + + def _check_histograms_result(self, plugin, tag_name, run_name, downsample): + if downsample: + downsample_to = 50 + expected_length = 50 + else: + downsample_to = None + expected_length = self._STEPS + + (data, mime_type) = plugin.histograms_impl( + tag_name, run_name, experiment="exp", downsample_to=downsample_to + ) + self.assertEqual("application/json", mime_type) + self.assertEqual( + expected_length, + len(data), + "expected %r, got %r (downsample=%r)" + % (expected_length, len(data), downsample), + ) + last_step_seen = None + for (i, datum) in enumerate(data): + [_unused_wall_time, step, buckets] = datum + if last_step_seen is not None: + self.assertGreater(step, last_step_seen) + last_step_seen = step + if not downsample: + self.assertEqual(i, step) + self.assertEqual(1 + step, buckets[0][0]) # first left-edge + self.assertEqual(3 + step, buckets[-1][1]) # last right-edge + self.assertAlmostEqual( + 3, # three items across all buckets + sum(bucket[2] for bucket in buckets), + ) + + def test_histograms_with_scalars(self): + self._test_histograms( + self._RUN_WITH_SCALARS, self._HISTOGRAM_TAG, should_work=False + ) + + def test_histograms_with_legacy_histogram(self): + self._test_histograms( + self._RUN_WITH_LEGACY_HISTOGRAM, self._LEGACY_HISTOGRAM_TAG + ) + + def test_histograms_with_histogram(self): + self._test_histograms( + self._RUN_WITH_HISTOGRAM, + "%s/histogram_summary" % self._HISTOGRAM_TAG, + ) + + @with_runs([_RUN_WITH_LEGACY_HISTOGRAM]) + def test_active_with_legacy_histogram(self, plugin): + self.assertTrue(plugin.is_active()) + + @with_runs([_RUN_WITH_HISTOGRAM]) + def test_active_with_histogram(self, plugin): + self.assertTrue(plugin.is_active()) + + @with_runs([_RUN_WITH_SCALARS]) + def test_active_with_scalars(self, plugin): + if plugin._data_provider: + # Hack, for now. + self.assertTrue(plugin.is_active()) + else: + self.assertFalse(plugin.is_active()) + + @with_runs( + [_RUN_WITH_SCALARS, _RUN_WITH_LEGACY_HISTOGRAM, _RUN_WITH_HISTOGRAM,] + ) + def test_active_with_all(self, plugin): + self.assertTrue(plugin.is_active()) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/histogram/metadata.py b/tensorboard/plugins/histogram/metadata.py index 456d372179..a3ce604fde 100644 --- a/tensorboard/plugins/histogram/metadata.py +++ b/tensorboard/plugins/histogram/metadata.py @@ -24,7 +24,7 @@ logger = tb_logging.get_logger() -PLUGIN_NAME = 'histograms' +PLUGIN_NAME = "histograms" # The most recent value for the `version` field of the # `HistogramPluginData` proto. @@ -32,42 +32,46 @@ def create_summary_metadata(display_name, description): - """Create a `summary_pb2.SummaryMetadata` proto for histogram plugin data. + """Create a `summary_pb2.SummaryMetadata` proto for histogram plugin data. - Returns: - A `summary_pb2.SummaryMetadata` protobuf object. - """ - content = plugin_data_pb2.HistogramPluginData(version=PROTO_VERSION) - return summary_pb2.SummaryMetadata( - display_name=display_name, - summary_description=description, - plugin_data=summary_pb2.SummaryMetadata.PluginData( - plugin_name=PLUGIN_NAME, - content=content.SerializeToString())) + Returns: + A `summary_pb2.SummaryMetadata` protobuf object. + """ + content = plugin_data_pb2.HistogramPluginData(version=PROTO_VERSION) + return summary_pb2.SummaryMetadata( + display_name=display_name, + summary_description=description, + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME, content=content.SerializeToString() + ), + ) def parse_plugin_metadata(content): - """Parse summary metadata to a Python object. + """Parse summary metadata to a Python object. - Arguments: - content: The `content` field of a `SummaryMetadata` proto - corresponding to the histogram plugin. + Arguments: + content: The `content` field of a `SummaryMetadata` proto + corresponding to the histogram plugin. - Returns: - A `HistogramPluginData` protobuf object. - """ - if not isinstance(content, bytes): - raise TypeError('Content type must be bytes') - if content == b'{}': - # Old-style JSON format. Equivalent to an all-default proto. - return plugin_data_pb2.HistogramPluginData() - else: - result = plugin_data_pb2.HistogramPluginData.FromString(content) - if result.version == 0: - return result + Returns: + A `HistogramPluginData` protobuf object. + """ + if not isinstance(content, bytes): + raise TypeError("Content type must be bytes") + if content == b"{}": + # Old-style JSON format. Equivalent to an all-default proto. + return plugin_data_pb2.HistogramPluginData() else: - logger.warn( - 'Unknown metadata version: %s. The latest version known to ' - 'this build of TensorBoard is %s; perhaps a newer build is ' - 'available?', result.version, PROTO_VERSION) - return result + result = plugin_data_pb2.HistogramPluginData.FromString(content) + if result.version == 0: + return result + else: + logger.warn( + "Unknown metadata version: %s. The latest version known to " + "this build of TensorBoard is %s; perhaps a newer build is " + "available?", + result.version, + PROTO_VERSION, + ) + return result diff --git a/tensorboard/plugins/histogram/summary.py b/tensorboard/plugins/histogram/summary.py index 3e6beac03f..36c144ded9 100644 --- a/tensorboard/plugins/histogram/summary.py +++ b/tensorboard/plugins/histogram/summary.py @@ -44,167 +44,191 @@ def _buckets(data, bucket_count=None): - """Create a TensorFlow op to group data into histogram buckets. - - Arguments: - data: A `Tensor` of any shape. Must be castable to `float64`. - bucket_count: Optional positive `int` or scalar `int32` `Tensor`. - Returns: - A `Tensor` of shape `[k, 3]` and type `float64`. The `i`th row is - a triple `[left_edge, right_edge, count]` for a single bucket. - The value of `k` is either `bucket_count` or `1` or `0`. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - if bucket_count is None: - bucket_count = summary_v2.DEFAULT_BUCKET_COUNT - with tf.name_scope('buckets', values=[data, bucket_count]), \ - tf.control_dependencies([tf.assert_scalar(bucket_count), - tf.assert_type(bucket_count, tf.int32)]): - data = tf.reshape(data, shape=[-1]) # flatten - data = tf.cast(data, tf.float64) - is_empty = tf.equal(tf.size(input=data), 0) - - def when_empty(): - return tf.constant([], shape=(0, 3), dtype=tf.float64) - - def when_nonempty(): - min_ = tf.reduce_min(input_tensor=data) - max_ = tf.reduce_max(input_tensor=data) - range_ = max_ - min_ - is_singular = tf.equal(range_, 0) - - def when_nonsingular(): - bucket_width = range_ / tf.cast(bucket_count, tf.float64) - offsets = data - min_ - bucket_indices = tf.cast(tf.floor(offsets / bucket_width), - dtype=tf.int32) - clamped_indices = tf.minimum(bucket_indices, bucket_count - 1) - one_hots = tf.one_hot(clamped_indices, depth=bucket_count) - bucket_counts = tf.cast(tf.reduce_sum(input_tensor=one_hots, axis=0), - dtype=tf.float64) - edges = tf.linspace(min_, max_, bucket_count + 1) - left_edges = edges[:-1] - right_edges = edges[1:] - return tf.transpose(a=tf.stack( - [left_edges, right_edges, bucket_counts])) - - def when_singular(): - center = min_ - bucket_starts = tf.stack([center - 0.5]) - bucket_ends = tf.stack([center + 0.5]) - bucket_counts = tf.stack([tf.cast(tf.size(input=data), tf.float64)]) - return tf.transpose( - a=tf.stack([bucket_starts, bucket_ends, bucket_counts])) - - return tf.cond(is_singular, when_singular, when_nonsingular) - - return tf.cond(is_empty, when_empty, when_nonempty) - - -def op(name, - data, - bucket_count=None, - display_name=None, - description=None, - collections=None): - """Create a legacy histogram summary op. - - Arguments: - name: A unique name for the generated summary node. - data: A `Tensor` of any shape. Must be castable to `float64`. - bucket_count: Optional positive `int`. The output will have this - many buckets, except in two edge cases. If there is no data, then - there are no buckets. If there is data but all points have the - same value, then there is one bucket whose left and right - endpoints are the same. - display_name: Optional name for this summary in TensorBoard, as a - constant `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - collections: Optional list of graph collections keys. The new - summary op is added to these collections. Defaults to - `[Graph Keys.SUMMARIES]`. - - Returns: - A TensorFlow summary op. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - if display_name is None: - display_name = name - summary_metadata = metadata.create_summary_metadata( - display_name=display_name, description=description) - with tf.name_scope(name): - tensor = _buckets(data, bucket_count=bucket_count) - return tf.summary.tensor_summary(name='histogram_summary', - tensor=tensor, - collections=collections, - summary_metadata=summary_metadata) + """Create a TensorFlow op to group data into histogram buckets. + + Arguments: + data: A `Tensor` of any shape. Must be castable to `float64`. + bucket_count: Optional positive `int` or scalar `int32` `Tensor`. + Returns: + A `Tensor` of shape `[k, 3]` and type `float64`. The `i`th row is + a triple `[left_edge, right_edge, count]` for a single bucket. + The value of `k` is either `bucket_count` or `1` or `0`. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if bucket_count is None: + bucket_count = summary_v2.DEFAULT_BUCKET_COUNT + with tf.name_scope( + "buckets", values=[data, bucket_count] + ), tf.control_dependencies( + [tf.assert_scalar(bucket_count), tf.assert_type(bucket_count, tf.int32)] + ): + data = tf.reshape(data, shape=[-1]) # flatten + data = tf.cast(data, tf.float64) + is_empty = tf.equal(tf.size(input=data), 0) + + def when_empty(): + return tf.constant([], shape=(0, 3), dtype=tf.float64) + + def when_nonempty(): + min_ = tf.reduce_min(input_tensor=data) + max_ = tf.reduce_max(input_tensor=data) + range_ = max_ - min_ + is_singular = tf.equal(range_, 0) + + def when_nonsingular(): + bucket_width = range_ / tf.cast(bucket_count, tf.float64) + offsets = data - min_ + bucket_indices = tf.cast( + tf.floor(offsets / bucket_width), dtype=tf.int32 + ) + clamped_indices = tf.minimum(bucket_indices, bucket_count - 1) + one_hots = tf.one_hot(clamped_indices, depth=bucket_count) + bucket_counts = tf.cast( + tf.reduce_sum(input_tensor=one_hots, axis=0), + dtype=tf.float64, + ) + edges = tf.linspace(min_, max_, bucket_count + 1) + left_edges = edges[:-1] + right_edges = edges[1:] + return tf.transpose( + a=tf.stack([left_edges, right_edges, bucket_counts]) + ) + + def when_singular(): + center = min_ + bucket_starts = tf.stack([center - 0.5]) + bucket_ends = tf.stack([center + 0.5]) + bucket_counts = tf.stack( + [tf.cast(tf.size(input=data), tf.float64)] + ) + return tf.transpose( + a=tf.stack([bucket_starts, bucket_ends, bucket_counts]) + ) + + return tf.cond(is_singular, when_singular, when_nonsingular) + + return tf.cond(is_empty, when_empty, when_nonempty) + + +def op( + name, + data, + bucket_count=None, + display_name=None, + description=None, + collections=None, +): + """Create a legacy histogram summary op. + + Arguments: + name: A unique name for the generated summary node. + data: A `Tensor` of any shape. Must be castable to `float64`. + bucket_count: Optional positive `int`. The output will have this + many buckets, except in two edge cases. If there is no data, then + there are no buckets. If there is data but all points have the + same value, then there is one bucket whose left and right + endpoints are the same. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A TensorFlow summary op. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description + ) + with tf.name_scope(name): + tensor = _buckets(data, bucket_count=bucket_count) + return tf.summary.tensor_summary( + name="histogram_summary", + tensor=tensor, + collections=collections, + summary_metadata=summary_metadata, + ) def pb(name, data, bucket_count=None, display_name=None, description=None): - """Create a legacy histogram summary protobuf. - - Arguments: - name: A unique name for the generated summary, including any desired - name scopes. - data: A `np.array` or array-like form of any shape. Must have type - castable to `float`. - bucket_count: Optional positive `int`. The output will have this - many buckets, except in two edge cases. If there is no data, then - there are no buckets. If there is data but all points have the - same value, then there is one bucket whose left and right - endpoints are the same. - display_name: Optional name for this summary in TensorBoard, as a - `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - `str`. Markdown is supported. Defaults to empty. - - Returns: - A `tf.Summary` protobuf object. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - if bucket_count is None: - bucket_count = summary_v2.DEFAULT_BUCKET_COUNT - data = np.array(data).flatten().astype(float) - if data.size == 0: - buckets = np.array([]).reshape((0, 3)) - else: - min_ = np.min(data) - max_ = np.max(data) - range_ = max_ - min_ - if range_ == 0: - center = min_ - buckets = np.array([[center - 0.5, center + 0.5, float(data.size)]]) + """Create a legacy histogram summary protobuf. + + Arguments: + name: A unique name for the generated summary, including any desired + name scopes. + data: A `np.array` or array-like form of any shape. Must have type + castable to `float`. + bucket_count: Optional positive `int`. The output will have this + many buckets, except in two edge cases. If there is no data, then + there are no buckets. If there is data but all points have the + same value, then there is one bucket whose left and right + endpoints are the same. + display_name: Optional name for this summary in TensorBoard, as a + `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + `str`. Markdown is supported. Defaults to empty. + + Returns: + A `tf.Summary` protobuf object. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if bucket_count is None: + bucket_count = summary_v2.DEFAULT_BUCKET_COUNT + data = np.array(data).flatten().astype(float) + if data.size == 0: + buckets = np.array([]).reshape((0, 3)) else: - bucket_width = range_ / bucket_count - offsets = data - min_ - bucket_indices = np.floor(offsets / bucket_width).astype(int) - clamped_indices = np.minimum(bucket_indices, bucket_count - 1) - one_hots = (np.array([clamped_indices]).transpose() - == np.arange(0, bucket_count)) # broadcast - assert one_hots.shape == (data.size, bucket_count), ( - one_hots.shape, (data.size, bucket_count)) - bucket_counts = np.sum(one_hots, axis=0) - edges = np.linspace(min_, max_, bucket_count + 1) - left_edges = edges[:-1] - right_edges = edges[1:] - buckets = np.array([left_edges, right_edges, bucket_counts]).transpose() - tensor = tf.make_tensor_proto(buckets, dtype=tf.float64) - - if display_name is None: - display_name = name - summary_metadata = metadata.create_summary_metadata( - display_name=display_name, description=description) - tf_summary_metadata = tf.SummaryMetadata.FromString( - summary_metadata.SerializeToString()) - - summary = tf.Summary() - summary.value.add(tag='%s/histogram_summary' % name, - metadata=tf_summary_metadata, - tensor=tensor) - return summary + min_ = np.min(data) + max_ = np.max(data) + range_ = max_ - min_ + if range_ == 0: + center = min_ + buckets = np.array([[center - 0.5, center + 0.5, float(data.size)]]) + else: + bucket_width = range_ / bucket_count + offsets = data - min_ + bucket_indices = np.floor(offsets / bucket_width).astype(int) + clamped_indices = np.minimum(bucket_indices, bucket_count - 1) + one_hots = np.array([clamped_indices]).transpose() == np.arange( + 0, bucket_count + ) # broadcast + assert one_hots.shape == (data.size, bucket_count), ( + one_hots.shape, + (data.size, bucket_count), + ) + bucket_counts = np.sum(one_hots, axis=0) + edges = np.linspace(min_, max_, bucket_count + 1) + left_edges = edges[:-1] + right_edges = edges[1:] + buckets = np.array( + [left_edges, right_edges, bucket_counts] + ).transpose() + tensor = tf.make_tensor_proto(buckets, dtype=tf.float64) + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description + ) + tf_summary_metadata = tf.SummaryMetadata.FromString( + summary_metadata.SerializeToString() + ) + + summary = tf.Summary() + summary.value.add( + tag="%s/histogram_summary" % name, + metadata=tf_summary_metadata, + tensor=tensor, + ) + return summary diff --git a/tensorboard/plugins/histogram/summary_test.py b/tensorboard/plugins/histogram/summary_test.py index 312ed413ad..b6b4d50640 100644 --- a/tensorboard/plugins/histogram/summary_test.py +++ b/tensorboard/plugins/histogram/summary_test.py @@ -33,198 +33,211 @@ try: - tf2.__version__ # Force lazy import to resolve + tf2.__version__ # Force lazy import to resolve except ImportError: - tf2 = None + tf2 = None try: - tf.compat.v1.enable_eager_execution() + tf.compat.v1.enable_eager_execution() except AttributeError: - # TF 2.0 doesn't have this symbol because eager is the default. - pass + # TF 2.0 doesn't have this symbol because eager is the default. + pass class SummaryBaseTest(object): - - def setUp(self): - super(SummaryBaseTest, self).setUp() - np.random.seed(0) - self.gaussian = np.random.normal(size=[100]) - - def histogram(self, *args, **kwargs): - raise NotImplementedError() - - def test_metadata(self): - pb = self.histogram('h', [], description='foo') - self.assertEqual(len(pb.value), 1) - summary_metadata = pb.value[0].metadata - self.assertEqual(summary_metadata.summary_description, 'foo') - plugin_data = summary_metadata.plugin_data - self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) - parsed = metadata.parse_plugin_metadata(plugin_data.content) - self.assertEqual(metadata.PROTO_VERSION, parsed.version) - - def test_empty_input(self): - pb = self.histogram('empty', []) - buckets = tensor_util.make_ndarray(pb.value[0].tensor) - np.testing.assert_allclose(buckets, np.array([]).reshape((0, 3))) - - def test_empty_input_of_high_rank(self): - pb = self.histogram('empty_but_fancy', [[[], []], [[], []]]) - buckets = tensor_util.make_ndarray(pb.value[0].tensor) - np.testing.assert_allclose(buckets, np.array([]).reshape((0, 3))) - - def test_singleton_input(self): - pb = self.histogram('twelve', [12]) - buckets = tensor_util.make_ndarray(pb.value[0].tensor) - np.testing.assert_allclose(buckets, np.array([[11.5, 12.5, 1]])) - - def test_input_with_all_same_values(self): - pb = self.histogram('twelven', [12, 12, 12]) - buckets = tensor_util.make_ndarray(pb.value[0].tensor) - np.testing.assert_allclose(buckets, np.array([[11.5, 12.5, 3]])) - - def test_fixed_input(self): - pass # TODO: test a small fixed input - - def test_normal_distribution_input(self): - bucket_count = 44 - pb = self.histogram( - 'normal', data=self.gaussian.reshape((5, -1)), buckets=bucket_count) - buckets = tensor_util.make_ndarray(pb.value[0].tensor) - self.assertEqual(buckets[:, 0].min(), self.gaussian.min()) - # Assert near, not equal, since TF's linspace op introduces floating point - # error in the upper bound of the result. - self.assertNear(buckets[:, 1].max(), self.gaussian.max(), 1.0**-10) - self.assertEqual(buckets[:, 2].sum(), self.gaussian.size) - np.testing.assert_allclose(buckets[1:, 0], buckets[:-1, 1]) - - def test_when_shape_not_statically_known(self): - self.skipTest('TODO: figure out how to test this') - placeholder = tf.compat.v1.placeholder(tf.float64, shape=None) - reshaped = self.gaussian.reshape((25, -1)) - self.histogram(data=reshaped, - data_tensor=placeholder, - feed_dict={placeholder: reshaped}) - # The proto-equality check is all we need. - - def test_when_bucket_count_not_statically_known(self): - self.skipTest('TODO: figure out how to test this') - placeholder = tf.compat.v1.placeholder(tf.int32, shape=()) - bucket_count = 44 - pb = self.histogram( - bucket_count=bucket_count, - bucket_count_tensor=placeholder, - feed_dict={placeholder: bucket_count}) - buckets = tensor_util.make_ndarray(pb.value[0].tensor) - self.assertEqual(buckets.shape, (bucket_count, 3)) + def setUp(self): + super(SummaryBaseTest, self).setUp() + np.random.seed(0) + self.gaussian = np.random.normal(size=[100]) + + def histogram(self, *args, **kwargs): + raise NotImplementedError() + + def test_metadata(self): + pb = self.histogram("h", [], description="foo") + self.assertEqual(len(pb.value), 1) + summary_metadata = pb.value[0].metadata + self.assertEqual(summary_metadata.summary_description, "foo") + plugin_data = summary_metadata.plugin_data + self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) + parsed = metadata.parse_plugin_metadata(plugin_data.content) + self.assertEqual(metadata.PROTO_VERSION, parsed.version) + + def test_empty_input(self): + pb = self.histogram("empty", []) + buckets = tensor_util.make_ndarray(pb.value[0].tensor) + np.testing.assert_allclose(buckets, np.array([]).reshape((0, 3))) + + def test_empty_input_of_high_rank(self): + pb = self.histogram("empty_but_fancy", [[[], []], [[], []]]) + buckets = tensor_util.make_ndarray(pb.value[0].tensor) + np.testing.assert_allclose(buckets, np.array([]).reshape((0, 3))) + + def test_singleton_input(self): + pb = self.histogram("twelve", [12]) + buckets = tensor_util.make_ndarray(pb.value[0].tensor) + np.testing.assert_allclose(buckets, np.array([[11.5, 12.5, 1]])) + + def test_input_with_all_same_values(self): + pb = self.histogram("twelven", [12, 12, 12]) + buckets = tensor_util.make_ndarray(pb.value[0].tensor) + np.testing.assert_allclose(buckets, np.array([[11.5, 12.5, 3]])) + + def test_fixed_input(self): + pass # TODO: test a small fixed input + + def test_normal_distribution_input(self): + bucket_count = 44 + pb = self.histogram( + "normal", data=self.gaussian.reshape((5, -1)), buckets=bucket_count + ) + buckets = tensor_util.make_ndarray(pb.value[0].tensor) + self.assertEqual(buckets[:, 0].min(), self.gaussian.min()) + # Assert near, not equal, since TF's linspace op introduces floating point + # error in the upper bound of the result. + self.assertNear(buckets[:, 1].max(), self.gaussian.max(), 1.0 ** -10) + self.assertEqual(buckets[:, 2].sum(), self.gaussian.size) + np.testing.assert_allclose(buckets[1:, 0], buckets[:-1, 1]) + + def test_when_shape_not_statically_known(self): + self.skipTest("TODO: figure out how to test this") + placeholder = tf.compat.v1.placeholder(tf.float64, shape=None) + reshaped = self.gaussian.reshape((25, -1)) + self.histogram( + data=reshaped, + data_tensor=placeholder, + feed_dict={placeholder: reshaped}, + ) + # The proto-equality check is all we need. + + def test_when_bucket_count_not_statically_known(self): + self.skipTest("TODO: figure out how to test this") + placeholder = tf.compat.v1.placeholder(tf.int32, shape=()) + bucket_count = 44 + pb = self.histogram( + bucket_count=bucket_count, + bucket_count_tensor=placeholder, + feed_dict={placeholder: bucket_count}, + ) + buckets = tensor_util.make_ndarray(pb.value[0].tensor) + self.assertEqual(buckets.shape, (bucket_count, 3)) class SummaryV1PbTest(SummaryBaseTest, tf.test.TestCase): - def histogram(self, *args, **kwargs): - # Map new name to the old name. - if 'buckets' in kwargs: - kwargs['bucket_count'] = kwargs.pop('buckets') - return summary.pb(*args, **kwargs) + def histogram(self, *args, **kwargs): + # Map new name to the old name. + if "buckets" in kwargs: + kwargs["bucket_count"] = kwargs.pop("buckets") + return summary.pb(*args, **kwargs) - def test_tag(self): - self.assertEqual('a/histogram_summary', - self.histogram('a', []).value[0].tag) - self.assertEqual('a/b/histogram_summary', - self.histogram('a/b', []).value[0].tag) + def test_tag(self): + self.assertEqual( + "a/histogram_summary", self.histogram("a", []).value[0].tag + ) + self.assertEqual( + "a/b/histogram_summary", self.histogram("a/b", []).value[0].tag + ) class SummaryV1OpTest(SummaryBaseTest, tf.test.TestCase): - def histogram(self, *args, **kwargs): - # Map new name to the old name. - if 'buckets' in kwargs: - kwargs['bucket_count'] = kwargs.pop('buckets') - return summary_pb2.Summary.FromString(summary.op(*args, **kwargs).numpy()) - - def test_tag(self): - self.assertEqual('a/histogram_summary', - self.histogram('a', []).value[0].tag) - self.assertEqual('a/b/histogram_summary', - self.histogram('a/b', []).value[0].tag) - - def test_scoped_tag(self): - with tf.name_scope('scope'): - self.assertEqual('scope/a/histogram_summary', - self.histogram('a', []).value[0].tag) + def histogram(self, *args, **kwargs): + # Map new name to the old name. + if "buckets" in kwargs: + kwargs["bucket_count"] = kwargs.pop("buckets") + return summary_pb2.Summary.FromString( + summary.op(*args, **kwargs).numpy() + ) + + def test_tag(self): + self.assertEqual( + "a/histogram_summary", self.histogram("a", []).value[0].tag + ) + self.assertEqual( + "a/b/histogram_summary", self.histogram("a/b", []).value[0].tag + ) + + def test_scoped_tag(self): + with tf.name_scope("scope"): + self.assertEqual( + "scope/a/histogram_summary", + self.histogram("a", []).value[0].tag, + ) class SummaryV2PbTest(SummaryBaseTest, tf.test.TestCase): - def histogram(self, *args, **kwargs): - return summary.histogram_pb(*args, **kwargs) + def histogram(self, *args, **kwargs): + return summary.histogram_pb(*args, **kwargs) class SummaryV2OpTest(SummaryBaseTest, tf.test.TestCase): - def setUp(self): - super(SummaryV2OpTest, self).setUp() - if tf2 is None: - self.skipTest('v2 summary API not available') - - def histogram(self, *args, **kwargs): - return self.histogram_event(*args, **kwargs).summary - - def histogram_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - summary.histogram(*args, **kwargs) - writer.close() - event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), '*'))) - self.assertEqual(len(event_files), 1) - events = list(tf.compat.v1.train.summary_iterator(event_files[0])) - # Expect a boilerplate event for the file_version, then the summary one. - self.assertEqual(len(events), 2) - # Delete the event file to reset to an empty directory for later calls. - # TODO(nickfelt): use a unique subdirectory per writer instead. - os.remove(event_files[0]) - return events[1] - - def write_histogram_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - summary.histogram(*args, **kwargs) - writer.close() - - def test_scoped_tag(self): - with tf.name_scope('scope'): - self.assertEqual('scope/a', self.histogram('a', []).value[0].tag) - - def test_step(self): - event = self.histogram_event('a', [], step=333) - self.assertEqual(333, event.step) - - def test_default_step(self): - try: - tf2.summary.experimental.set_step(333) - # TODO(nickfelt): change test logic so we can just omit `step` entirely. - event = self.histogram_event('a', [], step=None) - self.assertEqual(333, event.step) - finally: - # Reset to default state for other tests. - tf2.summary.experimental.set_step(None) + def setUp(self): + super(SummaryV2OpTest, self).setUp() + if tf2 is None: + self.skipTest("v2 summary API not available") + + def histogram(self, *args, **kwargs): + return self.histogram_event(*args, **kwargs).summary + + def histogram_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + summary.histogram(*args, **kwargs) + writer.close() + event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), "*"))) + self.assertEqual(len(event_files), 1) + events = list(tf.compat.v1.train.summary_iterator(event_files[0])) + # Expect a boilerplate event for the file_version, then the summary one. + self.assertEqual(len(events), 2) + # Delete the event file to reset to an empty directory for later calls. + # TODO(nickfelt): use a unique subdirectory per writer instead. + os.remove(event_files[0]) + return events[1] + + def write_histogram_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + summary.histogram(*args, **kwargs) + writer.close() + + def test_scoped_tag(self): + with tf.name_scope("scope"): + self.assertEqual("scope/a", self.histogram("a", []).value[0].tag) + + def test_step(self): + event = self.histogram_event("a", [], step=333) + self.assertEqual(333, event.step) + + def test_default_step(self): + try: + tf2.summary.experimental.set_step(333) + # TODO(nickfelt): change test logic so we can just omit `step` entirely. + event = self.histogram_event("a", [], step=None) + self.assertEqual(333, event.step) + finally: + # Reset to default state for other tests. + tf2.summary.experimental.set_step(None) class SummaryV2OpGraphTest(SummaryV2OpTest, tf.test.TestCase): - def write_histogram_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - # Hack to extract current scope since there's no direct API for it. - with tf.name_scope('_') as temp_scope: - scope = temp_scope.rstrip('/_') - @tf2.function - def graph_fn(): - # Recreate the active scope inside the defun since it won't propagate. - with tf.name_scope(scope): - summary.histogram(*args, **kwargs) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - graph_fn() - writer.close() - - -if __name__ == '__main__': - tf.test.main() + def write_histogram_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + # Hack to extract current scope since there's no direct API for it. + with tf.name_scope("_") as temp_scope: + scope = temp_scope.rstrip("/_") + + @tf2.function + def graph_fn(): + # Recreate the active scope inside the defun since it won't propagate. + with tf.name_scope(scope): + summary.histogram(*args, **kwargs) + + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + graph_fn() + writer.close() + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/histogram/summary_v2.py b/tensorboard/plugins/histogram/summary_v2.py index 172909f043..4877944773 100644 --- a/tensorboard/plugins/histogram/summary_v2.py +++ b/tensorboard/plugins/histogram/summary_v2.py @@ -42,170 +42,191 @@ def histogram(name, data, step=None, buckets=None, description=None): - """Write a histogram summary. - - Arguments: - name: A name for this summary. The summary tag used for TensorBoard will - be this name prefixed by any active name scopes. - data: A `Tensor` of any shape. Must be castable to `float64`. - step: Explicit `int64`-castable monotonic step value for this summary. If - omitted, this defaults to `tf.summary.experimental.get_step()`, which must - not be None. - buckets: Optional positive `int`. The output will have this - many buckets, except in two edge cases. If there is no data, then - there are no buckets. If there is data but all points have the - same value, then there is one bucket whose left and right - endpoints are the same. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - - Returns: - True on success, or false if no summary was emitted because no default - summary writer was available. - - Raises: - ValueError: if a default writer exists, but no step was provided and - `tf.summary.experimental.get_step()` is None. - """ - summary_metadata = metadata.create_summary_metadata( - display_name=None, description=description) - # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback - summary_scope = ( - getattr(tf.summary.experimental, 'summary_scope', None) or - tf.summary.summary_scope) - - def histogram_summary(data, buckets, histogram_metadata, step): - with summary_scope( - name, 'histogram_summary', values=[data, buckets, step]) as (tag, _): - # Defer histogram bucketing logic by passing it as a callable to write(), - # wrapped in a LazyTensorCreator for backwards compatibility, so that we - # only do this work when summaries are actually written. - @lazy_tensor_creator.LazyTensorCreator - def lazy_tensor(): - return _buckets(data, buckets) - return tf.summary.write( - tag=tag, tensor=lazy_tensor, step=step, metadata=summary_metadata) - - # `_buckets()` has dynamic output shapes which is not supported on TPU's. As so, place - # the bucketing ops on outside compilation cluster so that the function in executed on CPU. - # TODO(https://github.com/tensorflow/tensorboard/issues/2885): Remove this special - # handling once dynamic shapes are supported on TPU's. - if isinstance(tf.distribute.get_strategy(), - tf.distribute.experimental.TPUStrategy): - return tf.compat.v1.tpu.outside_compilation( - histogram_summary, data, buckets, summary_metadata, step) - return histogram_summary(data, buckets, summary_metadata, step) + """Write a histogram summary. + + Arguments: + name: A name for this summary. The summary tag used for TensorBoard will + be this name prefixed by any active name scopes. + data: A `Tensor` of any shape. Must be castable to `float64`. + step: Explicit `int64`-castable monotonic step value for this summary. If + omitted, this defaults to `tf.summary.experimental.get_step()`, which must + not be None. + buckets: Optional positive `int`. The output will have this + many buckets, except in two edge cases. If there is no data, then + there are no buckets. If there is data but all points have the + same value, then there is one bucket whose left and right + endpoints are the same. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + + Returns: + True on success, or false if no summary was emitted because no default + summary writer was available. + + Raises: + ValueError: if a default writer exists, but no step was provided and + `tf.summary.experimental.get_step()` is None. + """ + summary_metadata = metadata.create_summary_metadata( + display_name=None, description=description + ) + # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback + summary_scope = ( + getattr(tf.summary.experimental, "summary_scope", None) + or tf.summary.summary_scope + ) + + def histogram_summary(data, buckets, histogram_metadata, step): + with summary_scope( + name, "histogram_summary", values=[data, buckets, step] + ) as (tag, _): + # Defer histogram bucketing logic by passing it as a callable to write(), + # wrapped in a LazyTensorCreator for backwards compatibility, so that we + # only do this work when summaries are actually written. + @lazy_tensor_creator.LazyTensorCreator + def lazy_tensor(): + return _buckets(data, buckets) + + return tf.summary.write( + tag=tag, + tensor=lazy_tensor, + step=step, + metadata=summary_metadata, + ) + + # `_buckets()` has dynamic output shapes which is not supported on TPU's. As so, place + # the bucketing ops on outside compilation cluster so that the function in executed on CPU. + # TODO(https://github.com/tensorflow/tensorboard/issues/2885): Remove this special + # handling once dynamic shapes are supported on TPU's. + if isinstance( + tf.distribute.get_strategy(), tf.distribute.experimental.TPUStrategy + ): + return tf.compat.v1.tpu.outside_compilation( + histogram_summary, data, buckets, summary_metadata, step + ) + return histogram_summary(data, buckets, summary_metadata, step) def _buckets(data, bucket_count=None): - """Create a TensorFlow op to group data into histogram buckets. - - Arguments: - data: A `Tensor` of any shape. Must be castable to `float64`. - bucket_count: Optional positive `int` or scalar `int32` `Tensor`. - Returns: - A `Tensor` of shape `[k, 3]` and type `float64`. The `i`th row is - a triple `[left_edge, right_edge, count]` for a single bucket. - The value of `k` is either `bucket_count` or `1` or `0`. - """ - if bucket_count is None: - bucket_count = DEFAULT_BUCKET_COUNT - with tf.name_scope('buckets'): - tf.debugging.assert_scalar(bucket_count) - tf.debugging.assert_type(bucket_count, tf.int32) - data = tf.reshape(data, shape=[-1]) # flatten - data = tf.cast(data, tf.float64) - is_empty = tf.equal(tf.size(input=data), 0) - - def when_empty(): - return tf.constant([], shape=(0, 3), dtype=tf.float64) - - def when_nonempty(): - min_ = tf.reduce_min(input_tensor=data) - max_ = tf.reduce_max(input_tensor=data) - range_ = max_ - min_ - is_singular = tf.equal(range_, 0) - - def when_nonsingular(): - bucket_width = range_ / tf.cast(bucket_count, tf.float64) - offsets = data - min_ - bucket_indices = tf.cast(tf.floor(offsets / bucket_width), - dtype=tf.int32) - clamped_indices = tf.minimum(bucket_indices, bucket_count - 1) - one_hots = tf.one_hot(clamped_indices, depth=bucket_count) - bucket_counts = tf.cast(tf.reduce_sum(input_tensor=one_hots, axis=0), - dtype=tf.float64) - edges = tf.linspace(min_, max_, bucket_count + 1) - # Ensure edges[-1] == max_, which TF's linspace implementation does not - # do, leaving it subject to the whim of floating point rounding error. - edges = tf.concat([edges[:-1], [max_]], 0) - left_edges = edges[:-1] - right_edges = edges[1:] - return tf.transpose(a=tf.stack( - [left_edges, right_edges, bucket_counts])) - - def when_singular(): - center = min_ - bucket_starts = tf.stack([center - 0.5]) - bucket_ends = tf.stack([center + 0.5]) - bucket_counts = tf.stack([tf.cast(tf.size(input=data), tf.float64)]) - return tf.transpose( - a=tf.stack([bucket_starts, bucket_ends, bucket_counts])) - - return tf.cond(is_singular, when_singular, when_nonsingular) - - return tf.cond(is_empty, when_empty, when_nonempty) + """Create a TensorFlow op to group data into histogram buckets. + + Arguments: + data: A `Tensor` of any shape. Must be castable to `float64`. + bucket_count: Optional positive `int` or scalar `int32` `Tensor`. + Returns: + A `Tensor` of shape `[k, 3]` and type `float64`. The `i`th row is + a triple `[left_edge, right_edge, count]` for a single bucket. + The value of `k` is either `bucket_count` or `1` or `0`. + """ + if bucket_count is None: + bucket_count = DEFAULT_BUCKET_COUNT + with tf.name_scope("buckets"): + tf.debugging.assert_scalar(bucket_count) + tf.debugging.assert_type(bucket_count, tf.int32) + data = tf.reshape(data, shape=[-1]) # flatten + data = tf.cast(data, tf.float64) + is_empty = tf.equal(tf.size(input=data), 0) + + def when_empty(): + return tf.constant([], shape=(0, 3), dtype=tf.float64) + + def when_nonempty(): + min_ = tf.reduce_min(input_tensor=data) + max_ = tf.reduce_max(input_tensor=data) + range_ = max_ - min_ + is_singular = tf.equal(range_, 0) + + def when_nonsingular(): + bucket_width = range_ / tf.cast(bucket_count, tf.float64) + offsets = data - min_ + bucket_indices = tf.cast( + tf.floor(offsets / bucket_width), dtype=tf.int32 + ) + clamped_indices = tf.minimum(bucket_indices, bucket_count - 1) + one_hots = tf.one_hot(clamped_indices, depth=bucket_count) + bucket_counts = tf.cast( + tf.reduce_sum(input_tensor=one_hots, axis=0), + dtype=tf.float64, + ) + edges = tf.linspace(min_, max_, bucket_count + 1) + # Ensure edges[-1] == max_, which TF's linspace implementation does not + # do, leaving it subject to the whim of floating point rounding error. + edges = tf.concat([edges[:-1], [max_]], 0) + left_edges = edges[:-1] + right_edges = edges[1:] + return tf.transpose( + a=tf.stack([left_edges, right_edges, bucket_counts]) + ) + + def when_singular(): + center = min_ + bucket_starts = tf.stack([center - 0.5]) + bucket_ends = tf.stack([center + 0.5]) + bucket_counts = tf.stack( + [tf.cast(tf.size(input=data), tf.float64)] + ) + return tf.transpose( + a=tf.stack([bucket_starts, bucket_ends, bucket_counts]) + ) + + return tf.cond(is_singular, when_singular, when_nonsingular) + + return tf.cond(is_empty, when_empty, when_nonempty) def histogram_pb(tag, data, buckets=None, description=None): - """Create a histogram summary protobuf. - - Arguments: - tag: String tag for the summary. - data: A `np.array` or array-like form of any shape. Must have type - castable to `float`. - buckets: Optional positive `int`. The output will have this - many buckets, except in two edge cases. If there is no data, then - there are no buckets. If there is data but all points have the - same value, then there is one bucket whose left and right - endpoints are the same. - description: Optional long-form description for this summary, as a - `str`. Markdown is supported. Defaults to empty. - - Returns: - A `summary_pb2.Summary` protobuf object. - """ - bucket_count = DEFAULT_BUCKET_COUNT if buckets is None else buckets - data = np.array(data).flatten().astype(float) - if data.size == 0: - buckets = np.array([]).reshape((0, 3)) - else: - min_ = np.min(data) - max_ = np.max(data) - range_ = max_ - min_ - if range_ == 0: - center = min_ - buckets = np.array([[center - 0.5, center + 0.5, float(data.size)]]) + """Create a histogram summary protobuf. + + Arguments: + tag: String tag for the summary. + data: A `np.array` or array-like form of any shape. Must have type + castable to `float`. + buckets: Optional positive `int`. The output will have this + many buckets, except in two edge cases. If there is no data, then + there are no buckets. If there is data but all points have the + same value, then there is one bucket whose left and right + endpoints are the same. + description: Optional long-form description for this summary, as a + `str`. Markdown is supported. Defaults to empty. + + Returns: + A `summary_pb2.Summary` protobuf object. + """ + bucket_count = DEFAULT_BUCKET_COUNT if buckets is None else buckets + data = np.array(data).flatten().astype(float) + if data.size == 0: + buckets = np.array([]).reshape((0, 3)) else: - bucket_width = range_ / bucket_count - offsets = data - min_ - bucket_indices = np.floor(offsets / bucket_width).astype(int) - clamped_indices = np.minimum(bucket_indices, bucket_count - 1) - one_hots = (np.array([clamped_indices]).transpose() - == np.arange(0, bucket_count)) # broadcast - assert one_hots.shape == (data.size, bucket_count), ( - one_hots.shape, (data.size, bucket_count)) - bucket_counts = np.sum(one_hots, axis=0) - edges = np.linspace(min_, max_, bucket_count + 1) - left_edges = edges[:-1] - right_edges = edges[1:] - buckets = np.array([left_edges, right_edges, bucket_counts]).transpose() - tensor = tensor_util.make_tensor_proto(buckets, dtype=np.float64) - - summary_metadata = metadata.create_summary_metadata( - display_name=None, description=description) - summary = summary_pb2.Summary() - summary.value.add(tag=tag, - metadata=summary_metadata, - tensor=tensor) - return summary + min_ = np.min(data) + max_ = np.max(data) + range_ = max_ - min_ + if range_ == 0: + center = min_ + buckets = np.array([[center - 0.5, center + 0.5, float(data.size)]]) + else: + bucket_width = range_ / bucket_count + offsets = data - min_ + bucket_indices = np.floor(offsets / bucket_width).astype(int) + clamped_indices = np.minimum(bucket_indices, bucket_count - 1) + one_hots = np.array([clamped_indices]).transpose() == np.arange( + 0, bucket_count + ) # broadcast + assert one_hots.shape == (data.size, bucket_count), ( + one_hots.shape, + (data.size, bucket_count), + ) + bucket_counts = np.sum(one_hots, axis=0) + edges = np.linspace(min_, max_, bucket_count + 1) + left_edges = edges[:-1] + right_edges = edges[1:] + buckets = np.array( + [left_edges, right_edges, bucket_counts] + ).transpose() + tensor = tensor_util.make_tensor_proto(buckets, dtype=np.float64) + + summary_metadata = metadata.create_summary_metadata( + display_name=None, description=description + ) + summary = summary_pb2.Summary() + summary.value.add(tag=tag, metadata=summary_metadata, tensor=tensor) + return summary diff --git a/tensorboard/plugins/hparams/api_test.py b/tensorboard/plugins/hparams/api_test.py index cbb5e5a2a9..7c3a0dbd2d 100644 --- a/tensorboard/plugins/hparams/api_test.py +++ b/tensorboard/plugins/hparams/api_test.py @@ -24,13 +24,12 @@ class ApiTest(test.TestCase): + def test_has_core_attributes(self): + self.assertIs(api.HParam, summary_v2.HParam) - def test_has_core_attributes(self): - self.assertIs(api.HParam, summary_v2.HParam) - - def test_has_keras_dependent_attributes(self): - self.assertIs(api.KerasCallback, keras.Callback) + def test_has_keras_dependent_attributes(self): + self.assertIs(api.KerasCallback, keras.Callback) if __name__ == "__main__": - test.main() + test.main() diff --git a/tensorboard/plugins/hparams/backend_context.py b/tensorboard/plugins/hparams/backend_context.py index 1ccd7cd2c9..630c50caf2 100644 --- a/tensorboard/plugins/hparams/backend_context.py +++ b/tensorboard/plugins/hparams/backend_context.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Wraps the base_plugin.TBContext to stores additional data shared across - API handlers for the HParams plugin backend. -""" +"""Wraps the base_plugin.TBContext to stores additional data shared across API +handlers for the HParams plugin backend.""" from __future__ import absolute_import from __future__ import division @@ -33,277 +32,297 @@ class Context(object): - """Wraps the base_plugin.TBContext to stores additional data shared across - API handlers for the HParams plugin backend. - - Before adding fields to this class, carefully consider whether the field - truelly needs to be accessible to all API handlers or if it can be passed - separately to the handler constructor. - We want to avoid this class becoming a magic container of variables that - have no better place. See http://wiki.c2.com/?MagicContainer - """ - - def __init__(self, - tb_context, - max_domain_discrete_len=10): - """Instantiates a context. - - Args: - tb_context: base_plugin.TBContext. The "base" context we extend. - max_domain_discrete_len: int. Only used when computing the experiment - from the session runs. The maximum number of disticnt values a string - hyperparameter can have for us to populate its 'domain_discrete' field. - Typically, only tests should specify a value for this parameter. - """ - self._tb_context = tb_context - self._experiment_from_tag = None - self._experiment_from_tag_lock = threading.Lock() - self._max_domain_discrete_len = max_domain_discrete_len - - def experiment(self): - """Returns the experiment protobuffer defining the experiment. - - This method first attempts to find a metadata.EXPERIMENT_TAG tag and - retrieve the associated protobuffer. If no such tag is found, the method - will attempt to build a minimal experiment protobuffer by scanning for - all metadata.SESSION_START_INFO_TAG tags (to compute the hparam_infos - field of the experiment) and for all scalar tags (to compute the - metric_infos field of the experiment). - - Returns: - The experiment protobuffer. If no tags are found from which an experiment - protobuffer can be built (possibly, because the event data has not been - completely loaded yet), returns None. + """Wraps the base_plugin.TBContext to stores additional data shared across + API handlers for the HParams plugin backend. + + Before adding fields to this class, carefully consider whether the + field truelly needs to be accessible to all API handlers or if it + can be passed separately to the handler constructor. We want to + avoid this class becoming a magic container of variables that have + no better place. See http://wiki.c2.com/?MagicContainer """ - experiment = self._find_experiment_tag() - if experiment is None: - return self._compute_experiment_from_runs() - return experiment - @property - def multiplexer(self): - return self._tb_context.multiplexer + def __init__(self, tb_context, max_domain_discrete_len=10): + """Instantiates a context. + + Args: + tb_context: base_plugin.TBContext. The "base" context we extend. + max_domain_discrete_len: int. Only used when computing the experiment + from the session runs. The maximum number of disticnt values a string + hyperparameter can have for us to populate its 'domain_discrete' field. + Typically, only tests should specify a value for this parameter. + """ + self._tb_context = tb_context + self._experiment_from_tag = None + self._experiment_from_tag_lock = threading.Lock() + self._max_domain_discrete_len = max_domain_discrete_len + + def experiment(self): + """Returns the experiment protobuffer defining the experiment. + + This method first attempts to find a metadata.EXPERIMENT_TAG tag and + retrieve the associated protobuffer. If no such tag is found, the method + will attempt to build a minimal experiment protobuffer by scanning for + all metadata.SESSION_START_INFO_TAG tags (to compute the hparam_infos + field of the experiment) and for all scalar tags (to compute the + metric_infos field of the experiment). + + Returns: + The experiment protobuffer. If no tags are found from which an experiment + protobuffer can be built (possibly, because the event data has not been + completely loaded yet), returns None. + """ + experiment = self._find_experiment_tag() + if experiment is None: + return self._compute_experiment_from_runs() + return experiment + + @property + def multiplexer(self): + return self._tb_context.multiplexer + + @property + def tb_context(self): + return self._tb_context + + def _find_experiment_tag(self): + """Finds the experiment associcated with the metadata.EXPERIMENT_TAG + tag. + + Caches the experiment if it was found. + + Returns: + The experiment or None if no such experiment is found. + """ + with self._experiment_from_tag_lock: + if self._experiment_from_tag is None: + mapping = self.multiplexer.PluginRunToTagToContent( + metadata.PLUGIN_NAME + ) + for tag_to_content in mapping.values(): + if metadata.EXPERIMENT_TAG in tag_to_content: + self._experiment_from_tag = metadata.parse_experiment_plugin_data( + tag_to_content[metadata.EXPERIMENT_TAG] + ) + break + return self._experiment_from_tag + + def _compute_experiment_from_runs(self): + """Computes a minimal Experiment protocol buffer by scanning the + runs.""" + hparam_infos = self._compute_hparam_infos() + if not hparam_infos: + return None + metric_infos = self._compute_metric_infos() + return api_pb2.Experiment( + hparam_infos=hparam_infos, metric_infos=metric_infos + ) + + def _compute_hparam_infos(self): + """Computes a list of api_pb2.HParamInfo from the current run, tag + info. + + Finds all the SessionStartInfo messages and collects the hparams values + appearing in each one. For each hparam attempts to deduce a type that fits + all its values. Finally, sets the 'domain' of the resulting HParamInfo + to be discrete if the type is string and the number of distinct values is + small enough. + + Returns: + A list of api_pb2.HParamInfo messages. + """ + run_to_tag_to_content = self.multiplexer.PluginRunToTagToContent( + metadata.PLUGIN_NAME + ) + # Construct a dict mapping an hparam name to its list of values. + hparams = collections.defaultdict(list) + for tag_to_content in run_to_tag_to_content.values(): + if metadata.SESSION_START_INFO_TAG not in tag_to_content: + continue + start_info = metadata.parse_session_start_info_plugin_data( + tag_to_content[metadata.SESSION_START_INFO_TAG] + ) + for (name, value) in six.iteritems(start_info.hparams): + hparams[name].append(value) + + # Try to construct an HParamInfo for each hparam from its name and list + # of values. + result = [] + for (name, values) in six.iteritems(hparams): + hparam_info = self._compute_hparam_info_from_values(name, values) + if hparam_info is not None: + result.append(hparam_info) + return result + + def _compute_hparam_info_from_values(self, name, values): + """Builds an HParamInfo message from the hparam name and list of + values. + + Args: + name: string. The hparam name. + values: list of google.protobuf.Value messages. The list of values for the + hparam. + + Returns: + An api_pb2.HParamInfo message. + """ + # Figure out the type from the values. + # Ignore values whose type is not listed in api_pb2.DataType + # If all values have the same type, then that is the type used. + # Otherwise, the returned type is DATA_TYPE_STRING. + result = api_pb2.HParamInfo(name=name, type=api_pb2.DATA_TYPE_UNSET) + distinct_values = set( + _protobuf_value_to_string(v) + for v in values + if _protobuf_value_type(v) + ) + for v in values: + v_type = _protobuf_value_type(v) + if not v_type: + continue + if result.type == api_pb2.DATA_TYPE_UNSET: + result.type = v_type + elif result.type != v_type: + result.type = api_pb2.DATA_TYPE_STRING + if result.type == api_pb2.DATA_TYPE_STRING: + # A string result.type does not change, so we can exit the loop. + break + + # If we couldn't figure out a type, then we can't compute the hparam_info. + if result.type == api_pb2.DATA_TYPE_UNSET: + return None + + # If the result is a string, set the domain to be the distinct values if + # there aren't too many of them. + if ( + result.type == api_pb2.DATA_TYPE_STRING + and len(distinct_values) <= self._max_domain_discrete_len + ): + result.domain_discrete.extend(distinct_values) + + return result + + def _compute_metric_infos(self): + return ( + api_pb2.MetricInfo(name=api_pb2.MetricName(group=group, tag=tag)) + for tag, group in self._compute_metric_names() + ) + + def _compute_metric_names(self): + """Computes the list of metric names from all the scalar (run, tag) + pairs. + + The return value is a list of (tag, group) pairs representing the metric + names. The list is sorted in Python tuple-order (lexicographical). + + For example, if the scalar (run, tag) pairs are: + ("exp/session1", "loss") + ("exp/session2", "loss") + ("exp/session2/eval", "loss") + ("exp/session2/validation", "accuracy") + ("exp/no-session", "loss_2"), + and the runs corresponding to sessions are "exp/session1", "exp/session2", + this method will return [("loss", ""), ("loss", "/eval"), ("accuracy", + "/validation")] + + More precisely, each scalar (run, tag) pair is converted to a (tag, group) + metric name, where group is the suffix of run formed by removing the + longest prefix which is a session run. If no session run is a prefix of + 'run', the pair is skipped. + + Returns: + A python list containing pairs. Each pair is a (tag, group) pair + representing a metric name used in some session. + """ + session_runs = self._build_session_runs_set() + metric_names_set = set() + run_to_tag_to_content = self.multiplexer.PluginRunToTagToContent( + scalar_metadata.PLUGIN_NAME + ) + for (run, tag_to_content) in six.iteritems(run_to_tag_to_content): + session = _find_longest_parent_path(session_runs, run) + if not session: + continue + group = os.path.relpath(run, session) + # relpath() returns "." for the 'session' directory, we use an empty + # string. + if group == ".": + group = "" + metric_names_set.update( + (tag, group) for tag in tag_to_content.keys() + ) + metric_names_list = list(metric_names_set) + # Sort metrics for determinism. + metric_names_list.sort() + return metric_names_list + + def _build_session_runs_set(self): + result = set() + run_to_tag_to_content = self.multiplexer.PluginRunToTagToContent( + metadata.PLUGIN_NAME + ) + for (run, tag_to_content) in six.iteritems(run_to_tag_to_content): + if metadata.SESSION_START_INFO_TAG in tag_to_content: + result.add(run) + return result - @property - def tb_context(self): - return self._tb_context - def _find_experiment_tag(self): - """Finds the experiment associcated with the metadata.EXPERIMENT_TAG tag. +def _find_longest_parent_path(path_set, path): + """Finds the longest "parent-path" of 'path' in 'path_set'. - Caches the experiment if it was found. - - Returns: - The experiment or None if no such experiment is found. - """ - with self._experiment_from_tag_lock: - if self._experiment_from_tag is None: - mapping = self.multiplexer.PluginRunToTagToContent( - metadata.PLUGIN_NAME) - for tag_to_content in mapping.values(): - if metadata.EXPERIMENT_TAG in tag_to_content: - self._experiment_from_tag = metadata.parse_experiment_plugin_data( - tag_to_content[metadata.EXPERIMENT_TAG]) - break - return self._experiment_from_tag - - def _compute_experiment_from_runs(self): - """Computes a minimal Experiment protocol buffer by scanning the runs.""" - hparam_infos = self._compute_hparam_infos() - if not hparam_infos: - return None - metric_infos = self._compute_metric_infos() - return api_pb2.Experiment(hparam_infos=hparam_infos, - metric_infos=metric_infos) - - def _compute_hparam_infos(self): - """Computes a list of api_pb2.HParamInfo from the current run, tag info. - - Finds all the SessionStartInfo messages and collects the hparams values - appearing in each one. For each hparam attempts to deduce a type that fits - all its values. Finally, sets the 'domain' of the resulting HParamInfo - to be discrete if the type is string and the number of distinct values is - small enough. - - Returns: - A list of api_pb2.HParamInfo messages. - """ - run_to_tag_to_content = self.multiplexer.PluginRunToTagToContent( - metadata.PLUGIN_NAME) - # Construct a dict mapping an hparam name to its list of values. - hparams = collections.defaultdict(list) - for tag_to_content in run_to_tag_to_content.values(): - if metadata.SESSION_START_INFO_TAG not in tag_to_content: - continue - start_info = metadata.parse_session_start_info_plugin_data( - tag_to_content[metadata.SESSION_START_INFO_TAG]) - for (name, value) in six.iteritems(start_info.hparams): - hparams[name].append(value) - - # Try to construct an HParamInfo for each hparam from its name and list - # of values. - result = [] - for (name, values) in six.iteritems(hparams): - hparam_info = self._compute_hparam_info_from_values(name, values) - if hparam_info is not None: - result.append(hparam_info) - return result - - def _compute_hparam_info_from_values(self, name, values): - """Builds an HParamInfo message from the hparam name and list of values. + This function takes and returns "path-like" strings which are strings + made of strings separated by os.sep. No file access is performed here, so + these strings need not correspond to actual files in some file-system.. + This function returns the longest ancestor path + For example, for path_set=["/foo/bar", "/foo", "/bar/foo"] and + path="/foo/bar/sub_dir", returns "/foo/bar". Args: - name: string. The hparam name. - values: list of google.protobuf.Value messages. The list of values for the - hparam. + path_set: set of path-like strings -- e.g. a list of strings separated by + os.sep. No actual disk-access is performed here, so these need not + correspond to actual files. + path: a path-like string. Returns: - An api_pb2.HParamInfo message. + The element in path_set which is the longest parent directory of 'path'. """ - # Figure out the type from the values. - # Ignore values whose type is not listed in api_pb2.DataType - # If all values have the same type, then that is the type used. - # Otherwise, the returned type is DATA_TYPE_STRING. - result = api_pb2.HParamInfo(name=name, type=api_pb2.DATA_TYPE_UNSET) - distinct_values = set( - _protobuf_value_to_string(v) for v in values if _protobuf_value_type(v)) - for v in values: - v_type = _protobuf_value_type(v) - if not v_type: - continue - if result.type == api_pb2.DATA_TYPE_UNSET: - result.type = v_type - elif result.type != v_type: - result.type = api_pb2.DATA_TYPE_STRING - if result.type == api_pb2.DATA_TYPE_STRING: - # A string result.type does not change, so we can exit the loop. - break - - # If we couldn't figure out a type, then we can't compute the hparam_info. - if result.type == api_pb2.DATA_TYPE_UNSET: - return None - - # If the result is a string, set the domain to be the distinct values if - # there aren't too many of them. - if (result.type == api_pb2.DATA_TYPE_STRING - and len(distinct_values) <= self._max_domain_discrete_len): - result.domain_discrete.extend(distinct_values) - - return result - - def _compute_metric_infos(self): - return (api_pb2.MetricInfo(name=api_pb2.MetricName(group=group, tag=tag)) - for tag, group in self._compute_metric_names()) - - def _compute_metric_names(self): - """Computes the list of metric names from all the scalar (run, tag) pairs. - - The return value is a list of (tag, group) pairs representing the metric - names. The list is sorted in Python tuple-order (lexicographical). - - For example, if the scalar (run, tag) pairs are: - ("exp/session1", "loss") - ("exp/session2", "loss") - ("exp/session2/eval", "loss") - ("exp/session2/validation", "accuracy") - ("exp/no-session", "loss_2"), - and the runs corresponding to sessions are "exp/session1", "exp/session2", - this method will return [("loss", ""), ("loss", "/eval"), ("accuracy", - "/validation")] - - More precisely, each scalar (run, tag) pair is converted to a (tag, group) - metric name, where group is the suffix of run formed by removing the - longest prefix which is a session run. If no session run is a prefix of - 'run', the pair is skipped. - - Returns: - A python list containing pairs. Each pair is a (tag, group) pair - representing a metric name used in some session. - """ - session_runs = self._build_session_runs_set() - metric_names_set = set() - run_to_tag_to_content = self.multiplexer.PluginRunToTagToContent( - scalar_metadata.PLUGIN_NAME) - for (run, tag_to_content) in six.iteritems(run_to_tag_to_content): - session = _find_longest_parent_path(session_runs, run) - if not session: - continue - group = os.path.relpath(run, session) - # relpath() returns "." for the 'session' directory, we use an empty - # string. - if group == ".": - group = "" - metric_names_set.update((tag, group) for tag in tag_to_content.keys()) - metric_names_list = list(metric_names_set) - # Sort metrics for determinism. - metric_names_list.sort() - return metric_names_list - - def _build_session_runs_set(self): - result = set() - run_to_tag_to_content = self.multiplexer.PluginRunToTagToContent( - metadata.PLUGIN_NAME) - for (run, tag_to_content) in six.iteritems(run_to_tag_to_content): - if metadata.SESSION_START_INFO_TAG in tag_to_content: - result.add(run) - return result - - -def _find_longest_parent_path(path_set, path): - """Finds the longest "parent-path" of 'path' in 'path_set'. - - This function takes and returns "path-like" strings which are strings - made of strings separated by os.sep. No file access is performed here, so - these strings need not correspond to actual files in some file-system.. - This function returns the longest ancestor path - For example, for path_set=["/foo/bar", "/foo", "/bar/foo"] and - path="/foo/bar/sub_dir", returns "/foo/bar". - - Args: - path_set: set of path-like strings -- e.g. a list of strings separated by - os.sep. No actual disk-access is performed here, so these need not - correspond to actual files. - path: a path-like string. - - Returns: - The element in path_set which is the longest parent directory of 'path'. - """ - # This could likely be more efficiently implemented with a trie - # data-structure, but we don't want to add an extra dependency for that. - while path not in path_set: - if not path: - return None - path = os.path.dirname(path) - return path + # This could likely be more efficiently implemented with a trie + # data-structure, but we don't want to add an extra dependency for that. + while path not in path_set: + if not path: + return None + path = os.path.dirname(path) + return path def _protobuf_value_type(value): - """Returns the type of the google.protobuf.Value message as an api.DataType. + """Returns the type of the google.protobuf.Value message as an + api.DataType. - Returns None if the type of 'value' is not one of the types supported in - api_pb2.DataType. + Returns None if the type of 'value' is not one of the types supported in + api_pb2.DataType. - Args: - value: google.protobuf.Value message. - """ - if value.HasField("number_value"): - return api_pb2.DATA_TYPE_FLOAT64 - if value.HasField("string_value"): - return api_pb2.DATA_TYPE_STRING - if value.HasField("bool_value"): - return api_pb2.DATA_TYPE_BOOL - return None + Args: + value: google.protobuf.Value message. + """ + if value.HasField("number_value"): + return api_pb2.DATA_TYPE_FLOAT64 + if value.HasField("string_value"): + return api_pb2.DATA_TYPE_STRING + if value.HasField("bool_value"): + return api_pb2.DATA_TYPE_BOOL + return None def _protobuf_value_to_string(value): - """Returns a string representation of given google.protobuf.Value message. - - Args: - value: google.protobuf.Value message. Assumed to be of type 'number', - 'string' or 'bool'. - """ - value_in_json = json_format.MessageToJson(value) - if value.HasField("string_value"): - # Remove the quotations. - return value_in_json[1:-1] - return value_in_json + """Returns a string representation of given google.protobuf.Value message. + + Args: + value: google.protobuf.Value message. Assumed to be of type 'number', + 'string' or 'bool'. + """ + value_in_json = json_format.MessageToJson(value) + if value.HasField("string_value"): + # Remove the quotations. + return value_in_json[1:-1] + return value_in_json diff --git a/tensorboard/plugins/hparams/backend_context_test.py b/tensorboard/plugins/hparams/backend_context_test.py index e34133e80d..190a5c68f6 100644 --- a/tensorboard/plugins/hparams/backend_context_test.py +++ b/tensorboard/plugins/hparams/backend_context_test.py @@ -21,10 +21,10 @@ import operator try: - # python version >= 3.3 - from unittest import mock + # python version >= 3.3 + from unittest import mock except ImportError: - import mock # pylint: disable=unused-import + import mock # pylint: disable=unused-import import tensorflow as tf from google.protobuf import text_format @@ -36,130 +36,108 @@ from tensorboard.plugins.hparams import metadata from tensorboard.plugins.hparams import plugin_data_pb2 -DATA_TYPE_EXPERIMENT = 'experiment' -DATA_TYPE_SESSION_START_INFO = 'session_start_info' -DATA_TYPE_SESSION_END_INFO = 'session_end_info' +DATA_TYPE_EXPERIMENT = "experiment" +DATA_TYPE_SESSION_START_INFO = "session_start_info" +DATA_TYPE_SESSION_END_INFO = "session_end_info" class BackendContextTest(tf.test.TestCase): - # Make assertProtoEquals print all the diff. - maxDiff = None # pylint: disable=invalid-name + # Make assertProtoEquals print all the diff. + maxDiff = None # pylint: disable=invalid-name - def setUp(self): - self._mock_tb_context = mock.create_autospec(base_plugin.TBContext) - self._mock_multiplexer = mock.create_autospec( - plugin_event_multiplexer.EventMultiplexer) - self._mock_tb_context.multiplexer = self._mock_multiplexer - self._mock_multiplexer.PluginRunToTagToContent.side_effect = ( - self._mock_plugin_run_to_tag_to_content) - self.session_1_start_info_ = '' - self.session_2_start_info_ = '' - self.session_3_start_info_ = '' + def setUp(self): + self._mock_tb_context = mock.create_autospec(base_plugin.TBContext) + self._mock_multiplexer = mock.create_autospec( + plugin_event_multiplexer.EventMultiplexer + ) + self._mock_tb_context.multiplexer = self._mock_multiplexer + self._mock_multiplexer.PluginRunToTagToContent.side_effect = ( + self._mock_plugin_run_to_tag_to_content + ) + self.session_1_start_info_ = "" + self.session_2_start_info_ = "" + self.session_3_start_info_ = "" - def _mock_plugin_run_to_tag_to_content(self, plugin_name): - if plugin_name == metadata.PLUGIN_NAME: - return { - 'exp/session_1': { - metadata.SESSION_START_INFO_TAG: - self._serialized_plugin_data( - DATA_TYPE_SESSION_START_INFO, - self.session_1_start_info_ - ), - }, - 'exp/session_2': { - metadata.SESSION_START_INFO_TAG: - self._serialized_plugin_data( - DATA_TYPE_SESSION_START_INFO, - self.session_2_start_info_ - ), - }, - 'exp/session_3': { - metadata.SESSION_START_INFO_TAG: - self._serialized_plugin_data( - DATA_TYPE_SESSION_START_INFO, - self.session_3_start_info_ - ), - }, - } - SCALARS = event_accumulator.SCALARS # pylint: disable=invalid-name - if plugin_name == SCALARS: - return { - # We use None as the content here, since the content is not - # used in the test. - 'exp/session_1': { - 'loss': None, - 'accuracy': None - }, - 'exp/session_1/eval': { - 'loss': None, - }, - 'exp/session_1/train': { - 'loss': None, - }, - 'exp/session_2': { - 'loss': None, - 'accuracy': None, - }, - 'exp/session_2/eval': { - 'loss': None, - }, - 'exp/session_2/train': { - 'loss': None, - }, - 'exp/session_3': { - 'loss': None, - 'accuracy': None, - }, - 'exp/session_3/eval': { - 'loss': None, - }, - 'exp/session_3xyz/': { - 'loss2': None, - }, - } - self.fail("Unexpected plugin_name '%s' passed to" - ' EventMultiplexer.PluginRunToTagToContent' % plugin_name) + def _mock_plugin_run_to_tag_to_content(self, plugin_name): + if plugin_name == metadata.PLUGIN_NAME: + return { + "exp/session_1": { + metadata.SESSION_START_INFO_TAG: self._serialized_plugin_data( + DATA_TYPE_SESSION_START_INFO, self.session_1_start_info_ + ), + }, + "exp/session_2": { + metadata.SESSION_START_INFO_TAG: self._serialized_plugin_data( + DATA_TYPE_SESSION_START_INFO, self.session_2_start_info_ + ), + }, + "exp/session_3": { + metadata.SESSION_START_INFO_TAG: self._serialized_plugin_data( + DATA_TYPE_SESSION_START_INFO, self.session_3_start_info_ + ), + }, + } + SCALARS = event_accumulator.SCALARS # pylint: disable=invalid-name + if plugin_name == SCALARS: + return { + # We use None as the content here, since the content is not + # used in the test. + "exp/session_1": {"loss": None, "accuracy": None}, + "exp/session_1/eval": {"loss": None,}, + "exp/session_1/train": {"loss": None,}, + "exp/session_2": {"loss": None, "accuracy": None,}, + "exp/session_2/eval": {"loss": None,}, + "exp/session_2/train": {"loss": None,}, + "exp/session_3": {"loss": None, "accuracy": None,}, + "exp/session_3/eval": {"loss": None,}, + "exp/session_3xyz/": {"loss2": None,}, + } + self.fail( + "Unexpected plugin_name '%s' passed to" + " EventMultiplexer.PluginRunToTagToContent" % plugin_name + ) - def test_experiment_with_experiment_tag(self): - experiment = """ + def test_experiment_with_experiment_tag(self): + experiment = """ description: 'Test experiment' metric_infos: [ { name: { tag: 'current_temp' } } ] """ - self._mock_multiplexer.PluginRunToTagToContent.side_effect = None - self._mock_multiplexer.PluginRunToTagToContent.return_value = { - 'exp': { - metadata.EXPERIMENT_TAG: - self._serialized_plugin_data(DATA_TYPE_EXPERIMENT, experiment) + self._mock_multiplexer.PluginRunToTagToContent.side_effect = None + self._mock_multiplexer.PluginRunToTagToContent.return_value = { + "exp": { + metadata.EXPERIMENT_TAG: self._serialized_plugin_data( + DATA_TYPE_EXPERIMENT, experiment + ) + } } - } - ctxt = backend_context.Context(self._mock_tb_context) - self.assertProtoEquals(experiment, ctxt.experiment()) + ctxt = backend_context.Context(self._mock_tb_context) + self.assertProtoEquals(experiment, ctxt.experiment()) - def test_experiment_without_experiment_tag(self): - self.session_1_start_info_ = """ + def test_experiment_without_experiment_tag(self): + self.session_1_start_info_ = """ hparams:[ {key: 'batch_size' value: {number_value: 100}}, {key: 'lr' value: {number_value: 0.01}}, {key: 'model_type' value: {string_value: 'CNN'}} ] """ - self.session_2_start_info_ = """ + self.session_2_start_info_ = """ hparams:[ {key: 'batch_size' value: {number_value: 200}}, {key: 'lr' value: {number_value: 0.02}}, {key: 'model_type' value: {string_value: 'LATTICE'}} ] """ - self.session_3_start_info_ = """ + self.session_3_start_info_ = """ hparams:[ {key: 'batch_size' value: {number_value: 300}}, {key: 'lr' value: {number_value: 0.05}}, {key: 'model_type' value: {string_value: 'CNN'}} ] """ - expected_exp = """ + expected_exp = """ hparam_infos: { name: 'batch_size' type: DATA_TYPE_FLOAT64 @@ -189,31 +167,31 @@ def test_experiment_without_experiment_tag(self): name: {group: 'train', tag: 'loss'} } """ - ctxt = backend_context.Context(self._mock_tb_context) - actual_exp = ctxt.experiment() - _canonicalize_experiment(actual_exp) - self.assertProtoEquals(expected_exp, actual_exp) + ctxt = backend_context.Context(self._mock_tb_context) + actual_exp = ctxt.experiment() + _canonicalize_experiment(actual_exp) + self.assertProtoEquals(expected_exp, actual_exp) - def test_experiment_without_experiment_tag_different_hparam_types(self): - self.session_1_start_info_ = """ + def test_experiment_without_experiment_tag_different_hparam_types(self): + self.session_1_start_info_ = """ hparams:[ {key: 'batch_size' value: {number_value: 100}}, {key: 'lr' value: {string_value: '0.01'}} ] """ - self.session_2_start_info_ = """ + self.session_2_start_info_ = """ hparams:[ {key: 'lr' value: {number_value: 0.02}}, {key: 'model_type' value: {string_value: 'LATTICE'}} ] """ - self.session_3_start_info_ = """ + self.session_3_start_info_ = """ hparams:[ {key: 'batch_size' value: {bool_value: true}}, {key: 'model_type' value: {string_value: 'CNN'}} ] """ - expected_exp = """ + expected_exp = """ hparam_infos: { name: 'batch_size' type: DATA_TYPE_STRING @@ -251,31 +229,31 @@ def test_experiment_without_experiment_tag_different_hparam_types(self): name: {group: 'train', tag: 'loss'} } """ - ctxt = backend_context.Context(self._mock_tb_context) - actual_exp = ctxt.experiment() - _canonicalize_experiment(actual_exp) - self.assertProtoEquals(expected_exp, actual_exp) + ctxt = backend_context.Context(self._mock_tb_context) + actual_exp = ctxt.experiment() + _canonicalize_experiment(actual_exp) + self.assertProtoEquals(expected_exp, actual_exp) - def test_experiment_without_experiment_tag_many_distinct_values(self): - self.session_1_start_info_ = """ + def test_experiment_without_experiment_tag_many_distinct_values(self): + self.session_1_start_info_ = """ hparams:[ {key: 'batch_size' value: {number_value: 100}}, {key: 'lr' value: {string_value: '0.01'}} ] """ - self.session_2_start_info_ = """ + self.session_2_start_info_ = """ hparams:[ {key: 'lr' value: {number_value: 0.02}}, {key: 'model_type' value: {string_value: 'CNN'}} ] """ - self.session_3_start_info_ = """ + self.session_3_start_info_ = """ hparams:[ {key: 'batch_size' value: {bool_value: true}}, {key: 'model_type' value: {string_value: 'CNN'}} ] """ - expected_exp = """ + expected_exp = """ hparam_infos: { name: 'batch_size' type: DATA_TYPE_STRING @@ -304,33 +282,37 @@ def test_experiment_without_experiment_tag_many_distinct_values(self): name: {group: 'train', tag: 'loss'} } """ - ctxt = backend_context.Context(self._mock_tb_context, - max_domain_discrete_len=1) - actual_exp = ctxt.experiment() - _canonicalize_experiment(actual_exp) - self.assertProtoEquals(expected_exp, actual_exp) + ctxt = backend_context.Context( + self._mock_tb_context, max_domain_discrete_len=1 + ) + actual_exp = ctxt.experiment() + _canonicalize_experiment(actual_exp) + self.assertProtoEquals(expected_exp, actual_exp) - def _serialized_plugin_data(self, data_oneof_field, text_protobuffer): - oneof_type_dict = { - DATA_TYPE_EXPERIMENT: api_pb2.Experiment, - DATA_TYPE_SESSION_START_INFO: plugin_data_pb2.SessionStartInfo, - DATA_TYPE_SESSION_END_INFO: plugin_data_pb2.SessionEndInfo - } - protobuffer = text_format.Merge(text_protobuffer, - oneof_type_dict[data_oneof_field]()) - plugin_data = plugin_data_pb2.HParamsPluginData() - getattr(plugin_data, data_oneof_field).CopyFrom(protobuffer) - return metadata.create_summary_metadata(plugin_data).plugin_data.content + def _serialized_plugin_data(self, data_oneof_field, text_protobuffer): + oneof_type_dict = { + DATA_TYPE_EXPERIMENT: api_pb2.Experiment, + DATA_TYPE_SESSION_START_INFO: plugin_data_pb2.SessionStartInfo, + DATA_TYPE_SESSION_END_INFO: plugin_data_pb2.SessionEndInfo, + } + protobuffer = text_format.Merge( + text_protobuffer, oneof_type_dict[data_oneof_field]() + ) + plugin_data = plugin_data_pb2.HParamsPluginData() + getattr(plugin_data, data_oneof_field).CopyFrom(protobuffer) + return metadata.create_summary_metadata(plugin_data).plugin_data.content def _canonicalize_experiment(exp): - """Sorts the repeated fields of an Experiment message.""" - exp.hparam_infos.sort(key=operator.attrgetter('name')) - exp.metric_infos.sort(key=operator.attrgetter('name.group', 'name.tag')) - for hparam_info in exp.hparam_infos: - if hparam_info.HasField('domain_discrete'): - hparam_info.domain_discrete.values.sort( - key=operator.attrgetter('string_value')) + """Sorts the repeated fields of an Experiment message.""" + exp.hparam_infos.sort(key=operator.attrgetter("name")) + exp.metric_infos.sort(key=operator.attrgetter("name.group", "name.tag")) + for hparam_info in exp.hparam_infos: + if hparam_info.HasField("domain_discrete"): + hparam_info.domain_discrete.values.sort( + key=operator.attrgetter("string_value") + ) + -if __name__ == '__main__': - tf.test.main() +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/hparams/error.py b/tensorboard/plugins/hparams/error.py index b88446a825..02c7b89658 100644 --- a/tensorboard/plugins/hparams/error.py +++ b/tensorboard/plugins/hparams/error.py @@ -16,9 +16,11 @@ class HParamsError(Exception): - """Represents an error that is meaningful to the end-user. Such an error - should have a meaningful error message. Other errors, (such as resulting - from some internal invariants being violated) should be represented by - other exceptions. - """ - pass + """Represents an error that is meaningful to the end-user. + + Such an error should have a meaningful error message. Other errors, + (such as resulting from some internal invariants being violated) + should be represented by other exceptions. + """ + + pass diff --git a/tensorboard/plugins/hparams/get_experiment.py b/tensorboard/plugins/hparams/get_experiment.py index b690111d99..91dbf29f52 100644 --- a/tensorboard/plugins/hparams/get_experiment.py +++ b/tensorboard/plugins/hparams/get_experiment.py @@ -23,30 +23,30 @@ class Handler(object): - """Handles a GetExperiment request. """ - - def __init__(self, context): - """Constructor. - - Args: - context: A backend_context.Context instance. - """ - self._context = context - - def run(self): - """Handles the request specified on construction. - - Returns: - An Experiment object. - - """ - experiment = self._context.experiment() - if experiment is None: - raise error.HParamsError( - "Can't find an HParams-plugin experiment data in" - " the log directory. Note that it takes some time to" - " scan the log directory; if you just started" - " Tensorboard it could be that we haven't finished" - " scanning it yet. Consider trying again in a" - " few seconds.") - return experiment + """Handles a GetExperiment request.""" + + def __init__(self, context): + """Constructor. + + Args: + context: A backend_context.Context instance. + """ + self._context = context + + def run(self): + """Handles the request specified on construction. + + Returns: + An Experiment object. + """ + experiment = self._context.experiment() + if experiment is None: + raise error.HParamsError( + "Can't find an HParams-plugin experiment data in" + " the log directory. Note that it takes some time to" + " scan the log directory; if you just started" + " Tensorboard it could be that we haven't finished" + " scanning it yet. Consider trying again in a" + " few seconds." + ) + return experiment diff --git a/tensorboard/plugins/hparams/hparams_demo.py b/tensorboard/plugins/hparams/hparams_demo.py index ac4e762bcb..0d669f33d4 100644 --- a/tensorboard/plugins/hparams/hparams_demo.py +++ b/tensorboard/plugins/hparams/hparams_demo.py @@ -39,10 +39,10 @@ if int(tf.__version__.split(".")[0]) < 2: - # The tag names emitted for Keras metrics changed from "acc" (in 1.x) - # to "accuracy" (in 2.x), so this demo does not work properly in - # TensorFlow 1.x (even with `tf.enable_eager_execution()`). - raise ImportError("TensorFlow 2.x is required to run this demo.") + # The tag names emitted for Keras metrics changed from "acc" (in 1.x) + # to "accuracy" (in 2.x), so this demo does not work properly in + # TensorFlow 1.x (even with `tf.enable_eager_execution()`). + raise ImportError("TensorFlow 2.x is required to run this demo.") flags.DEFINE_integer( @@ -59,12 +59,10 @@ "summary_freq", 600, "Summaries will be written every n steps, where n is the value of " - "this flag.", + "this flag.", ) flags.DEFINE_integer( - "num_epochs", - 5, - "Number of epochs per trial.", + "num_epochs", 5, "Number of epochs per trial.", ) @@ -89,164 +87,154 @@ METRICS = [ hp.Metric( - "epoch_accuracy", - group="validation", - display_name="accuracy (val.)", + "epoch_accuracy", group="validation", display_name="accuracy (val.)", ), + hp.Metric("epoch_loss", group="validation", display_name="loss (val.)",), hp.Metric( - "epoch_loss", - group="validation", - display_name="loss (val.)", - ), - hp.Metric( - "batch_accuracy", - group="train", - display_name="accuracy (train)", - ), - hp.Metric( - "batch_loss", - group="train", - display_name="loss (train)", + "batch_accuracy", group="train", display_name="accuracy (train)", ), + hp.Metric("batch_loss", group="train", display_name="loss (train)",), ] def model_fn(hparams, seed): - """Create a Keras model with the given hyperparameters. - - Args: - hparams: A dict mapping hyperparameters in `HPARAMS` to values. - seed: A hashable object to be used as a random seed (e.g., to - construct dropout layers in the model). - - Returns: - A compiled Keras model. - """ - rng = random.Random(seed) - - model = tf.keras.models.Sequential() - model.add(tf.keras.layers.Input(INPUT_SHAPE)) - model.add(tf.keras.layers.Reshape(INPUT_SHAPE + (1,))) # grayscale channel - - # Add convolutional layers. - conv_filters = 8 - for _ in xrange(hparams[HP_CONV_LAYERS]): - model.add(tf.keras.layers.Conv2D( - filters=conv_filters, - kernel_size=hparams[HP_CONV_KERNEL_SIZE], - padding="same", - activation="relu", - )) - model.add(tf.keras.layers.MaxPool2D(pool_size=2, padding="same")) - conv_filters *= 2 - - model.add(tf.keras.layers.Flatten()) - model.add(tf.keras.layers.Dropout(hparams[HP_DROPOUT], seed=rng.random())) - - # Add fully connected layers. - dense_neurons = 32 - for _ in xrange(hparams[HP_DENSE_LAYERS]): - model.add(tf.keras.layers.Dense(dense_neurons, activation="relu")) - dense_neurons *= 2 - - # Add the final output layer. - model.add(tf.keras.layers.Dense(OUTPUT_CLASSES, activation="softmax")) - - model.compile( - loss="sparse_categorical_crossentropy", - optimizer=hparams[HP_OPTIMIZER], - metrics=["accuracy"], - ) - return model + """Create a Keras model with the given hyperparameters. + + Args: + hparams: A dict mapping hyperparameters in `HPARAMS` to values. + seed: A hashable object to be used as a random seed (e.g., to + construct dropout layers in the model). + + Returns: + A compiled Keras model. + """ + rng = random.Random(seed) + + model = tf.keras.models.Sequential() + model.add(tf.keras.layers.Input(INPUT_SHAPE)) + model.add(tf.keras.layers.Reshape(INPUT_SHAPE + (1,))) # grayscale channel + + # Add convolutional layers. + conv_filters = 8 + for _ in xrange(hparams[HP_CONV_LAYERS]): + model.add( + tf.keras.layers.Conv2D( + filters=conv_filters, + kernel_size=hparams[HP_CONV_KERNEL_SIZE], + padding="same", + activation="relu", + ) + ) + model.add(tf.keras.layers.MaxPool2D(pool_size=2, padding="same")) + conv_filters *= 2 + + model.add(tf.keras.layers.Flatten()) + model.add(tf.keras.layers.Dropout(hparams[HP_DROPOUT], seed=rng.random())) + + # Add fully connected layers. + dense_neurons = 32 + for _ in xrange(hparams[HP_DENSE_LAYERS]): + model.add(tf.keras.layers.Dense(dense_neurons, activation="relu")) + dense_neurons *= 2 + + # Add the final output layer. + model.add(tf.keras.layers.Dense(OUTPUT_CLASSES, activation="softmax")) + + model.compile( + loss="sparse_categorical_crossentropy", + optimizer=hparams[HP_OPTIMIZER], + metrics=["accuracy"], + ) + return model def run(data, base_logdir, session_id, hparams): - """Run a training/validation session. - - Flags must have been parsed for this function to behave. - - Args: - data: The data as loaded by `prepare_data()`. - base_logdir: The top-level logdir to which to write summary data. - session_id: A unique string ID for this session. - hparams: A dict mapping hyperparameters in `HPARAMS` to values. - """ - model = model_fn(hparams=hparams, seed=session_id) - logdir = os.path.join(base_logdir, session_id) - - callback = tf.keras.callbacks.TensorBoard( - logdir, - update_freq=flags.FLAGS.summary_freq, - profile_batch=0, # workaround for issue #2084 - ) - hparams_callback = hp.KerasCallback(logdir, hparams) - ((x_train, y_train), (x_test, y_test)) = data - result = model.fit( - x=x_train, - y=y_train, - epochs=flags.FLAGS.num_epochs, - shuffle=False, - validation_data=(x_test, y_test), - callbacks=[callback, hparams_callback], - ) + """Run a training/validation session. + + Flags must have been parsed for this function to behave. + + Args: + data: The data as loaded by `prepare_data()`. + base_logdir: The top-level logdir to which to write summary data. + session_id: A unique string ID for this session. + hparams: A dict mapping hyperparameters in `HPARAMS` to values. + """ + model = model_fn(hparams=hparams, seed=session_id) + logdir = os.path.join(base_logdir, session_id) + + callback = tf.keras.callbacks.TensorBoard( + logdir, + update_freq=flags.FLAGS.summary_freq, + profile_batch=0, # workaround for issue #2084 + ) + hparams_callback = hp.KerasCallback(logdir, hparams) + ((x_train, y_train), (x_test, y_test)) = data + result = model.fit( + x=x_train, + y=y_train, + epochs=flags.FLAGS.num_epochs, + shuffle=False, + validation_data=(x_test, y_test), + callbacks=[callback, hparams_callback], + ) def prepare_data(): - """Load and normalize data.""" - ((x_train, y_train), (x_test, y_test)) = DATASET.load_data() - x_train = x_train.astype("float32") - x_test = x_test.astype("float32") - x_train /= 255.0 - x_test /= 255.0 - return ((x_train, y_train), (x_test, y_test)) + """Load and normalize data.""" + ((x_train, y_train), (x_test, y_test)) = DATASET.load_data() + x_train = x_train.astype("float32") + x_test = x_test.astype("float32") + x_train /= 255.0 + x_test /= 255.0 + return ((x_train, y_train), (x_test, y_test)) def run_all(logdir, verbose=False): - """Perform random search over the hyperparameter space. - - Arguments: - logdir: The top-level directory into which to write data. This - directory should be empty or nonexistent. - verbose: If true, print out each run's name as it begins. - """ - data = prepare_data() - rng = random.Random(0) - - with tf.summary.create_file_writer(logdir).as_default(): - hp.hparams_config(hparams=HPARAMS, metrics=METRICS) - - sessions_per_group = 2 - num_sessions = flags.FLAGS.num_session_groups * sessions_per_group - session_index = 0 # across all session groups - for group_index in xrange(flags.FLAGS.num_session_groups): - hparams = {h: h.domain.sample_uniform(rng) for h in HPARAMS} - hparams_string = str(hparams) - for repeat_index in xrange(sessions_per_group): - session_id = str(session_index) - session_index += 1 - if verbose: - print( - "--- Running training session %d/%d" - % (session_index, num_sessions) - ) - print(hparams_string) - print("--- repeat #: %d" % (repeat_index + 1)) - run( - data=data, - base_logdir=logdir, - session_id=session_id, - hparams=hparams, - ) + """Perform random search over the hyperparameter space. + + Arguments: + logdir: The top-level directory into which to write data. This + directory should be empty or nonexistent. + verbose: If true, print out each run's name as it begins. + """ + data = prepare_data() + rng = random.Random(0) + + with tf.summary.create_file_writer(logdir).as_default(): + hp.hparams_config(hparams=HPARAMS, metrics=METRICS) + + sessions_per_group = 2 + num_sessions = flags.FLAGS.num_session_groups * sessions_per_group + session_index = 0 # across all session groups + for group_index in xrange(flags.FLAGS.num_session_groups): + hparams = {h: h.domain.sample_uniform(rng) for h in HPARAMS} + hparams_string = str(hparams) + for repeat_index in xrange(sessions_per_group): + session_id = str(session_index) + session_index += 1 + if verbose: + print( + "--- Running training session %d/%d" + % (session_index, num_sessions) + ) + print(hparams_string) + print("--- repeat #: %d" % (repeat_index + 1)) + run( + data=data, + base_logdir=logdir, + session_id=session_id, + hparams=hparams, + ) def main(unused_argv): - np.random.seed(0) - logdir = flags.FLAGS.logdir - shutil.rmtree(logdir, ignore_errors=True) - print("Saving output to %s." % logdir) - run_all(logdir=logdir, verbose=True) - print("Done. Output saved to %s." % logdir) + np.random.seed(0) + logdir = flags.FLAGS.logdir + shutil.rmtree(logdir, ignore_errors=True) + print("Saving output to %s." % logdir) + run_all(logdir=logdir, verbose=True) + print("Done. Output saved to %s." % logdir) if __name__ == "__main__": - app.run(main) + app.run(main) diff --git a/tensorboard/plugins/hparams/hparams_minimal_demo.py b/tensorboard/plugins/hparams/hparams_minimal_demo.py index ffd648d82d..6fc8e640ba 100644 --- a/tensorboard/plugins/hparams/hparams_minimal_demo.py +++ b/tensorboard/plugins/hparams/hparams_minimal_demo.py @@ -55,23 +55,27 @@ FLAGS = flags.FLAGS -flags.DEFINE_integer('num_session_groups', 50, - 'The approximate number of session groups to create.') -flags.DEFINE_string('logdir', '/tmp/hparams_minimal_demo', - 'The directory to write the summary information to.') -flags.DEFINE_integer('summary_freq', 1, - 'Summaries will be every n steps, ' - 'where n is the value of this flag.') -flags.DEFINE_integer('num_steps', 100, - 'Number of steps per trial.') +flags.DEFINE_integer( + "num_session_groups", + 50, + "The approximate number of session groups to create.", +) +flags.DEFINE_string( + "logdir", + "/tmp/hparams_minimal_demo", + "The directory to write the summary information to.", +) +flags.DEFINE_integer( + "summary_freq", + 1, + "Summaries will be every n steps, " "where n is the value of this flag.", +) +flags.DEFINE_integer("num_steps", 100, "Number of steps per trial.") # Total number of sessions is given by: # len(TEMPERATURE_LIST)^2 * len(HEAT_COEFFICIENTS) * 2 -HEAT_COEFFICIENTS = { - 'water': 0.001, - 'air': 0.003 -} +HEAT_COEFFICIENTS = {"water": 0.001, "air": 0.003} TEMPERATURE_LIST = [] @@ -79,191 +83,223 @@ # depends on a flag and flag parsing hasn't happened yet. Instead, we use # a function that we call in main() below. def init_temperature_list(): - global TEMPERATURE_LIST - TEMPERATURE_LIST = [ - 270+i*50.0 - for i in xrange( - 0, int(math.sqrt(FLAGS.num_session_groups/len(HEAT_COEFFICIENTS))))] + global TEMPERATURE_LIST + TEMPERATURE_LIST = [ + 270 + i * 50.0 + for i in xrange( + 0, int(math.sqrt(FLAGS.num_session_groups / len(HEAT_COEFFICIENTS))) + ) + ] def fingerprint(string): - m = hashlib.md5() - m.update(string.encode('utf-8')) - return m.hexdigest() + m = hashlib.md5() + m.update(string.encode("utf-8")) + return m.hexdigest() def create_experiment_summary(): - """Returns a summary proto buffer holding this experiment.""" - - # Convert TEMPERATURE_LIST to google.protobuf.ListValue - temperature_list = struct_pb2.ListValue() - temperature_list.extend(TEMPERATURE_LIST) - materials = struct_pb2.ListValue() - materials.extend(HEAT_COEFFICIENTS.keys()) - return summary.experiment_pb( - hparam_infos=[ - api_pb2.HParamInfo(name='initial_temperature', - display_name='Initial temperature', - type=api_pb2.DATA_TYPE_FLOAT64, - domain_discrete=temperature_list), - api_pb2.HParamInfo(name='ambient_temperature', - display_name='Ambient temperature', - type=api_pb2.DATA_TYPE_FLOAT64, - domain_discrete=temperature_list), - api_pb2.HParamInfo(name='material', - display_name='Material', - type=api_pb2.DATA_TYPE_STRING, - domain_discrete=materials) - ], - metric_infos=[ - api_pb2.MetricInfo( - name=api_pb2.MetricName( - tag='temperature/current/scalar_summary'), - display_name='Current Temp.'), - api_pb2.MetricInfo( - name=api_pb2.MetricName( - tag='temperature/difference_to_ambient/scalar_summary'), - display_name='Difference To Ambient Temp.'), - api_pb2.MetricInfo( - name=api_pb2.MetricName( - tag='delta/scalar_summary'), - display_name='Delta T') - ] - ) + """Returns a summary proto buffer holding this experiment.""" + + # Convert TEMPERATURE_LIST to google.protobuf.ListValue + temperature_list = struct_pb2.ListValue() + temperature_list.extend(TEMPERATURE_LIST) + materials = struct_pb2.ListValue() + materials.extend(HEAT_COEFFICIENTS.keys()) + return summary.experiment_pb( + hparam_infos=[ + api_pb2.HParamInfo( + name="initial_temperature", + display_name="Initial temperature", + type=api_pb2.DATA_TYPE_FLOAT64, + domain_discrete=temperature_list, + ), + api_pb2.HParamInfo( + name="ambient_temperature", + display_name="Ambient temperature", + type=api_pb2.DATA_TYPE_FLOAT64, + domain_discrete=temperature_list, + ), + api_pb2.HParamInfo( + name="material", + display_name="Material", + type=api_pb2.DATA_TYPE_STRING, + domain_discrete=materials, + ), + ], + metric_infos=[ + api_pb2.MetricInfo( + name=api_pb2.MetricName( + tag="temperature/current/scalar_summary" + ), + display_name="Current Temp.", + ), + api_pb2.MetricInfo( + name=api_pb2.MetricName( + tag="temperature/difference_to_ambient/scalar_summary" + ), + display_name="Difference To Ambient Temp.", + ), + api_pb2.MetricInfo( + name=api_pb2.MetricName(tag="delta/scalar_summary"), + display_name="Delta T", + ), + ], + ) def run(logdir, session_id, hparams, group_name): - """Runs a temperature simulation. - - This will simulate an object at temperature `initial_temperature` - sitting at rest in a large room at temperature `ambient_temperature`. - The object has some intrinsic `heat_coefficient`, which indicates - how much thermal conductivity it has: for instance, metals have high - thermal conductivity, while the thermal conductivity of water is low. - - Over time, the object's temperature will adjust to match the - temperature of its environment. We'll track the object's temperature, - how far it is from the room's temperature, and how much it changes at - each time step. - - Arguments: - logdir: the top-level directory into which to write summary data - session_id: an id for the session. - hparams: A dictionary mapping a hyperparameter name to its value. - group_name: an id for the session group this session belongs to. - """ - tf.reset_default_graph() - tf.set_random_seed(0) - - initial_temperature = hparams['initial_temperature'] - ambient_temperature = hparams['ambient_temperature'] - heat_coefficient = HEAT_COEFFICIENTS[hparams['material']] - session_dir = os.path.join(logdir, session_id) - writer = tf.summary.FileWriter(session_dir) - writer.add_summary(summary.session_start_pb(hparams=hparams, - group_name=group_name)) - writer.flush() - with tf.name_scope('temperature'): - # Create a mutable variable to hold the object's temperature, and - # create a scalar summary to track its value over time. The name of - # the summary will appear as 'temperature/current' due to the - # name-scope above. - temperature = tf.Variable( - tf.constant(initial_temperature), - name='temperature') - scalar_summary.op('current', temperature, - display_name='Temperature', - description='The temperature of the object under ' - 'simulation, in Kelvins.') - - # Compute how much the object's temperature differs from that of its - # environment, and track this, too: likewise, as - # 'temperature/difference_to_ambient'. - ambient_difference = temperature - ambient_temperature - scalar_summary.op('difference_to_ambient', ambient_difference, - display_name='Difference to ambient temperature', - description=('The difference between the ambient ' - 'temperature and the temperature of the ' - 'object under simulation, in Kelvins.')) - - # Newton suggested that the rate of change of the temperature of an - # object is directly proportional to this `ambient_difference` above, - # where the proportionality constant is what we called the heat - # coefficient. But in real life, not everything is quite so clean, so - # we'll add in some noise. (The value of 50 is arbitrary, chosen to - # make the data look somewhat interesting. :-) ) - noise = 50 * tf.random.normal([]) - delta = -heat_coefficient * (ambient_difference + noise) - scalar_summary.op('delta', delta, - description='The change in temperature from the previous ' - 'step, in Kelvins.') - - # Collect all the scalars that we want to keep track of. - summ = tf.summary.merge_all() - - # Now, augment the current temperature by this delta that we computed, - # blocking the assignment on summary collection to avoid race conditions - # and ensure that the summary always reports the pre-update value. - with tf.control_dependencies([summ]): - update_step = temperature.assign_add(delta) - - sess = tf.Session() - sess.run(tf.global_variables_initializer()) - for step in xrange(FLAGS.num_steps): - # By asking TensorFlow to compute the update step, we force it to - # change the value of the temperature variable. We don't actually - # care about this value, so we discard it; instead, we grab the - # summary data computed along the way. - (s, _) = sess.run([summ, update_step]) - if (step % FLAGS.summary_freq) == 0: - writer.add_summary(s, global_step=step) - writer.add_summary(summary.session_end_pb(api_pb2.STATUS_SUCCESS)) - writer.close() + """Runs a temperature simulation. + + This will simulate an object at temperature `initial_temperature` + sitting at rest in a large room at temperature `ambient_temperature`. + The object has some intrinsic `heat_coefficient`, which indicates + how much thermal conductivity it has: for instance, metals have high + thermal conductivity, while the thermal conductivity of water is low. + + Over time, the object's temperature will adjust to match the + temperature of its environment. We'll track the object's temperature, + how far it is from the room's temperature, and how much it changes at + each time step. + + Arguments: + logdir: the top-level directory into which to write summary data + session_id: an id for the session. + hparams: A dictionary mapping a hyperparameter name to its value. + group_name: an id for the session group this session belongs to. + """ + tf.reset_default_graph() + tf.set_random_seed(0) + + initial_temperature = hparams["initial_temperature"] + ambient_temperature = hparams["ambient_temperature"] + heat_coefficient = HEAT_COEFFICIENTS[hparams["material"]] + session_dir = os.path.join(logdir, session_id) + writer = tf.summary.FileWriter(session_dir) + writer.add_summary( + summary.session_start_pb(hparams=hparams, group_name=group_name) + ) + writer.flush() + with tf.name_scope("temperature"): + # Create a mutable variable to hold the object's temperature, and + # create a scalar summary to track its value over time. The name of + # the summary will appear as 'temperature/current' due to the + # name-scope above. + temperature = tf.Variable( + tf.constant(initial_temperature), name="temperature" + ) + scalar_summary.op( + "current", + temperature, + display_name="Temperature", + description="The temperature of the object under " + "simulation, in Kelvins.", + ) + + # Compute how much the object's temperature differs from that of its + # environment, and track this, too: likewise, as + # 'temperature/difference_to_ambient'. + ambient_difference = temperature - ambient_temperature + scalar_summary.op( + "difference_to_ambient", + ambient_difference, + display_name="Difference to ambient temperature", + description=( + "The difference between the ambient " + "temperature and the temperature of the " + "object under simulation, in Kelvins." + ), + ) + + # Newton suggested that the rate of change of the temperature of an + # object is directly proportional to this `ambient_difference` above, + # where the proportionality constant is what we called the heat + # coefficient. But in real life, not everything is quite so clean, so + # we'll add in some noise. (The value of 50 is arbitrary, chosen to + # make the data look somewhat interesting. :-) ) + noise = 50 * tf.random.normal([]) + delta = -heat_coefficient * (ambient_difference + noise) + scalar_summary.op( + "delta", + delta, + description="The change in temperature from the previous " + "step, in Kelvins.", + ) + + # Collect all the scalars that we want to keep track of. + summ = tf.summary.merge_all() + + # Now, augment the current temperature by this delta that we computed, + # blocking the assignment on summary collection to avoid race conditions + # and ensure that the summary always reports the pre-update value. + with tf.control_dependencies([summ]): + update_step = temperature.assign_add(delta) + + sess = tf.Session() + sess.run(tf.global_variables_initializer()) + for step in xrange(FLAGS.num_steps): + # By asking TensorFlow to compute the update step, we force it to + # change the value of the temperature variable. We don't actually + # care about this value, so we discard it; instead, we grab the + # summary data computed along the way. + (s, _) = sess.run([summ, update_step]) + if (step % FLAGS.summary_freq) == 0: + writer.add_summary(s, global_step=step) + writer.add_summary(summary.session_end_pb(api_pb2.STATUS_SUCCESS)) + writer.close() def run_all(logdir, verbose=False): - """Run simulations on a reasonable set of parameters. - - Arguments: - logdir: the directory into which to store all the runs' data - verbose: if true, print out each run's name as it begins. - """ - writer = tf.summary.FileWriter(logdir) - writer.add_summary(create_experiment_summary()) - writer.close() - session_num = 0 - num_sessions = (len(TEMPERATURE_LIST)*len(TEMPERATURE_LIST)* - len(HEAT_COEFFICIENTS)*2) - for initial_temperature in TEMPERATURE_LIST: - for ambient_temperature in TEMPERATURE_LIST: - for material in HEAT_COEFFICIENTS: - hparams = {u'initial_temperature': initial_temperature, - u'ambient_temperature': ambient_temperature, - u'material': material} - hparam_str = str(hparams) - group_name = fingerprint(hparam_str) - for repeat_idx in xrange(2): - session_id = str(session_num) - if verbose: - print('--- Running training session %d/%d' % (session_num + 1, - num_sessions)) - print(hparam_str) - print('--- repeat #: %d' % (repeat_idx+1)) - run(logdir, session_id, hparams, group_name) - session_num += 1 + """Run simulations on a reasonable set of parameters. + + Arguments: + logdir: the directory into which to store all the runs' data + verbose: if true, print out each run's name as it begins. + """ + writer = tf.summary.FileWriter(logdir) + writer.add_summary(create_experiment_summary()) + writer.close() + session_num = 0 + num_sessions = ( + len(TEMPERATURE_LIST) + * len(TEMPERATURE_LIST) + * len(HEAT_COEFFICIENTS) + * 2 + ) + for initial_temperature in TEMPERATURE_LIST: + for ambient_temperature in TEMPERATURE_LIST: + for material in HEAT_COEFFICIENTS: + hparams = { + u"initial_temperature": initial_temperature, + u"ambient_temperature": ambient_temperature, + u"material": material, + } + hparam_str = str(hparams) + group_name = fingerprint(hparam_str) + for repeat_idx in xrange(2): + session_id = str(session_num) + if verbose: + print( + "--- Running training session %d/%d" + % (session_num + 1, num_sessions) + ) + print(hparam_str) + print("--- repeat #: %d" % (repeat_idx + 1)) + run(logdir, session_id, hparams, group_name) + session_num += 1 def main(unused_argv): - if tf.executing_eagerly(): - print('Sorry, this demo currently can\'t be run in eager mode.') - return + if tf.executing_eagerly(): + print("Sorry, this demo currently can't be run in eager mode.") + return - init_temperature_list() - shutil.rmtree(FLAGS.logdir, ignore_errors=True) - print('Saving output to %s.' % FLAGS.logdir) - run_all(FLAGS.logdir, verbose=True) - print('Done. Output saved to %s.' % FLAGS.logdir) + init_temperature_list() + shutil.rmtree(FLAGS.logdir, ignore_errors=True) + print("Saving output to %s." % FLAGS.logdir) + run_all(FLAGS.logdir, verbose=True) + print("Done. Output saved to %s." % FLAGS.logdir) -if __name__ == '__main__': - app.run(main) +if __name__ == "__main__": + app.run(main) diff --git a/tensorboard/plugins/hparams/hparams_plugin.py b/tensorboard/plugins/hparams/hparams_plugin.py index a04e2e885b..89ca26f3bb 100644 --- a/tensorboard/plugins/hparams/hparams_plugin.py +++ b/tensorboard/plugins/hparams/hparams_plugin.py @@ -14,8 +14,8 @@ # ============================================================================== """The TensorBoard HParams plugin. -See `http_api.md` in this directory for specifications of the routes for this -plugin. +See `http_api.md` in this directory for specifications of the routes for +this plugin. """ from __future__ import absolute_import @@ -45,119 +45,138 @@ class HParamsPlugin(base_plugin.TBPlugin): - """HParams Plugin for TensorBoard. - It supports both GETs and POSTs. See 'http_api.md' for more details. - """ + """HParams Plugin for TensorBoard. - plugin_name = metadata.PLUGIN_NAME - - def __init__(self, context): - """Instantiates HParams plugin via TensorBoard core. - - Args: - context: A base_plugin.TBContext instance. + It supports both GETs and POSTs. See 'http_api.md' for more details. """ - self._context = backend_context.Context(context) - - def get_plugin_apps(self): - """See base class.""" - - return { - '/experiment': self.get_experiment_route, - '/session_groups': self.list_session_groups_route, - '/metric_evals': self.list_metric_evals_route, - } - def is_active(self): - """Returns True if the hparams plugin is active. - - The hparams plugin is active iff there is a tag with - the hparams plugin name as its plugin name and the scalars plugin is - registered and active. - """ - if not self._context.multiplexer: - return False - scalars_plugin = self._get_scalars_plugin() - if not scalars_plugin or not scalars_plugin.is_active(): - return False - return bool(self._context.multiplexer.PluginRunToTagToContent( - metadata.PLUGIN_NAME)) - - def frontend_metadata(self): - return base_plugin.FrontendMetadata(element_name='tf-hparams-dashboard') - - # ---- /experiment ----------------------------------------------------------- - @wrappers.Request.application - def get_experiment_route(self, request): - try: - # This backend currently ignores the request parameters, but (for a POST) - # we must advance the input stream to skip them -- otherwise the next HTTP - # request will be parsed incorrectly. - _ = _parse_request_argument(request, api_pb2.GetExperimentRequest) - return http_util.Respond( - request, - json_format.MessageToJson( - get_experiment.Handler(self._context).run(), - including_default_value_fields=True, - ), 'application/json') - except error.HParamsError as e: - logger.error('HParams error: %s' % e) - raise werkzeug.exceptions.BadRequest(description=str(e)) - - # ---- /session_groups ------------------------------------------------------- - @wrappers.Request.application - def list_session_groups_route(self, request): - try: - request_proto = _parse_request_argument( - request, api_pb2.ListSessionGroupsRequest) - return http_util.Respond( - request, - json_format.MessageToJson( - list_session_groups.Handler(self._context, request_proto).run(), - including_default_value_fields=True, - ), - 'application/json') - except error.HParamsError as e: - logger.error('HParams error: %s' % e) - raise werkzeug.exceptions.BadRequest(description=str(e)) - - # ---- /metric_evals --------------------------------------------------------- - @wrappers.Request.application - def list_metric_evals_route(self, request): - try: - request_proto = _parse_request_argument( - request, api_pb2.ListMetricEvalsRequest) - scalars_plugin = self._get_scalars_plugin() - if not scalars_plugin: - raise error.HParamsError('Internal error: the scalars plugin is not' - ' registered; yet, the hparams plugin is' - ' active.') - return http_util.Respond( - request, - json.dumps( - list_metric_evals.Handler(request_proto, scalars_plugin).run()), - 'application/json') - except error.HParamsError as e: - logger.error('HParams error: %s' % e) - raise werkzeug.exceptions.BadRequest(description=str(e)) - - def _get_scalars_plugin(self): - """Tries to get the scalars plugin. - - Returns: - The scalars plugin or None if it is not yet registered. - """ - return self._context.tb_context.plugin_name_to_instance.get( - scalars_metadata.PLUGIN_NAME) + plugin_name = metadata.PLUGIN_NAME + + def __init__(self, context): + """Instantiates HParams plugin via TensorBoard core. + + Args: + context: A base_plugin.TBContext instance. + """ + self._context = backend_context.Context(context) + + def get_plugin_apps(self): + """See base class.""" + + return { + "/experiment": self.get_experiment_route, + "/session_groups": self.list_session_groups_route, + "/metric_evals": self.list_metric_evals_route, + } + + def is_active(self): + """Returns True if the hparams plugin is active. + + The hparams plugin is active iff there is a tag with the hparams + plugin name as its plugin name and the scalars plugin is + registered and active. + """ + if not self._context.multiplexer: + return False + scalars_plugin = self._get_scalars_plugin() + if not scalars_plugin or not scalars_plugin.is_active(): + return False + return bool( + self._context.multiplexer.PluginRunToTagToContent( + metadata.PLUGIN_NAME + ) + ) + + def frontend_metadata(self): + return base_plugin.FrontendMetadata(element_name="tf-hparams-dashboard") + + # ---- /experiment ----------------------------------------------------------- + @wrappers.Request.application + def get_experiment_route(self, request): + try: + # This backend currently ignores the request parameters, but (for a POST) + # we must advance the input stream to skip them -- otherwise the next HTTP + # request will be parsed incorrectly. + _ = _parse_request_argument(request, api_pb2.GetExperimentRequest) + return http_util.Respond( + request, + json_format.MessageToJson( + get_experiment.Handler(self._context).run(), + including_default_value_fields=True, + ), + "application/json", + ) + except error.HParamsError as e: + logger.error("HParams error: %s" % e) + raise werkzeug.exceptions.BadRequest(description=str(e)) + + # ---- /session_groups ------------------------------------------------------- + @wrappers.Request.application + def list_session_groups_route(self, request): + try: + request_proto = _parse_request_argument( + request, api_pb2.ListSessionGroupsRequest + ) + return http_util.Respond( + request, + json_format.MessageToJson( + list_session_groups.Handler( + self._context, request_proto + ).run(), + including_default_value_fields=True, + ), + "application/json", + ) + except error.HParamsError as e: + logger.error("HParams error: %s" % e) + raise werkzeug.exceptions.BadRequest(description=str(e)) + + # ---- /metric_evals --------------------------------------------------------- + @wrappers.Request.application + def list_metric_evals_route(self, request): + try: + request_proto = _parse_request_argument( + request, api_pb2.ListMetricEvalsRequest + ) + scalars_plugin = self._get_scalars_plugin() + if not scalars_plugin: + raise error.HParamsError( + "Internal error: the scalars plugin is not" + " registered; yet, the hparams plugin is" + " active." + ) + return http_util.Respond( + request, + json.dumps( + list_metric_evals.Handler( + request_proto, scalars_plugin + ).run() + ), + "application/json", + ) + except error.HParamsError as e: + logger.error("HParams error: %s" % e) + raise werkzeug.exceptions.BadRequest(description=str(e)) + + def _get_scalars_plugin(self): + """Tries to get the scalars plugin. + + Returns: + The scalars plugin or None if it is not yet registered. + """ + return self._context.tb_context.plugin_name_to_instance.get( + scalars_metadata.PLUGIN_NAME + ) def _parse_request_argument(request, proto_class): - if request.method == 'POST': - return json_format.Parse(request.data, proto_class()) - - # args.get() returns the request URI-unescaped. - request_json = request.args.get('request') - if request_json is None: - raise error.HParamsError( - 'Expected a JSON-formatted \'request\' arg of type: %s' % proto_class) - return json_format.Parse(request_json, proto_class()) + if request.method == "POST": + return json_format.Parse(request.data, proto_class()) + + # args.get() returns the request URI-unescaped. + request_json = request.args.get("request") + if request_json is None: + raise error.HParamsError( + "Expected a JSON-formatted 'request' arg of type: %s" % proto_class + ) + return json_format.Parse(request_json, proto_class()) diff --git a/tensorboard/plugins/hparams/hparams_util.py b/tensorboard/plugins/hparams/hparams_util.py index 7390254daf..0df4a83763 100644 --- a/tensorboard/plugins/hparams/hparams_util.py +++ b/tensorboard/plugins/hparams/hparams_util.py @@ -60,135 +60,185 @@ FLAGS = flags.FLAGS flags.DEFINE_string( - "action", "", "The action to perform. One of {'create_experiment'," - " 'start_session', 'end_session'}.") + "action", + "", + "The action to perform. One of {'create_experiment'," + " 'start_session', 'end_session'}.", +) flags.DEFINE_string("logdir", "", "The log directory to write the summary to.") # --action=create_experiment flags. flags.DEFINE_string( - "hparam_infos", "", "Only used when --action=create_experiment." + "hparam_infos", + "", + "Only used when --action=create_experiment." " A text-formatted HParamsInfoList protobuf describing" - " the hyperparameters used in the experiment.") + " the hyperparameters used in the experiment.", +) flags.DEFINE_string( - "metric_infos", "", "Only used when --action=create_experiment." + "metric_infos", + "", + "Only used when --action=create_experiment." " A text-formatted MetricInfosList protobuf describing" - " the metrics used in the experiment.") + " the metrics used in the experiment.", +) flags.DEFINE_string( - "description", "", "(Optional) only used when --action=create_experiment." - " The description for the experiment.") + "description", + "", + "(Optional) only used when --action=create_experiment." + " The description for the experiment.", +) flags.DEFINE_string( - "user", getpass.getuser(), + "user", + getpass.getuser(), "(Optional) only used when --action=create_experiment." - " The name of the user creating the experiment.") + " The name of the user creating the experiment.", +) flags.DEFINE_float( - "time_created_secs", time.time(), + "time_created_secs", + time.time(), "(Optional) only used when --action=create_experiment." " The creation time of the experiment in seconds since" - " epoch.") + " epoch.", +) # --action=start_session flags. flags.DEFINE_string( - "hparams", "", "Only used when --action=start_session." + "hparams", + "", + "Only used when --action=start_session." " A text-formatted HParams protobuf describing" - " the hyperparameter values used in the session.") + " the hyperparameter values used in the session.", +) flags.DEFINE_string( - "model_uri", "", "(Optional) only used when --action=start_session." + "model_uri", + "", + "(Optional) only used when --action=start_session." " A uri describing the location where model checkpoints" - " are saved.") + " are saved.", +) flags.DEFINE_string( - "monitor_url", "", "(Optional) only used when --action=start_session." + "monitor_url", + "", + "(Optional) only used when --action=start_session." " A url for a webpage showing monitoring information on" - " the session job.") + " the session job.", +) flags.DEFINE_string( - "group_name", "", "(Optional) only used when --action=start_session." + "group_name", + "", + "(Optional) only used when --action=start_session." " The name of the group this session belongs to" " (empty group means the session is the only one in its" - " group).") + " group).", +) flags.DEFINE_float( - "start_time_secs", time.time(), + "start_time_secs", + time.time(), "(Optional) only used when --action=start_session." - " The time the session started in seconds since epoch.") + " The time the session started in seconds since epoch.", +) # --action=end_session flags. flags.DEFINE_string( - "status", "", "Only used when --action=end_session." + "status", + "", + "Only used when --action=end_session." " A string representation of a member of the Status enum." - " The status the session ended at.") + " The status the session ended at.", +) flags.DEFINE_float( - "end_time_secs", time.time(), + "end_time_secs", + time.time(), "(Optional) only used when --action=end_session." - " The time the session ended in seconds since epoch.") + " The time the session ended in seconds since epoch.", +) def main(argv): - del argv # Unused. - if FLAGS.action == "create_experiment": - create_experiment() - elif FLAGS.action == "start_session": - start_session() - elif FLAGS.action == "end_session": - end_session() - else: - raise ValueError("Invalid action requested: '%s'" % FLAGS.action) + del argv # Unused. + if FLAGS.action == "create_experiment": + create_experiment() + elif FLAGS.action == "start_session": + start_session() + elif FLAGS.action == "end_session": + end_session() + else: + raise ValueError("Invalid action requested: '%s'" % FLAGS.action) def create_experiment(): - hparam_infos = hparams_util_pb2.HParamInfosList() - text_format.Merge(FLAGS.hparam_infos, hparam_infos) - metric_infos = hparams_util_pb2.MetricInfosList() - text_format.Merge(FLAGS.metric_infos, metric_infos) - write_summary( - summary.experiment_pb(hparam_infos.hparam_infos, - metric_infos.metric_infos, FLAGS.user, - FLAGS.description, FLAGS.time_created_secs)) + hparam_infos = hparams_util_pb2.HParamInfosList() + text_format.Merge(FLAGS.hparam_infos, hparam_infos) + metric_infos = hparams_util_pb2.MetricInfosList() + text_format.Merge(FLAGS.metric_infos, metric_infos) + write_summary( + summary.experiment_pb( + hparam_infos.hparam_infos, + metric_infos.metric_infos, + FLAGS.user, + FLAGS.description, + FLAGS.time_created_secs, + ) + ) def start_session(): - hparams = hparams_util_pb2.HParams() - text_format.Merge(FLAGS.hparams, hparams) - # Convert hparams.hparams values from google.protobuf.Value to Python native - # objects. - hparams = { - key: value_to_python(value) - for (key, value) in six.iteritems(hparams.hparams) - } - write_summary( - summary.session_start_pb(hparams, FLAGS.model_uri, FLAGS.monitor_url, - FLAGS.group_name, FLAGS.start_time_secs)) + hparams = hparams_util_pb2.HParams() + text_format.Merge(FLAGS.hparams, hparams) + # Convert hparams.hparams values from google.protobuf.Value to Python native + # objects. + hparams = { + key: value_to_python(value) + for (key, value) in six.iteritems(hparams.hparams) + } + write_summary( + summary.session_start_pb( + hparams, + FLAGS.model_uri, + FLAGS.monitor_url, + FLAGS.group_name, + FLAGS.start_time_secs, + ) + ) def value_to_python(value): - """Converts a google.protobuf.Value to a native python object.""" + """Converts a google.protobuf.Value to a native python object.""" - # We use the ListValue Well Known Type Value-to-native Python conversion - # logic to avoid depending on value's protobuf representation. - l = struct_pb2.ListValue(values=[value]) - return l[0] + # We use the ListValue Well Known Type Value-to-native Python conversion + # logic to avoid depending on value's protobuf representation. + l = struct_pb2.ListValue(values=[value]) + return l[0] def end_session(): - write_summary( - summary.session_end_pb( - api_pb2.Status.Value(FLAGS.status), FLAGS.end_time_secs)) + write_summary( + summary.session_end_pb( + api_pb2.Status.Value(FLAGS.status), FLAGS.end_time_secs + ) + ) def write_summary(summary_pb): - tf.compat.v1.enable_eager_execution() - writer = tf.compat.v2.summary.create_file_writer(FLAGS.logdir) - with writer.as_default(): - if hasattr(tf.compat.v2.summary.experimental, "write_raw_pb"): - tf.compat.v2.summary.experimental.write_raw_pb( - summary_pb.SerializeToString(), step=0) - else: - # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove the - # fallback to import_event(). - event = tf.compat.v1.Event(summary=summary_pb) - tf.compat.v2.summary.import_event( - tf.constant(event.SerializeToString(), dtype=tf.string)) - # The following may not be required since the context manager may - # already flush on __exit__, but it doesn't hurt to do it here, as well. - tf.compat.v2.summary.flush() + tf.compat.v1.enable_eager_execution() + writer = tf.compat.v2.summary.create_file_writer(FLAGS.logdir) + with writer.as_default(): + if hasattr(tf.compat.v2.summary.experimental, "write_raw_pb"): + tf.compat.v2.summary.experimental.write_raw_pb( + summary_pb.SerializeToString(), step=0 + ) + else: + # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove the + # fallback to import_event(). + event = tf.compat.v1.Event(summary=summary_pb) + tf.compat.v2.summary.import_event( + tf.constant(event.SerializeToString(), dtype=tf.string) + ) + # The following may not be required since the context manager may + # already flush on __exit__, but it doesn't hurt to do it here, as well. + tf.compat.v2.summary.flush() if __name__ == "__main__": - app.run(main) + app.run(main) diff --git a/tensorboard/plugins/hparams/keras.py b/tensorboard/plugins/hparams/keras.py index 1fce3bd0f7..889d0b7c77 100644 --- a/tensorboard/plugins/hparams/keras.py +++ b/tensorboard/plugins/hparams/keras.py @@ -31,67 +31,69 @@ class Callback(tf.keras.callbacks.Callback): - """Callback for logging hyperparameters to TensorBoard. + """Callback for logging hyperparameters to TensorBoard. - NOTE: This callback only works in TensorFlow eager mode. - """ + NOTE: This callback only works in TensorFlow eager mode. + """ - def __init__(self, writer, hparams, trial_id=None): - """Create a callback for logging hyperparameters to TensorBoard. + def __init__(self, writer, hparams, trial_id=None): + """Create a callback for logging hyperparameters to TensorBoard. - As with the standard `tf.keras.callbacks.TensorBoard` class, each - callback object is valid for only one call to `model.fit`. + As with the standard `tf.keras.callbacks.TensorBoard` class, each + callback object is valid for only one call to `model.fit`. - Args: - writer: The `SummaryWriter` object to which hparams should be - written, or a logdir (as a `str`) to be passed to - `tf.summary.create_file_writer` to create such a writer. - hparams: A `dict` mapping hyperparameters to the values used in - this session. Keys should be the names of `HParam` objects used - in an experiment, or the `HParam` objects themselves. Values - should be Python `bool`, `int`, `float`, or `string` values, - depending on the type of the hyperparameter. - trial_id: An optional `str` ID for the set of hyperparameter - values used in this trial. Defaults to a hash of the - hyperparameters. + Args: + writer: The `SummaryWriter` object to which hparams should be + written, or a logdir (as a `str`) to be passed to + `tf.summary.create_file_writer` to create such a writer. + hparams: A `dict` mapping hyperparameters to the values used in + this session. Keys should be the names of `HParam` objects used + in an experiment, or the `HParam` objects themselves. Values + should be Python `bool`, `int`, `float`, or `string` values, + depending on the type of the hyperparameter. + trial_id: An optional `str` ID for the set of hyperparameter + values used in this trial. Defaults to a hash of the + hyperparameters. - Raises: - ValueError: If two entries in `hparams` share the same - hyperparameter name. - """ - # Defer creating the actual summary until we write it, so that the - # timestamp is correct. But create a "dry-run" first to fail fast in - # case the `hparams` are invalid. - self._hparams = dict(hparams) - self._trial_id = trial_id - summary_v2.hparams_pb(self._hparams, trial_id=self._trial_id) - if writer is None: - raise TypeError("writer must be a `SummaryWriter` or `str`, not None") - elif isinstance(writer, str): - self._writer = tf.compat.v2.summary.create_file_writer(writer) - else: - self._writer = writer + Raises: + ValueError: If two entries in `hparams` share the same + hyperparameter name. + """ + # Defer creating the actual summary until we write it, so that the + # timestamp is correct. But create a "dry-run" first to fail fast in + # case the `hparams` are invalid. + self._hparams = dict(hparams) + self._trial_id = trial_id + summary_v2.hparams_pb(self._hparams, trial_id=self._trial_id) + if writer is None: + raise TypeError( + "writer must be a `SummaryWriter` or `str`, not None" + ) + elif isinstance(writer, str): + self._writer = tf.compat.v2.summary.create_file_writer(writer) + else: + self._writer = writer - def _get_writer(self): - if self._writer is None: - raise RuntimeError( - "hparams Keras callback cannot be reused across training sessions" - ) - if not tf.executing_eagerly(): - raise RuntimeError( - "hparams Keras callback only supported in TensorFlow eager mode" - ) - return self._writer + def _get_writer(self): + if self._writer is None: + raise RuntimeError( + "hparams Keras callback cannot be reused across training sessions" + ) + if not tf.executing_eagerly(): + raise RuntimeError( + "hparams Keras callback only supported in TensorFlow eager mode" + ) + return self._writer - def on_train_begin(self, logs=None): - del logs # unused - with self._get_writer().as_default(): - summary_v2.hparams(self._hparams, trial_id=self._trial_id) + def on_train_begin(self, logs=None): + del logs # unused + with self._get_writer().as_default(): + summary_v2.hparams(self._hparams, trial_id=self._trial_id) - def on_train_end(self, logs=None): - del logs # unused - with self._get_writer().as_default(): - pb = summary.session_end_pb(api_pb2.STATUS_SUCCESS) - raw_pb = pb.SerializeToString() - tf.compat.v2.summary.experimental.write_raw_pb(raw_pb, step=0) - self._writer = None + def on_train_end(self, logs=None): + del logs # unused + with self._get_writer().as_default(): + pb = summary.session_end_pb(api_pb2.STATUS_SUCCESS) + raw_pb = pb.SerializeToString() + tf.compat.v2.summary.experimental.write_raw_pb(raw_pb, step=0) + self._writer = None diff --git a/tensorboard/plugins/hparams/keras_test.py b/tensorboard/plugins/hparams/keras_test.py index 6c3ddb761c..152611eed7 100644 --- a/tensorboard/plugins/hparams/keras_test.py +++ b/tensorboard/plugins/hparams/keras_test.py @@ -25,10 +25,10 @@ import tensorflow as tf try: - # python version >= 3.3 - from unittest import mock + # python version >= 3.3 + from unittest import mock except ImportError: - import mock # pylint: disable=unused-import + import mock # pylint: disable=unused-import from tensorboard.plugins.hparams import keras from tensorboard.plugins.hparams import metadata @@ -40,69 +40,76 @@ class CallbackTest(tf.test.TestCase): - - def setUp(self): - super(CallbackTest, self).setUp() - self.logdir = os.path.join(self.get_temp_dir(), "logs") - - def _initialize_model(self, writer): - HP_DENSE_NEURONS = hp.HParam("dense_neurons", hp.IntInterval(4, 16)) - self.hparams = { - "optimizer": "adam", - HP_DENSE_NEURONS: 8, - } - self.model = tf.keras.models.Sequential([ - tf.keras.layers.Dense(self.hparams[HP_DENSE_NEURONS], input_shape=(1,)), - tf.keras.layers.Dense(1, activation="sigmoid"), - ]) - self.model.compile(loss="mse", optimizer=self.hparams["optimizer"]) - self.trial_id = "my_trial" - self.callback = keras.Callback(writer, self.hparams, trial_id=self.trial_id) - - def test_eager(self): - def mock_time(): - mock_time.time += 1 - return mock_time.time - mock_time.time = 1556227801.875 - initial_time = mock_time.time - with mock.patch("time.time", mock_time): - self._initialize_model(writer=self.logdir) - self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback]) - final_time = mock_time.time - - files = os.listdir(self.logdir) - self.assertEqual(len(files), 1, files) - events_file = os.path.join(self.logdir, files[0]) - plugin_data = [] - for event in tf.compat.v1.train.summary_iterator(events_file): - if event.WhichOneof("what") != "summary": - continue - self.assertEqual(len(event.summary.value), 1, event.summary.value) - value = event.summary.value[0] - self.assertEqual( - value.metadata.plugin_data.plugin_name, - metadata.PLUGIN_NAME, - ) - plugin_data.append(value.metadata.plugin_data.content) - - self.assertEqual(len(plugin_data), 2, plugin_data) - (start_plugin_data, end_plugin_data) = plugin_data - start_pb = metadata.parse_session_start_info_plugin_data(start_plugin_data) - end_pb = metadata.parse_session_end_info_plugin_data(end_plugin_data) - - # We're not the only callers of `time.time`; Keras calls it - # internally an unspecified number of times, so we're not guaranteed - # to know the exact values. Instead, we perform relative checks... - self.assertGreater(start_pb.start_time_secs, initial_time) - self.assertLess(start_pb.start_time_secs, end_pb.end_time_secs) - self.assertLessEqual(start_pb.start_time_secs, final_time) - # ...and then stub out the times for proto equality checks below. - start_pb.start_time_secs = 1234.5 - end_pb.end_time_secs = 6789.0 - - expected_start_pb = plugin_data_pb2.SessionStartInfo() - text_format.Merge( - """ + def setUp(self): + super(CallbackTest, self).setUp() + self.logdir = os.path.join(self.get_temp_dir(), "logs") + + def _initialize_model(self, writer): + HP_DENSE_NEURONS = hp.HParam("dense_neurons", hp.IntInterval(4, 16)) + self.hparams = { + "optimizer": "adam", + HP_DENSE_NEURONS: 8, + } + self.model = tf.keras.models.Sequential( + [ + tf.keras.layers.Dense( + self.hparams[HP_DENSE_NEURONS], input_shape=(1,) + ), + tf.keras.layers.Dense(1, activation="sigmoid"), + ] + ) + self.model.compile(loss="mse", optimizer=self.hparams["optimizer"]) + self.trial_id = "my_trial" + self.callback = keras.Callback( + writer, self.hparams, trial_id=self.trial_id + ) + + def test_eager(self): + def mock_time(): + mock_time.time += 1 + return mock_time.time + + mock_time.time = 1556227801.875 + initial_time = mock_time.time + with mock.patch("time.time", mock_time): + self._initialize_model(writer=self.logdir) + self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback]) + final_time = mock_time.time + + files = os.listdir(self.logdir) + self.assertEqual(len(files), 1, files) + events_file = os.path.join(self.logdir, files[0]) + plugin_data = [] + for event in tf.compat.v1.train.summary_iterator(events_file): + if event.WhichOneof("what") != "summary": + continue + self.assertEqual(len(event.summary.value), 1, event.summary.value) + value = event.summary.value[0] + self.assertEqual( + value.metadata.plugin_data.plugin_name, metadata.PLUGIN_NAME, + ) + plugin_data.append(value.metadata.plugin_data.content) + + self.assertEqual(len(plugin_data), 2, plugin_data) + (start_plugin_data, end_plugin_data) = plugin_data + start_pb = metadata.parse_session_start_info_plugin_data( + start_plugin_data + ) + end_pb = metadata.parse_session_end_info_plugin_data(end_plugin_data) + + # We're not the only callers of `time.time`; Keras calls it + # internally an unspecified number of times, so we're not guaranteed + # to know the exact values. Instead, we perform relative checks... + self.assertGreater(start_pb.start_time_secs, initial_time) + self.assertLess(start_pb.start_time_secs, end_pb.end_time_secs) + self.assertLessEqual(start_pb.start_time_secs, final_time) + # ...and then stub out the times for proto equality checks below. + start_pb.start_time_secs = 1234.5 + end_pb.end_time_secs = 6789.0 + + expected_start_pb = plugin_data_pb2.SessionStartInfo() + text_format.Merge( + """ start_time_secs: 1234.5 group_name: "my_trial" hparams { @@ -118,78 +125,85 @@ def mock_time(): } } """, - expected_start_pb, - ) - self.assertEqual(start_pb, expected_start_pb) + expected_start_pb, + ) + self.assertEqual(start_pb, expected_start_pb) - expected_end_pb = plugin_data_pb2.SessionEndInfo() - text_format.Merge( - """ + expected_end_pb = plugin_data_pb2.SessionEndInfo() + text_format.Merge( + """ end_time_secs: 6789.0 status: STATUS_SUCCESS """, - expected_end_pb, - ) - self.assertEqual(end_pb, expected_end_pb) - - def test_explicit_writer(self): - writer = tf.compat.v2.summary.create_file_writer( - self.logdir, - filename_suffix=".magic", - ) - self._initialize_model(writer=writer) - self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback]) - - files = os.listdir(self.logdir) - self.assertEqual(len(files), 1, files) - filename = files[0] - self.assertTrue(filename.endswith(".magic"), filename) - # We'll assume that the contents are correct, as in the case where - # the file writer was constructed implicitly. - - def test_non_eager_failure(self): - with tf.compat.v1.Graph().as_default(): - assert not tf.executing_eagerly() - self._initialize_model(writer=self.logdir) - with six.assertRaisesRegex( - self, RuntimeError, "only supported in TensorFlow eager mode"): + expected_end_pb, + ) + self.assertEqual(end_pb, expected_end_pb) + + def test_explicit_writer(self): + writer = tf.compat.v2.summary.create_file_writer( + self.logdir, filename_suffix=".magic", + ) + self._initialize_model(writer=writer) + self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback]) + + files = os.listdir(self.logdir) + self.assertEqual(len(files), 1, files) + filename = files[0] + self.assertTrue(filename.endswith(".magic"), filename) + # We'll assume that the contents are correct, as in the case where + # the file writer was constructed implicitly. + + def test_non_eager_failure(self): + with tf.compat.v1.Graph().as_default(): + assert not tf.executing_eagerly() + self._initialize_model(writer=self.logdir) + with six.assertRaisesRegex( + self, RuntimeError, "only supported in TensorFlow eager mode" + ): + self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback]) + + def test_reuse_failure(self): + self._initialize_model(writer=self.logdir) self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback]) + with six.assertRaisesRegex( + self, RuntimeError, "cannot be reused across training sessions" + ): + self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback]) + + def test_invalid_writer(self): + with six.assertRaisesRegex( + self, + TypeError, + "writer must be a `SummaryWriter` or `str`, not None", + ): + keras.Callback(writer=None, hparams={}) + + def test_duplicate_hparam_names_across_object_and_string(self): + hparams = { + "foo": 1, + hp.HParam("foo"): 1, + } + with six.assertRaisesRegex( + self, ValueError, "multiple values specified for hparam 'foo'" + ): + keras.Callback(self.get_temp_dir(), hparams) + + def test_duplicate_hparam_names_from_two_objects(self): + hparams = { + hp.HParam("foo"): 1, + hp.HParam("foo"): 1, + } + with six.assertRaisesRegex( + self, ValueError, "multiple values specified for hparam 'foo'" + ): + keras.Callback(self.get_temp_dir(), hparams) - def test_reuse_failure(self): - self._initialize_model(writer=self.logdir) - self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback]) - with six.assertRaisesRegex( - self, RuntimeError, "cannot be reused across training sessions"): - self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback]) - - def test_invalid_writer(self): - with six.assertRaisesRegex( - self, TypeError, "writer must be a `SummaryWriter` or `str`, not None"): - keras.Callback(writer=None, hparams={}) - - def test_duplicate_hparam_names_across_object_and_string(self): - hparams = { - "foo": 1, - hp.HParam("foo"): 1, - } - with six.assertRaisesRegex( - self, ValueError, "multiple values specified for hparam 'foo'"): - keras.Callback(self.get_temp_dir(), hparams) - - def test_duplicate_hparam_names_from_two_objects(self): - hparams = { - hp.HParam("foo"): 1, - hp.HParam("foo"): 1, - } - with six.assertRaisesRegex( - self, ValueError, "multiple values specified for hparam 'foo'"): - keras.Callback(self.get_temp_dir(), hparams) - - def test_invalid_trial_id(self): - with six.assertRaisesRegex( - self, TypeError, "`trial_id` should be a `str`, but got: 12"): - keras.Callback(self.get_temp_dir(), {}, trial_id=12) + def test_invalid_trial_id(self): + with six.assertRaisesRegex( + self, TypeError, "`trial_id` should be a `str`, but got: 12" + ): + keras.Callback(self.get_temp_dir(), {}, trial_id=12) if __name__ == "__main__": - tf.test.main() + tf.test.main() diff --git a/tensorboard/plugins/hparams/list_metric_evals.py b/tensorboard/plugins/hparams/list_metric_evals.py index dc54ce20f0..1d22ede1ed 100644 --- a/tensorboard/plugins/hparams/list_metric_evals.py +++ b/tensorboard/plugins/hparams/list_metric_evals.py @@ -23,27 +23,29 @@ class Handler(object): - """Handles a ListMetricEvals request. """ - - def __init__(self, request, scalars_plugin_instance): - """Constructor. - - Args: - request: A ListSessionGroupsRequest protobuf. - scalars_plugin_instance: A scalars_plugin.ScalarsPlugin. - """ - self._request = request - self._scalars_plugin_instance = scalars_plugin_instance - - def run(self): - """Executes the request. - - Returns: - An array of tuples representing the metric evaluations--each of the form - (how do you pronounce that, anyway?
", - "samples": 1, + def setUp(self): + self.log_dir = tempfile.mkdtemp() + + # We use numpy.random to generate images. We seed to avoid non-determinism + # in this test. + numpy.random.seed(42) + + # Create old-style image summaries for run "foo". + tf.compat.v1.reset_default_graph() + sess = tf.compat.v1.Session() + placeholder = tf.compat.v1.placeholder(tf.uint8) + tf.compat.v1.summary.image(name="baz", tensor=placeholder) + merged_summary_op = tf.compat.v1.summary.merge_all() + foo_directory = os.path.join(self.log_dir, "foo") + with test_util.FileWriterCache.get(foo_directory) as writer: + writer.add_graph(sess.graph) + for step in xrange(2): + writer.add_summary( + sess.run( + merged_summary_op, + feed_dict={ + placeholder: ( + numpy.random.rand(1, 16, 42, 3) * 255 + ).astype(numpy.uint8) + }, + ), + global_step=step, + ) + + # Create new-style image summaries for run bar. + tf.compat.v1.reset_default_graph() + sess = tf.compat.v1.Session() + placeholder = tf.compat.v1.placeholder(tf.uint8) + summary.op( + name="quux", + images=placeholder, + description="how do you pronounce that, anyway?", + ) + merged_summary_op = tf.compat.v1.summary.merge_all() + bar_directory = os.path.join(self.log_dir, "bar") + with test_util.FileWriterCache.get(bar_directory) as writer: + writer.add_graph(sess.graph) + for step in xrange(2): + writer.add_summary( + sess.run( + merged_summary_op, + feed_dict={ + placeholder: ( + numpy.random.rand(1, 8, 6, 3) * 255 + ).astype(numpy.uint8) + }, + ), + global_step=step, + ) + + # Start a server with the plugin. + multiplexer = event_multiplexer.EventMultiplexer( + {"foo": foo_directory, "bar": bar_directory,} + ) + multiplexer.Reload() + context = base_plugin.TBContext( + logdir=self.log_dir, multiplexer=multiplexer + ) + plugin = images_plugin.ImagesPlugin(context) + wsgi_app = application.TensorBoardWSGI([plugin]) + self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) + self.routes = plugin.get_plugin_apps() + + def tearDown(self): + shutil.rmtree(self.log_dir, ignore_errors=True) + + def _DeserializeResponse(self, byte_content): + """Deserializes byte content that is a JSON encoding. + + Args: + byte_content: The byte content of a response. + + Returns: + The deserialized python object decoded from JSON. + """ + return json.loads(byte_content.decode("utf-8")) + + def testRoutesProvided(self): + """Tests that the plugin offers the correct routes.""" + self.assertIsInstance(self.routes["/images"], collections.Callable) + self.assertIsInstance( + self.routes["/individualImage"], collections.Callable + ) + self.assertIsInstance(self.routes["/tags"], collections.Callable) + + def testOldStyleImagesRoute(self): + """Tests that the /images routes returns correct old-style data.""" + response = self.server.get( + "/data/plugin/images/images?run=foo&tag=baz/image/0&sample=0" + ) + self.assertEqual(200, response.status_code) + + # Verify that the correct entries are returned. + entries = self._DeserializeResponse(response.get_data()) + self.assertEqual(2, len(entries)) + + # Verify that the 1st entry is correct. + entry = entries[0] + self.assertEqual(42, entry["width"]) + self.assertEqual(16, entry["height"]) + self.assertEqual(0, entry["step"]) + parsed_query = urllib.parse.parse_qs(entry["query"]) + self.assertListEqual(["foo"], parsed_query["run"]) + self.assertListEqual(["baz/image/0"], parsed_query["tag"]) + self.assertListEqual(["0"], parsed_query["sample"]) + self.assertListEqual(["0"], parsed_query["index"]) + + # Verify that the 2nd entry is correct. + entry = entries[1] + self.assertEqual(42, entry["width"]) + self.assertEqual(16, entry["height"]) + self.assertEqual(1, entry["step"]) + parsed_query = urllib.parse.parse_qs(entry["query"]) + self.assertListEqual(["foo"], parsed_query["run"]) + self.assertListEqual(["baz/image/0"], parsed_query["tag"]) + self.assertListEqual(["0"], parsed_query["sample"]) + self.assertListEqual(["1"], parsed_query["index"]) + + def testNewStyleImagesRoute(self): + """Tests that the /images routes returns correct new-style data.""" + response = self.server.get( + "/data/plugin/images/images?run=bar&tag=quux/image_summary&sample=0" + ) + self.assertEqual(200, response.status_code) + + # Verify that the correct entries are returned. + entries = self._DeserializeResponse(response.get_data()) + self.assertEqual(2, len(entries)) + + # Verify that the 1st entry is correct. + entry = entries[0] + self.assertEqual(6, entry["width"]) + self.assertEqual(8, entry["height"]) + self.assertEqual(0, entry["step"]) + parsed_query = urllib.parse.parse_qs(entry["query"]) + self.assertListEqual(["bar"], parsed_query["run"]) + self.assertListEqual(["quux/image_summary"], parsed_query["tag"]) + self.assertListEqual(["0"], parsed_query["sample"]) + self.assertListEqual(["0"], parsed_query["index"]) + + # Verify that the 2nd entry is correct. + entry = entries[1] + self.assertEqual(6, entry["width"]) + self.assertEqual(8, entry["height"]) + self.assertEqual(1, entry["step"]) + parsed_query = urllib.parse.parse_qs(entry["query"]) + self.assertListEqual(["bar"], parsed_query["run"]) + self.assertListEqual(["quux/image_summary"], parsed_query["tag"]) + self.assertListEqual(["0"], parsed_query["sample"]) + self.assertListEqual(["1"], parsed_query["index"]) + + def testOldStyleIndividualImageRoute(self): + """Tests fetching an individual image from an old-style summary.""" + response = self.server.get( + "/data/plugin/images/individualImage" + "?run=foo&tag=baz/image/0&sample=0&index=0" + ) + self.assertEqual(200, response.status_code) + self.assertEqual("image/png", response.headers.get("content-type")) + + def testNewStyleIndividualImageRoute(self): + """Tests fetching an individual image from a new-style summary.""" + response = self.server.get( + "/data/plugin/images/individualImage" + "?run=bar&tag=quux/image_summary&sample=0&index=0" + ) + self.assertEqual(200, response.status_code) + self.assertEqual("image/png", response.headers.get("content-type")) + + def testRunsRoute(self): + """Tests that the /runs route offers the correct run to tag mapping.""" + response = self.server.get("/data/plugin/images/tags") + self.assertEqual(200, response.status_code) + self.assertDictEqual( + { + "foo": { + "baz/image/0": { + "displayName": "baz/image/0", + "description": "", + "samples": 1, + }, + }, + "bar": { + "quux/image_summary": { + "displayName": "quux", + "description": "how do you pronounce that, anyway?
", + "samples": 1, + }, + }, }, - }, - }, self._DeserializeResponse(response.get_data())) + self._DeserializeResponse(response.get_data()), + ) if __name__ == "__main__": - tf.test.main() + tf.test.main() diff --git a/tensorboard/plugins/image/metadata.py b/tensorboard/plugins/image/metadata.py index f96d9b252e..93f6512579 100644 --- a/tensorboard/plugins/image/metadata.py +++ b/tensorboard/plugins/image/metadata.py @@ -24,7 +24,7 @@ logger = tb_logging.get_logger() -PLUGIN_NAME = 'images' +PLUGIN_NAME = "images" # The most recent value for the `version` field of the `ImagePluginData` # proto. @@ -32,39 +32,43 @@ def create_summary_metadata(display_name, description): - """Create a `summary_pb2.SummaryMetadata` proto for image plugin data. + """Create a `summary_pb2.SummaryMetadata` proto for image plugin data. - Returns: - A `summary_pb2.SummaryMetadata` protobuf object. - """ - content = plugin_data_pb2.ImagePluginData(version=PROTO_VERSION) - metadata = summary_pb2.SummaryMetadata( - display_name=display_name, - summary_description=description, - plugin_data=summary_pb2.SummaryMetadata.PluginData( - plugin_name=PLUGIN_NAME, - content=content.SerializeToString())) - return metadata + Returns: + A `summary_pb2.SummaryMetadata` protobuf object. + """ + content = plugin_data_pb2.ImagePluginData(version=PROTO_VERSION) + metadata = summary_pb2.SummaryMetadata( + display_name=display_name, + summary_description=description, + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME, content=content.SerializeToString() + ), + ) + return metadata def parse_plugin_metadata(content): - """Parse summary metadata to a Python object. + """Parse summary metadata to a Python object. - Arguments: - content: The `content` field of a `SummaryMetadata` proto - corresponding to the image plugin. + Arguments: + content: The `content` field of a `SummaryMetadata` proto + corresponding to the image plugin. - Returns: - An `ImagePluginData` protobuf object. - """ - if not isinstance(content, bytes): - raise TypeError('Content type must be bytes') - result = plugin_data_pb2.ImagePluginData.FromString(content) - if result.version == 0: - return result - else: - logger.warn( - 'Unknown metadata version: %s. The latest version known to ' - 'this build of TensorBoard is %s; perhaps a newer build is ' - 'available?', result.version, PROTO_VERSION) - return result + Returns: + An `ImagePluginData` protobuf object. + """ + if not isinstance(content, bytes): + raise TypeError("Content type must be bytes") + result = plugin_data_pb2.ImagePluginData.FromString(content) + if result.version == 0: + return result + else: + logger.warn( + "Unknown metadata version: %s. The latest version known to " + "this build of TensorBoard is %s; perhaps a newer build is " + "available?", + result.version, + PROTO_VERSION, + ) + return result diff --git a/tensorboard/plugins/image/summary.py b/tensorboard/plugins/image/summary.py index 22fc5829b3..1f52acd7f9 100644 --- a/tensorboard/plugins/image/summary.py +++ b/tensorboard/plugins/image/summary.py @@ -36,110 +36,129 @@ image = summary_v2.image -def op(name, - images, - max_outputs=3, - display_name=None, - description=None, - collections=None): - """Create a legacy image summary op for use in a TensorFlow graph. - - Arguments: - name: A unique name for the generated summary node. - images: A `Tensor` representing pixel data with shape `[k, h, w, c]`, - where `k` is the number of images, `h` and `w` are the height and - width of the images, and `c` is the number of channels, which - should be 1, 3, or 4. Any of the dimensions may be statically - unknown (i.e., `None`). - max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this - many images will be emitted at each step. When more than - `max_outputs` many images are provided, the first `max_outputs` many - images will be used and the rest silently discarded. - display_name: Optional name for this summary in TensorBoard, as a - constant `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - collections: Optional list of graph collections keys. The new - summary op is added to these collections. Defaults to - `[Graph Keys.SUMMARIES]`. - - Returns: - A TensorFlow summary op. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - if display_name is None: - display_name = name - summary_metadata = metadata.create_summary_metadata( - display_name=display_name, description=description) - with tf.name_scope(name), \ - tf.control_dependencies([tf.assert_rank(images, 4), - tf.assert_type(images, tf.uint8), - tf.assert_non_negative(max_outputs)]): - limited_images = images[:max_outputs] - encoded_images = tf.map_fn(tf.image.encode_png, limited_images, - dtype=tf.string, - name='encode_each_image') - image_shape = tf.shape(input=images) - dimensions = tf.stack([tf.as_string(image_shape[2], name='width'), - tf.as_string(image_shape[1], name='height')], - name='dimensions') - tensor = tf.concat([dimensions, encoded_images], axis=0) - return tf.summary.tensor_summary(name='image_summary', - tensor=tensor, - collections=collections, - summary_metadata=summary_metadata) +def op( + name, + images, + max_outputs=3, + display_name=None, + description=None, + collections=None, +): + """Create a legacy image summary op for use in a TensorFlow graph. + + Arguments: + name: A unique name for the generated summary node. + images: A `Tensor` representing pixel data with shape `[k, h, w, c]`, + where `k` is the number of images, `h` and `w` are the height and + width of the images, and `c` is the number of channels, which + should be 1, 3, or 4. Any of the dimensions may be statically + unknown (i.e., `None`). + max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this + many images will be emitted at each step. When more than + `max_outputs` many images are provided, the first `max_outputs` many + images will be used and the rest silently discarded. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A TensorFlow summary op. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description + ) + with tf.name_scope(name), tf.control_dependencies( + [ + tf.assert_rank(images, 4), + tf.assert_type(images, tf.uint8), + tf.assert_non_negative(max_outputs), + ] + ): + limited_images = images[:max_outputs] + encoded_images = tf.map_fn( + tf.image.encode_png, + limited_images, + dtype=tf.string, + name="encode_each_image", + ) + image_shape = tf.shape(input=images) + dimensions = tf.stack( + [ + tf.as_string(image_shape[2], name="width"), + tf.as_string(image_shape[1], name="height"), + ], + name="dimensions", + ) + tensor = tf.concat([dimensions, encoded_images], axis=0) + return tf.summary.tensor_summary( + name="image_summary", + tensor=tensor, + collections=collections, + summary_metadata=summary_metadata, + ) def pb(name, images, max_outputs=3, display_name=None, description=None): - """Create a legacy image summary protobuf. - - This behaves as if you were to create an `op` with the same arguments - (wrapped with constant tensors where appropriate) and then execute - that summary op in a TensorFlow session. - - Arguments: - name: A unique name for the generated summary, including any desired - name scopes. - images: An `np.array` representing pixel data with shape - `[k, h, w, c]`, where `k` is the number of images, `w` and `h` are - the width and height of the images, and `c` is the number of - channels, which should be 1, 3, or 4. - max_outputs: Optional `int`. At most this many images will be - emitted. If more than this many images are provided, the first - `max_outputs` many images will be used and the rest silently - discarded. - display_name: Optional name for this summary in TensorBoard, as a - `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - `str`. Markdown is supported. Defaults to empty. - - Returns: - A `tf.Summary` protobuf object. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - images = np.array(images).astype(np.uint8) - if images.ndim != 4: - raise ValueError('Shape %r must have rank 4' % (images.shape, )) - - limited_images = images[:max_outputs] - encoded_images = [encoder.encode_png(image) for image in limited_images] - (width, height) = (images.shape[2], images.shape[1]) - content = [str(width), str(height)] + encoded_images - tensor = tf.make_tensor_proto(content, dtype=tf.string) - - if display_name is None: - display_name = name - summary_metadata = metadata.create_summary_metadata( - display_name=display_name, description=description) - tf_summary_metadata = tf.SummaryMetadata.FromString( - summary_metadata.SerializeToString()) - - summary = tf.Summary() - summary.value.add(tag='%s/image_summary' % name, - metadata=tf_summary_metadata, - tensor=tensor) - return summary + """Create a legacy image summary protobuf. + + This behaves as if you were to create an `op` with the same arguments + (wrapped with constant tensors where appropriate) and then execute + that summary op in a TensorFlow session. + + Arguments: + name: A unique name for the generated summary, including any desired + name scopes. + images: An `np.array` representing pixel data with shape + `[k, h, w, c]`, where `k` is the number of images, `w` and `h` are + the width and height of the images, and `c` is the number of + channels, which should be 1, 3, or 4. + max_outputs: Optional `int`. At most this many images will be + emitted. If more than this many images are provided, the first + `max_outputs` many images will be used and the rest silently + discarded. + display_name: Optional name for this summary in TensorBoard, as a + `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + `str`. Markdown is supported. Defaults to empty. + + Returns: + A `tf.Summary` protobuf object. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + images = np.array(images).astype(np.uint8) + if images.ndim != 4: + raise ValueError("Shape %r must have rank 4" % (images.shape,)) + + limited_images = images[:max_outputs] + encoded_images = [encoder.encode_png(image) for image in limited_images] + (width, height) = (images.shape[2], images.shape[1]) + content = [str(width), str(height)] + encoded_images + tensor = tf.make_tensor_proto(content, dtype=tf.string) + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description + ) + tf_summary_metadata = tf.SummaryMetadata.FromString( + summary_metadata.SerializeToString() + ) + + summary = tf.Summary() + summary.value.add( + tag="%s/image_summary" % name, + metadata=tf_summary_metadata, + tensor=tensor, + ) + return summary diff --git a/tensorboard/plugins/image/summary_test.py b/tensorboard/plugins/image/summary_test.py index af7e58ca48..7efa02d3fb 100644 --- a/tensorboard/plugins/image/summary_test.py +++ b/tensorboard/plugins/image/summary_test.py @@ -32,208 +32,215 @@ from tensorboard.plugins.image import summary try: - tf2.__version__ # Force lazy import to resolve + tf2.__version__ # Force lazy import to resolve except ImportError: - tf2 = None + tf2 = None try: - tf.compat.v1.enable_eager_execution() + tf.compat.v1.enable_eager_execution() except AttributeError: - # TF 2.0 doesn't have this symbol because eager is the default. - pass + # TF 2.0 doesn't have this symbol because eager is the default. + pass class SummaryBaseTest(object): - - def setUp(self): - super(SummaryBaseTest, self).setUp() - np.random.seed(0) - self.image_width = 20 - self.image_height = 15 - self.image_count = 1 - self.image_channels = 3 - - def _generate_images(self, **kwargs): - size = [ - kwargs.get('n', self.image_count), - kwargs.get('h', self.image_height), - kwargs.get('w', self.image_width), - kwargs.get('c', self.image_channels), - ] - return np.random.uniform(low=0, high=255, size=size).astype(np.uint8) - - def image(self, *args, **kwargs): - raise NotImplementedError() - - def test_tag(self): - data = np.array(1, np.uint8, ndmin=4) - self.assertEqual('a', self.image('a', data).value[0].tag) - self.assertEqual('a/b', self.image('a/b', data).value[0].tag) - - def test_metadata(self): - data = np.array(1, np.uint8, ndmin=4) - description = 'By Leonardo da Vinci' - pb = self.image('mona_lisa', data, description=description) - summary_metadata = pb.value[0].metadata - self.assertEqual(summary_metadata.summary_description, description) - plugin_data = summary_metadata.plugin_data - self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) - content = summary_metadata.plugin_data.content - # There's no content, so successfully parsing is fine. - metadata.parse_plugin_metadata(content) - - def test_png_format_roundtrip(self): - images = self._generate_images(c=1) - pb = self.image('mona_lisa', images) - encoded = pb.value[0].tensor.string_val[2] # skip width, height - self.assertAllEqual(images[0], tf.image.decode_png(encoded)) - - def _test_dimensions(self, images): - pb = self.image('mona_lisa', images) - self.assertEqual(1, len(pb.value)) - result = pb.value[0].tensor.string_val - # Check annotated dimensions. - self.assertEqual(tf.compat.as_bytes(str(self.image_width)), result[0]) - self.assertEqual(tf.compat.as_bytes(str(self.image_height)), result[1]) - for i, encoded in enumerate(result[2:]): - decoded = tf.image.decode_png(encoded) - self.assertEqual(images[i].shape, decoded.shape) - - def test_dimensions(self): - self._test_dimensions(self._generate_images(c=1)) - self._test_dimensions(self._generate_images(c=2)) - self._test_dimensions(self._generate_images(c=3)) - self._test_dimensions(self._generate_images(c=4)) - - def test_image_count_zero(self): - shape = (0, self.image_height, self.image_width, 3) - data = np.array([], np.uint8).reshape(shape) - pb = self.image('mona_lisa', data, max_outputs=3) - self.assertEqual(1, len(pb.value)) - result = pb.value[0].tensor.string_val - self.assertEqual(tf.compat.as_bytes(str(self.image_width)), result[0]) - self.assertEqual(tf.compat.as_bytes(str(self.image_height)), result[1]) - self.assertEqual(2, len(result)) - - def test_image_count_less_than_max_outputs(self): - max_outputs = 3 - data = self._generate_images(n=(max_outputs - 1)) - pb = self.image('mona_lisa', data, max_outputs=max_outputs) - self.assertEqual(1, len(pb.value)) - result = pb.value[0].tensor.string_val - image_results = result[2:] # skip width, height - self.assertEqual(len(data), len(image_results)) - - def test_image_count_more_than_max_outputs(self): - max_outputs = 3 - data = self._generate_images(n=(max_outputs + 1)) - pb = self.image('mona_lisa', data, max_outputs=max_outputs) - self.assertEqual(1, len(pb.value)) - result = pb.value[0].tensor.string_val - image_results = result[2:] # skip width, height - self.assertEqual(max_outputs, len(image_results)) - - def test_requires_nonnegative_max_outputs(self): - data = np.array(1, np.uint8, ndmin=4) - with six.assertRaisesRegex( - self, (ValueError, tf.errors.InvalidArgumentError), '>= 0'): - self.image('mona_lisa', data, max_outputs=-1) - - def test_requires_rank_4(self): - with six.assertRaisesRegex(self, ValueError, 'must have rank 4'): - self.image('mona_lisa', [[[1], [2]], [[3], [4]]]) + def setUp(self): + super(SummaryBaseTest, self).setUp() + np.random.seed(0) + self.image_width = 20 + self.image_height = 15 + self.image_count = 1 + self.image_channels = 3 + + def _generate_images(self, **kwargs): + size = [ + kwargs.get("n", self.image_count), + kwargs.get("h", self.image_height), + kwargs.get("w", self.image_width), + kwargs.get("c", self.image_channels), + ] + return np.random.uniform(low=0, high=255, size=size).astype(np.uint8) + + def image(self, *args, **kwargs): + raise NotImplementedError() + + def test_tag(self): + data = np.array(1, np.uint8, ndmin=4) + self.assertEqual("a", self.image("a", data).value[0].tag) + self.assertEqual("a/b", self.image("a/b", data).value[0].tag) + + def test_metadata(self): + data = np.array(1, np.uint8, ndmin=4) + description = "By Leonardo da Vinci" + pb = self.image("mona_lisa", data, description=description) + summary_metadata = pb.value[0].metadata + self.assertEqual(summary_metadata.summary_description, description) + plugin_data = summary_metadata.plugin_data + self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) + content = summary_metadata.plugin_data.content + # There's no content, so successfully parsing is fine. + metadata.parse_plugin_metadata(content) + + def test_png_format_roundtrip(self): + images = self._generate_images(c=1) + pb = self.image("mona_lisa", images) + encoded = pb.value[0].tensor.string_val[2] # skip width, height + self.assertAllEqual(images[0], tf.image.decode_png(encoded)) + + def _test_dimensions(self, images): + pb = self.image("mona_lisa", images) + self.assertEqual(1, len(pb.value)) + result = pb.value[0].tensor.string_val + # Check annotated dimensions. + self.assertEqual(tf.compat.as_bytes(str(self.image_width)), result[0]) + self.assertEqual(tf.compat.as_bytes(str(self.image_height)), result[1]) + for i, encoded in enumerate(result[2:]): + decoded = tf.image.decode_png(encoded) + self.assertEqual(images[i].shape, decoded.shape) + + def test_dimensions(self): + self._test_dimensions(self._generate_images(c=1)) + self._test_dimensions(self._generate_images(c=2)) + self._test_dimensions(self._generate_images(c=3)) + self._test_dimensions(self._generate_images(c=4)) + + def test_image_count_zero(self): + shape = (0, self.image_height, self.image_width, 3) + data = np.array([], np.uint8).reshape(shape) + pb = self.image("mona_lisa", data, max_outputs=3) + self.assertEqual(1, len(pb.value)) + result = pb.value[0].tensor.string_val + self.assertEqual(tf.compat.as_bytes(str(self.image_width)), result[0]) + self.assertEqual(tf.compat.as_bytes(str(self.image_height)), result[1]) + self.assertEqual(2, len(result)) + + def test_image_count_less_than_max_outputs(self): + max_outputs = 3 + data = self._generate_images(n=(max_outputs - 1)) + pb = self.image("mona_lisa", data, max_outputs=max_outputs) + self.assertEqual(1, len(pb.value)) + result = pb.value[0].tensor.string_val + image_results = result[2:] # skip width, height + self.assertEqual(len(data), len(image_results)) + + def test_image_count_more_than_max_outputs(self): + max_outputs = 3 + data = self._generate_images(n=(max_outputs + 1)) + pb = self.image("mona_lisa", data, max_outputs=max_outputs) + self.assertEqual(1, len(pb.value)) + result = pb.value[0].tensor.string_val + image_results = result[2:] # skip width, height + self.assertEqual(max_outputs, len(image_results)) + + def test_requires_nonnegative_max_outputs(self): + data = np.array(1, np.uint8, ndmin=4) + with six.assertRaisesRegex( + self, (ValueError, tf.errors.InvalidArgumentError), ">= 0" + ): + self.image("mona_lisa", data, max_outputs=-1) + + def test_requires_rank_4(self): + with six.assertRaisesRegex(self, ValueError, "must have rank 4"): + self.image("mona_lisa", [[[1], [2]], [[3], [4]]]) class SummaryV1PbTest(SummaryBaseTest, tf.test.TestCase): - def image(self, *args, **kwargs): - return summary.pb(*args, **kwargs) + def image(self, *args, **kwargs): + return summary.pb(*args, **kwargs) - def test_tag(self): - data = np.array(1, np.uint8, ndmin=4) - self.assertEqual('a/image_summary', self.image('a', data).value[0].tag) - self.assertEqual('a/b/image_summary', self.image('a/b', data).value[0].tag) + def test_tag(self): + data = np.array(1, np.uint8, ndmin=4) + self.assertEqual("a/image_summary", self.image("a", data).value[0].tag) + self.assertEqual( + "a/b/image_summary", self.image("a/b", data).value[0].tag + ) - def test_requires_nonnegative_max_outputs(self): - self.skipTest('summary V1 pb does not actually enforce this') + def test_requires_nonnegative_max_outputs(self): + self.skipTest("summary V1 pb does not actually enforce this") class SummaryV1OpTest(SummaryBaseTest, tf.test.TestCase): - def image(self, *args, **kwargs): - args = list(args) - # Force first argument to tf.uint8 since the V1 version requires this. - args[1] = tf.cast(tf.constant(args[1]), tf.uint8) - return summary_pb2.Summary.FromString(summary.op(*args, **kwargs).numpy()) - - def test_tag(self): - data = np.array(1, np.uint8, ndmin=4) - self.assertEqual('a/image_summary', self.image('a', data).value[0].tag) - self.assertEqual('a/b/image_summary', self.image('a/b', data).value[0].tag) - - def test_scoped_tag(self): - data = np.array(1, np.uint8, ndmin=4) - with tf.name_scope('scope'): - self.assertEqual('scope/a/image_summary', - self.image('a', data).value[0].tag) - - def test_image_count_zero(self): - self.skipTest('fails under eager because map_fn() returns float dtype') + def image(self, *args, **kwargs): + args = list(args) + # Force first argument to tf.uint8 since the V1 version requires this. + args[1] = tf.cast(tf.constant(args[1]), tf.uint8) + return summary_pb2.Summary.FromString( + summary.op(*args, **kwargs).numpy() + ) + + def test_tag(self): + data = np.array(1, np.uint8, ndmin=4) + self.assertEqual("a/image_summary", self.image("a", data).value[0].tag) + self.assertEqual( + "a/b/image_summary", self.image("a/b", data).value[0].tag + ) + + def test_scoped_tag(self): + data = np.array(1, np.uint8, ndmin=4) + with tf.name_scope("scope"): + self.assertEqual( + "scope/a/image_summary", self.image("a", data).value[0].tag + ) + + def test_image_count_zero(self): + self.skipTest("fails under eager because map_fn() returns float dtype") class SummaryV2OpTest(SummaryBaseTest, tf.test.TestCase): - def setUp(self): - super(SummaryV2OpTest, self).setUp() - if tf2 is None: - self.skipTest('TF v2 summary API not available') - - def image(self, *args, **kwargs): - return self.image_event(*args, **kwargs).summary - - def image_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - summary.image(*args, **kwargs) - writer.close() - event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), '*'))) - self.assertEqual(len(event_files), 1) - events = list(tf.compat.v1.train.summary_iterator(event_files[0])) - # Expect a boilerplate event for the file_version, then the summary one. - self.assertEqual(len(events), 2) - # Delete the event file to reset to an empty directory for later calls. - # TODO(nickfelt): use a unique subdirectory per writer instead. - os.remove(event_files[0]) - return events[1] - - def test_scoped_tag(self): - data = np.array(1, np.uint8, ndmin=4) - with tf.name_scope('scope'): - self.assertEqual('scope/a', self.image('a', data).value[0].tag) - - def test_step(self): - data = np.array(1, np.uint8, ndmin=4) - event = self.image_event('a', data, step=333) - self.assertEqual(333, event.step) - - def test_default_step(self): - data = np.array(1, np.uint8, ndmin=4) - try: - tf2.summary.experimental.set_step(333) - # TODO(nickfelt): change test logic so we can just omit `step` entirely. - event = self.image_event('a', data, step=None) - self.assertEqual(333, event.step) - finally: - # Reset to default state for other tests. - tf2.summary.experimental.set_step(None) - - def test_floating_point_data(self): - data = np.array([-0.01, 0.0, 0.9, 1.0, 1.1]).reshape((1, -1, 1, 1)) - pb = self.image('mona_lisa', data) - encoded = pb.value[0].tensor.string_val[2] # skip width, height - decoded = tf.image.decode_png(encoded).numpy() - # Float values outside [0, 1) are truncated, and everything is scaled to the - # range [0, 255] with 229 = 0.9 * 255, truncated. - self.assertAllEqual([0, 0, 229, 255, 255], list(decoded.flat)) - - -if __name__ == '__main__': - tf.test.main() + def setUp(self): + super(SummaryV2OpTest, self).setUp() + if tf2 is None: + self.skipTest("TF v2 summary API not available") + + def image(self, *args, **kwargs): + return self.image_event(*args, **kwargs).summary + + def image_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + summary.image(*args, **kwargs) + writer.close() + event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), "*"))) + self.assertEqual(len(event_files), 1) + events = list(tf.compat.v1.train.summary_iterator(event_files[0])) + # Expect a boilerplate event for the file_version, then the summary one. + self.assertEqual(len(events), 2) + # Delete the event file to reset to an empty directory for later calls. + # TODO(nickfelt): use a unique subdirectory per writer instead. + os.remove(event_files[0]) + return events[1] + + def test_scoped_tag(self): + data = np.array(1, np.uint8, ndmin=4) + with tf.name_scope("scope"): + self.assertEqual("scope/a", self.image("a", data).value[0].tag) + + def test_step(self): + data = np.array(1, np.uint8, ndmin=4) + event = self.image_event("a", data, step=333) + self.assertEqual(333, event.step) + + def test_default_step(self): + data = np.array(1, np.uint8, ndmin=4) + try: + tf2.summary.experimental.set_step(333) + # TODO(nickfelt): change test logic so we can just omit `step` entirely. + event = self.image_event("a", data, step=None) + self.assertEqual(333, event.step) + finally: + # Reset to default state for other tests. + tf2.summary.experimental.set_step(None) + + def test_floating_point_data(self): + data = np.array([-0.01, 0.0, 0.9, 1.0, 1.1]).reshape((1, -1, 1, 1)) + pb = self.image("mona_lisa", data) + encoded = pb.value[0].tensor.string_val[2] # skip width, height + decoded = tf.image.decode_png(encoded).numpy() + # Float values outside [0, 1) are truncated, and everything is scaled to the + # range [0, 255] with 229 = 0.9 * 255, truncated. + self.assertAllEqual([0, 0, 229, 255, 255], list(decoded.flat)) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/image/summary_v2.py b/tensorboard/plugins/image/summary_v2.py index 2b87278f89..a3f146fd0e 100644 --- a/tensorboard/plugins/image/summary_v2.py +++ b/tensorboard/plugins/image/summary_v2.py @@ -27,71 +27,80 @@ from tensorboard.util import lazy_tensor_creator -def image(name, - data, - step=None, - max_outputs=3, - description=None): - """Write an image summary. +def image(name, data, step=None, max_outputs=3, description=None): + """Write an image summary. - Arguments: - name: A name for this summary. The summary tag used for TensorBoard will - be this name prefixed by any active name scopes. - data: A `Tensor` representing pixel data with shape `[k, h, w, c]`, - where `k` is the number of images, `h` and `w` are the height and - width of the images, and `c` is the number of channels, which - should be 1, 2, 3, or 4 (grayscale, grayscale with alpha, RGB, RGBA). - Any of the dimensions may be statically unknown (i.e., `None`). - Floating point data will be clipped to the range [0,1). - step: Explicit `int64`-castable monotonic step value for this summary. If - omitted, this defaults to `tf.summary.experimental.get_step()`, which must - not be None. - max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this - many images will be emitted at each step. When more than - `max_outputs` many images are provided, the first `max_outputs` many - images will be used and the rest silently discarded. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. + Arguments: + name: A name for this summary. The summary tag used for TensorBoard will + be this name prefixed by any active name scopes. + data: A `Tensor` representing pixel data with shape `[k, h, w, c]`, + where `k` is the number of images, `h` and `w` are the height and + width of the images, and `c` is the number of channels, which + should be 1, 2, 3, or 4 (grayscale, grayscale with alpha, RGB, RGBA). + Any of the dimensions may be statically unknown (i.e., `None`). + Floating point data will be clipped to the range [0,1). + step: Explicit `int64`-castable monotonic step value for this summary. If + omitted, this defaults to `tf.summary.experimental.get_step()`, which must + not be None. + max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this + many images will be emitted at each step. When more than + `max_outputs` many images are provided, the first `max_outputs` many + images will be used and the rest silently discarded. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. - Returns: - True on success, or false if no summary was emitted because no default - summary writer was available. + Returns: + True on success, or false if no summary was emitted because no default + summary writer was available. - Raises: - ValueError: if a default writer exists, but no step was provided and - `tf.summary.experimental.get_step()` is None. - """ - summary_metadata = metadata.create_summary_metadata( - display_name=None, description=description) - # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback - summary_scope = ( - getattr(tf.summary.experimental, 'summary_scope', None) or - tf.summary.summary_scope) - with summary_scope( - name, 'image_summary', values=[data, max_outputs, step]) as (tag, _): - # Defer image encoding preprocessing by passing it as a callable to write(), - # wrapped in a LazyTensorCreator for backwards compatibility, so that we - # only do this work when summaries are actually written. - @lazy_tensor_creator.LazyTensorCreator - def lazy_tensor(): - tf.debugging.assert_rank(data, 4) - tf.debugging.assert_non_negative(max_outputs) - images = tf.image.convert_image_dtype(data, tf.uint8, saturate=True) - limited_images = images[:max_outputs] - encoded_images = tf.map_fn(tf.image.encode_png, limited_images, - dtype=tf.string, - name='encode_each_image') - # Workaround for map_fn returning float dtype for an empty elems input. - encoded_images = tf.cond( - tf.shape(input=encoded_images)[0] > 0, - lambda: encoded_images, lambda: tf.constant([], tf.string)) - image_shape = tf.shape(input=images) - dimensions = tf.stack([tf.as_string(image_shape[2], name='width'), - tf.as_string(image_shape[1], name='height')], - name='dimensions') - return tf.concat([dimensions, encoded_images], axis=0) + Raises: + ValueError: if a default writer exists, but no step was provided and + `tf.summary.experimental.get_step()` is None. + """ + summary_metadata = metadata.create_summary_metadata( + display_name=None, description=description + ) + # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback + summary_scope = ( + getattr(tf.summary.experimental, "summary_scope", None) + or tf.summary.summary_scope + ) + with summary_scope( + name, "image_summary", values=[data, max_outputs, step] + ) as (tag, _): + # Defer image encoding preprocessing by passing it as a callable to write(), + # wrapped in a LazyTensorCreator for backwards compatibility, so that we + # only do this work when summaries are actually written. + @lazy_tensor_creator.LazyTensorCreator + def lazy_tensor(): + tf.debugging.assert_rank(data, 4) + tf.debugging.assert_non_negative(max_outputs) + images = tf.image.convert_image_dtype(data, tf.uint8, saturate=True) + limited_images = images[:max_outputs] + encoded_images = tf.map_fn( + tf.image.encode_png, + limited_images, + dtype=tf.string, + name="encode_each_image", + ) + # Workaround for map_fn returning float dtype for an empty elems input. + encoded_images = tf.cond( + tf.shape(input=encoded_images)[0] > 0, + lambda: encoded_images, + lambda: tf.constant([], tf.string), + ) + image_shape = tf.shape(input=images) + dimensions = tf.stack( + [ + tf.as_string(image_shape[2], name="width"), + tf.as_string(image_shape[1], name="height"), + ], + name="dimensions", + ) + return tf.concat([dimensions, encoded_images], axis=0) - # To ensure that image encoding logic is only executed when summaries - # are written, we pass callable to `tensor` parameter. - return tf.summary.write( - tag=tag, tensor=lazy_tensor, step=step, metadata=summary_metadata) + # To ensure that image encoding logic is only executed when summaries + # are written, we pass callable to `tensor` parameter. + return tf.summary.write( + tag=tag, tensor=lazy_tensor, step=step, metadata=summary_metadata + ) diff --git a/tensorboard/plugins/interactive_inference/interactive_inference_plugin.py b/tensorboard/plugins/interactive_inference/interactive_inference_plugin.py index 1ba875aad5..edd203678b 100644 --- a/tensorboard/plugins/interactive_inference/interactive_inference_plugin.py +++ b/tensorboard/plugins/interactive_inference/interactive_inference_plugin.py @@ -48,380 +48,494 @@ class InteractiveInferencePlugin(base_plugin.TBPlugin): - """Plugin for understanding/debugging model inference. - """ - - # This string field is used by TensorBoard to generate the paths for routes - # provided by this plugin. It must thus be URL-friendly. This field is also - # used to uniquely identify this plugin throughout TensorBoard. See BasePlugin - # for details. - plugin_name = 'whatif' - examples = [] - updated_example_indices = set() - sprite = None - example_class = tf.train.Example - - # The standard name for encoded image features in TensorFlow. - image_feature_name = 'image/encoded' - - # The width and height of the thumbnail for any images for Facets Dive. - sprite_thumbnail_dim_px = 32 - - # The vocab of inference class indices to label names for the model. - label_vocab = [] - - def __init__(self, context): - """Constructs an interactive inference plugin for TensorBoard. - - Args: - context: A base_plugin.TBContext instance. - """ - self._logdir = context.logdir - self._has_auth_group = (context.flags and - 'authorized_groups' in context.flags and - context.flags.authorized_groups != '') - - def get_plugin_apps(self): - """Obtains a mapping between routes and handlers. Stores the logdir. - - Returns: - A mapping between routes and handlers (functions that respond to - requests). - """ - return { - '/infer': self._infer, - '/update_example': self._update_example, - '/examples_from_path': self._examples_from_path_handler, - '/sprite': self._serve_sprite, - '/duplicate_example': self._duplicate_example, - '/delete_example': self._delete_example, - '/infer_mutants': self._infer_mutants_handler, - '/eligible_features': self._eligible_features_from_example_handler, - '/sort_eligible_features': self._sort_eligible_features_handler, - } - - def is_active(self): - """Determines whether this plugin is active. - - Returns: - A boolean. Whether this plugin is active. - """ - # TODO(jameswex): Maybe enable if config flags were specified? - return False - - def frontend_metadata(self): - # TODO(#2338): Keep this in sync with the `registerDashboard` call - # on the frontend until that call is removed. - return base_plugin.FrontendMetadata( - element_name='tf-interactive-inference-dashboard', - tab_name='What-If Tool', - ) - - def generate_sprite(self, example_strings): - # Generate a sprite image for the examples if the examples contain the - # standard encoded image feature. - feature_list = (self.examples[0].features.feature - if self.example_class == tf.train.Example - else self.examples[0].context.feature) - self.sprite = ( - inference_utils.create_sprite_image(example_strings) - if (len(self.examples) and self.image_feature_name in feature_list) else - None) - - @wrappers.Request.application - def _examples_from_path_handler(self, request): - """Returns JSON of the specified examples. - - Args: - request: A request that should contain 'examples_path' and 'max_examples'. - - Returns: - JSON of up to max_examlpes of the examples in the path. - """ - examples_count = int(request.args.get('max_examples')) - examples_path = request.args.get('examples_path') - sampling_odds = float(request.args.get('sampling_odds')) - self.example_class = (tf.train.SequenceExample - if request.args.get('sequence_examples') == 'true' - else tf.train.Example) - try: - platform_utils.throw_if_file_access_not_allowed(examples_path, - self._logdir, - self._has_auth_group) - example_strings = platform_utils.example_protos_from_path( - examples_path, examples_count, parse_examples=False, - sampling_odds=sampling_odds, example_class=self.example_class) - self.examples = [ - self.example_class.FromString(ex) for ex in example_strings] - self.generate_sprite(example_strings) - json_examples = [ - json_format.MessageToJson(example) for example in self.examples - ] - self.updated_example_indices = set(range(len(json_examples))) - return http_util.Respond( - request, - {'examples': json_examples, - 'sprite': True if self.sprite else False}, 'application/json') - except common_utils.InvalidUserInputError as e: - return http_util.Respond(request, {'error': e.message}, - 'application/json', code=400) - - @wrappers.Request.application - def _serve_sprite(self, request): - return http_util.Respond(request, self.sprite, 'image/png') - - @wrappers.Request.application - def _update_example(self, request): - """Updates the specified example. - - Args: - request: A request that should contain 'index' and 'example'. - - Returns: - An empty response. - """ - if request.method != 'POST': - return http_util.Respond(request, {'error': 'invalid non-POST request'}, - 'application/json', code=405) - example_json = request.form['example'] - index = int(request.form['index']) - if index >= len(self.examples): - return http_util.Respond(request, {'error': 'invalid index provided'}, - 'application/json', code=400) - new_example = self.example_class() - json_format.Parse(example_json, new_example) - self.examples[index] = new_example - self.updated_example_indices.add(index) - self.generate_sprite([ex.SerializeToString() for ex in self.examples]) - return http_util.Respond(request, {}, 'application/json') - - @wrappers.Request.application - def _duplicate_example(self, request): - """Duplicates the specified example. - - Args: - request: A request that should contain 'index'. - - Returns: - An empty response. - """ - index = int(request.args.get('index')) - if index >= len(self.examples): - return http_util.Respond(request, {'error': 'invalid index provided'}, - 'application/json', code=400) - new_example = self.example_class() - new_example.CopyFrom(self.examples[index]) - self.examples.append(new_example) - self.updated_example_indices.add(len(self.examples) - 1) - self.generate_sprite([ex.SerializeToString() for ex in self.examples]) - return http_util.Respond(request, {}, 'application/json') - - @wrappers.Request.application - def _delete_example(self, request): - """Deletes the specified example. - - Args: - request: A request that should contain 'index'. - - Returns: - An empty response. - """ - index = int(request.args.get('index')) - if index >= len(self.examples): - return http_util.Respond(request, {'error': 'invalid index provided'}, - 'application/json', code=400) - del self.examples[index] - self.updated_example_indices = set([ - i if i < index else i - 1 for i in self.updated_example_indices]) - self.generate_sprite([ex.SerializeToString() for ex in self.examples]) - return http_util.Respond(request, {}, 'application/json') - - def _parse_request_arguments(self, request): - """Parses comma separated request arguments - - Args: - request: A request that should contain 'inference_address', 'model_name', - 'model_version', 'model_signature'. - - Returns: - A tuple of lists for model parameters - """ - inference_addresses = request.args.get('inference_address').split(',') - model_names = request.args.get('model_name').split(',') - model_versions = request.args.get('model_version').split(',') - model_signatures = request.args.get('model_signature').split(',') - if len(model_names) != len(inference_addresses): - raise common_utils.InvalidUserInputError('Every model should have a ' + - 'name and address.') - return inference_addresses, model_names, model_versions, model_signatures - - @wrappers.Request.application - def _infer(self, request): - """Returns JSON for the `vz-line-chart`s for a feature. - - Args: - request: A request that should contain 'inference_address', 'model_name', - 'model_type, 'model_version', 'model_signature' and 'label_vocab_path'. - - Returns: - A list of JSON objects, one for each chart. - """ - label_vocab = inference_utils.get_label_vocab( - request.args.get('label_vocab_path')) - - try: - if request.method != 'GET': - logger.error('%s requests are forbidden.', request.method) - return http_util.Respond(request, {'error': 'invalid non-GET request'}, - 'application/json', code=405) - - (inference_addresses, model_names, model_versions, - model_signatures) = self._parse_request_arguments(request) - - indices_to_infer = sorted(self.updated_example_indices) - examples_to_infer = [self.examples[index] for index in indices_to_infer] - infer_objs = [] - for model_num in xrange(len(inference_addresses)): - serving_bundle = inference_utils.ServingBundle( - inference_addresses[model_num], - model_names[model_num], - request.args.get('model_type'), - model_versions[model_num], - model_signatures[model_num], - request.args.get('use_predict') == 'true', - request.args.get('predict_input_tensor'), - request.args.get('predict_output_tensor')) - (predictions, _) = inference_utils.run_inference_for_inference_results( - examples_to_infer, serving_bundle) - infer_objs.append(predictions) - - resp = {'indices': indices_to_infer, 'results': infer_objs} - self.updated_example_indices = set() - return http_util.Respond(request, {'inferences': json.dumps(resp), - 'vocab': json.dumps(label_vocab)}, - 'application/json') - except common_utils.InvalidUserInputError as e: - return http_util.Respond(request, {'error': e.message}, - 'application/json', code=400) - except AbortionError as e: - return http_util.Respond(request, {'error': e.details}, - 'application/json', code=400) - - @wrappers.Request.application - def _eligible_features_from_example_handler(self, request): - """Returns a list of JSON objects for each feature in the example. - - Args: - request: A request for features. - - Returns: - A list with a JSON object for each feature. - Numeric features are represented as {name: observedMin: observedMax:}. - Categorical features are repesented as {name: samples:[]}. - """ - features_list = inference_utils.get_eligible_features( - self.examples[0: NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS) - return http_util.Respond(request, features_list, 'application/json') - - @wrappers.Request.application - def _sort_eligible_features_handler(self, request): - """Returns a sorted list of JSON objects for each feature in the example. - - The list is sorted by interestingness in terms of the resulting change in - inference values across feature values, for partial dependence plots. - - Args: - request: A request for sorted features. - - Returns: - A sorted list with a JSON object for each feature. - Numeric features are represented as - {name: observedMin: observedMax: interestingness:}. - Categorical features are repesented as - {name: samples:[] interestingness:}. - """ - try: - features_list = inference_utils.get_eligible_features( - self.examples[0: NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS) - example_index = int(request.args.get('example_index', '0')) - (inference_addresses, model_names, model_versions, - model_signatures) = self._parse_request_arguments(request) - chart_data = {} - for feat in features_list: - chart_data[feat['name']] = self._infer_mutants_impl( - feat['name'], example_index, - inference_addresses, model_names, request.args.get('model_type'), - model_versions, model_signatures, - request.args.get('use_predict') == 'true', - request.args.get('predict_input_tensor'), - request.args.get('predict_output_tensor'), - feat['observedMin'] if 'observedMin' in feat else 0, - feat['observedMax'] if 'observedMin' in feat else 0, - None) - features_list = inference_utils.sort_eligible_features( - features_list, chart_data) - return http_util.Respond(request, features_list, 'application/json') - except common_utils.InvalidUserInputError as e: - return http_util.Respond(request, {'error': e.message}, - 'application/json', code=400) - - @wrappers.Request.application - def _infer_mutants_handler(self, request): - """Returns JSON for the partial dependence plots for a feature. - - Args: - request: A request that should contain 'feature_name', 'example_index', - 'inference_address', 'model_name', 'model_type', 'model_version', and - 'model_signature'. - - Returns: - A list of JSON objects, one for each chart. - """ - try: - if request.method != 'GET': - logger.error('%s requests are forbidden.', request.method) - return http_util.Respond(request, {'error': 'invalid non-GET request'}, - 'application/json', code=405) - - example_index = int(request.args.get('example_index', '0')) - feature_name = request.args.get('feature_name') - (inference_addresses, model_names, model_versions, - model_signatures) = self._parse_request_arguments(request) - json_mapping = self._infer_mutants_impl(feature_name, example_index, - inference_addresses, model_names, request.args.get('model_type'), - model_versions, model_signatures, - request.args.get('use_predict') == 'true', - request.args.get('predict_input_tensor'), - request.args.get('predict_output_tensor'), - request.args.get('x_min'), request.args.get('x_max'), - request.args.get('feature_index_pattern')) - return http_util.Respond(request, json_mapping, 'application/json') - except common_utils.InvalidUserInputError as e: - return http_util.Respond(request, {'error': e.message}, - 'application/json', code=400) - - def _infer_mutants_impl(self, feature_name, example_index, inference_addresses, - model_names, model_type, model_versions, model_signatures, use_predict, - predict_input_tensor, predict_output_tensor, x_min, x_max, - feature_index_pattern): - """Helper for generating PD plots for a feature.""" - examples = (self.examples if example_index == -1 - else [self.examples[example_index]]) - serving_bundles = [] - for model_num in xrange(len(inference_addresses)): - serving_bundles.append(inference_utils.ServingBundle( - inference_addresses[model_num], - model_names[model_num], - model_type, - model_versions[model_num], - model_signatures[model_num], - use_predict, - predict_input_tensor, - predict_output_tensor)) - - viz_params = inference_utils.VizParams( - x_min, x_max, - self.examples[0:NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS, - feature_index_pattern) - return inference_utils.mutant_charts_for_feature( - examples, feature_name, serving_bundles, viz_params) + """Plugin for understanding/debugging model inference.""" + + # This string field is used by TensorBoard to generate the paths for routes + # provided by this plugin. It must thus be URL-friendly. This field is also + # used to uniquely identify this plugin throughout TensorBoard. See BasePlugin + # for details. + plugin_name = "whatif" + examples = [] + updated_example_indices = set() + sprite = None + example_class = tf.train.Example + + # The standard name for encoded image features in TensorFlow. + image_feature_name = "image/encoded" + + # The width and height of the thumbnail for any images for Facets Dive. + sprite_thumbnail_dim_px = 32 + + # The vocab of inference class indices to label names for the model. + label_vocab = [] + + def __init__(self, context): + """Constructs an interactive inference plugin for TensorBoard. + + Args: + context: A base_plugin.TBContext instance. + """ + self._logdir = context.logdir + self._has_auth_group = ( + context.flags + and "authorized_groups" in context.flags + and context.flags.authorized_groups != "" + ) + + def get_plugin_apps(self): + """Obtains a mapping between routes and handlers. Stores the logdir. + + Returns: + A mapping between routes and handlers (functions that respond to + requests). + """ + return { + "/infer": self._infer, + "/update_example": self._update_example, + "/examples_from_path": self._examples_from_path_handler, + "/sprite": self._serve_sprite, + "/duplicate_example": self._duplicate_example, + "/delete_example": self._delete_example, + "/infer_mutants": self._infer_mutants_handler, + "/eligible_features": self._eligible_features_from_example_handler, + "/sort_eligible_features": self._sort_eligible_features_handler, + } + + def is_active(self): + """Determines whether this plugin is active. + + Returns: + A boolean. Whether this plugin is active. + """ + # TODO(jameswex): Maybe enable if config flags were specified? + return False + + def frontend_metadata(self): + # TODO(#2338): Keep this in sync with the `registerDashboard` call + # on the frontend until that call is removed. + return base_plugin.FrontendMetadata( + element_name="tf-interactive-inference-dashboard", + tab_name="What-If Tool", + ) + + def generate_sprite(self, example_strings): + # Generate a sprite image for the examples if the examples contain the + # standard encoded image feature. + feature_list = ( + self.examples[0].features.feature + if self.example_class == tf.train.Example + else self.examples[0].context.feature + ) + self.sprite = ( + inference_utils.create_sprite_image(example_strings) + if (len(self.examples) and self.image_feature_name in feature_list) + else None + ) + + @wrappers.Request.application + def _examples_from_path_handler(self, request): + """Returns JSON of the specified examples. + + Args: + request: A request that should contain 'examples_path' and 'max_examples'. + + Returns: + JSON of up to max_examlpes of the examples in the path. + """ + examples_count = int(request.args.get("max_examples")) + examples_path = request.args.get("examples_path") + sampling_odds = float(request.args.get("sampling_odds")) + self.example_class = ( + tf.train.SequenceExample + if request.args.get("sequence_examples") == "true" + else tf.train.Example + ) + try: + platform_utils.throw_if_file_access_not_allowed( + examples_path, self._logdir, self._has_auth_group + ) + example_strings = platform_utils.example_protos_from_path( + examples_path, + examples_count, + parse_examples=False, + sampling_odds=sampling_odds, + example_class=self.example_class, + ) + self.examples = [ + self.example_class.FromString(ex) for ex in example_strings + ] + self.generate_sprite(example_strings) + json_examples = [ + json_format.MessageToJson(example) for example in self.examples + ] + self.updated_example_indices = set(range(len(json_examples))) + return http_util.Respond( + request, + { + "examples": json_examples, + "sprite": True if self.sprite else False, + }, + "application/json", + ) + except common_utils.InvalidUserInputError as e: + return http_util.Respond( + request, {"error": e.message}, "application/json", code=400 + ) + + @wrappers.Request.application + def _serve_sprite(self, request): + return http_util.Respond(request, self.sprite, "image/png") + + @wrappers.Request.application + def _update_example(self, request): + """Updates the specified example. + + Args: + request: A request that should contain 'index' and 'example'. + + Returns: + An empty response. + """ + if request.method != "POST": + return http_util.Respond( + request, + {"error": "invalid non-POST request"}, + "application/json", + code=405, + ) + example_json = request.form["example"] + index = int(request.form["index"]) + if index >= len(self.examples): + return http_util.Respond( + request, + {"error": "invalid index provided"}, + "application/json", + code=400, + ) + new_example = self.example_class() + json_format.Parse(example_json, new_example) + self.examples[index] = new_example + self.updated_example_indices.add(index) + self.generate_sprite([ex.SerializeToString() for ex in self.examples]) + return http_util.Respond(request, {}, "application/json") + + @wrappers.Request.application + def _duplicate_example(self, request): + """Duplicates the specified example. + + Args: + request: A request that should contain 'index'. + + Returns: + An empty response. + """ + index = int(request.args.get("index")) + if index >= len(self.examples): + return http_util.Respond( + request, + {"error": "invalid index provided"}, + "application/json", + code=400, + ) + new_example = self.example_class() + new_example.CopyFrom(self.examples[index]) + self.examples.append(new_example) + self.updated_example_indices.add(len(self.examples) - 1) + self.generate_sprite([ex.SerializeToString() for ex in self.examples]) + return http_util.Respond(request, {}, "application/json") + + @wrappers.Request.application + def _delete_example(self, request): + """Deletes the specified example. + + Args: + request: A request that should contain 'index'. + + Returns: + An empty response. + """ + index = int(request.args.get("index")) + if index >= len(self.examples): + return http_util.Respond( + request, + {"error": "invalid index provided"}, + "application/json", + code=400, + ) + del self.examples[index] + self.updated_example_indices = set( + [i if i < index else i - 1 for i in self.updated_example_indices] + ) + self.generate_sprite([ex.SerializeToString() for ex in self.examples]) + return http_util.Respond(request, {}, "application/json") + + def _parse_request_arguments(self, request): + """Parses comma separated request arguments. + + Args: + request: A request that should contain 'inference_address', 'model_name', + 'model_version', 'model_signature'. + + Returns: + A tuple of lists for model parameters + """ + inference_addresses = request.args.get("inference_address").split(",") + model_names = request.args.get("model_name").split(",") + model_versions = request.args.get("model_version").split(",") + model_signatures = request.args.get("model_signature").split(",") + if len(model_names) != len(inference_addresses): + raise common_utils.InvalidUserInputError( + "Every model should have a " + "name and address." + ) + return ( + inference_addresses, + model_names, + model_versions, + model_signatures, + ) + + @wrappers.Request.application + def _infer(self, request): + """Returns JSON for the `vz-line-chart`s for a feature. + + Args: + request: A request that should contain 'inference_address', 'model_name', + 'model_type, 'model_version', 'model_signature' and 'label_vocab_path'. + + Returns: + A list of JSON objects, one for each chart. + """ + label_vocab = inference_utils.get_label_vocab( + request.args.get("label_vocab_path") + ) + + try: + if request.method != "GET": + logger.error("%s requests are forbidden.", request.method) + return http_util.Respond( + request, + {"error": "invalid non-GET request"}, + "application/json", + code=405, + ) + + ( + inference_addresses, + model_names, + model_versions, + model_signatures, + ) = self._parse_request_arguments(request) + + indices_to_infer = sorted(self.updated_example_indices) + examples_to_infer = [ + self.examples[index] for index in indices_to_infer + ] + infer_objs = [] + for model_num in xrange(len(inference_addresses)): + serving_bundle = inference_utils.ServingBundle( + inference_addresses[model_num], + model_names[model_num], + request.args.get("model_type"), + model_versions[model_num], + model_signatures[model_num], + request.args.get("use_predict") == "true", + request.args.get("predict_input_tensor"), + request.args.get("predict_output_tensor"), + ) + ( + predictions, + _, + ) = inference_utils.run_inference_for_inference_results( + examples_to_infer, serving_bundle + ) + infer_objs.append(predictions) + + resp = {"indices": indices_to_infer, "results": infer_objs} + self.updated_example_indices = set() + return http_util.Respond( + request, + { + "inferences": json.dumps(resp), + "vocab": json.dumps(label_vocab), + }, + "application/json", + ) + except common_utils.InvalidUserInputError as e: + return http_util.Respond( + request, {"error": e.message}, "application/json", code=400 + ) + except AbortionError as e: + return http_util.Respond( + request, {"error": e.details}, "application/json", code=400 + ) + + @wrappers.Request.application + def _eligible_features_from_example_handler(self, request): + """Returns a list of JSON objects for each feature in the example. + + Args: + request: A request for features. + + Returns: + A list with a JSON object for each feature. + Numeric features are represented as {name: observedMin: observedMax:}. + Categorical features are repesented as {name: samples:[]}. + """ + features_list = inference_utils.get_eligible_features( + self.examples[0:NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS + ) + return http_util.Respond(request, features_list, "application/json") + + @wrappers.Request.application + def _sort_eligible_features_handler(self, request): + """Returns a sorted list of JSON objects for each feature in the + example. + + The list is sorted by interestingness in terms of the resulting change in + inference values across feature values, for partial dependence plots. + + Args: + request: A request for sorted features. + + Returns: + A sorted list with a JSON object for each feature. + Numeric features are represented as + {name: observedMin: observedMax: interestingness:}. + Categorical features are repesented as + {name: samples:[] interestingness:}. + """ + try: + features_list = inference_utils.get_eligible_features( + self.examples[0:NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS + ) + example_index = int(request.args.get("example_index", "0")) + ( + inference_addresses, + model_names, + model_versions, + model_signatures, + ) = self._parse_request_arguments(request) + chart_data = {} + for feat in features_list: + chart_data[feat["name"]] = self._infer_mutants_impl( + feat["name"], + example_index, + inference_addresses, + model_names, + request.args.get("model_type"), + model_versions, + model_signatures, + request.args.get("use_predict") == "true", + request.args.get("predict_input_tensor"), + request.args.get("predict_output_tensor"), + feat["observedMin"] if "observedMin" in feat else 0, + feat["observedMax"] if "observedMin" in feat else 0, + None, + ) + features_list = inference_utils.sort_eligible_features( + features_list, chart_data + ) + return http_util.Respond(request, features_list, "application/json") + except common_utils.InvalidUserInputError as e: + return http_util.Respond( + request, {"error": e.message}, "application/json", code=400 + ) + + @wrappers.Request.application + def _infer_mutants_handler(self, request): + """Returns JSON for the partial dependence plots for a feature. + + Args: + request: A request that should contain 'feature_name', 'example_index', + 'inference_address', 'model_name', 'model_type', 'model_version', and + 'model_signature'. + + Returns: + A list of JSON objects, one for each chart. + """ + try: + if request.method != "GET": + logger.error("%s requests are forbidden.", request.method) + return http_util.Respond( + request, + {"error": "invalid non-GET request"}, + "application/json", + code=405, + ) + + example_index = int(request.args.get("example_index", "0")) + feature_name = request.args.get("feature_name") + ( + inference_addresses, + model_names, + model_versions, + model_signatures, + ) = self._parse_request_arguments(request) + json_mapping = self._infer_mutants_impl( + feature_name, + example_index, + inference_addresses, + model_names, + request.args.get("model_type"), + model_versions, + model_signatures, + request.args.get("use_predict") == "true", + request.args.get("predict_input_tensor"), + request.args.get("predict_output_tensor"), + request.args.get("x_min"), + request.args.get("x_max"), + request.args.get("feature_index_pattern"), + ) + return http_util.Respond(request, json_mapping, "application/json") + except common_utils.InvalidUserInputError as e: + return http_util.Respond( + request, {"error": e.message}, "application/json", code=400 + ) + + def _infer_mutants_impl( + self, + feature_name, + example_index, + inference_addresses, + model_names, + model_type, + model_versions, + model_signatures, + use_predict, + predict_input_tensor, + predict_output_tensor, + x_min, + x_max, + feature_index_pattern, + ): + """Helper for generating PD plots for a feature.""" + examples = ( + self.examples + if example_index == -1 + else [self.examples[example_index]] + ) + serving_bundles = [] + for model_num in xrange(len(inference_addresses)): + serving_bundles.append( + inference_utils.ServingBundle( + inference_addresses[model_num], + model_names[model_num], + model_type, + model_versions[model_num], + model_signatures[model_num], + use_predict, + predict_input_tensor, + predict_output_tensor, + ) + ) + + viz_params = inference_utils.VizParams( + x_min, + x_max, + self.examples[0:NUM_EXAMPLES_TO_SCAN], + NUM_MUTANTS, + feature_index_pattern, + ) + return inference_utils.mutant_charts_for_feature( + examples, feature_name, serving_bundles, viz_params + ) diff --git a/tensorboard/plugins/interactive_inference/interactive_inference_plugin_loader.py b/tensorboard/plugins/interactive_inference/interactive_inference_plugin_loader.py index eceaf1ad62..72e71daea3 100644 --- a/tensorboard/plugins/interactive_inference/interactive_inference_plugin_loader.py +++ b/tensorboard/plugins/interactive_inference/interactive_inference_plugin_loader.py @@ -22,24 +22,27 @@ class InteractiveInferencePluginLoader(base_plugin.TBLoader): - """InteractiveInferencePlugin factory. + """InteractiveInferencePlugin factory. - This class checks for `tensorflow` install and dependency. - """ - - def load(self, context): - """Returns the plugin, if possible. - - Args: - context: The TBContext flags. - - Returns: - A InteractiveInferencePlugin instance or None if it couldn't be loaded. + This class checks for `tensorflow` install and dependency. """ - try: - # pylint: disable=unused-import - import tensorflow - except ImportError: - return - from tensorboard.plugins.interactive_inference.interactive_inference_plugin import InteractiveInferencePlugin - return InteractiveInferencePlugin(context) + + def load(self, context): + """Returns the plugin, if possible. + + Args: + context: The TBContext flags. + + Returns: + A InteractiveInferencePlugin instance or None if it couldn't be loaded. + """ + try: + # pylint: disable=unused-import + import tensorflow + except ImportError: + return + from tensorboard.plugins.interactive_inference.interactive_inference_plugin import ( + InteractiveInferencePlugin, + ) + + return InteractiveInferencePlugin(context) diff --git a/tensorboard/plugins/interactive_inference/interactive_inference_plugin_test.py b/tensorboard/plugins/interactive_inference/interactive_inference_plugin_test.py index 417dd17687..a18ae484ee 100644 --- a/tensorboard/plugins/interactive_inference/interactive_inference_plugin_test.py +++ b/tensorboard/plugins/interactive_inference/interactive_inference_plugin_test.py @@ -26,10 +26,10 @@ import tensorflow as tf try: - # python version >= 3.3 - from unittest import mock + # python version >= 3.3 + from unittest import mock except ImportError: - import mock # pylint: disable=unused-import + import mock # pylint: disable=unused-import from six.moves import urllib_parse from google.protobuf import json_format @@ -38,233 +38,286 @@ from werkzeug import wrappers from tensorboard.backend import application -from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer +from tensorboard.backend.event_processing import ( + plugin_event_multiplexer as event_multiplexer, +) from tensorboard.plugins import base_plugin from tensorboard.plugins.interactive_inference.utils import inference_utils from tensorboard.plugins.interactive_inference.utils import platform_utils from tensorboard.plugins.interactive_inference.utils import test_utils -from tensorboard.plugins.interactive_inference import interactive_inference_plugin +from tensorboard.plugins.interactive_inference import ( + interactive_inference_plugin, +) class InferencePluginTest(tf.test.TestCase): - - def setUp(self): - self.logdir = tf.compat.v1.test.get_temp_dir() - - self.context = base_plugin.TBContext(logdir=self.logdir) - self.plugin = interactive_inference_plugin.InteractiveInferencePlugin( - self.context) - wsgi_app = application.TensorBoardWSGI([self.plugin]) - self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) - - def get_fake_example(self, single_int_value=0): - example = tf.train.Example() - example.features.feature['single_int'].int64_list.value.extend( - [single_int_value]) - return example - - def test_examples_from_path(self): - examples = [self.get_fake_example(0), self.get_fake_example(1)] - examples_path = os.path.join(self.logdir, 'test_examples.rio') - test_utils.write_out_examples(examples, examples_path) - - response = self.server.get( - '/data/plugin/whatif/examples_from_path?' + - urllib_parse.urlencode({ - 'examples_path': examples_path, - 'max_examples': 2, - 'sampling_odds': 1, - })) - self.assertEqual(200, response.status_code) - example_strings = json.loads(response.get_data().decode('utf-8'))['examples'] - received_examples = [json.loads(x) for x in example_strings] - self.assertEqual(2, len(received_examples)) - self.assertEqual(0, - int(received_examples[0]['features']['feature'][ - 'single_int']['int64List']['value'][0])) - self.assertEqual(1, - int(received_examples[1]['features']['feature'][ - 'single_int']['int64List']['value'][0])) - - def test_examples_from_path_if_path_does_not_exist(self): - response = self.server.get( - '/data/plugin/whatif/examples_from_path?' + - urllib_parse.urlencode({ - 'examples_path': 'does_not_exist', - 'max_examples': 2, - 'sampling_odds': 1, - })) - error = json.loads(response.get_data().decode('utf-8'))['error'] - self.assertTrue(error) - - def test_update_example(self): - self.plugin.examples = [tf.train.Example()] - example = self.get_fake_example() - response = self.server.post( - '/data/plugin/whatif/update_example', - data=dict(example=json_format.MessageToJson(example), index='0')) - self.assertEqual(200, response.status_code) - self.assertEqual(example, self.plugin.examples[0]) - self.assertTrue(0 in self.plugin.updated_example_indices) - - def test_update_example_invalid_index(self): - self.plugin.examples = [tf.train.Example()] - example = self.get_fake_example() - response = self.server.post( - '/data/plugin/whatif/update_example', - data=dict(example=json_format.MessageToJson(example), index='1')) - error = json.loads(response.get_data().decode('utf-8'))['error'] - self.assertTrue(error) - - @mock.patch.object(platform_utils, 'call_servo') - def test_infer(self, mock_call_servo): - self.plugin.examples = [ - self.get_fake_example(0), - self.get_fake_example(1), - self.get_fake_example(2) - ] - self.plugin.updated_example_indices = set([0, 2]) - - inference_result_proto = regression_pb2.RegressionResponse() - regression = inference_result_proto.result.regressions.add() - regression.value = 0.45 - regression = inference_result_proto.result.regressions.add() - regression.value = 0.55 - mock_call_servo.return_value = inference_result_proto - - response = self.server.get( - '/data/plugin/whatif/infer?' + urllib_parse.urlencode({ - 'inference_address': 'addr', - 'model_name': 'name', - 'model_type': 'regression', - 'model_version': ',', - 'model_signature': ',', - })) - - self.assertEqual(200, response.status_code) - self.assertEqual(0, len(self.plugin.updated_example_indices)) - inferences = json.loads(json.loads(response.get_data().decode('utf-8'))[ - 'inferences']) - self.assertTrue(0 in inferences['indices']) - self.assertFalse(1 in inferences['indices']) - self.assertTrue(2 in inferences['indices']) - - def _DeserializeResponse(self, byte_content): - """Deserializes byte content that is a JSON encoding. - - Args: - byte_content: The byte content of a JSON response. - - Returns: - The deserialized python object decoded from JSON. - """ - return json.loads(byte_content.decode('utf-8')) - - def test_eligible_features_from_example_proto(self): - example = test_utils.make_fake_example(single_int_val=2) - self.plugin.examples = [example] - - response = self.server.get('/data/plugin/whatif/eligible_features') - self.assertEqual(200, response.status_code) - - # Returns a list of dict objects that have been sorted by feature_name. - data = self._DeserializeResponse(response.get_data()) - - sorted_feature_names = [ - 'non_numeric', 'repeated_float', 'repeated_int', 'single_float', - 'single_int' - ] - self.assertEqual(sorted_feature_names, [d['name'] for d in data]) - np.testing.assert_almost_equal([-1, 1., 10, 24.5, 2.], - [d.get('observedMin', -1) for d in data]) - np.testing.assert_almost_equal([-1, 4., 20, 24.5, 2.], - [d.get('observedMax', -1) for d in data]) - - # Test that only non_numeric feature has samples. - self.assertFalse(any(d.get('samples') for d in data[1:])) - self.assertEqual(['cat'], data[0]['samples']) - - @mock.patch.object(inference_utils, 'mutant_charts_for_feature') - def test_infer_mutants_handler(self, mock_mutant_charts_for_feature): - - # A no-op that just passes the example passed to mutant_charts_for_feature - # back through. This tests that the URL parameters get processed properly - # within infer_mutants_handler. - def pass_through(example, feature_name, serving_bundles, viz_params): - return { - 'example': str(example), - 'feature_name': feature_name, - 'serving_bundles': [{ - 'inference_address': serving_bundles[0].inference_address, - 'model_name': serving_bundles[0].model_name, - 'model_type': serving_bundles[0].model_type, - }], - 'viz_params': { - 'x_min': viz_params.x_min, - 'x_max': viz_params.x_max - } - } - - mock_mutant_charts_for_feature.side_effect = pass_through - - example = test_utils.make_fake_example() - self.plugin.examples = [example] - - response = self.server.get( - '/data/plugin/whatif/infer_mutants?' + urllib_parse.urlencode({ - 'feature_name': 'single_int', - 'model_name': '/ml/cassandrax/iris_classification', - 'inference_address': 'ml-serving-temp.prediction', - 'model_type': 'classification', - 'model_version': ',', - 'model_signature': ',', - 'x_min': '-10', - 'x_max': '10', - })) - result = self._DeserializeResponse(response.get_data()) - self.assertEqual(str([example]), result['example']) - self.assertEqual('single_int', result['feature_name']) - self.assertEqual('ml-serving-temp.prediction', - result['serving_bundles'][0]['inference_address']) - self.assertEqual('/ml/cassandrax/iris_classification', - result['serving_bundles'][0]['model_name']) - self.assertEqual('classification', result['serving_bundles'][0]['model_type']) - self.assertAlmostEqual(-10, result['viz_params']['x_min']) - self.assertAlmostEqual(10, result['viz_params']['x_max']) - - @mock.patch.object(inference_utils, 'sort_eligible_features') - @mock.patch.object(inference_utils, 'mutant_charts_for_feature') - def test_infer( - self, mock_mutant_charts_for_feature, mock_sort_eligible_features): - self.plugin.examples = [ - self.get_fake_example(0), - self.get_fake_example(1), - self.get_fake_example(2) - ] - - mock_mutant_charts_for_feature.return_value = [] - sorted_features_list = [ - {'name': 'feat1', 'interestingness': .2}, - {'name': 'feat2', 'interestingness': .1} - ] - mock_sort_eligible_features.return_value = sorted_features_list - - url_options = urllib_parse.urlencode({ - 'inference_address': 'addr', - 'model_name': 'name', - 'model_type': 'regression', - 'model_version': '', - 'model_signature': '', - }) - response = self.server.get( - '/data/plugin/whatif/sort_eligible_features?' + url_options) - - self.assertEqual(200, response.status_code) - self.assertEqual(0, len(self.plugin.updated_example_indices)) - output_list = json.loads(response.get_data().decode('utf-8')) - self.assertEquals('feat1', output_list[0]['name']) - self.assertEquals('feat2', output_list[1]['name']) - - -if __name__ == '__main__': - tf.test.main() + def setUp(self): + self.logdir = tf.compat.v1.test.get_temp_dir() + + self.context = base_plugin.TBContext(logdir=self.logdir) + self.plugin = interactive_inference_plugin.InteractiveInferencePlugin( + self.context + ) + wsgi_app = application.TensorBoardWSGI([self.plugin]) + self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) + + def get_fake_example(self, single_int_value=0): + example = tf.train.Example() + example.features.feature["single_int"].int64_list.value.extend( + [single_int_value] + ) + return example + + def test_examples_from_path(self): + examples = [self.get_fake_example(0), self.get_fake_example(1)] + examples_path = os.path.join(self.logdir, "test_examples.rio") + test_utils.write_out_examples(examples, examples_path) + + response = self.server.get( + "/data/plugin/whatif/examples_from_path?" + + urllib_parse.urlencode( + { + "examples_path": examples_path, + "max_examples": 2, + "sampling_odds": 1, + } + ) + ) + self.assertEqual(200, response.status_code) + example_strings = json.loads(response.get_data().decode("utf-8"))[ + "examples" + ] + received_examples = [json.loads(x) for x in example_strings] + self.assertEqual(2, len(received_examples)) + self.assertEqual( + 0, + int( + received_examples[0]["features"]["feature"]["single_int"][ + "int64List" + ]["value"][0] + ), + ) + self.assertEqual( + 1, + int( + received_examples[1]["features"]["feature"]["single_int"][ + "int64List" + ]["value"][0] + ), + ) + + def test_examples_from_path_if_path_does_not_exist(self): + response = self.server.get( + "/data/plugin/whatif/examples_from_path?" + + urllib_parse.urlencode( + { + "examples_path": "does_not_exist", + "max_examples": 2, + "sampling_odds": 1, + } + ) + ) + error = json.loads(response.get_data().decode("utf-8"))["error"] + self.assertTrue(error) + + def test_update_example(self): + self.plugin.examples = [tf.train.Example()] + example = self.get_fake_example() + response = self.server.post( + "/data/plugin/whatif/update_example", + data=dict(example=json_format.MessageToJson(example), index="0"), + ) + self.assertEqual(200, response.status_code) + self.assertEqual(example, self.plugin.examples[0]) + self.assertTrue(0 in self.plugin.updated_example_indices) + + def test_update_example_invalid_index(self): + self.plugin.examples = [tf.train.Example()] + example = self.get_fake_example() + response = self.server.post( + "/data/plugin/whatif/update_example", + data=dict(example=json_format.MessageToJson(example), index="1"), + ) + error = json.loads(response.get_data().decode("utf-8"))["error"] + self.assertTrue(error) + + @mock.patch.object(platform_utils, "call_servo") + def test_infer(self, mock_call_servo): + self.plugin.examples = [ + self.get_fake_example(0), + self.get_fake_example(1), + self.get_fake_example(2), + ] + self.plugin.updated_example_indices = set([0, 2]) + + inference_result_proto = regression_pb2.RegressionResponse() + regression = inference_result_proto.result.regressions.add() + regression.value = 0.45 + regression = inference_result_proto.result.regressions.add() + regression.value = 0.55 + mock_call_servo.return_value = inference_result_proto + + response = self.server.get( + "/data/plugin/whatif/infer?" + + urllib_parse.urlencode( + { + "inference_address": "addr", + "model_name": "name", + "model_type": "regression", + "model_version": ",", + "model_signature": ",", + } + ) + ) + + self.assertEqual(200, response.status_code) + self.assertEqual(0, len(self.plugin.updated_example_indices)) + inferences = json.loads( + json.loads(response.get_data().decode("utf-8"))["inferences"] + ) + self.assertTrue(0 in inferences["indices"]) + self.assertFalse(1 in inferences["indices"]) + self.assertTrue(2 in inferences["indices"]) + + def _DeserializeResponse(self, byte_content): + """Deserializes byte content that is a JSON encoding. + + Args: + byte_content: The byte content of a JSON response. + + Returns: + The deserialized python object decoded from JSON. + """ + return json.loads(byte_content.decode("utf-8")) + + def test_eligible_features_from_example_proto(self): + example = test_utils.make_fake_example(single_int_val=2) + self.plugin.examples = [example] + + response = self.server.get("/data/plugin/whatif/eligible_features") + self.assertEqual(200, response.status_code) + + # Returns a list of dict objects that have been sorted by feature_name. + data = self._DeserializeResponse(response.get_data()) + + sorted_feature_names = [ + "non_numeric", + "repeated_float", + "repeated_int", + "single_float", + "single_int", + ] + self.assertEqual(sorted_feature_names, [d["name"] for d in data]) + np.testing.assert_almost_equal( + [-1, 1.0, 10, 24.5, 2.0], [d.get("observedMin", -1) for d in data] + ) + np.testing.assert_almost_equal( + [-1, 4.0, 20, 24.5, 2.0], [d.get("observedMax", -1) for d in data] + ) + + # Test that only non_numeric feature has samples. + self.assertFalse(any(d.get("samples") for d in data[1:])) + self.assertEqual(["cat"], data[0]["samples"]) + + @mock.patch.object(inference_utils, "mutant_charts_for_feature") + def test_infer_mutants_handler(self, mock_mutant_charts_for_feature): + + # A no-op that just passes the example passed to mutant_charts_for_feature + # back through. This tests that the URL parameters get processed properly + # within infer_mutants_handler. + def pass_through(example, feature_name, serving_bundles, viz_params): + return { + "example": str(example), + "feature_name": feature_name, + "serving_bundles": [ + { + "inference_address": serving_bundles[ + 0 + ].inference_address, + "model_name": serving_bundles[0].model_name, + "model_type": serving_bundles[0].model_type, + } + ], + "viz_params": { + "x_min": viz_params.x_min, + "x_max": viz_params.x_max, + }, + } + + mock_mutant_charts_for_feature.side_effect = pass_through + + example = test_utils.make_fake_example() + self.plugin.examples = [example] + + response = self.server.get( + "/data/plugin/whatif/infer_mutants?" + + urllib_parse.urlencode( + { + "feature_name": "single_int", + "model_name": "/ml/cassandrax/iris_classification", + "inference_address": "ml-serving-temp.prediction", + "model_type": "classification", + "model_version": ",", + "model_signature": ",", + "x_min": "-10", + "x_max": "10", + } + ) + ) + result = self._DeserializeResponse(response.get_data()) + self.assertEqual(str([example]), result["example"]) + self.assertEqual("single_int", result["feature_name"]) + self.assertEqual( + "ml-serving-temp.prediction", + result["serving_bundles"][0]["inference_address"], + ) + self.assertEqual( + "/ml/cassandrax/iris_classification", + result["serving_bundles"][0]["model_name"], + ) + self.assertEqual( + "classification", result["serving_bundles"][0]["model_type"] + ) + self.assertAlmostEqual(-10, result["viz_params"]["x_min"]) + self.assertAlmostEqual(10, result["viz_params"]["x_max"]) + + @mock.patch.object(inference_utils, "sort_eligible_features") + @mock.patch.object(inference_utils, "mutant_charts_for_feature") + def test_infer( + self, mock_mutant_charts_for_feature, mock_sort_eligible_features + ): + self.plugin.examples = [ + self.get_fake_example(0), + self.get_fake_example(1), + self.get_fake_example(2), + ] + + mock_mutant_charts_for_feature.return_value = [] + sorted_features_list = [ + {"name": "feat1", "interestingness": 0.2}, + {"name": "feat2", "interestingness": 0.1}, + ] + mock_sort_eligible_features.return_value = sorted_features_list + + url_options = urllib_parse.urlencode( + { + "inference_address": "addr", + "model_name": "name", + "model_type": "regression", + "model_version": "", + "model_signature": "", + } + ) + response = self.server.get( + "/data/plugin/whatif/sort_eligible_features?" + url_options + ) + + self.assertEqual(200, response.status_code) + self.assertEqual(0, len(self.plugin.updated_example_indices)) + output_list = json.loads(response.get_data().decode("utf-8")) + self.assertEquals("feat1", output_list[0]["name"]) + self.assertEquals("feat2", output_list[1]["name"]) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/interactive_inference/utils/common_utils.py b/tensorboard/plugins/interactive_inference/utils/common_utils.py index ce916a0bc7..3ca6b21ba3 100644 --- a/tensorboard/plugins/interactive_inference/utils/common_utils.py +++ b/tensorboard/plugins/interactive_inference/utils/common_utils.py @@ -19,73 +19,78 @@ class InvalidUserInputError(Exception): - """An exception to throw if user input is detected to be invalid. + """An exception to throw if user input is detected to be invalid. - Attributes: - original_exception: The triggering `Exception` object to be wrapped, or - a string. - """ + Attributes: + original_exception: The triggering `Exception` object to be wrapped, or + a string. + """ - def __init__(self, original_exception): - """Inits InvalidUserInputError.""" - self.original_exception = original_exception - Exception.__init__(self) + def __init__(self, original_exception): + """Inits InvalidUserInputError.""" + self.original_exception = original_exception + Exception.__init__(self) - @property - def message(self): - return 'InvalidUserInputError: ' + str(self.original_exception) + @property + def message(self): + return "InvalidUserInputError: " + str(self.original_exception) def convert_predict_response(pred, serving_bundle): - """Converts a PredictResponse to ClassificationResponse or RegressionResponse. + """Converts a PredictResponse to ClassificationResponse or + RegressionResponse. - Args: - pred: PredictResponse to convert. - serving_bundle: A `ServingBundle` object that contains the information about - the serving request that the response was generated by. + Args: + pred: PredictResponse to convert. + serving_bundle: A `ServingBundle` object that contains the information about + the serving request that the response was generated by. + + Returns: + A ClassificationResponse or RegressionResponse. + """ + output = pred.outputs[serving_bundle.predict_output_tensor] + raw_output = output.float_val + if serving_bundle.model_type == "classification": + values = [] + for example_index in range(output.tensor_shape.dim[0].size): + start = example_index * output.tensor_shape.dim[1].size + values.append( + raw_output[start : start + output.tensor_shape.dim[1].size] + ) + else: + values = raw_output + return convert_prediction_values(values, serving_bundle, pred.model_spec) - Returns: - A ClassificationResponse or RegressionResponse. - """ - output = pred.outputs[serving_bundle.predict_output_tensor] - raw_output = output.float_val - if serving_bundle.model_type == 'classification': - values = [] - for example_index in range(output.tensor_shape.dim[0].size): - start = example_index * output.tensor_shape.dim[1].size - values.append(raw_output[start:start + output.tensor_shape.dim[1].size]) - else: - values = raw_output - return convert_prediction_values(values, serving_bundle, pred.model_spec) def convert_prediction_values(values, serving_bundle, model_spec=None): - """Converts tensor values into ClassificationResponse or RegressionResponse. + """Converts tensor values into ClassificationResponse or + RegressionResponse. - Args: - values: For classification, a 2D list of numbers. The first dimension is for - each example being predicted. The second dimension are the probabilities - for each class ID in the prediction. For regression, a 1D list of numbers, - with a regression score for each example being predicted. - serving_bundle: A `ServingBundle` object that contains the information about - the serving request that the response was generated by. - model_spec: Optional model spec to put into the response. + Args: + values: For classification, a 2D list of numbers. The first dimension is for + each example being predicted. The second dimension are the probabilities + for each class ID in the prediction. For regression, a 1D list of numbers, + with a regression score for each example being predicted. + serving_bundle: A `ServingBundle` object that contains the information about + the serving request that the response was generated by. + model_spec: Optional model spec to put into the response. - Returns: - A ClassificationResponse or RegressionResponse. - """ - if serving_bundle.model_type == 'classification': - response = classification_pb2.ClassificationResponse() - for example_index in range(len(values)): - classification = response.result.classifications.add() - for class_index in range(len(values[example_index])): - class_score = classification.classes.add() - class_score.score = values[example_index][class_index] - class_score.label = str(class_index) - else: - response = regression_pb2.RegressionResponse() - for example_index in range(len(values)): - regression = response.result.regressions.add() - regression.value = values[example_index] - if model_spec: - response.model_spec.CopyFrom(model_spec) - return response + Returns: + A ClassificationResponse or RegressionResponse. + """ + if serving_bundle.model_type == "classification": + response = classification_pb2.ClassificationResponse() + for example_index in range(len(values)): + classification = response.result.classifications.add() + for class_index in range(len(values[example_index])): + class_score = classification.classes.add() + class_score.score = values[example_index][class_index] + class_score.label = str(class_index) + else: + response = regression_pb2.RegressionResponse() + for example_index in range(len(values)): + regression = response.result.regressions.add() + regression.value = values[example_index] + if model_spec: + response.model_spec.CopyFrom(model_spec) + return response diff --git a/tensorboard/plugins/interactive_inference/utils/inference_utils.py b/tensorboard/plugins/interactive_inference/utils/inference_utils.py index 6b552b6767..c612052db6 100644 --- a/tensorboard/plugins/interactive_inference/utils/inference_utils.py +++ b/tensorboard/plugins/interactive_inference/utils/inference_utils.py @@ -37,607 +37,690 @@ class VizParams(object): - """Light-weight class for holding UI state. - - Attributes: - x_min: The minimum value to use to generate mutants for the feature - (as specified the user on the UI). - x_max: The maximum value to use to generate mutants for the feature - (as specified the user on the UI). - examples: A list of examples to scan in order to generate statistics for - mutants. - num_mutants: Int number of mutants to generate per chart. - feature_index_pattern: String that specifies a restricted set of indices - of the feature to generate mutants for (useful for features that is a - long repeated field. See `convert_pattern_to_indices` for more details. - """ - - def __init__(self, x_min, x_max, examples, num_mutants, - feature_index_pattern): - """Inits VizParams may raise InvalidUserInputError for bad user inputs.""" - - def to_float_or_none(x): - try: - return float(x) - except (ValueError, TypeError): - return None - - def to_int(x): - try: - return int(x) - except (ValueError, TypeError) as e: - raise common_utils.InvalidUserInputError(e) - - def convert_pattern_to_indices(pattern): - """Converts a printer-page-style pattern and returns a list of indices. - - Args: - pattern: A printer-page-style pattern with only numeric characters, - commas, dashes, and optionally spaces. - - For example, a pattern of '0,2,4-6' would yield [0, 2, 4, 5, 6]. - - Returns: - A list of indices represented by the pattern. - """ - pieces = [token.strip() for token in pattern.split(',')] - indices = [] - for piece in pieces: - if '-' in piece: - lower, upper = [int(x.strip()) for x in piece.split('-', 1)] - indices.extend(range(lower, upper + 1)) - else: - indices.append(int(piece.strip())) - return sorted(indices) - - self.x_min = to_float_or_none(x_min) - self.x_max = to_float_or_none(x_max) - self.examples = examples - self.num_mutants = to_int(num_mutants) + """Light-weight class for holding UI state. + + Attributes: + x_min: The minimum value to use to generate mutants for the feature + (as specified the user on the UI). + x_max: The maximum value to use to generate mutants for the feature + (as specified the user on the UI). + examples: A list of examples to scan in order to generate statistics for + mutants. + num_mutants: Int number of mutants to generate per chart. + feature_index_pattern: String that specifies a restricted set of indices + of the feature to generate mutants for (useful for features that is a + long repeated field. See `convert_pattern_to_indices` for more details. + """ - # By default, there are no specific user-requested feature indices. - self.feature_indices = [] - if feature_index_pattern: - try: - self.feature_indices = convert_pattern_to_indices( - feature_index_pattern) - except ValueError as e: - # If the user-requested range is invalid, use the default range. - pass + def __init__( + self, x_min, x_max, examples, num_mutants, feature_index_pattern + ): + """Inits VizParams may raise InvalidUserInputError for bad user + inputs.""" + + def to_float_or_none(x): + try: + return float(x) + except (ValueError, TypeError): + return None + + def to_int(x): + try: + return int(x) + except (ValueError, TypeError) as e: + raise common_utils.InvalidUserInputError(e) + + def convert_pattern_to_indices(pattern): + """Converts a printer-page-style pattern and returns a list of + indices. + + Args: + pattern: A printer-page-style pattern with only numeric characters, + commas, dashes, and optionally spaces. + + For example, a pattern of '0,2,4-6' would yield [0, 2, 4, 5, 6]. + + Returns: + A list of indices represented by the pattern. + """ + pieces = [token.strip() for token in pattern.split(",")] + indices = [] + for piece in pieces: + if "-" in piece: + lower, upper = [int(x.strip()) for x in piece.split("-", 1)] + indices.extend(range(lower, upper + 1)) + else: + indices.append(int(piece.strip())) + return sorted(indices) + + self.x_min = to_float_or_none(x_min) + self.x_max = to_float_or_none(x_max) + self.examples = examples + self.num_mutants = to_int(num_mutants) + + # By default, there are no specific user-requested feature indices. + self.feature_indices = [] + if feature_index_pattern: + try: + self.feature_indices = convert_pattern_to_indices( + feature_index_pattern + ) + except ValueError as e: + # If the user-requested range is invalid, use the default range. + pass class OriginalFeatureList(object): - """Light-weight class for holding the original values in the example. + """Light-weight class for holding the original values in the example. - Should not be created by hand, but rather generated via - `parse_original_feature_from_example`. Just used to hold inferred info - about the example. + Should not be created by hand, but rather generated via + `parse_original_feature_from_example`. Just used to hold inferred info + about the example. - Attributes: - feature_name: String name of the feature. - original_value: The value of the feature in the original example. - feature_type: One of ['int64_list', 'float_list']. + Attributes: + feature_name: String name of the feature. + original_value: The value of the feature in the original example. + feature_type: One of ['int64_list', 'float_list']. - Raises: - ValueError: If OriginalFeatureList fails init validation. - """ + Raises: + ValueError: If OriginalFeatureList fails init validation. + """ - def __init__(self, feature_name, original_value, feature_type): - """Inits OriginalFeatureList.""" - self.feature_name = feature_name - self.original_value = [ - ensure_not_binary(value) for value in original_value] - self.feature_type = feature_type + def __init__(self, feature_name, original_value, feature_type): + """Inits OriginalFeatureList.""" + self.feature_name = feature_name + self.original_value = [ + ensure_not_binary(value) for value in original_value + ] + self.feature_type = feature_type - # Derived attributes. - self.length = sum(1 for _ in original_value) + # Derived attributes. + self.length = sum(1 for _ in original_value) class MutantFeatureValue(object): - """Light-weight class for holding mutated values in the example. - - Should not be created by hand but rather generated via `make_mutant_features`. + """Light-weight class for holding mutated values in the example. - Used to represent a "mutant example": an example that is mostly identical to - the user-provided original example, but has one feature that is different. + Should not be created by hand but rather generated via `make_mutant_features`. - Attributes: - original_feature: An `OriginalFeatureList` object representing the feature - to create mutants for. - index: The index of the feature to create mutants for. The feature can be - a repeated field, and we want to plot mutations of its various indices. - mutant_value: The proposed mutant value for the given index. + Used to represent a "mutant example": an example that is mostly identical to + the user-provided original example, but has one feature that is different. - Raises: - ValueError: If MutantFeatureValue fails init validation. - """ + Attributes: + original_feature: An `OriginalFeatureList` object representing the feature + to create mutants for. + index: The index of the feature to create mutants for. The feature can be + a repeated field, and we want to plot mutations of its various indices. + mutant_value: The proposed mutant value for the given index. - def __init__(self, original_feature, index, mutant_value): - """Inits MutantFeatureValue.""" - if not isinstance(original_feature, OriginalFeatureList): - raise ValueError( - 'original_feature should be `OriginalFeatureList`, but had ' - 'unexpected type: {}'.format(type(original_feature))) - self.original_feature = original_feature + Raises: + ValueError: If MutantFeatureValue fails init validation. + """ - if index is not None and not isinstance(index, integer_types): - raise ValueError( - 'index should be None or int, but had unexpected type: {}'.format( - type(index))) - self.index = index - self.mutant_value = (mutant_value.encode() - if isinstance(mutant_value, string_types) else mutant_value) + def __init__(self, original_feature, index, mutant_value): + """Inits MutantFeatureValue.""" + if not isinstance(original_feature, OriginalFeatureList): + raise ValueError( + "original_feature should be `OriginalFeatureList`, but had " + "unexpected type: {}".format(type(original_feature)) + ) + self.original_feature = original_feature + + if index is not None and not isinstance(index, integer_types): + raise ValueError( + "index should be None or int, but had unexpected type: {}".format( + type(index) + ) + ) + self.index = index + self.mutant_value = ( + mutant_value.encode() + if isinstance(mutant_value, string_types) + else mutant_value + ) class ServingBundle(object): - """Light-weight class for holding info to make the inference request. - - Attributes: - inference_address: An address (such as "hostname:port") to send inference - requests to. - model_name: The Servo model name. - model_type: One of ['classification', 'regression']. - model_version: The version number of the model as a string. If set to an - empty string, the latest model will be used. - signature: The signature of the model to infer. If set to an empty string, - the default signuature will be used. - use_predict: If true then use the servo Predict API as opposed to - Classification or Regression. - predict_input_tensor: The name of the input tensor to parse when using the - Predict API. - predict_output_tensor: The name of the output tensor to parse when using the - Predict API. - estimator: An estimator to use instead of calling an external model. - feature_spec: A feature spec for use with the estimator. - custom_predict_fn: A custom prediction function. - - Raises: - ValueError: If ServingBundle fails init validation. - """ - - def __init__(self, inference_address, model_name, model_type, model_version, - signature, use_predict, predict_input_tensor, - predict_output_tensor, estimator=None, feature_spec=None, - custom_predict_fn=None): - """Inits ServingBundle.""" - if not isinstance(inference_address, string_types): - raise ValueError('Invalid inference_address has type: {}'.format( - type(inference_address))) - # Clean the inference_address so that SmartStub likes it. - self.inference_address = inference_address.replace('http://', '').replace( - 'https://', '') - - if not isinstance(model_name, string_types): - raise ValueError('Invalid model_name has type: {}'.format( - type(model_name))) - self.model_name = model_name - - if model_type not in ['classification', 'regression']: - raise ValueError('Invalid model_type: {}'.format(model_type)) - self.model_type = model_type - - self.model_version = int(model_version) if model_version else None - - self.signature = signature if signature else None - - self.use_predict = use_predict - self.predict_input_tensor = predict_input_tensor - self.predict_output_tensor = predict_output_tensor - self.estimator = estimator - self.feature_spec = feature_spec - self.custom_predict_fn = custom_predict_fn + """Light-weight class for holding info to make the inference request. + + Attributes: + inference_address: An address (such as "hostname:port") to send inference + requests to. + model_name: The Servo model name. + model_type: One of ['classification', 'regression']. + model_version: The version number of the model as a string. If set to an + empty string, the latest model will be used. + signature: The signature of the model to infer. If set to an empty string, + the default signuature will be used. + use_predict: If true then use the servo Predict API as opposed to + Classification or Regression. + predict_input_tensor: The name of the input tensor to parse when using the + Predict API. + predict_output_tensor: The name of the output tensor to parse when using the + Predict API. + estimator: An estimator to use instead of calling an external model. + feature_spec: A feature spec for use with the estimator. + custom_predict_fn: A custom prediction function. + + Raises: + ValueError: If ServingBundle fails init validation. + """ + + def __init__( + self, + inference_address, + model_name, + model_type, + model_version, + signature, + use_predict, + predict_input_tensor, + predict_output_tensor, + estimator=None, + feature_spec=None, + custom_predict_fn=None, + ): + """Inits ServingBundle.""" + if not isinstance(inference_address, string_types): + raise ValueError( + "Invalid inference_address has type: {}".format( + type(inference_address) + ) + ) + # Clean the inference_address so that SmartStub likes it. + self.inference_address = inference_address.replace( + "http://", "" + ).replace("https://", "") + + if not isinstance(model_name, string_types): + raise ValueError( + "Invalid model_name has type: {}".format(type(model_name)) + ) + self.model_name = model_name + + if model_type not in ["classification", "regression"]: + raise ValueError("Invalid model_type: {}".format(model_type)) + self.model_type = model_type + + self.model_version = int(model_version) if model_version else None + + self.signature = signature if signature else None + + self.use_predict = use_predict + self.predict_input_tensor = predict_input_tensor + self.predict_output_tensor = predict_output_tensor + self.estimator = estimator + self.feature_spec = feature_spec + self.custom_predict_fn = custom_predict_fn def ensure_not_binary(value): - """Return non-binary version of value.""" - try: - return value.decode() if isinstance(value, binary_type) else value - except UnicodeDecodeError: - # If the value cannot be decoded as a string (such as an encoded image), - # then just return the value. - return value + """Return non-binary version of value.""" + try: + return value.decode() if isinstance(value, binary_type) else value + except UnicodeDecodeError: + # If the value cannot be decoded as a string (such as an encoded image), + # then just return the value. + return value def proto_value_for_feature(example, feature_name): - """Get the value of a feature from Example regardless of feature type.""" - feature = get_example_features(example)[feature_name] - if feature is None: - raise ValueError('Feature {} is not on example proto.'.format(feature_name)) - feature_type = feature.WhichOneof('kind') - if feature_type is None: - raise ValueError('Feature {} on example proto has no declared type.'.format( - feature_name)) - return getattr(feature, feature_type).value + """Get the value of a feature from Example regardless of feature type.""" + feature = get_example_features(example)[feature_name] + if feature is None: + raise ValueError( + "Feature {} is not on example proto.".format(feature_name) + ) + feature_type = feature.WhichOneof("kind") + if feature_type is None: + raise ValueError( + "Feature {} on example proto has no declared type.".format( + feature_name + ) + ) + return getattr(feature, feature_type).value def parse_original_feature_from_example(example, feature_name): - """Returns an `OriginalFeatureList` for the specified feature_name. + """Returns an `OriginalFeatureList` for the specified feature_name. - Args: - example: An example. - feature_name: A string feature name. + Args: + example: An example. + feature_name: A string feature name. - Returns: - A filled in `OriginalFeatureList` object representing the feature. - """ - feature = get_example_features(example)[feature_name] - feature_type = feature.WhichOneof('kind') - original_value = proto_value_for_feature(example, feature_name) + Returns: + A filled in `OriginalFeatureList` object representing the feature. + """ + feature = get_example_features(example)[feature_name] + feature_type = feature.WhichOneof("kind") + original_value = proto_value_for_feature(example, feature_name) - return OriginalFeatureList(feature_name, original_value, feature_type) + return OriginalFeatureList(feature_name, original_value, feature_type) def wrap_inference_results(inference_result_proto): - """Returns packaged inference results from the provided proto. + """Returns packaged inference results from the provided proto. - Args: - inference_result_proto: The classification or regression response proto. + Args: + inference_result_proto: The classification or regression response proto. - Returns: - An InferenceResult proto with the result from the response. - """ - inference_proto = inference_pb2.InferenceResult() - if isinstance(inference_result_proto, - classification_pb2.ClassificationResponse): - inference_proto.classification_result.CopyFrom( - inference_result_proto.result) - elif isinstance(inference_result_proto, regression_pb2.RegressionResponse): - inference_proto.regression_result.CopyFrom(inference_result_proto.result) - return inference_proto + Returns: + An InferenceResult proto with the result from the response. + """ + inference_proto = inference_pb2.InferenceResult() + if isinstance( + inference_result_proto, classification_pb2.ClassificationResponse + ): + inference_proto.classification_result.CopyFrom( + inference_result_proto.result + ) + elif isinstance(inference_result_proto, regression_pb2.RegressionResponse): + inference_proto.regression_result.CopyFrom( + inference_result_proto.result + ) + return inference_proto def get_numeric_feature_names(example): - """Returns a list of feature names for float and int64 type features. + """Returns a list of feature names for float and int64 type features. - Args: - example: An example. + Args: + example: An example. - Returns: - A list of strings of the names of numeric features. - """ - numeric_features = ('float_list', 'int64_list') - features = get_example_features(example) - return sorted([ - feature_name for feature_name in features - if features[feature_name].WhichOneof('kind') in numeric_features - ]) + Returns: + A list of strings of the names of numeric features. + """ + numeric_features = ("float_list", "int64_list") + features = get_example_features(example) + return sorted( + [ + feature_name + for feature_name in features + if features[feature_name].WhichOneof("kind") in numeric_features + ] + ) def get_categorical_feature_names(example): - """Returns a list of feature names for byte type features. + """Returns a list of feature names for byte type features. - Args: - example: An example. + Args: + example: An example. - Returns: - A list of categorical feature names (e.g. ['education', 'marital_status'] ) - """ - features = get_example_features(example) - return sorted([ - feature_name for feature_name in features - if features[feature_name].WhichOneof('kind') == 'bytes_list' - ]) + Returns: + A list of categorical feature names (e.g. ['education', 'marital_status'] ) + """ + features = get_example_features(example) + return sorted( + [ + feature_name + for feature_name in features + if features[feature_name].WhichOneof("kind") == "bytes_list" + ] + ) def get_numeric_features_to_observed_range(examples): - """Returns numerical features and their observed ranges. - - Args: - examples: Examples to read to get ranges. - - Returns: - A dict mapping feature_name -> {'observedMin': 'observedMax': } dicts, - with a key for each numerical feature. - """ - observed_features = collections.defaultdict(list) # name -> [value, ] - for example in examples: - for feature_name in get_numeric_feature_names(example): - original_feature = parse_original_feature_from_example( - example, feature_name) - observed_features[feature_name].extend(original_feature.original_value) - return { - feature_name: { - 'observedMin': min(feature_values), - 'observedMax': max(feature_values), - } - for feature_name, feature_values in iteritems(observed_features) - } + """Returns numerical features and their observed ranges. + + Args: + examples: Examples to read to get ranges. + + Returns: + A dict mapping feature_name -> {'observedMin': 'observedMax': } dicts, + with a key for each numerical feature. + """ + observed_features = collections.defaultdict(list) # name -> [value, ] + for example in examples: + for feature_name in get_numeric_feature_names(example): + original_feature = parse_original_feature_from_example( + example, feature_name + ) + observed_features[feature_name].extend( + original_feature.original_value + ) + return { + feature_name: { + "observedMin": min(feature_values), + "observedMax": max(feature_values), + } + for feature_name, feature_values in iteritems(observed_features) + } def get_categorical_features_to_sampling(examples, top_k): - """Returns categorical features and a sampling of their most-common values. - - The results of this slow function are used by the visualization repeatedly, - so the results are cached. - - Args: - examples: Examples to read to get feature samples. - top_k: Max number of samples to return per feature. - - Returns: - A dict of feature_name -> {'samples': ['Married-civ-spouse', - 'Never-married', 'Divorced']}. - - There is one key for each categorical feature. - - Currently, the inner dict just has one key, but this structure leaves room - for further expansion, and mirrors the structure used by - `get_numeric_features_to_observed_range`. - """ - observed_features = collections.defaultdict(list) # name -> [value, ] - for example in examples: - for feature_name in get_categorical_feature_names(example): - original_feature = parse_original_feature_from_example( - example, feature_name) - observed_features[feature_name].extend(original_feature.original_value) - - result = {} - for feature_name, feature_values in sorted(iteritems(observed_features)): - samples = [ - word - for word, count in collections.Counter(feature_values).most_common( - top_k) if count > 1 - ] - if samples: - result[feature_name] = {'samples': samples} - return result + """Returns categorical features and a sampling of their most-common values. + The results of this slow function are used by the visualization repeatedly, + so the results are cached. -def make_mutant_features(original_feature, index_to_mutate, viz_params): - """Return a list of `MutantFeatureValue`s that are variants of original.""" - lower = viz_params.x_min - upper = viz_params.x_max - examples = viz_params.examples - num_mutants = viz_params.num_mutants - - if original_feature.feature_type == 'float_list': - return [ - MutantFeatureValue(original_feature, index_to_mutate, value) - for value in np.linspace(lower, upper, num_mutants) - ] - elif original_feature.feature_type == 'int64_list': - mutant_values = np.linspace(int(lower), int(upper), - num_mutants).astype(int).tolist() - # Remove duplicates that can occur due to integer constraint. - mutant_values = sorted(set(mutant_values)) - return [ - MutantFeatureValue(original_feature, index_to_mutate, value) - for value in mutant_values - ] - elif original_feature.feature_type == 'bytes_list': - feature_to_samples = get_categorical_features_to_sampling( - examples, num_mutants) - - # `mutant_values` looks like: - # [['Married-civ-spouse'], ['Never-married'], ['Divorced'], ['Separated']] - mutant_values = feature_to_samples[original_feature.feature_name]['samples'] - return [ - MutantFeatureValue(original_feature, None, value) - for value in mutant_values - ] - else: - raise ValueError('Malformed original feature had type of: ' + - original_feature.feature_type) - - -def make_mutant_tuples(example_protos, original_feature, index_to_mutate, - viz_params): - """Return a list of `MutantFeatureValue`s and a list of mutant Examples. - - Args: - example_protos: The examples to mutate. - original_feature: A `OriginalFeatureList` that encapsulates the feature to - mutate. - index_to_mutate: The index of the int64_list or float_list to mutate. - viz_params: A `VizParams` object that contains the UI state of the request. - - Returns: - A list of `MutantFeatureValue`s and a list of mutant examples. - """ - mutant_features = make_mutant_features(original_feature, index_to_mutate, - viz_params) - mutant_examples = [] - for example_proto in example_protos: - for mutant_feature in mutant_features: - copied_example = copy.deepcopy(example_proto) - feature_name = mutant_feature.original_feature.feature_name - - try: - feature_list = proto_value_for_feature(copied_example, feature_name) - if index_to_mutate is None: - new_values = mutant_feature.mutant_value - else: - new_values = list(feature_list) - new_values[index_to_mutate] = mutant_feature.mutant_value - - del feature_list[:] - feature_list.extend(new_values) - mutant_examples.append(copied_example) - except (ValueError, IndexError): - # If the mutant value can't be set, still add the example to the - # mutant_example even though no change was made. This is necessary to - # allow for computation of global PD plots when not all examples have - # the same number of feature values for a feature. - mutant_examples.append(copied_example) - - return mutant_features, mutant_examples - - -def mutant_charts_for_feature(example_protos, feature_name, serving_bundles, - viz_params): - """Returns JSON formatted for rendering all charts for a feature. - - Args: - example_proto: The example protos to mutate. - feature_name: The string feature name to mutate. - serving_bundles: One `ServingBundle` object per model, that contains the - information to make the serving request. - viz_params: A `VizParams` object that contains the UI state of the request. - - Raises: - InvalidUserInputError if `viz_params.feature_index_pattern` requests out of - range indices for `feature_name` within `example_proto`. - - Returns: - A JSON-able dict for rendering a single mutant chart. parsed in - `tf-inference-dashboard.html`. - { - 'chartType': 'numeric', # oneof('numeric', 'categorical') - 'data': [A list of data] # parseable by vz-line-chart or vz-bar-chart - } - """ - - def chart_for_index(index_to_mutate): - mutant_features, mutant_examples = make_mutant_tuples( - example_protos, original_feature, index_to_mutate, viz_params) - - charts = [] - for serving_bundle in serving_bundles: - (inference_result_proto, _) = run_inference( - mutant_examples, serving_bundle) - charts.append(make_json_formatted_for_single_chart( - mutant_features, inference_result_proto, index_to_mutate)) - return charts - try: - original_feature = parse_original_feature_from_example( - example_protos[0], feature_name) - except ValueError as e: - return { - 'chartType': 'categorical', - 'data': [] - } + Args: + examples: Examples to read to get feature samples. + top_k: Max number of samples to return per feature. - indices_to_mutate = viz_params.feature_indices or range( - original_feature.length) - chart_type = ('categorical' if original_feature.feature_type == 'bytes_list' - else 'numeric') + Returns: + A dict of feature_name -> {'samples': ['Married-civ-spouse', + 'Never-married', 'Divorced']}. - try: - return { - 'chartType': chart_type, - 'data': [ - chart_for_index(index_to_mutate) - for index_to_mutate in indices_to_mutate + There is one key for each categorical feature. + + Currently, the inner dict just has one key, but this structure leaves room + for further expansion, and mirrors the structure used by + `get_numeric_features_to_observed_range`. + """ + observed_features = collections.defaultdict(list) # name -> [value, ] + for example in examples: + for feature_name in get_categorical_feature_names(example): + original_feature = parse_original_feature_from_example( + example, feature_name + ) + observed_features[feature_name].extend( + original_feature.original_value + ) + + result = {} + for feature_name, feature_values in sorted(iteritems(observed_features)): + samples = [ + word + for word, count in collections.Counter(feature_values).most_common( + top_k + ) + if count > 1 ] - } - except IndexError as e: - raise common_utils.InvalidUserInputError(e) - - -def make_json_formatted_for_single_chart(mutant_features, - inference_result_proto, - index_to_mutate): - """Returns JSON formatted for a single mutant chart. - - Args: - mutant_features: An iterable of `MutantFeatureValue`s representing the - X-axis. - inference_result_proto: A ClassificationResponse or RegressionResponse - returned by Servo, representing the Y-axis. - It contains one 'classification' or 'regression' for every Example that - was sent for inference. The length of that field should be the same length - of mutant_features. - index_to_mutate: The index of the feature being mutated for this chart. - - Returns: - A JSON-able dict for rendering a single mutant chart, parseable by - `vz-line-chart` or `vz-bar-chart`. - """ - x_label = 'step' - y_label = 'scalar' - - if isinstance(inference_result_proto, - classification_pb2.ClassificationResponse): - # classification_label -> [{x_label: y_label:}] - series = {} - - # ClassificationResponse has a separate probability for each label - for idx, classification in enumerate( - inference_result_proto.result.classifications): - # For each example to use for mutant inference, we create a copied example - # with the feature in question changed to each possible mutant value. So - # when we get the inferences back, we get num_examples*num_mutants - # results. So, modding by len(mutant_features) allows us to correctly - # lookup the mutant value for each inference. - mutant_feature = mutant_features[idx % len(mutant_features)] - for class_index, classification_class in enumerate( - classification.classes): - # Fill in class index when labels are missing - if classification_class.label == '': - classification_class.label = str(class_index) - # Special case to not include the "0" class in binary classification. - # Since that just results in a chart that is symmetric around 0.5. - if len( - classification.classes) == 2 and classification_class.label == '0': - continue - key = classification_class.label - if index_to_mutate: - key += ' (index %d)' % index_to_mutate - if not key in series: - series[key] = {} - mutant_val = ensure_not_binary(mutant_feature.mutant_value) - if not mutant_val in series[key]: - series[key][mutant_val] = [] - series[key][mutant_val].append( - classification_class.score) - - # Post-process points to have separate list for each class - return_series = collections.defaultdict(list) - for key, mutant_values in iteritems(series): - for value, y_list in iteritems(mutant_values): - return_series[key].append({ - x_label: value, - y_label: sum(y_list) / float(len(y_list)) - }) - return_series[key].sort(key=lambda p: p[x_label]) - return return_series - - elif isinstance(inference_result_proto, regression_pb2.RegressionResponse): - points = {} - - for idx, regression in enumerate(inference_result_proto.result.regressions): - # For each example to use for mutant inference, we create a copied example - # with the feature in question changed to each possible mutant value. So - # when we get the inferences back, we get num_examples*num_mutants - # results. So, modding by len(mutant_features) allows us to correctly - # lookup the mutant value for each inference. - mutant_feature = mutant_features[idx % len(mutant_features)] - mutant_val = ensure_not_binary(mutant_feature.mutant_value) - if not mutant_val in points: - points[mutant_val] = [] - points[mutant_val].append(regression.value) - key = 'value' - if (index_to_mutate != 0): - key += ' (index %d)' % index_to_mutate - list_of_points = [] - for value, y_list in iteritems(points): - list_of_points.append({ - x_label: value, - y_label: sum(y_list) / float(len(y_list)) - }) - list_of_points.sort(key=lambda p: p[x_label]) - return {key: list_of_points} - - else: - raise NotImplementedError('Only classification and regression implemented.') + if samples: + result[feature_name] = {"samples": samples} + return result + + +def make_mutant_features(original_feature, index_to_mutate, viz_params): + """Return a list of `MutantFeatureValue`s that are variants of original.""" + lower = viz_params.x_min + upper = viz_params.x_max + examples = viz_params.examples + num_mutants = viz_params.num_mutants + + if original_feature.feature_type == "float_list": + return [ + MutantFeatureValue(original_feature, index_to_mutate, value) + for value in np.linspace(lower, upper, num_mutants) + ] + elif original_feature.feature_type == "int64_list": + mutant_values = ( + np.linspace(int(lower), int(upper), num_mutants) + .astype(int) + .tolist() + ) + # Remove duplicates that can occur due to integer constraint. + mutant_values = sorted(set(mutant_values)) + return [ + MutantFeatureValue(original_feature, index_to_mutate, value) + for value in mutant_values + ] + elif original_feature.feature_type == "bytes_list": + feature_to_samples = get_categorical_features_to_sampling( + examples, num_mutants + ) + + # `mutant_values` looks like: + # [['Married-civ-spouse'], ['Never-married'], ['Divorced'], ['Separated']] + mutant_values = feature_to_samples[original_feature.feature_name][ + "samples" + ] + return [ + MutantFeatureValue(original_feature, None, value) + for value in mutant_values + ] + else: + raise ValueError( + "Malformed original feature had type of: " + + original_feature.feature_type + ) + + +def make_mutant_tuples( + example_protos, original_feature, index_to_mutate, viz_params +): + """Return a list of `MutantFeatureValue`s and a list of mutant Examples. + + Args: + example_protos: The examples to mutate. + original_feature: A `OriginalFeatureList` that encapsulates the feature to + mutate. + index_to_mutate: The index of the int64_list or float_list to mutate. + viz_params: A `VizParams` object that contains the UI state of the request. + + Returns: + A list of `MutantFeatureValue`s and a list of mutant examples. + """ + mutant_features = make_mutant_features( + original_feature, index_to_mutate, viz_params + ) + mutant_examples = [] + for example_proto in example_protos: + for mutant_feature in mutant_features: + copied_example = copy.deepcopy(example_proto) + feature_name = mutant_feature.original_feature.feature_name + + try: + feature_list = proto_value_for_feature( + copied_example, feature_name + ) + if index_to_mutate is None: + new_values = mutant_feature.mutant_value + else: + new_values = list(feature_list) + new_values[index_to_mutate] = mutant_feature.mutant_value + + del feature_list[:] + feature_list.extend(new_values) + mutant_examples.append(copied_example) + except (ValueError, IndexError): + # If the mutant value can't be set, still add the example to the + # mutant_example even though no change was made. This is necessary to + # allow for computation of global PD plots when not all examples have + # the same number of feature values for a feature. + mutant_examples.append(copied_example) + + return mutant_features, mutant_examples + + +def mutant_charts_for_feature( + example_protos, feature_name, serving_bundles, viz_params +): + """Returns JSON formatted for rendering all charts for a feature. + + Args: + example_proto: The example protos to mutate. + feature_name: The string feature name to mutate. + serving_bundles: One `ServingBundle` object per model, that contains the + information to make the serving request. + viz_params: A `VizParams` object that contains the UI state of the request. + + Raises: + InvalidUserInputError if `viz_params.feature_index_pattern` requests out of + range indices for `feature_name` within `example_proto`. + + Returns: + A JSON-able dict for rendering a single mutant chart. parsed in + `tf-inference-dashboard.html`. + { + 'chartType': 'numeric', # oneof('numeric', 'categorical') + 'data': [A list of data] # parseable by vz-line-chart or vz-bar-chart + } + """ + + def chart_for_index(index_to_mutate): + mutant_features, mutant_examples = make_mutant_tuples( + example_protos, original_feature, index_to_mutate, viz_params + ) + + charts = [] + for serving_bundle in serving_bundles: + (inference_result_proto, _) = run_inference( + mutant_examples, serving_bundle + ) + charts.append( + make_json_formatted_for_single_chart( + mutant_features, inference_result_proto, index_to_mutate + ) + ) + return charts + + try: + original_feature = parse_original_feature_from_example( + example_protos[0], feature_name + ) + except ValueError as e: + return {"chartType": "categorical", "data": []} + + indices_to_mutate = viz_params.feature_indices or range( + original_feature.length + ) + chart_type = ( + "categorical" + if original_feature.feature_type == "bytes_list" + else "numeric" + ) + + try: + return { + "chartType": chart_type, + "data": [ + chart_for_index(index_to_mutate) + for index_to_mutate in indices_to_mutate + ], + } + except IndexError as e: + raise common_utils.InvalidUserInputError(e) + + +def make_json_formatted_for_single_chart( + mutant_features, inference_result_proto, index_to_mutate +): + """Returns JSON formatted for a single mutant chart. + + Args: + mutant_features: An iterable of `MutantFeatureValue`s representing the + X-axis. + inference_result_proto: A ClassificationResponse or RegressionResponse + returned by Servo, representing the Y-axis. + It contains one 'classification' or 'regression' for every Example that + was sent for inference. The length of that field should be the same length + of mutant_features. + index_to_mutate: The index of the feature being mutated for this chart. + + Returns: + A JSON-able dict for rendering a single mutant chart, parseable by + `vz-line-chart` or `vz-bar-chart`. + """ + x_label = "step" + y_label = "scalar" + + if isinstance( + inference_result_proto, classification_pb2.ClassificationResponse + ): + # classification_label -> [{x_label: y_label:}] + series = {} + + # ClassificationResponse has a separate probability for each label + for idx, classification in enumerate( + inference_result_proto.result.classifications + ): + # For each example to use for mutant inference, we create a copied example + # with the feature in question changed to each possible mutant value. So + # when we get the inferences back, we get num_examples*num_mutants + # results. So, modding by len(mutant_features) allows us to correctly + # lookup the mutant value for each inference. + mutant_feature = mutant_features[idx % len(mutant_features)] + for class_index, classification_class in enumerate( + classification.classes + ): + # Fill in class index when labels are missing + if classification_class.label == "": + classification_class.label = str(class_index) + # Special case to not include the "0" class in binary classification. + # Since that just results in a chart that is symmetric around 0.5. + if ( + len(classification.classes) == 2 + and classification_class.label == "0" + ): + continue + key = classification_class.label + if index_to_mutate: + key += " (index %d)" % index_to_mutate + if not key in series: + series[key] = {} + mutant_val = ensure_not_binary(mutant_feature.mutant_value) + if not mutant_val in series[key]: + series[key][mutant_val] = [] + series[key][mutant_val].append(classification_class.score) + + # Post-process points to have separate list for each class + return_series = collections.defaultdict(list) + for key, mutant_values in iteritems(series): + for value, y_list in iteritems(mutant_values): + return_series[key].append( + {x_label: value, y_label: sum(y_list) / float(len(y_list))} + ) + return_series[key].sort(key=lambda p: p[x_label]) + return return_series + + elif isinstance(inference_result_proto, regression_pb2.RegressionResponse): + points = {} + + for idx, regression in enumerate( + inference_result_proto.result.regressions + ): + # For each example to use for mutant inference, we create a copied example + # with the feature in question changed to each possible mutant value. So + # when we get the inferences back, we get num_examples*num_mutants + # results. So, modding by len(mutant_features) allows us to correctly + # lookup the mutant value for each inference. + mutant_feature = mutant_features[idx % len(mutant_features)] + mutant_val = ensure_not_binary(mutant_feature.mutant_value) + if not mutant_val in points: + points[mutant_val] = [] + points[mutant_val].append(regression.value) + key = "value" + if index_to_mutate != 0: + key += " (index %d)" % index_to_mutate + list_of_points = [] + for value, y_list in iteritems(points): + list_of_points.append( + {x_label: value, y_label: sum(y_list) / float(len(y_list))} + ) + list_of_points.sort(key=lambda p: p[x_label]) + return {key: list_of_points} + + else: + raise NotImplementedError( + "Only classification and regression implemented." + ) def get_example_features(example): - """Returns the non-sequence features from the provided example.""" - return (example.features.feature if isinstance(example, tf.train.Example) - else example.context.feature) + """Returns the non-sequence features from the provided example.""" + return ( + example.features.feature + if isinstance(example, tf.train.Example) + else example.context.feature + ) + def run_inference_for_inference_results(examples, serving_bundle): - """Calls servo and wraps the inference results.""" - (inference_result_proto, extra_results) = run_inference( - examples, serving_bundle) - inferences = wrap_inference_results(inference_result_proto) - infer_json = json_format.MessageToJson( - inferences, including_default_value_fields=True) - return json.loads(infer_json), extra_results + """Calls servo and wraps the inference results.""" + (inference_result_proto, extra_results) = run_inference( + examples, serving_bundle + ) + inferences = wrap_inference_results(inference_result_proto) + infer_json = json_format.MessageToJson( + inferences, including_default_value_fields=True + ) + return json.loads(infer_json), extra_results + def get_eligible_features(examples, num_mutants): - """Returns a list of JSON objects for each feature in the examples. + """Returns a list of JSON objects for each feature in the examples. This list is used to drive partial dependence plots in the plugin. @@ -650,84 +733,88 @@ def get_eligible_features(examples, num_mutants): Numeric features are represented as {name: observedMin: observedMax:}. Categorical features are repesented as {name: samples:[]}. """ - features_dict = ( - get_numeric_features_to_observed_range( - examples)) - - features_dict.update( - get_categorical_features_to_sampling( - examples, num_mutants)) - - # Massage the features_dict into a sorted list before returning because - # Polymer dom-repeat needs a list. - features_list = [] - for k, v in sorted(features_dict.items()): - v['name'] = k - features_list.append(v) - return features_list + features_dict = get_numeric_features_to_observed_range(examples) + + features_dict.update( + get_categorical_features_to_sampling(examples, num_mutants) + ) + + # Massage the features_dict into a sorted list before returning because + # Polymer dom-repeat needs a list. + features_list = [] + for k, v in sorted(features_dict.items()): + v["name"] = k + features_list.append(v) + return features_list + def sort_eligible_features(features_list, chart_data): - """Returns a sorted list of objects representing each feature. - - The list is sorted by interestingness in terms of the resulting change in - inference values across feature values, for partial dependence plots. - - Args: - features_list: A list of eligible features in the format of the return - from the get_eligible_features function. - chart_data: A dict of feature names to chart data, formatted as the - output from the mutant_charts_for_feature function. - - Returns: - A sorted list of the inputted features_list, with the addition of - an 'interestingness' key with a calculated number for feature feature. - The list is sorted with the feature with highest interestingness first. - """ - sorted_features_list = copy.deepcopy(features_list) - for feature in sorted_features_list: - name = feature['name'] - charts = chart_data[name] - max_measure = 0 - is_numeric = charts['chartType'] == 'numeric' - for models in charts['data']: - for chart in models: - for series in chart.values(): - if is_numeric: - # For numeric features, interestingness is the total Y distance - # traveled across the line chart. - measure = 0 - for i in range(len(series) - 1): - measure += abs(series[i]['scalar'] - series[i + 1]['scalar']) - else: - # For categorical features, interestingness is the difference - # between the min and max Y values in the chart, as interestingness - # for categorical charts shouldn't depend on the order of items - # being charted. - min_y = float("inf") - max_y = float("-inf") - for i in range(len(series)): - val = series[i]['scalar'] - if val < min_y: - min_y = val - if val > max_y: - max_y = val - measure = max_y - min_y - if measure > max_measure: - max_measure = measure - feature['interestingness'] = max_measure - - return sorted( - sorted_features_list, key=lambda x: x['interestingness'], reverse=True) + """Returns a sorted list of objects representing each feature. + + The list is sorted by interestingness in terms of the resulting change in + inference values across feature values, for partial dependence plots. + + Args: + features_list: A list of eligible features in the format of the return + from the get_eligible_features function. + chart_data: A dict of feature names to chart data, formatted as the + output from the mutant_charts_for_feature function. + + Returns: + A sorted list of the inputted features_list, with the addition of + an 'interestingness' key with a calculated number for feature feature. + The list is sorted with the feature with highest interestingness first. + """ + sorted_features_list = copy.deepcopy(features_list) + for feature in sorted_features_list: + name = feature["name"] + charts = chart_data[name] + max_measure = 0 + is_numeric = charts["chartType"] == "numeric" + for models in charts["data"]: + for chart in models: + for series in chart.values(): + if is_numeric: + # For numeric features, interestingness is the total Y distance + # traveled across the line chart. + measure = 0 + for i in range(len(series) - 1): + measure += abs( + series[i]["scalar"] - series[i + 1]["scalar"] + ) + else: + # For categorical features, interestingness is the difference + # between the min and max Y values in the chart, as interestingness + # for categorical charts shouldn't depend on the order of items + # being charted. + min_y = float("inf") + max_y = float("-inf") + for i in range(len(series)): + val = series[i]["scalar"] + if val < min_y: + min_y = val + if val > max_y: + max_y = val + measure = max_y - min_y + if measure > max_measure: + max_measure = measure + feature["interestingness"] = max_measure + + return sorted( + sorted_features_list, key=lambda x: x["interestingness"], reverse=True + ) + def get_label_vocab(vocab_path): - """Returns a list of label strings loaded from the provided path.""" - if vocab_path: - try: - with tf.io.gfile.GFile(vocab_path, 'r') as f: - return [line.rstrip('\n') for line in f] - except tf.errors.NotFoundError as err: - logging.error('error reading vocab file: %s', err) - return [] + """Returns a list of label strings loaded from the provided path.""" + if vocab_path: + try: + with tf.io.gfile.GFile(vocab_path, "r") as f: + return [line.rstrip("\n") for line in f] + except tf.errors.NotFoundError as err: + logging.error("error reading vocab file: %s", err) + return [] + def create_sprite_image(examples): """Returns an encoded sprite image for use in Facets Dive. @@ -740,123 +827,141 @@ def create_sprite_image(examples): """ def generate_image_from_thubnails(thumbnails, thumbnail_dims): - """Generates a sprite atlas image from a set of thumbnails.""" - num_thumbnails = tf.shape(thumbnails)[0].eval() - images_per_row = int(math.ceil(math.sqrt(num_thumbnails))) - thumb_height = thumbnail_dims[0] - thumb_width = thumbnail_dims[1] - master_height = images_per_row * thumb_height - master_width = images_per_row * thumb_width - num_channels = 3 - master = np.zeros([master_height, master_width, num_channels]) - for idx, image in enumerate(thumbnails.eval()): - left_idx = idx % images_per_row - top_idx = int(math.floor(idx / images_per_row)) - left_start = left_idx * thumb_width - left_end = left_start + thumb_width - top_start = top_idx * thumb_height - top_end = top_start + thumb_height - master[top_start:top_end, left_start:left_end, :] = image - return tf.image.encode_png(master) - - image_feature_name = 'image/encoded' + """Generates a sprite atlas image from a set of thumbnails.""" + num_thumbnails = tf.shape(thumbnails)[0].eval() + images_per_row = int(math.ceil(math.sqrt(num_thumbnails))) + thumb_height = thumbnail_dims[0] + thumb_width = thumbnail_dims[1] + master_height = images_per_row * thumb_height + master_width = images_per_row * thumb_width + num_channels = 3 + master = np.zeros([master_height, master_width, num_channels]) + for idx, image in enumerate(thumbnails.eval()): + left_idx = idx % images_per_row + top_idx = int(math.floor(idx / images_per_row)) + left_start = left_idx * thumb_width + left_end = left_start + thumb_width + top_start = top_idx * thumb_height + top_end = top_start + thumb_height + master[top_start:top_end, left_start:left_end, :] = image + return tf.image.encode_png(master) + + image_feature_name = "image/encoded" sprite_thumbnail_dim_px = 32 with tf.compat.v1.Session(): - keys_to_features = { - image_feature_name: - tf.io.FixedLenFeature((), tf.string, default_value=''), - } - parsed = tf.io.parse_example(examples, keys_to_features) - images = tf.zeros([1, 1, 1, 1], tf.float32) - i = tf.constant(0) - thumbnail_dims = (sprite_thumbnail_dim_px, - sprite_thumbnail_dim_px) - num_examples = tf.constant(len(examples)) - encoded_images = parsed[image_feature_name] - - # Loop over all examples, decoding the image feature value, resizing - # and appending to a list of all images. - def loop_body(i, encoded_images, images): - encoded_image = encoded_images[i] - image = tf.image.decode_jpeg(encoded_image, channels=3) - resized_image = tf.image.resize(image, thumbnail_dims) - expanded_image = tf.expand_dims(resized_image, 0) - images = tf.cond( - tf.equal(i, 0), lambda: expanded_image, - lambda: tf.concat([images, expanded_image], 0)) - return i + 1, encoded_images, images - - loop_out = tf.while_loop( - lambda i, encoded_images, images: tf.less(i, num_examples), - loop_body, [i, encoded_images, images], - shape_invariants=[ - i.get_shape(), - encoded_images.get_shape(), - tf.TensorShape(None) - ]) - - # Create the single sprite atlas image from these thumbnails. - sprite = generate_image_from_thubnails(loop_out[2], thumbnail_dims) - return sprite.eval() + keys_to_features = { + image_feature_name: tf.io.FixedLenFeature( + (), tf.string, default_value="" + ), + } + parsed = tf.io.parse_example(examples, keys_to_features) + images = tf.zeros([1, 1, 1, 1], tf.float32) + i = tf.constant(0) + thumbnail_dims = (sprite_thumbnail_dim_px, sprite_thumbnail_dim_px) + num_examples = tf.constant(len(examples)) + encoded_images = parsed[image_feature_name] + + # Loop over all examples, decoding the image feature value, resizing + # and appending to a list of all images. + def loop_body(i, encoded_images, images): + encoded_image = encoded_images[i] + image = tf.image.decode_jpeg(encoded_image, channels=3) + resized_image = tf.image.resize(image, thumbnail_dims) + expanded_image = tf.expand_dims(resized_image, 0) + images = tf.cond( + tf.equal(i, 0), + lambda: expanded_image, + lambda: tf.concat([images, expanded_image], 0), + ) + return i + 1, encoded_images, images + + loop_out = tf.while_loop( + lambda i, encoded_images, images: tf.less(i, num_examples), + loop_body, + [i, encoded_images, images], + shape_invariants=[ + i.get_shape(), + encoded_images.get_shape(), + tf.TensorShape(None), + ], + ) + + # Create the single sprite atlas image from these thumbnails. + sprite = generate_image_from_thubnails(loop_out[2], thumbnail_dims) + return sprite.eval() + def run_inference(examples, serving_bundle): - """Run inference on examples given model information - - Args: - examples: A list of examples that matches the model spec. - serving_bundle: A `ServingBundle` object that contains the information to - make the inference request. - - Returns: - A tuple with the first entry being the ClassificationResponse or - RegressionResponse proto and the second entry being a dictionary of extra - data for each example, such as attributions, or None if no data exists. - """ - batch_size = 64 - if serving_bundle.estimator and serving_bundle.feature_spec: - # If provided an estimator and feature spec then run inference locally. - preds = serving_bundle.estimator.predict( - lambda: tf.data.Dataset.from_tensor_slices( - tf.io.parse_example([ex.SerializeToString() for ex in examples], - serving_bundle.feature_spec)).batch(batch_size)) - - # Use the specified key if one is provided. - key_to_use = (serving_bundle.predict_output_tensor - if serving_bundle.use_predict else None) - - values = [] - for pred in preds: - if key_to_use is None: - # If the prediction dictionary only contains one key, use it. - returned_keys = list(pred.keys()) - if len(returned_keys) == 1: - key_to_use = returned_keys[0] - # Use default keys if necessary. - elif serving_bundle.model_type == 'classification': - key_to_use = 'probabilities' + """Run inference on examples given model information. + + Args: + examples: A list of examples that matches the model spec. + serving_bundle: A `ServingBundle` object that contains the information to + make the inference request. + + Returns: + A tuple with the first entry being the ClassificationResponse or + RegressionResponse proto and the second entry being a dictionary of extra + data for each example, such as attributions, or None if no data exists. + """ + batch_size = 64 + if serving_bundle.estimator and serving_bundle.feature_spec: + # If provided an estimator and feature spec then run inference locally. + preds = serving_bundle.estimator.predict( + lambda: tf.data.Dataset.from_tensor_slices( + tf.io.parse_example( + [ex.SerializeToString() for ex in examples], + serving_bundle.feature_spec, + ) + ).batch(batch_size) + ) + + # Use the specified key if one is provided. + key_to_use = ( + serving_bundle.predict_output_tensor + if serving_bundle.use_predict + else None + ) + + values = [] + for pred in preds: + if key_to_use is None: + # If the prediction dictionary only contains one key, use it. + returned_keys = list(pred.keys()) + if len(returned_keys) == 1: + key_to_use = returned_keys[0] + # Use default keys if necessary. + elif serving_bundle.model_type == "classification": + key_to_use = "probabilities" + else: + key_to_use = "predictions" + if key_to_use not in pred: + raise KeyError( + '"%s" not found in model predictions dictionary' + % key_to_use + ) + + values.append(pred[key_to_use]) + return ( + common_utils.convert_prediction_values(values, serving_bundle), + None, + ) + elif serving_bundle.custom_predict_fn: + # If custom_predict_fn is provided, pass examples directly for local + # inference. + values = serving_bundle.custom_predict_fn(examples) + extra_results = None + # If the custom prediction function returned a dict, then parse out the + # prediction scores. If it is just a list, then the results are the + # prediction results without attributions or other data. + if isinstance(values, dict): + preds = values.pop("predictions") + extra_results = values else: - key_to_use = 'predictions' - if key_to_use not in pred: - raise KeyError( - '"%s" not found in model predictions dictionary' % key_to_use) - - values.append(pred[key_to_use]) - return (common_utils.convert_prediction_values(values, serving_bundle), - None) - elif serving_bundle.custom_predict_fn: - # If custom_predict_fn is provided, pass examples directly for local - # inference. - values = serving_bundle.custom_predict_fn(examples) - extra_results = None - # If the custom prediction function returned a dict, then parse out the - # prediction scores. If it is just a list, then the results are the - # prediction results without attributions or other data. - if isinstance(values, dict): - preds = values.pop('predictions') - extra_results = values + preds = values + return ( + common_utils.convert_prediction_values(preds, serving_bundle), + extra_results, + ) else: - preds = values - return (common_utils.convert_prediction_values(preds, serving_bundle), - extra_results) - else: - return (platform_utils.call_servo(examples, serving_bundle), None) + return (platform_utils.call_servo(examples, serving_bundle), None) diff --git a/tensorboard/plugins/interactive_inference/utils/inference_utils_test.py b/tensorboard/plugins/interactive_inference/utils/inference_utils_test.py index 6598a4940a..16abcc7bee 100644 --- a/tensorboard/plugins/interactive_inference/utils/inference_utils_test.py +++ b/tensorboard/plugins/interactive_inference/utils/inference_utils_test.py @@ -25,10 +25,10 @@ import tensorflow as tf try: - # python version >= 3.3 - from unittest import mock + # python version >= 3.3 + from unittest import mock except ImportError: - import mock # pylint: disable=unused-import + import mock # pylint: disable=unused-import from tensorflow_serving.apis import classification_pb2 from tensorflow_serving.apis import predict_pb2 @@ -41,518 +41,630 @@ class InferenceUtilsTest(tf.test.TestCase): - - def setUp(self): - self.logdir = tf.compat.v1.test.get_temp_dir() - self.examples_path = os.path.join(self.logdir, 'example.pb') - - def tearDown(self): - try: - os.remove(self.examples_path) - except EnvironmentError: - pass - - def make_and_write_fake_example(self): - """Make example and write it to self.examples_path.""" - example = test_utils.make_fake_example() - test_utils.write_out_examples([example], self.examples_path) - return example - - def test_parse_original_feature_from_example(self): - example = test_utils.make_fake_example() - original_feature = inference_utils.parse_original_feature_from_example( - example, 'repeated_float') - self.assertEqual('repeated_float', original_feature.feature_name) - self.assertEqual([1.0, 2.0, 3.0, 4.0], original_feature.original_value) - self.assertEqual('float_list', original_feature.feature_type) - self.assertEqual(4, original_feature.length) - - original_feature = inference_utils.parse_original_feature_from_example( - example, 'repeated_int') - self.assertEqual('repeated_int', original_feature.feature_name) - self.assertEqual([10, 20], original_feature.original_value) - self.assertEqual('int64_list', original_feature.feature_type) - self.assertEqual(2, original_feature.length) - - original_feature = inference_utils.parse_original_feature_from_example( - example, 'single_int') - self.assertEqual('single_int', original_feature.feature_name) - self.assertEqual([0], original_feature.original_value) - self.assertEqual('int64_list', original_feature.feature_type) - self.assertEqual(1, original_feature.length) - - def test_parse_original_feature_from_example_binary(self): - example = tf.train.Example() - example.features.feature['img'].bytes_list.value.extend([b'\xef']) - original_feature = inference_utils.parse_original_feature_from_example( - example, 'img') - self.assertEqual('img', original_feature.feature_name) - self.assertEqual([b'\xef'], original_feature.original_value) - - def test_example_protos_from_path_get_all_in_file(self): - cns_path = os.path.join(tf.compat.v1.test.get_temp_dir(), - 'dummy_example') - example = test_utils.make_fake_example() - test_utils.write_out_examples([example], cns_path) - dummy_examples = platform_utils.example_protos_from_path(cns_path) - self.assertEqual(1, len(dummy_examples)) - self.assertEqual(example, dummy_examples[0]) - - def test_example_protos_from_path_get_two(self): - cns_path = os.path.join(tf.compat.v1.test.get_temp_dir(), - 'dummy_example') - example_one = test_utils.make_fake_example(1) - example_two = test_utils.make_fake_example(2) - example_three = test_utils.make_fake_example(3) - test_utils.write_out_examples([example_one, example_two, example_three], - cns_path) - dummy_examples = platform_utils.example_protos_from_path(cns_path, 2) - self.assertEqual(2, len(dummy_examples)) - self.assertEqual(example_one, dummy_examples[0]) - self.assertEqual(example_two, dummy_examples[1]) - - def test_example_protos_from_path_use_wildcard(self): - cns_path = os.path.join(tf.compat.v1.test.get_temp_dir(), - 'wildcard_example1') - example1 = test_utils.make_fake_example(1) - test_utils.write_out_examples([example1], cns_path) - cns_path = os.path.join(tf.compat.v1.test.get_temp_dir(), - 'wildcard_example2') - example2 = test_utils.make_fake_example(2) - test_utils.write_out_examples([example2], cns_path) - - wildcard_path = os.path.join(tf.compat.v1.test.get_temp_dir(), - 'wildcard_example*') - dummy_examples = platform_utils.example_protos_from_path( - wildcard_path) - self.assertEqual(2, len(dummy_examples)) - - def test_example_proto_from_path_if_does_not_exist(self): - cns_path = os.path.join(tf.compat.v1.test.get_temp_dir(), 'does_not_exist') - with self.assertRaises(common_utils.InvalidUserInputError): - platform_utils.example_protos_from_path(cns_path) - - def test_get_numeric_features(self): - example = test_utils.make_fake_example(single_int_val=2) - data = inference_utils.get_numeric_feature_names(example) - self.assertEqual( - ['repeated_float', 'repeated_int', 'single_float', 'single_int'], data) - - def test_get_numeric_features_to_observed_range(self): - example = test_utils.make_fake_example(single_int_val=2) - - data = inference_utils.get_numeric_features_to_observed_range( - [example]) - - # Returns a sorted list by feature_name. - self.assertDictEqual({ - 'repeated_float': { - 'observedMin': 1., - 'observedMax': 4., - }, - 'repeated_int': { - 'observedMin': 10, - 'observedMax': 20, - }, - 'single_float': { - 'observedMin': 24.5, - 'observedMax': 24.5, - }, - 'single_int': { - 'observedMin': 2., - 'observedMax': 2., - }, - }, data) - - def test_get_categorical_features_to_sampling(self): - cat_example = tf.train.Example() - cat_example.features.feature['non_numeric'].bytes_list.value.extend( - [b'cat']) - - cow_example = tf.train.Example() - cow_example.features.feature['non_numeric'].bytes_list.value.extend( - [b'cow']) - - pony_example = tf.train.Example() - pony_example.features.feature['non_numeric'].bytes_list.value.extend( - [b'pony']) - - examples = [cat_example] * 4 + [cow_example] * 5 + [pony_example] * 10 - - # If we stop sampling at the first 3 examples, the only example should be - # cat example. - data = inference_utils.get_categorical_features_to_sampling( - examples[0: 3], top_k=1) - self.assertDictEqual({ - 'non_numeric': { - 'samples': ['cat'] - } - }, data) - - # If we sample more examples, the top 2 examples should be cow and pony. - data = inference_utils.get_categorical_features_to_sampling( - examples[0: 20], top_k=2) - self.assertDictEqual({ - 'non_numeric': { - 'samples': ['pony', 'cow'] - } - }, data) - - def test_wrap_inference_results_classification(self): - """Test wrapping a classification result.""" - inference_result_proto = classification_pb2.ClassificationResponse() - classification = inference_result_proto.result.classifications.add() - inference_class = classification.classes.add() - inference_class.label = 'class_b' - inference_class.score = 0.3 - inference_class = classification.classes.add() - inference_class.label = 'class_a' - inference_class.score = 0.7 - - wrapped = inference_utils.wrap_inference_results(inference_result_proto) - self.assertEqual(1, len(wrapped.classification_result.classifications)) - self.assertEqual( - 2, len(wrapped.classification_result.classifications[0].classes)) - - def test_wrap_inference_results_regression(self): - """Test wrapping a regression result.""" - inference_result_proto = regression_pb2.RegressionResponse() - regression = inference_result_proto.result.regressions.add() - regression.value = 0.45 - regression = inference_result_proto.result.regressions.add() - regression.value = 0.55 - - wrapped = inference_utils.wrap_inference_results(inference_result_proto) - self.assertEqual(2, len(wrapped.regression_result.regressions)) - - @mock.patch.object(inference_utils, 'make_json_formatted_for_single_chart') - @mock.patch.object(platform_utils, 'call_servo') - def test_mutant_charts_for_feature(self, mock_call_servo, - mock_make_json_formatted_for_single_chart): - example = self.make_and_write_fake_example() - serving_bundles = [inference_utils.ServingBundle('', '', 'classification', - '', '', False, '', '')] - num_mutants = 10 - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples=[example], - num_mutants=num_mutants, - feature_index_pattern=None) - - mock_call_servo = lambda _, __: None - mock_make_json_formatted_for_single_chart = lambda _, __: {} - charts = inference_utils.mutant_charts_for_feature( - [example], 'repeated_float', serving_bundles, viz_params) - self.assertEqual('numeric', charts['chartType']) - self.assertEqual(4, len(charts['data'])) - charts = inference_utils.mutant_charts_for_feature( - [example], 'repeated_int', serving_bundles, viz_params) - self.assertEqual('numeric', charts['chartType']) - self.assertEqual(2, len(charts['data'])) - charts = inference_utils.mutant_charts_for_feature( - [example], 'single_int', serving_bundles, viz_params) - self.assertEqual('numeric', charts['chartType']) - self.assertEqual(1, len(charts['data'])) - charts = inference_utils.mutant_charts_for_feature( - [example], 'non_numeric', serving_bundles, viz_params) - self.assertEqual('categorical', charts['chartType']) - self.assertEqual(3, len(charts['data'])) - - @mock.patch.object(inference_utils, 'make_json_formatted_for_single_chart') - @mock.patch.object(platform_utils, 'call_servo') - def test_mutant_charts_for_feature_with_feature_index_pattern( - self, mock_call_servo, mock_make_json_formatted_for_single_chart): - example = self.make_and_write_fake_example() - serving_bundles = [inference_utils.ServingBundle('', '', 'classification', - '', '', False, '', '')] - num_mutants = 10 - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples=[example], - num_mutants=num_mutants, - feature_index_pattern='0 , 2-3') - - mock_call_servo = lambda _, __: None - mock_make_json_formatted_for_single_chart = lambda _, __: {} - charts = inference_utils.mutant_charts_for_feature( - [example], 'repeated_float', serving_bundles, viz_params) - self.assertEqual('numeric', charts['chartType']) - self.assertEqual(3, len(charts['data'])) - - # These should return 3 charts even though all fields from the index - # pattern don't exist for the example. - charts = inference_utils.mutant_charts_for_feature( - [example], 'repeated_int', serving_bundles, viz_params) - self.assertEqual('numeric', charts['chartType']) - self.assertEqual(3, len(charts['data'])) - - charts = inference_utils.mutant_charts_for_feature( - [example], 'single_int', serving_bundles, viz_params) - self.assertEqual('numeric', charts['chartType']) - self.assertEqual(3, len(charts['data'])) - - def test_make_mutant_tuples_float_list(self): - example = self.make_and_write_fake_example() - index_to_mutate = 1 - num_mutants = 10 - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples = [example], - num_mutants=num_mutants, - feature_index_pattern=None) - - original_feature = inference_utils.parse_original_feature_from_example( - example, 'repeated_float') - mutant_features, mutant_examples = inference_utils.make_mutant_tuples( - [example], - original_feature, - index_to_mutate=index_to_mutate, - viz_params=viz_params) - - # Check that values in mutant_features and mutant_examples are as expected. - expected_values = np.linspace(1, 10, num_mutants) - np.testing.assert_almost_equal( - expected_values, - [mutant_feature.mutant_value for mutant_feature in mutant_features]) - np.testing.assert_almost_equal(expected_values, [ - mutant_example.features.feature['repeated_float'] - .float_list.value[index_to_mutate] for mutant_example in mutant_examples - ]) - - # Check that the example (other than the mutant value) is the same. - for expected_value, mutant_example in zip(expected_values, mutant_examples): - mutant_values = test_utils.value_from_example(mutant_example, - 'repeated_float') - original_values = test_utils.value_from_example(example, 'repeated_float') - original_values[index_to_mutate] = expected_value - self.assertEqual(original_values, mutant_values) - - def test_make_mutant_tuples_int_list(self): - example = self.make_and_write_fake_example() - index_to_mutate = 1 - num_mutants = 10 - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples = [example], - num_mutants=num_mutants, - feature_index_pattern=None) - original_feature = inference_utils.parse_original_feature_from_example( - example, 'repeated_int') - mutant_features, mutant_examples = inference_utils.make_mutant_tuples( - [example], - original_feature, - index_to_mutate=index_to_mutate, - viz_params=viz_params) - - # Check that values in mutant_features and mutant_examples are as expected. - expected_values = np.linspace(1, 10, num_mutants) - np.testing.assert_almost_equal( - expected_values, - [mutant_feature.mutant_value for mutant_feature in mutant_features]) - np.testing.assert_almost_equal(expected_values, [ - mutant_example.features.feature['repeated_int'] - .int64_list.value[index_to_mutate] for mutant_example in mutant_examples - ]) - - # Check that the example (other than the mutant value) is the same. - for expected_value, mutant_example in zip(expected_values, mutant_examples): - mutant_values = test_utils.value_from_example(mutant_example, - 'repeated_int') - original_values = test_utils.value_from_example(example, 'repeated_int') - original_values[index_to_mutate] = expected_value - self.assertEqual(original_values, mutant_values) - - def test_make_json_formatted_for_single_chart_classification(self): - """Test making a classification chart with a single point on it.""" - inference_result_proto = classification_pb2.ClassificationResponse() - classification = inference_result_proto.result.classifications.add() - inference_class = classification.classes.add() - inference_class.label = 'class_a' - inference_class.score = 0.7 - - inference_class = classification.classes.add() - inference_class.label = 'class_b' - inference_class.score = 0.3 - - original_feature = inference_utils.OriginalFeatureList( - 'feature_name', [2.], 'float_list') - mutant_feature = inference_utils.MutantFeatureValue( - original_feature, index=0, mutant_value=20) - - jsonable = inference_utils.make_json_formatted_for_single_chart( - [mutant_feature], inference_result_proto, 0) - - self.assertEqual(['class_a', 'class_b'], sorted(jsonable.keys())) - self.assertEqual(1, len(jsonable['class_a'])) - self.assertEqual(20, jsonable['class_a'][0]['step']) - self.assertAlmostEqual(0.7, jsonable['class_a'][0]['scalar']) - - self.assertEqual(1, len(jsonable['class_b'])) - self.assertEqual(20, jsonable['class_b'][0]['step']) - self.assertAlmostEqual(0.3, jsonable['class_b'][0]['scalar']) - - def test_make_json_formatted_for_single_chart_regression(self): - """Test making a regression chart with a single point on it.""" - inference_result_proto = regression_pb2.RegressionResponse() - regression = inference_result_proto.result.regressions.add() - regression.value = 0.45 - regression = inference_result_proto.result.regressions.add() - regression.value = 0.55 - - original_feature = inference_utils.OriginalFeatureList( - 'feature_name', [2.], 'float_list') - mutant_feature = inference_utils.MutantFeatureValue( - original_feature, index=0, mutant_value=20) - mutant_feature_2 = inference_utils.MutantFeatureValue( - original_feature, index=0, mutant_value=10) - - jsonable = inference_utils.make_json_formatted_for_single_chart( - [mutant_feature, mutant_feature_2], inference_result_proto, 0) - - self.assertEqual(['value'], list(jsonable.keys())) - self.assertEqual(2, len(jsonable['value'])) - self.assertEqual(10, jsonable['value'][0]['step']) - self.assertAlmostEqual(0.55, jsonable['value'][0]['scalar']) - self.assertEqual(20, jsonable['value'][1]['step']) - self.assertAlmostEqual(0.45, jsonable['value'][1]['scalar']) - - def test_convert_predict_response_regression(self): - """Test converting a PredictResponse to a RegressionResponse.""" - predict = predict_pb2.PredictResponse() - output = predict.outputs['scores'] - dim = output.tensor_shape.dim.add() - dim.size = 2 - output.float_val.extend([0.1, 0.2]) - - bundle = inference_utils.ServingBundle( - '', '', 'regression', '', '', True, '', 'scores') - converted = common_utils.convert_predict_response(predict, bundle) - - self.assertAlmostEqual(0.1, converted.result.regressions[0].value) - self.assertAlmostEqual(0.2, converted.result.regressions[1].value) - - def test_convert_predict_response_classification(self): - """Test converting a PredictResponse to a ClassificationResponse.""" - predict = predict_pb2.PredictResponse() - output = predict.outputs['probabilities'] - dim = output.tensor_shape.dim.add() - dim.size = 3 - dim = output.tensor_shape.dim.add() - dim.size = 2 - output.float_val.extend([1., 0., .9, .1, .8, .2]) - - bundle = inference_utils.ServingBundle( - '', '', 'classification', '', '', True, '', 'probabilities') - converted = common_utils.convert_predict_response(predict, bundle) - - self.assertEqual("0", converted.result.classifications[0].classes[0].label) - self.assertAlmostEqual( - 1, converted.result.classifications[0].classes[0].score) - self.assertEqual("1", converted.result.classifications[0].classes[1].label) - self.assertAlmostEqual( - 0, converted.result.classifications[0].classes[1].score) - - self.assertEqual("0", converted.result.classifications[1].classes[0].label) - self.assertAlmostEqual( - .9, converted.result.classifications[1].classes[0].score) - self.assertEqual("1", converted.result.classifications[1].classes[1].label) - self.assertAlmostEqual( - .1, converted.result.classifications[1].classes[1].score) - - self.assertEqual("0", converted.result.classifications[2].classes[0].label) - self.assertAlmostEqual( - .8, converted.result.classifications[2].classes[0].score) - self.assertEqual("1", converted.result.classifications[2].classes[1].label) - self.assertAlmostEqual( - .2, converted.result.classifications[2].classes[1].score) - - def test_vizparams_pattern_parser(self): - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples=[], - num_mutants=0, - feature_index_pattern=None) - self.assertEqual([], viz_params.feature_indices) - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples=[], - num_mutants=0, - feature_index_pattern='1-3') - self.assertEqual([1, 2, 3], viz_params.feature_indices) - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples=[], - num_mutants=0, - feature_index_pattern='3-1') - self.assertEqual([], viz_params.feature_indices) - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples=[], - num_mutants=0, - feature_index_pattern='1-1') - self.assertEqual([1], viz_params.feature_indices) - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples=[], - num_mutants=0, - feature_index_pattern='3, 1') - self.assertEqual([1, 3], viz_params.feature_indices) - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples=[], - num_mutants=0, - feature_index_pattern='0-') - self.assertEqual([], viz_params.feature_indices) - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples=[], - num_mutants=0, - feature_index_pattern='0-error') - self.assertEqual([], viz_params.feature_indices) - viz_params = inference_utils.VizParams( - x_min=1, - x_max=10, - examples=[], - num_mutants=0, - feature_index_pattern='0-3-5') - self.assertEqual([], viz_params.feature_indices) - - def test_sort_eligible_features(self): - features_list = [{'name': 'feat1'}, {'name': 'feat2'}] - chart_data = { - 'feat1': { - 'chartType': 'numeric', - 'data': [[ - {'series1': [{'scalar': .2}, {'scalar': .1}, {'scalar': .3}]}, - {'series2': [{'scalar': .2}, {'scalar': .1}, {'scalar': .4}]}, - ]] - }, - 'feat2': { - 'chartType': 'categorical', - 'data': [[ - {'series1': [{'scalar': .2}, {'scalar': .1}, {'scalar': .3}]}, - {'series2': [{'scalar': .2}, {'scalar': .1}, {'scalar': .9}]}, - ]] + def setUp(self): + self.logdir = tf.compat.v1.test.get_temp_dir() + self.examples_path = os.path.join(self.logdir, "example.pb") + + def tearDown(self): + try: + os.remove(self.examples_path) + except EnvironmentError: + pass + + def make_and_write_fake_example(self): + """Make example and write it to self.examples_path.""" + example = test_utils.make_fake_example() + test_utils.write_out_examples([example], self.examples_path) + return example + + def test_parse_original_feature_from_example(self): + example = test_utils.make_fake_example() + original_feature = inference_utils.parse_original_feature_from_example( + example, "repeated_float" + ) + self.assertEqual("repeated_float", original_feature.feature_name) + self.assertEqual([1.0, 2.0, 3.0, 4.0], original_feature.original_value) + self.assertEqual("float_list", original_feature.feature_type) + self.assertEqual(4, original_feature.length) + + original_feature = inference_utils.parse_original_feature_from_example( + example, "repeated_int" + ) + self.assertEqual("repeated_int", original_feature.feature_name) + self.assertEqual([10, 20], original_feature.original_value) + self.assertEqual("int64_list", original_feature.feature_type) + self.assertEqual(2, original_feature.length) + + original_feature = inference_utils.parse_original_feature_from_example( + example, "single_int" + ) + self.assertEqual("single_int", original_feature.feature_name) + self.assertEqual([0], original_feature.original_value) + self.assertEqual("int64_list", original_feature.feature_type) + self.assertEqual(1, original_feature.length) + + def test_parse_original_feature_from_example_binary(self): + example = tf.train.Example() + example.features.feature["img"].bytes_list.value.extend([b"\xef"]) + original_feature = inference_utils.parse_original_feature_from_example( + example, "img" + ) + self.assertEqual("img", original_feature.feature_name) + self.assertEqual([b"\xef"], original_feature.original_value) + + def test_example_protos_from_path_get_all_in_file(self): + cns_path = os.path.join( + tf.compat.v1.test.get_temp_dir(), "dummy_example" + ) + example = test_utils.make_fake_example() + test_utils.write_out_examples([example], cns_path) + dummy_examples = platform_utils.example_protos_from_path(cns_path) + self.assertEqual(1, len(dummy_examples)) + self.assertEqual(example, dummy_examples[0]) + + def test_example_protos_from_path_get_two(self): + cns_path = os.path.join( + tf.compat.v1.test.get_temp_dir(), "dummy_example" + ) + example_one = test_utils.make_fake_example(1) + example_two = test_utils.make_fake_example(2) + example_three = test_utils.make_fake_example(3) + test_utils.write_out_examples( + [example_one, example_two, example_three], cns_path + ) + dummy_examples = platform_utils.example_protos_from_path(cns_path, 2) + self.assertEqual(2, len(dummy_examples)) + self.assertEqual(example_one, dummy_examples[0]) + self.assertEqual(example_two, dummy_examples[1]) + + def test_example_protos_from_path_use_wildcard(self): + cns_path = os.path.join( + tf.compat.v1.test.get_temp_dir(), "wildcard_example1" + ) + example1 = test_utils.make_fake_example(1) + test_utils.write_out_examples([example1], cns_path) + cns_path = os.path.join( + tf.compat.v1.test.get_temp_dir(), "wildcard_example2" + ) + example2 = test_utils.make_fake_example(2) + test_utils.write_out_examples([example2], cns_path) + + wildcard_path = os.path.join( + tf.compat.v1.test.get_temp_dir(), "wildcard_example*" + ) + dummy_examples = platform_utils.example_protos_from_path(wildcard_path) + self.assertEqual(2, len(dummy_examples)) + + def test_example_proto_from_path_if_does_not_exist(self): + cns_path = os.path.join( + tf.compat.v1.test.get_temp_dir(), "does_not_exist" + ) + with self.assertRaises(common_utils.InvalidUserInputError): + platform_utils.example_protos_from_path(cns_path) + + def test_get_numeric_features(self): + example = test_utils.make_fake_example(single_int_val=2) + data = inference_utils.get_numeric_feature_names(example) + self.assertEqual( + ["repeated_float", "repeated_int", "single_float", "single_int"], + data, + ) + + def test_get_numeric_features_to_observed_range(self): + example = test_utils.make_fake_example(single_int_val=2) + + data = inference_utils.get_numeric_features_to_observed_range([example]) + + # Returns a sorted list by feature_name. + self.assertDictEqual( + { + "repeated_float": {"observedMin": 1.0, "observedMax": 4.0,}, + "repeated_int": {"observedMin": 10, "observedMax": 20,}, + "single_float": {"observedMin": 24.5, "observedMax": 24.5,}, + "single_int": {"observedMin": 2.0, "observedMax": 2.0,}, + }, + data, + ) + + def test_get_categorical_features_to_sampling(self): + cat_example = tf.train.Example() + cat_example.features.feature["non_numeric"].bytes_list.value.extend( + [b"cat"] + ) + + cow_example = tf.train.Example() + cow_example.features.feature["non_numeric"].bytes_list.value.extend( + [b"cow"] + ) + + pony_example = tf.train.Example() + pony_example.features.feature["non_numeric"].bytes_list.value.extend( + [b"pony"] + ) + + examples = [cat_example] * 4 + [cow_example] * 5 + [pony_example] * 10 + + # If we stop sampling at the first 3 examples, the only example should be + # cat example. + data = inference_utils.get_categorical_features_to_sampling( + examples[0:3], top_k=1 + ) + self.assertDictEqual({"non_numeric": {"samples": ["cat"]}}, data) + + # If we sample more examples, the top 2 examples should be cow and pony. + data = inference_utils.get_categorical_features_to_sampling( + examples[0:20], top_k=2 + ) + self.assertDictEqual( + {"non_numeric": {"samples": ["pony", "cow"]}}, data + ) + + def test_wrap_inference_results_classification(self): + """Test wrapping a classification result.""" + inference_result_proto = classification_pb2.ClassificationResponse() + classification = inference_result_proto.result.classifications.add() + inference_class = classification.classes.add() + inference_class.label = "class_b" + inference_class.score = 0.3 + inference_class = classification.classes.add() + inference_class.label = "class_a" + inference_class.score = 0.7 + + wrapped = inference_utils.wrap_inference_results(inference_result_proto) + self.assertEqual(1, len(wrapped.classification_result.classifications)) + self.assertEqual( + 2, len(wrapped.classification_result.classifications[0].classes) + ) + + def test_wrap_inference_results_regression(self): + """Test wrapping a regression result.""" + inference_result_proto = regression_pb2.RegressionResponse() + regression = inference_result_proto.result.regressions.add() + regression.value = 0.45 + regression = inference_result_proto.result.regressions.add() + regression.value = 0.55 + + wrapped = inference_utils.wrap_inference_results(inference_result_proto) + self.assertEqual(2, len(wrapped.regression_result.regressions)) + + @mock.patch.object(inference_utils, "make_json_formatted_for_single_chart") + @mock.patch.object(platform_utils, "call_servo") + def test_mutant_charts_for_feature( + self, mock_call_servo, mock_make_json_formatted_for_single_chart + ): + example = self.make_and_write_fake_example() + serving_bundles = [ + inference_utils.ServingBundle( + "", "", "classification", "", "", False, "", "" + ) + ] + num_mutants = 10 + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[example], + num_mutants=num_mutants, + feature_index_pattern=None, + ) + + mock_call_servo = lambda _, __: None + mock_make_json_formatted_for_single_chart = lambda _, __: {} + charts = inference_utils.mutant_charts_for_feature( + [example], "repeated_float", serving_bundles, viz_params + ) + self.assertEqual("numeric", charts["chartType"]) + self.assertEqual(4, len(charts["data"])) + charts = inference_utils.mutant_charts_for_feature( + [example], "repeated_int", serving_bundles, viz_params + ) + self.assertEqual("numeric", charts["chartType"]) + self.assertEqual(2, len(charts["data"])) + charts = inference_utils.mutant_charts_for_feature( + [example], "single_int", serving_bundles, viz_params + ) + self.assertEqual("numeric", charts["chartType"]) + self.assertEqual(1, len(charts["data"])) + charts = inference_utils.mutant_charts_for_feature( + [example], "non_numeric", serving_bundles, viz_params + ) + self.assertEqual("categorical", charts["chartType"]) + self.assertEqual(3, len(charts["data"])) + + @mock.patch.object(inference_utils, "make_json_formatted_for_single_chart") + @mock.patch.object(platform_utils, "call_servo") + def test_mutant_charts_for_feature_with_feature_index_pattern( + self, mock_call_servo, mock_make_json_formatted_for_single_chart + ): + example = self.make_and_write_fake_example() + serving_bundles = [ + inference_utils.ServingBundle( + "", "", "classification", "", "", False, "", "" + ) + ] + num_mutants = 10 + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[example], + num_mutants=num_mutants, + feature_index_pattern="0 , 2-3", + ) + + mock_call_servo = lambda _, __: None + mock_make_json_formatted_for_single_chart = lambda _, __: {} + charts = inference_utils.mutant_charts_for_feature( + [example], "repeated_float", serving_bundles, viz_params + ) + self.assertEqual("numeric", charts["chartType"]) + self.assertEqual(3, len(charts["data"])) + + # These should return 3 charts even though all fields from the index + # pattern don't exist for the example. + charts = inference_utils.mutant_charts_for_feature( + [example], "repeated_int", serving_bundles, viz_params + ) + self.assertEqual("numeric", charts["chartType"]) + self.assertEqual(3, len(charts["data"])) + + charts = inference_utils.mutant_charts_for_feature( + [example], "single_int", serving_bundles, viz_params + ) + self.assertEqual("numeric", charts["chartType"]) + self.assertEqual(3, len(charts["data"])) + + def test_make_mutant_tuples_float_list(self): + example = self.make_and_write_fake_example() + index_to_mutate = 1 + num_mutants = 10 + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[example], + num_mutants=num_mutants, + feature_index_pattern=None, + ) + + original_feature = inference_utils.parse_original_feature_from_example( + example, "repeated_float" + ) + mutant_features, mutant_examples = inference_utils.make_mutant_tuples( + [example], + original_feature, + index_to_mutate=index_to_mutate, + viz_params=viz_params, + ) + + # Check that values in mutant_features and mutant_examples are as expected. + expected_values = np.linspace(1, 10, num_mutants) + np.testing.assert_almost_equal( + expected_values, + [mutant_feature.mutant_value for mutant_feature in mutant_features], + ) + np.testing.assert_almost_equal( + expected_values, + [ + mutant_example.features.feature[ + "repeated_float" + ].float_list.value[index_to_mutate] + for mutant_example in mutant_examples + ], + ) + + # Check that the example (other than the mutant value) is the same. + for expected_value, mutant_example in zip( + expected_values, mutant_examples + ): + mutant_values = test_utils.value_from_example( + mutant_example, "repeated_float" + ) + original_values = test_utils.value_from_example( + example, "repeated_float" + ) + original_values[index_to_mutate] = expected_value + self.assertEqual(original_values, mutant_values) + + def test_make_mutant_tuples_int_list(self): + example = self.make_and_write_fake_example() + index_to_mutate = 1 + num_mutants = 10 + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[example], + num_mutants=num_mutants, + feature_index_pattern=None, + ) + original_feature = inference_utils.parse_original_feature_from_example( + example, "repeated_int" + ) + mutant_features, mutant_examples = inference_utils.make_mutant_tuples( + [example], + original_feature, + index_to_mutate=index_to_mutate, + viz_params=viz_params, + ) + + # Check that values in mutant_features and mutant_examples are as expected. + expected_values = np.linspace(1, 10, num_mutants) + np.testing.assert_almost_equal( + expected_values, + [mutant_feature.mutant_value for mutant_feature in mutant_features], + ) + np.testing.assert_almost_equal( + expected_values, + [ + mutant_example.features.feature[ + "repeated_int" + ].int64_list.value[index_to_mutate] + for mutant_example in mutant_examples + ], + ) + + # Check that the example (other than the mutant value) is the same. + for expected_value, mutant_example in zip( + expected_values, mutant_examples + ): + mutant_values = test_utils.value_from_example( + mutant_example, "repeated_int" + ) + original_values = test_utils.value_from_example( + example, "repeated_int" + ) + original_values[index_to_mutate] = expected_value + self.assertEqual(original_values, mutant_values) + + def test_make_json_formatted_for_single_chart_classification(self): + """Test making a classification chart with a single point on it.""" + inference_result_proto = classification_pb2.ClassificationResponse() + classification = inference_result_proto.result.classifications.add() + inference_class = classification.classes.add() + inference_class.label = "class_a" + inference_class.score = 0.7 + + inference_class = classification.classes.add() + inference_class.label = "class_b" + inference_class.score = 0.3 + + original_feature = inference_utils.OriginalFeatureList( + "feature_name", [2.0], "float_list" + ) + mutant_feature = inference_utils.MutantFeatureValue( + original_feature, index=0, mutant_value=20 + ) + + jsonable = inference_utils.make_json_formatted_for_single_chart( + [mutant_feature], inference_result_proto, 0 + ) + + self.assertEqual(["class_a", "class_b"], sorted(jsonable.keys())) + self.assertEqual(1, len(jsonable["class_a"])) + self.assertEqual(20, jsonable["class_a"][0]["step"]) + self.assertAlmostEqual(0.7, jsonable["class_a"][0]["scalar"]) + + self.assertEqual(1, len(jsonable["class_b"])) + self.assertEqual(20, jsonable["class_b"][0]["step"]) + self.assertAlmostEqual(0.3, jsonable["class_b"][0]["scalar"]) + + def test_make_json_formatted_for_single_chart_regression(self): + """Test making a regression chart with a single point on it.""" + inference_result_proto = regression_pb2.RegressionResponse() + regression = inference_result_proto.result.regressions.add() + regression.value = 0.45 + regression = inference_result_proto.result.regressions.add() + regression.value = 0.55 + + original_feature = inference_utils.OriginalFeatureList( + "feature_name", [2.0], "float_list" + ) + mutant_feature = inference_utils.MutantFeatureValue( + original_feature, index=0, mutant_value=20 + ) + mutant_feature_2 = inference_utils.MutantFeatureValue( + original_feature, index=0, mutant_value=10 + ) + + jsonable = inference_utils.make_json_formatted_for_single_chart( + [mutant_feature, mutant_feature_2], inference_result_proto, 0 + ) + + self.assertEqual(["value"], list(jsonable.keys())) + self.assertEqual(2, len(jsonable["value"])) + self.assertEqual(10, jsonable["value"][0]["step"]) + self.assertAlmostEqual(0.55, jsonable["value"][0]["scalar"]) + self.assertEqual(20, jsonable["value"][1]["step"]) + self.assertAlmostEqual(0.45, jsonable["value"][1]["scalar"]) + + def test_convert_predict_response_regression(self): + """Test converting a PredictResponse to a RegressionResponse.""" + predict = predict_pb2.PredictResponse() + output = predict.outputs["scores"] + dim = output.tensor_shape.dim.add() + dim.size = 2 + output.float_val.extend([0.1, 0.2]) + + bundle = inference_utils.ServingBundle( + "", "", "regression", "", "", True, "", "scores" + ) + converted = common_utils.convert_predict_response(predict, bundle) + + self.assertAlmostEqual(0.1, converted.result.regressions[0].value) + self.assertAlmostEqual(0.2, converted.result.regressions[1].value) + + def test_convert_predict_response_classification(self): + """Test converting a PredictResponse to a ClassificationResponse.""" + predict = predict_pb2.PredictResponse() + output = predict.outputs["probabilities"] + dim = output.tensor_shape.dim.add() + dim.size = 3 + dim = output.tensor_shape.dim.add() + dim.size = 2 + output.float_val.extend([1.0, 0.0, 0.9, 0.1, 0.8, 0.2]) + + bundle = inference_utils.ServingBundle( + "", "", "classification", "", "", True, "", "probabilities" + ) + converted = common_utils.convert_predict_response(predict, bundle) + + self.assertEqual( + "0", converted.result.classifications[0].classes[0].label + ) + self.assertAlmostEqual( + 1, converted.result.classifications[0].classes[0].score + ) + self.assertEqual( + "1", converted.result.classifications[0].classes[1].label + ) + self.assertAlmostEqual( + 0, converted.result.classifications[0].classes[1].score + ) + + self.assertEqual( + "0", converted.result.classifications[1].classes[0].label + ) + self.assertAlmostEqual( + 0.9, converted.result.classifications[1].classes[0].score + ) + self.assertEqual( + "1", converted.result.classifications[1].classes[1].label + ) + self.assertAlmostEqual( + 0.1, converted.result.classifications[1].classes[1].score + ) + + self.assertEqual( + "0", converted.result.classifications[2].classes[0].label + ) + self.assertAlmostEqual( + 0.8, converted.result.classifications[2].classes[0].score + ) + self.assertEqual( + "1", converted.result.classifications[2].classes[1].label + ) + self.assertAlmostEqual( + 0.2, converted.result.classifications[2].classes[1].score + ) + + def test_vizparams_pattern_parser(self): + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[], + num_mutants=0, + feature_index_pattern=None, + ) + self.assertEqual([], viz_params.feature_indices) + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[], + num_mutants=0, + feature_index_pattern="1-3", + ) + self.assertEqual([1, 2, 3], viz_params.feature_indices) + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[], + num_mutants=0, + feature_index_pattern="3-1", + ) + self.assertEqual([], viz_params.feature_indices) + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[], + num_mutants=0, + feature_index_pattern="1-1", + ) + self.assertEqual([1], viz_params.feature_indices) + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[], + num_mutants=0, + feature_index_pattern="3, 1", + ) + self.assertEqual([1, 3], viz_params.feature_indices) + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[], + num_mutants=0, + feature_index_pattern="0-", + ) + self.assertEqual([], viz_params.feature_indices) + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[], + num_mutants=0, + feature_index_pattern="0-error", + ) + self.assertEqual([], viz_params.feature_indices) + viz_params = inference_utils.VizParams( + x_min=1, + x_max=10, + examples=[], + num_mutants=0, + feature_index_pattern="0-3-5", + ) + self.assertEqual([], viz_params.feature_indices) + + def test_sort_eligible_features(self): + features_list = [{"name": "feat1"}, {"name": "feat2"}] + chart_data = { + "feat1": { + "chartType": "numeric", + "data": [ + [ + { + "series1": [ + {"scalar": 0.2}, + {"scalar": 0.1}, + {"scalar": 0.3}, + ] + }, + { + "series2": [ + {"scalar": 0.2}, + {"scalar": 0.1}, + {"scalar": 0.4}, + ] + }, + ] + ], + }, + "feat2": { + "chartType": "categorical", + "data": [ + [ + { + "series1": [ + {"scalar": 0.2}, + {"scalar": 0.1}, + {"scalar": 0.3}, + ] + }, + { + "series2": [ + {"scalar": 0.2}, + {"scalar": 0.1}, + {"scalar": 0.9}, + ] + }, + ] + ], + }, } - } - sorted_list = inference_utils.sort_eligible_features( - features_list, chart_data) - print(sorted_list) - self.assertEqual('feat2', sorted_list[0]['name']) - self.assertEqual(.8, sorted_list[0]['interestingness']) - self.assertEqual('feat1', sorted_list[1]['name']) - self.assertEqual(.4, sorted_list[1]['interestingness']) - -if __name__ == '__main__': - tf.test.main() + sorted_list = inference_utils.sort_eligible_features( + features_list, chart_data + ) + print(sorted_list) + self.assertEqual("feat2", sorted_list[0]["name"]) + self.assertEqual(0.8, sorted_list[0]["interestingness"]) + self.assertEqual("feat1", sorted_list[1]["name"]) + self.assertEqual(0.4, sorted_list[1]["interestingness"]) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/interactive_inference/utils/platform_utils.py b/tensorboard/plugins/interactive_inference/utils/platform_utils.py index bc87b03694..dbfd80374b 100644 --- a/tensorboard/plugins/interactive_inference/utils/platform_utils.py +++ b/tensorboard/plugins/interactive_inference/utils/platform_utils.py @@ -30,176 +30,194 @@ def filepath_to_filepath_list(file_path): - """Returns a list of files given by a filepath. + """Returns a list of files given by a filepath. - Args: - file_path: A path, possibly representing a single file, or containing a - wildcard or sharded path. + Args: + file_path: A path, possibly representing a single file, or containing a + wildcard or sharded path. - Returns: - A list of files represented by the provided path. - """ - file_path = file_path.strip() - if '*' in file_path: - return glob(file_path) - else: - return [file_path] + Returns: + A list of files represented by the provided path. + """ + file_path = file_path.strip() + if "*" in file_path: + return glob(file_path) + else: + return [file_path] def throw_if_file_access_not_allowed(file_path, logdir, has_auth_group): - """Throws an error if a file cannot be loaded for inference. - - Args: - file_path: A file path. - logdir: The path to the logdir of the TensorBoard context. - has_auth_group: True if TensorBoard was started with an authorized group, - in which case we allow access to all visible files. - - Raises: - InvalidUserInputError: If the file is not in the logdir and is not globally - readable. - """ - return - - -def example_protos_from_path(path, - num_examples=10, - start_index=0, - parse_examples=True, - sampling_odds=1, - example_class=tf.train.Example): - """Returns a number of examples from the provided path. - - Args: - path: A string path to the examples. - num_examples: The maximum number of examples to return from the path. - parse_examples: If true then parses the serialized proto from the path into - proto objects. Defaults to True. - sampling_odds: Odds of loading an example, used for sampling. When >= 1 - (the default), then all examples are loaded. - example_class: tf.train.Example or tf.train.SequenceExample class to load. - Defaults to tf.train.Example. - - Returns: - A list of Example protos or serialized proto strings at the path. - - Raises: - InvalidUserInputError: If examples cannot be procured from the path. - """ - - def append_examples_from_iterable(iterable, examples): - for value in iterable: - if sampling_odds >= 1 or random.random() < sampling_odds: - examples.append( - example_class.FromString(value) if parse_examples else value) - if len(examples) >= num_examples: - return - - examples = [] - - if path.endswith('.csv'): - def are_floats(values): - for value in values: + """Throws an error if a file cannot be loaded for inference. + + Args: + file_path: A file path. + logdir: The path to the logdir of the TensorBoard context. + has_auth_group: True if TensorBoard was started with an authorized group, + in which case we allow access to all visible files. + + Raises: + InvalidUserInputError: If the file is not in the logdir and is not globally + readable. + """ + return + + +def example_protos_from_path( + path, + num_examples=10, + start_index=0, + parse_examples=True, + sampling_odds=1, + example_class=tf.train.Example, +): + """Returns a number of examples from the provided path. + + Args: + path: A string path to the examples. + num_examples: The maximum number of examples to return from the path. + parse_examples: If true then parses the serialized proto from the path into + proto objects. Defaults to True. + sampling_odds: Odds of loading an example, used for sampling. When >= 1 + (the default), then all examples are loaded. + example_class: tf.train.Example or tf.train.SequenceExample class to load. + Defaults to tf.train.Example. + + Returns: + A list of Example protos or serialized proto strings at the path. + + Raises: + InvalidUserInputError: If examples cannot be procured from the path. + """ + + def append_examples_from_iterable(iterable, examples): + for value in iterable: + if sampling_odds >= 1 or random.random() < sampling_odds: + examples.append( + example_class.FromString(value) if parse_examples else value + ) + if len(examples) >= num_examples: + return + + examples = [] + + if path.endswith(".csv"): + + def are_floats(values): + for value in values: + try: + float(value) + except ValueError: + return False + return True + + csv.register_dialect("CsvDialect", skipinitialspace=True) + rows = csv.DictReader(open(path), dialect="CsvDialect") + for row in rows: + if sampling_odds < 1 and random.random() > sampling_odds: + continue + example = tf.train.Example() + for col in row.keys(): + # Parse out individual values from vertical-bar-delimited lists + values = [val.strip() for val in row[col].split("|")] + if are_floats(values): + example.features.feature[col].float_list.value.extend( + [float(val) for val in values] + ) + else: + example.features.feature[col].bytes_list.value.extend( + [val.encode("utf-8") for val in values] + ) + examples.append( + example if parse_examples else example.SerializeToString() + ) + if len(examples) >= num_examples: + break + return examples + + filenames = filepath_to_filepath_list(path) + compression_types = [ + "", # no compression (distinct from `None`!) + "GZIP", + "ZLIB", + ] + current_compression_idx = 0 + current_file_index = 0 + while current_file_index < len(filenames) and current_compression_idx < len( + compression_types + ): try: - float(value) - except ValueError: - return False - return True - csv.register_dialect('CsvDialect', skipinitialspace=True) - rows = csv.DictReader(open(path), dialect='CsvDialect') - for row in rows: - if sampling_odds < 1 and random.random() > sampling_odds: - continue - example = tf.train.Example() - for col in row.keys(): - # Parse out individual values from vertical-bar-delimited lists - values = [val.strip() for val in row[col].split('|')] - if are_floats(values): - example.features.feature[col].float_list.value.extend( - [float(val) for val in values]) - else: - example.features.feature[col].bytes_list.value.extend( - [val.encode('utf-8') for val in values]) - examples.append( - example if parse_examples else example.SerializeToString()) - if len(examples) >= num_examples: - break - return examples - - filenames = filepath_to_filepath_list(path) - compression_types = [ - '', # no compression (distinct from `None`!) - 'GZIP', - 'ZLIB', - ] - current_compression_idx = 0 - current_file_index = 0 - while (current_file_index < len(filenames) and - current_compression_idx < len(compression_types)): - try: - record_iterator = tf.compat.v1.python_io.tf_record_iterator( - path=filenames[current_file_index], - options=tf.io.TFRecordOptions( - compression_types[current_compression_idx])) - append_examples_from_iterable(record_iterator, examples) - current_file_index += 1 - if len(examples) >= num_examples: - break - except tf.errors.DataLossError: - current_compression_idx += 1 - except (IOError, tf.errors.NotFoundError) as e: - raise common_utils.InvalidUserInputError(e) - - if examples: - return examples - else: - raise common_utils.InvalidUserInputError( - 'No examples found at ' + path + - '. Valid formats are TFRecord files.') + record_iterator = tf.compat.v1.python_io.tf_record_iterator( + path=filenames[current_file_index], + options=tf.io.TFRecordOptions( + compression_types[current_compression_idx] + ), + ) + append_examples_from_iterable(record_iterator, examples) + current_file_index += 1 + if len(examples) >= num_examples: + break + except tf.errors.DataLossError: + current_compression_idx += 1 + except (IOError, tf.errors.NotFoundError) as e: + raise common_utils.InvalidUserInputError(e) + + if examples: + return examples + else: + raise common_utils.InvalidUserInputError( + "No examples found at " + + path + + ". Valid formats are TFRecord files." + ) + def call_servo(examples, serving_bundle): - """Send an RPC request to the Servomatic prediction service. - - Args: - examples: A list of examples that matches the model spec. - serving_bundle: A `ServingBundle` object that contains the information to - make the serving request. - - Returns: - A ClassificationResponse or RegressionResponse proto. - """ - parsed_url = urlparse('http://' + serving_bundle.inference_address) - channel = implementations.insecure_channel(parsed_url.hostname, - parsed_url.port) - stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) - - if serving_bundle.use_predict: - request = predict_pb2.PredictRequest() - elif serving_bundle.model_type == 'classification': - request = classification_pb2.ClassificationRequest() - else: - request = regression_pb2.RegressionRequest() - request.model_spec.name = serving_bundle.model_name - if serving_bundle.model_version is not None: - request.model_spec.version.value = serving_bundle.model_version - if serving_bundle.signature is not None: - request.model_spec.signature_name = serving_bundle.signature - - if serving_bundle.use_predict: - # tf.compat.v1 API used here to convert tf.example into proto. This - # utility file is bundled in the witwidget pip package which has a dep - # on TensorFlow. - request.inputs[serving_bundle.predict_input_tensor].CopyFrom( - tf.compat.v1.make_tensor_proto( - values=[ex.SerializeToString() for ex in examples], - dtype=types_pb2.DT_STRING)) - else: - request.input.example_list.examples.extend(examples) - - if serving_bundle.use_predict: - return common_utils.convert_predict_response( - stub.Predict(request, 30.0), serving_bundle) # 30 secs timeout - elif serving_bundle.model_type == 'classification': - return stub.Classify(request, 30.0) # 30 secs timeout - else: - return stub.Regress(request, 30.0) # 30 secs timeout + """Send an RPC request to the Servomatic prediction service. + + Args: + examples: A list of examples that matches the model spec. + serving_bundle: A `ServingBundle` object that contains the information to + make the serving request. + + Returns: + A ClassificationResponse or RegressionResponse proto. + """ + parsed_url = urlparse("http://" + serving_bundle.inference_address) + channel = implementations.insecure_channel( + parsed_url.hostname, parsed_url.port + ) + stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) + + if serving_bundle.use_predict: + request = predict_pb2.PredictRequest() + elif serving_bundle.model_type == "classification": + request = classification_pb2.ClassificationRequest() + else: + request = regression_pb2.RegressionRequest() + request.model_spec.name = serving_bundle.model_name + if serving_bundle.model_version is not None: + request.model_spec.version.value = serving_bundle.model_version + if serving_bundle.signature is not None: + request.model_spec.signature_name = serving_bundle.signature + + if serving_bundle.use_predict: + # tf.compat.v1 API used here to convert tf.example into proto. This + # utility file is bundled in the witwidget pip package which has a dep + # on TensorFlow. + request.inputs[serving_bundle.predict_input_tensor].CopyFrom( + tf.compat.v1.make_tensor_proto( + values=[ex.SerializeToString() for ex in examples], + dtype=types_pb2.DT_STRING, + ) + ) + else: + request.input.example_list.examples.extend(examples) + + if serving_bundle.use_predict: + return common_utils.convert_predict_response( + stub.Predict(request, 30.0), serving_bundle + ) # 30 secs timeout + elif serving_bundle.model_type == "classification": + return stub.Classify(request, 30.0) # 30 secs timeout + else: + return stub.Regress(request, 30.0) # 30 secs timeout diff --git a/tensorboard/plugins/interactive_inference/utils/test_utils.py b/tensorboard/plugins/interactive_inference/utils/test_utils.py index d2715427c8..f5cb660d10 100644 --- a/tensorboard/plugins/interactive_inference/utils/test_utils.py +++ b/tensorboard/plugins/interactive_inference/utils/test_utils.py @@ -22,30 +22,33 @@ def make_fake_example(single_int_val=0): - """Make a fake example with numeric and string features.""" - example = tf.train.Example() - example.features.feature['repeated_float'].float_list.value.extend( - [1.0, 2.0, 3.0, 4.0]) - example.features.feature['repeated_int'].int64_list.value.extend([10, 20]) - - example.features.feature['single_int'].int64_list.value.extend( - [single_int_val]) - example.features.feature['single_float'].float_list.value.extend([24.5]) - example.features.feature['non_numeric'].bytes_list.value.extend( - [b'cat', b'cat', b'woof']) - return example + """Make a fake example with numeric and string features.""" + example = tf.train.Example() + example.features.feature["repeated_float"].float_list.value.extend( + [1.0, 2.0, 3.0, 4.0] + ) + example.features.feature["repeated_int"].int64_list.value.extend([10, 20]) + + example.features.feature["single_int"].int64_list.value.extend( + [single_int_val] + ) + example.features.feature["single_float"].float_list.value.extend([24.5]) + example.features.feature["non_numeric"].bytes_list.value.extend( + [b"cat", b"cat", b"woof"] + ) + return example def write_out_examples(examples, path): - """Writes protos to the CNS path.""" + """Writes protos to the CNS path.""" - writer = tf.io.TFRecordWriter(path) - for example in examples: - writer.write(example.SerializeToString()) + writer = tf.io.TFRecordWriter(path) + for example in examples: + writer.write(example.SerializeToString()) def value_from_example(example, feature_name): - """Returns the feature as a Python list.""" - feature = example.features.feature[feature_name] - feature_type = feature.WhichOneof('kind') - return getattr(feature, feature_type).value[:] + """Returns the feature as a Python list.""" + feature = example.features.feature[feature_name] + feature_type = feature.WhichOneof("kind") + return getattr(feature, feature_type).value[:] diff --git a/tensorboard/plugins/interactive_inference/witwidget/__init__.py b/tensorboard/plugins/interactive_inference/witwidget/__init__.py index c809332077..eed38f9112 100644 --- a/tensorboard/plugins/interactive_inference/witwidget/__init__.py +++ b/tensorboard/plugins/interactive_inference/witwidget/__init__.py @@ -14,10 +14,13 @@ from witwidget.notebook.visualization import * + def _jupyter_nbextension_paths(): - return [{ - 'section': 'notebook', - 'src': 'static', - 'dest': 'wit-widget', - 'require': 'wit-widget/extension' - }] + return [ + { + "section": "notebook", + "src": "static", + "dest": "wit-widget", + "require": "wit-widget/extension", + } + ] diff --git a/tensorboard/plugins/interactive_inference/witwidget/notebook/base.py b/tensorboard/plugins/interactive_inference/witwidget/notebook/base.py index 17fbde4ac2..98a9ee0454 100644 --- a/tensorboard/plugins/interactive_inference/witwidget/notebook/base.py +++ b/tensorboard/plugins/interactive_inference/witwidget/notebook/base.py @@ -32,582 +32,707 @@ NUM_EXAMPLES_FOR_MUTANT_ANALYSIS = 50 # Custom user agent for tracking number of calls to Cloud AI Platform. -USER_AGENT_FOR_CAIP_TRACKING = 'WhatIfTool' +USER_AGENT_FOR_CAIP_TRACKING = "WhatIfTool" try: - POOL_SIZE = max(multiprocessing.cpu_count() - 1, 1) + POOL_SIZE = max(multiprocessing.cpu_count() - 1, 1) except Exception: - POOL_SIZE = 1 + POOL_SIZE = 1 class WitWidgetBase(object): - """WIT widget base class for common code between Jupyter and Colab.""" - - def __init__(self, config_builder): - """Constructor for WitWidgetBase. - - Args: - config_builder: WitConfigBuilder object containing settings for WIT. - """ - tf.get_logger().setLevel(logging.WARNING) - config = config_builder.build() - copied_config = dict(config) - self.estimator_and_spec = ( - dict(config.get('estimator_and_spec')) - if 'estimator_and_spec' in config else {}) - self.compare_estimator_and_spec = ( - dict(config.get('compare_estimator_and_spec')) - if 'compare_estimator_and_spec' in config else {}) - if 'estimator_and_spec' in copied_config: - del copied_config['estimator_and_spec'] - if 'compare_estimator_and_spec' in copied_config: - del copied_config['compare_estimator_and_spec'] - - self.custom_predict_fn = config.get('custom_predict_fn') - self.compare_custom_predict_fn = config.get('compare_custom_predict_fn') - self.custom_distance_fn = config.get('custom_distance_fn') - self.adjust_prediction_fn = config.get('adjust_prediction') - self.compare_adjust_prediction_fn = config.get('compare_adjust_prediction') - self.adjust_example_fn = config.get('adjust_example') - self.compare_adjust_example_fn = config.get('compare_adjust_example') - self.adjust_attribution_fn = config.get('adjust_attribution') - self.compare_adjust_attribution_fn = config.get('compare_adjust_attribution') - - if 'custom_predict_fn' in copied_config: - del copied_config['custom_predict_fn'] - if 'compare_custom_predict_fn' in copied_config: - del copied_config['compare_custom_predict_fn'] - if 'custom_distance_fn' in copied_config: - del copied_config['custom_distance_fn'] - copied_config['uses_custom_distance_fn'] = True - if 'adjust_prediction' in copied_config: - del copied_config['adjust_prediction'] - if 'compare_adjust_prediction' in copied_config: - del copied_config['compare_adjust_prediction'] - if 'adjust_example' in copied_config: - del copied_config['adjust_example'] - if 'compare_adjust_example' in copied_config: - del copied_config['compare_adjust_example'] - if 'adjust_attribution' in copied_config: - del copied_config['adjust_attribution'] - if 'compare_adjust_attribution' in copied_config: - del copied_config['compare_adjust_attribution'] - - examples = copied_config.pop('examples') - self.config = copied_config - self.set_examples(examples) - - # This tracks whether mutant inference is running in order to - # skip calling for explanations for CAIP models when inferring - # for mutant inference, for performance reasons. - self.running_mutant_infer = False - - # If using AI Platform for prediction, set the correct custom prediction - # functions. - if self.config.get('use_aip'): - self.custom_predict_fn = self._predict_aip_model - if self.config.get('compare_use_aip'): - self.compare_custom_predict_fn = self._predict_aip_compare_model - - # If using JSON input (not Example protos) and a custom predict - # function, then convert examples to JSON before sending to the - # custom predict function. - if self.config.get('uses_json_input'): - if self.custom_predict_fn is not None and not self.config.get('use_aip'): - user_predict = self.custom_predict_fn - def wrapped_custom_predict_fn(examples): - return user_predict(self._json_from_tf_examples(examples)) - self.custom_predict_fn = wrapped_custom_predict_fn - if (self.compare_custom_predict_fn is not None and - not self.config.get('compare_use_aip')): - compare_user_predict = self.compare_custom_predict_fn - def wrapped_compare_custom_predict_fn(examples): - return compare_user_predict(self._json_from_tf_examples(examples)) - self.compare_custom_predict_fn = wrapped_compare_custom_predict_fn - - def _get_element_html(self): - return """ + """WIT widget base class for common code between Jupyter and Colab.""" + + def __init__(self, config_builder): + """Constructor for WitWidgetBase. + + Args: + config_builder: WitConfigBuilder object containing settings for WIT. + """ + tf.get_logger().setLevel(logging.WARNING) + config = config_builder.build() + copied_config = dict(config) + self.estimator_and_spec = ( + dict(config.get("estimator_and_spec")) + if "estimator_and_spec" in config + else {} + ) + self.compare_estimator_and_spec = ( + dict(config.get("compare_estimator_and_spec")) + if "compare_estimator_and_spec" in config + else {} + ) + if "estimator_and_spec" in copied_config: + del copied_config["estimator_and_spec"] + if "compare_estimator_and_spec" in copied_config: + del copied_config["compare_estimator_and_spec"] + + self.custom_predict_fn = config.get("custom_predict_fn") + self.compare_custom_predict_fn = config.get("compare_custom_predict_fn") + self.custom_distance_fn = config.get("custom_distance_fn") + self.adjust_prediction_fn = config.get("adjust_prediction") + self.compare_adjust_prediction_fn = config.get( + "compare_adjust_prediction" + ) + self.adjust_example_fn = config.get("adjust_example") + self.compare_adjust_example_fn = config.get("compare_adjust_example") + self.adjust_attribution_fn = config.get("adjust_attribution") + self.compare_adjust_attribution_fn = config.get( + "compare_adjust_attribution" + ) + + if "custom_predict_fn" in copied_config: + del copied_config["custom_predict_fn"] + if "compare_custom_predict_fn" in copied_config: + del copied_config["compare_custom_predict_fn"] + if "custom_distance_fn" in copied_config: + del copied_config["custom_distance_fn"] + copied_config["uses_custom_distance_fn"] = True + if "adjust_prediction" in copied_config: + del copied_config["adjust_prediction"] + if "compare_adjust_prediction" in copied_config: + del copied_config["compare_adjust_prediction"] + if "adjust_example" in copied_config: + del copied_config["adjust_example"] + if "compare_adjust_example" in copied_config: + del copied_config["compare_adjust_example"] + if "adjust_attribution" in copied_config: + del copied_config["adjust_attribution"] + if "compare_adjust_attribution" in copied_config: + del copied_config["compare_adjust_attribution"] + + examples = copied_config.pop("examples") + self.config = copied_config + self.set_examples(examples) + + # This tracks whether mutant inference is running in order to + # skip calling for explanations for CAIP models when inferring + # for mutant inference, for performance reasons. + self.running_mutant_infer = False + + # If using AI Platform for prediction, set the correct custom prediction + # functions. + if self.config.get("use_aip"): + self.custom_predict_fn = self._predict_aip_model + if self.config.get("compare_use_aip"): + self.compare_custom_predict_fn = self._predict_aip_compare_model + + # If using JSON input (not Example protos) and a custom predict + # function, then convert examples to JSON before sending to the + # custom predict function. + if self.config.get("uses_json_input"): + if self.custom_predict_fn is not None and not self.config.get( + "use_aip" + ): + user_predict = self.custom_predict_fn + + def wrapped_custom_predict_fn(examples): + return user_predict(self._json_from_tf_examples(examples)) + + self.custom_predict_fn = wrapped_custom_predict_fn + if ( + self.compare_custom_predict_fn is not None + and not self.config.get("compare_use_aip") + ): + compare_user_predict = self.compare_custom_predict_fn + + def wrapped_compare_custom_predict_fn(examples): + return compare_user_predict( + self._json_from_tf_examples(examples) + ) + + self.compare_custom_predict_fn = ( + wrapped_compare_custom_predict_fn + ) + + def _get_element_html(self): + return """ """ - def set_examples(self, examples): - """Sets the examples shown in WIT. - - The examples are initially set by the examples specified in the config - builder during construction. This method can change which examples WIT - displays. - """ - if self.config.get('uses_json_input'): - tf_examples = self._json_to_tf_examples(examples) - self.examples = [json_format.MessageToJson(ex) for ex in tf_examples] - else: - self.examples = [json_format.MessageToJson(ex) for ex in examples] - self.updated_example_indices = set(range(len(examples))) - - def compute_custom_distance_impl(self, index, params=None): - exs_for_distance = [ - self.json_to_proto(example) for example in self.examples] - selected_ex = exs_for_distance[index] - return self.custom_distance_fn(selected_ex, exs_for_distance, params) - - def json_to_proto(self, json): - ex = (tf.train.SequenceExample() - if self.config.get('are_sequence_examples') - else tf.train.Example()) - json_format.Parse(json, ex) - return ex - - def infer_impl(self): - """Performs inference on examples that require inference.""" - indices_to_infer = sorted(self.updated_example_indices) - examples_to_infer = [ - self.json_to_proto(self.examples[index]) for index in indices_to_infer] - infer_objs = [] - extra_output_objs = [] - serving_bundle = inference_utils.ServingBundle( - self.config.get('inference_address'), - self.config.get('model_name'), - self.config.get('model_type'), - self.config.get('model_version'), - self.config.get('model_signature'), - self.config.get('uses_predict_api'), - self.config.get('predict_input_tensor'), - self.config.get('predict_output_tensor'), - self.estimator_and_spec.get('estimator'), - self.estimator_and_spec.get('feature_spec'), - self.custom_predict_fn) - (predictions, extra_output) = ( - inference_utils.run_inference_for_inference_results( - examples_to_infer, serving_bundle)) - infer_objs.append(predictions) - extra_output_objs.append(extra_output) - if ('inference_address_2' in self.config or - self.compare_estimator_and_spec.get('estimator') or - self.compare_custom_predict_fn): - serving_bundle = inference_utils.ServingBundle( - self.config.get('inference_address_2'), - self.config.get('model_name_2'), - self.config.get('model_type'), - self.config.get('model_version_2'), - self.config.get('model_signature_2'), - self.config.get('uses_predict_api'), - self.config.get('predict_input_tensor'), - self.config.get('predict_output_tensor'), - self.compare_estimator_and_spec.get('estimator'), - self.compare_estimator_and_spec.get('feature_spec'), - self.compare_custom_predict_fn) - (predictions, extra_output) = ( - inference_utils.run_inference_for_inference_results( - examples_to_infer, serving_bundle)) - infer_objs.append(predictions) - extra_output_objs.append(extra_output) - self.updated_example_indices = set() - return { - 'inferences': {'indices': indices_to_infer, 'results': infer_objs}, - 'label_vocab': self.config.get('label_vocab'), - 'extra_outputs': extra_output_objs} - - def infer_mutants_impl(self, info): - """Performs mutant inference on specified examples.""" - example_index = int(info['example_index']) - feature_name = info['feature_name'] - examples = (self.examples if example_index == -1 - else [self.examples[example_index]]) - examples = [self.json_to_proto(ex) for ex in examples] - scan_examples = [self.json_to_proto(ex) for ex in self.examples[0:50]] - serving_bundles = [] - serving_bundles.append(inference_utils.ServingBundle( - self.config.get('inference_address'), - self.config.get('model_name'), - self.config.get('model_type'), - self.config.get('model_version'), - self.config.get('model_signature'), - self.config.get('uses_predict_api'), - self.config.get('predict_input_tensor'), - self.config.get('predict_output_tensor'), - self.estimator_and_spec.get('estimator'), - self.estimator_and_spec.get('feature_spec'), - self.custom_predict_fn)) - if ('inference_address_2' in self.config or - self.compare_estimator_and_spec.get('estimator') or - self.compare_custom_predict_fn): - serving_bundles.append(inference_utils.ServingBundle( - self.config.get('inference_address_2'), - self.config.get('model_name_2'), - self.config.get('model_type'), - self.config.get('model_version_2'), - self.config.get('model_signature_2'), - self.config.get('uses_predict_api'), - self.config.get('predict_input_tensor'), - self.config.get('predict_output_tensor'), - self.compare_estimator_and_spec.get('estimator'), - self.compare_estimator_and_spec.get('feature_spec'), - self.compare_custom_predict_fn)) - viz_params = inference_utils.VizParams( - info['x_min'], info['x_max'], - scan_examples, 10, - info['feature_index_pattern']) - self.running_mutant_infer = True - charts = inference_utils.mutant_charts_for_feature( - examples, feature_name, serving_bundles, viz_params) - self.running_mutant_infer = False - return charts - - def get_eligible_features_impl(self): - """Returns information about features eligible for mutant inference.""" - examples = [self.json_to_proto(ex) for ex in self.examples[ - 0:NUM_EXAMPLES_FOR_MUTANT_ANALYSIS]] - return inference_utils.get_eligible_features( - examples, NUM_MUTANTS_TO_GENERATE) - - def sort_eligible_features_impl(self, info): - """Returns sorted list of interesting features for mutant inference.""" - features_list = info['features'] - chart_data = {} - for feat in features_list: - chart_data[feat['name']] = self.infer_mutants_impl({ - 'x_min': feat['observedMin'] if 'observedMin' in feat else 0, - 'x_max': feat['observedMax'] if 'observedMin' in feat else 0, - 'feature_index_pattern': None, - 'feature_name': feat['name'], - 'example_index': info['example_index'], - }) - return inference_utils.sort_eligible_features( - features_list, chart_data) - - def create_sprite(self): - """Returns an encoded image of thumbnails for image examples.""" - # Generate a sprite image for the examples if the examples contain the - # standard encoded image feature. - if not self.examples: - return None - example_to_check = self.json_to_proto(self.examples[0]) - feature_list = (example_to_check.context.feature - if self.config.get('are_sequence_examples') - else example_to_check.features.feature) - if 'image/encoded' in feature_list: - example_strings = [ - self.json_to_proto(ex).SerializeToString() - for ex in self.examples] - encoded = ensure_str(base64.b64encode( - inference_utils.create_sprite_image(example_strings))) - return 'data:image/png;base64,{}'.format(encoded) - else: - return None - - def _json_from_tf_examples(self, tf_examples): - json_exs = [] - feature_names = self.config.get('feature_names') - for ex in tf_examples: - # Create a JSON list or dict for each example depending on settings. - # Strip out any explicitly-labeled target feature from the example. - # This is needed because AI Platform models that accept JSON cannot handle - # when non-input features are provided as part of the object to run - # prediction on. - if self.config.get('uses_json_list'): - json_ex = [] - for feat in ex.features.feature: - if feature_names and feat in feature_names: - feat_idx = feature_names.index(feat) - else: - feat_idx = int(feat) - if (feat == self.config.get('target_feature') or - feat_idx == self.config.get('target_feature')): - continue - # Ensure the example value list is long enough to add the next feature - # from the tf.Example. - if feat_idx >= len(json_ex): - json_ex.extend([None] * (feat_idx - len(json_ex) + 1)) - if ex.features.feature[feat].HasField('int64_list'): - json_ex[feat_idx] = ex.features.feature[feat].int64_list.value[0] - elif ex.features.feature[feat].HasField('float_list'): - json_ex[feat_idx] = ex.features.feature[feat].float_list.value[0] - else: - json_ex[feat_idx] = ensure_str( - ex.features.feature[feat].bytes_list.value[0]) - else: - json_ex = {} - for feat in ex.features.feature: - if feat == self.config.get('target_feature'): - continue - if ex.features.feature[feat].HasField('int64_list'): - json_ex[feat] = ex.features.feature[feat].int64_list.value[0] - elif ex.features.feature[feat].HasField('float_list'): - json_ex[feat] = ex.features.feature[feat].float_list.value[0] - else: - json_ex[feat] = ensure_str( - ex.features.feature[feat].bytes_list.value[0]) - json_exs.append(json_ex) - return json_exs - - def _json_to_tf_examples(self, examples): - def add_single_feature(feat, value, ex): - if isinstance(value, integer_types): - ex.features.feature[feat].int64_list.value.append(value) - elif isinstance(value, Number): - ex.features.feature[feat].float_list.value.append(value) - else: - ex.features.feature[feat].bytes_list.value.append(value.encode('utf-8')) - - tf_examples = [] - for json_ex in examples: - ex = tf.train.Example() - # JSON examples can be lists of values (for xgboost models for instance), - # or dicts of key/value pairs. - if self.config.get('uses_json_list'): - feature_names = self.config.get('feature_names') - for (i, value) in enumerate(json_ex): - # If feature names have been provided, use those feature names instead - # of list indices for feature name when storing as tf.Example. - if feature_names and len(feature_names) > i: - feat = feature_names[i] - else: - feat = str(i) - add_single_feature(feat, value, ex) - tf_examples.append(ex) - else: - for feat in json_ex: - add_single_feature(feat, json_ex[feat], ex) - tf_examples.append(ex) - return tf_examples - - def _predict_aip_model(self, examples): - return self._predict_aip_impl( - examples, - self.config.get('inference_address'), - self.config.get('model_name'), - self.config.get('model_signature'), - self.config.get('force_json_input'), - self.adjust_example_fn, - self.adjust_prediction_fn, - self.adjust_attribution_fn, - self.config.get('aip_service_name'), - self.config.get('aip_service_version'), - self.config.get('get_explanations'), - self.config.get('aip_batch_size'), - self.config.get('aip_api_key')) - - def _predict_aip_compare_model(self, examples): - return self._predict_aip_impl( - examples, - self.config.get('inference_address_2'), - self.config.get('model_name_2'), - self.config.get('model_signature_2'), - self.config.get('compare_force_json_input'), - self.compare_adjust_example_fn, - self.compare_adjust_prediction_fn, - self.compare_adjust_attribution_fn, - self.config.get('compare_aip_service_name'), - self.config.get('compare_aip_service_version'), - self.config.get('compare_get_explanations'), - self.config.get('compare_aip_batch_size'), - self.config.get('compare_aip_api_key')) - - def _predict_aip_impl(self, examples, project, model, version, force_json, - adjust_example, adjust_prediction, adjust_attribution, - service_name, service_version, get_explanations, - batch_size, api_key): - """Custom prediction function for running inference through AI Platform.""" - - # Set up environment for GCP call for specified project. - os.environ['GOOGLE_CLOUD_PROJECT'] = project - - should_explain = get_explanations and not self.running_mutant_infer - - def predict(exs): - """Run prediction on a list of examples and return results.""" - # Properly package the examples to send for prediction. - discovery_url = None - error_during_prediction = False - if api_key is not None: - discovery_url = ( - ('https://%s.googleapis.com/$discovery/rest' - '?labels=GOOGLE_INTERNAL&key=%s&version=%s') - % (service_name, api_key, 'v1')) - credentials = GoogleCredentials.get_application_default() - service = googleapiclient.discovery.build( - service_name, service_version, cache_discovery=False, - developerKey=api_key, discoveryServiceUrl=discovery_url, - credentials=credentials) - else: - service = googleapiclient.discovery.build( - service_name, service_version, cache_discovery=False) - - name = 'projects/{}/models/{}'.format(project, model) - if version is not None: - name += '/versions/{}'.format(version) - - if self.config.get('uses_json_input') or force_json: - examples_for_predict = self._json_from_tf_examples(exs) - else: - examples_for_predict = [{'b64': base64.b64encode( - example.SerializeToString()).decode('utf-8') } - for example in exs] - - # If there is a user-specified input example adjustment to make, make it. - if adjust_example: - examples_for_predict = [ - adjust_example(ex) for ex in examples_for_predict] - - # Send request, including custom user-agent for tracking. - request_builder = service.projects().predict( - name=name, - body={'instances': examples_for_predict} - ) - user_agent = request_builder.headers.get('user-agent') - request_builder.headers['user-agent'] = ( - USER_AGENT_FOR_CAIP_TRACKING + - ('-' + user_agent if user_agent else '')) - try: - response = request_builder.execute() - except Exception as e: - error_during_prediction = True - response = {'error': str(e)} - - # Get the attributions and baseline score if explaination is enabled. - if should_explain and not error_during_prediction: - try: - request_builder = service.projects().explain( - name=name, - body={'instances': examples_for_predict} - ) - request_builder.headers['user-agent'] = ( - USER_AGENT_FOR_CAIP_TRACKING + - ('-' + user_agent if user_agent else '')) - explain_response = request_builder.execute() - explanations = ([explain['attributions_by_label'][0]['attributions'] - for explain in explain_response['explanations']]) - baseline_scores = [] - for i, explain in enumerate(explanations): - baseline_scores.append( - explain_response['explanations'][i][ - 'attributions_by_label'][0]['baseline_score']) - response.update( - {'explanations': explanations, 'baseline_scores': baseline_scores}) - except Exception as e: - pass - return response - - def chunks(l, n): - """Yield successive n-sized chunks from l.""" - for i in range(0, len(l), n): - yield l[i:i + n] - - # Run prediction in batches in threads. - if batch_size is None: - batch_size = len(examples) - batched_examples = list(chunks(examples, batch_size)) - - pool = multiprocessing.pool.ThreadPool(processes=POOL_SIZE) - responses = pool.map(predict, batched_examples) - pool.close() - pool.join() - - for response in responses: - if 'error' in response: - raise RuntimeError(response['error']) - - # Parse the results from the responses and return them. - all_predictions = [] - all_baseline_scores = [] - all_attributions = [] - - for response in responses: - if 'explanations' in response: - # If an attribution adjustment function was provided, use it to adjust - # the attributions. - if adjust_attribution is not None: - all_attributions.extend([ - adjust_attribution(attr) for attr in response['explanations']]) + def set_examples(self, examples): + """Sets the examples shown in WIT. + + The examples are initially set by the examples specified in the + config builder during construction. This method can change which + examples WIT displays. + """ + if self.config.get("uses_json_input"): + tf_examples = self._json_to_tf_examples(examples) + self.examples = [ + json_format.MessageToJson(ex) for ex in tf_examples + ] else: - all_attributions.extend(response['explanations']) - - if 'baseline_scores' in response: - all_baseline_scores.extend(response['baseline_scores']) - - # Use the specified key if one is provided. - key_to_use = self.config.get('predict_output_tensor') - - for pred in response['predictions']: - # If the prediction contains a key to fetch the prediction, use it. - if isinstance(pred, dict): - if key_to_use is None: - # If the dictionary only contains one key, use it. - returned_keys = list(pred.keys()) - if len(returned_keys) == 1: - key_to_use = returned_keys[0] - # Use a default key if necessary. - elif self.config.get('model_type') == 'classification': - key_to_use = 'probabilities' - else: - key_to_use = 'outputs' - - if key_to_use not in pred: - raise KeyError( - '"%s" not found in model predictions dictionary' % key_to_use) - - pred = pred[key_to_use] - - # If the model is regression and the response is a list, extract the - # score by taking the first element. - if (self.config.get('model_type') == 'regression' and - isinstance(pred, list)): - pred = pred[0] - - # If an prediction adjustment function was provided, use it to adjust - # the prediction. - if adjust_prediction: - pred = adjust_prediction(pred) - - # If the model is classification and the response is a single number, - # treat that as the positive class score for a binary classification - # and convert it into a list of those two class scores. WIT only - # accepts lists of class scores as results from classification models. - if (self.config.get('model_type') == 'classification'): - if not isinstance(pred, list): - pred = [pred] - if len(pred) == 1: - pred = [1 - pred[0], pred[0]] - - all_predictions.append(pred) - - results = {'predictions': all_predictions} - if all_attributions: - results.update({'attributions': all_attributions}) - if all_baseline_scores: - results.update({'baseline_score': all_baseline_scores}) - return results - - def create_selection_callback(self, examples, max_examples): - """Returns an example selection callback for use with TFMA. - - The returned function can be provided as an event handler for a TFMA - visualization to dynamically load examples matching a selected slice into - WIT. - - Args: - examples: A list of tf.Examples to filter and use with WIT. - max_examples: The maximum number of examples to create. - """ - def handle_selection(selected): - def extract_values(feat): - if feat.HasField('bytes_list'): - return [v.decode('utf-8') for v in feat.bytes_list.value] - elif feat.HasField('int64_list'): - return feat.int64_list.value - elif feat.HasField('float_list'): - return feat.float_list.value - return None - - filtered_examples = [] - for ex in examples: - if selected['sliceName'] == 'Overall': - filtered_examples.append(ex) + self.examples = [json_format.MessageToJson(ex) for ex in examples] + self.updated_example_indices = set(range(len(examples))) + + def compute_custom_distance_impl(self, index, params=None): + exs_for_distance = [ + self.json_to_proto(example) for example in self.examples + ] + selected_ex = exs_for_distance[index] + return self.custom_distance_fn(selected_ex, exs_for_distance, params) + + def json_to_proto(self, json): + ex = ( + tf.train.SequenceExample() + if self.config.get("are_sequence_examples") + else tf.train.Example() + ) + json_format.Parse(json, ex) + return ex + + def infer_impl(self): + """Performs inference on examples that require inference.""" + indices_to_infer = sorted(self.updated_example_indices) + examples_to_infer = [ + self.json_to_proto(self.examples[index]) + for index in indices_to_infer + ] + infer_objs = [] + extra_output_objs = [] + serving_bundle = inference_utils.ServingBundle( + self.config.get("inference_address"), + self.config.get("model_name"), + self.config.get("model_type"), + self.config.get("model_version"), + self.config.get("model_signature"), + self.config.get("uses_predict_api"), + self.config.get("predict_input_tensor"), + self.config.get("predict_output_tensor"), + self.estimator_and_spec.get("estimator"), + self.estimator_and_spec.get("feature_spec"), + self.custom_predict_fn, + ) + ( + predictions, + extra_output, + ) = inference_utils.run_inference_for_inference_results( + examples_to_infer, serving_bundle + ) + infer_objs.append(predictions) + extra_output_objs.append(extra_output) + if ( + "inference_address_2" in self.config + or self.compare_estimator_and_spec.get("estimator") + or self.compare_custom_predict_fn + ): + serving_bundle = inference_utils.ServingBundle( + self.config.get("inference_address_2"), + self.config.get("model_name_2"), + self.config.get("model_type"), + self.config.get("model_version_2"), + self.config.get("model_signature_2"), + self.config.get("uses_predict_api"), + self.config.get("predict_input_tensor"), + self.config.get("predict_output_tensor"), + self.compare_estimator_and_spec.get("estimator"), + self.compare_estimator_and_spec.get("feature_spec"), + self.compare_custom_predict_fn, + ) + ( + predictions, + extra_output, + ) = inference_utils.run_inference_for_inference_results( + examples_to_infer, serving_bundle + ) + infer_objs.append(predictions) + extra_output_objs.append(extra_output) + self.updated_example_indices = set() + return { + "inferences": {"indices": indices_to_infer, "results": infer_objs}, + "label_vocab": self.config.get("label_vocab"), + "extra_outputs": extra_output_objs, + } + + def infer_mutants_impl(self, info): + """Performs mutant inference on specified examples.""" + example_index = int(info["example_index"]) + feature_name = info["feature_name"] + examples = ( + self.examples + if example_index == -1 + else [self.examples[example_index]] + ) + examples = [self.json_to_proto(ex) for ex in examples] + scan_examples = [self.json_to_proto(ex) for ex in self.examples[0:50]] + serving_bundles = [] + serving_bundles.append( + inference_utils.ServingBundle( + self.config.get("inference_address"), + self.config.get("model_name"), + self.config.get("model_type"), + self.config.get("model_version"), + self.config.get("model_signature"), + self.config.get("uses_predict_api"), + self.config.get("predict_input_tensor"), + self.config.get("predict_output_tensor"), + self.estimator_and_spec.get("estimator"), + self.estimator_and_spec.get("feature_spec"), + self.custom_predict_fn, + ) + ) + if ( + "inference_address_2" in self.config + or self.compare_estimator_and_spec.get("estimator") + or self.compare_custom_predict_fn + ): + serving_bundles.append( + inference_utils.ServingBundle( + self.config.get("inference_address_2"), + self.config.get("model_name_2"), + self.config.get("model_type"), + self.config.get("model_version_2"), + self.config.get("model_signature_2"), + self.config.get("uses_predict_api"), + self.config.get("predict_input_tensor"), + self.config.get("predict_output_tensor"), + self.compare_estimator_and_spec.get("estimator"), + self.compare_estimator_and_spec.get("feature_spec"), + self.compare_custom_predict_fn, + ) + ) + viz_params = inference_utils.VizParams( + info["x_min"], + info["x_max"], + scan_examples, + 10, + info["feature_index_pattern"], + ) + self.running_mutant_infer = True + charts = inference_utils.mutant_charts_for_feature( + examples, feature_name, serving_bundles, viz_params + ) + self.running_mutant_infer = False + return charts + + def get_eligible_features_impl(self): + """Returns information about features eligible for mutant inference.""" + examples = [ + self.json_to_proto(ex) + for ex in self.examples[0:NUM_EXAMPLES_FOR_MUTANT_ANALYSIS] + ] + return inference_utils.get_eligible_features( + examples, NUM_MUTANTS_TO_GENERATE + ) + + def sort_eligible_features_impl(self, info): + """Returns sorted list of interesting features for mutant inference.""" + features_list = info["features"] + chart_data = {} + for feat in features_list: + chart_data[feat["name"]] = self.infer_mutants_impl( + { + "x_min": feat["observedMin"] + if "observedMin" in feat + else 0, + "x_max": feat["observedMax"] + if "observedMin" in feat + else 0, + "feature_index_pattern": None, + "feature_name": feat["name"], + "example_index": info["example_index"], + } + ) + return inference_utils.sort_eligible_features(features_list, chart_data) + + def create_sprite(self): + """Returns an encoded image of thumbnails for image examples.""" + # Generate a sprite image for the examples if the examples contain the + # standard encoded image feature. + if not self.examples: + return None + example_to_check = self.json_to_proto(self.examples[0]) + feature_list = ( + example_to_check.context.feature + if self.config.get("are_sequence_examples") + else example_to_check.features.feature + ) + if "image/encoded" in feature_list: + example_strings = [ + self.json_to_proto(ex).SerializeToString() + for ex in self.examples + ] + encoded = ensure_str( + base64.b64encode( + inference_utils.create_sprite_image(example_strings) + ) + ) + return "data:image/png;base64,{}".format(encoded) else: - values = extract_values(ex.features.feature[selected['sliceName']]) - if selected['sliceValue'] in values: - filtered_examples.append(ex) - if len(filtered_examples) == max_examples: - break - - self.set_examples(filtered_examples) - return handle_selection + return None + + def _json_from_tf_examples(self, tf_examples): + json_exs = [] + feature_names = self.config.get("feature_names") + for ex in tf_examples: + # Create a JSON list or dict for each example depending on settings. + # Strip out any explicitly-labeled target feature from the example. + # This is needed because AI Platform models that accept JSON cannot handle + # when non-input features are provided as part of the object to run + # prediction on. + if self.config.get("uses_json_list"): + json_ex = [] + for feat in ex.features.feature: + if feature_names and feat in feature_names: + feat_idx = feature_names.index(feat) + else: + feat_idx = int(feat) + if feat == self.config.get( + "target_feature" + ) or feat_idx == self.config.get("target_feature"): + continue + # Ensure the example value list is long enough to add the next feature + # from the tf.Example. + if feat_idx >= len(json_ex): + json_ex.extend([None] * (feat_idx - len(json_ex) + 1)) + if ex.features.feature[feat].HasField("int64_list"): + json_ex[feat_idx] = ex.features.feature[ + feat + ].int64_list.value[0] + elif ex.features.feature[feat].HasField("float_list"): + json_ex[feat_idx] = ex.features.feature[ + feat + ].float_list.value[0] + else: + json_ex[feat_idx] = ensure_str( + ex.features.feature[feat].bytes_list.value[0] + ) + else: + json_ex = {} + for feat in ex.features.feature: + if feat == self.config.get("target_feature"): + continue + if ex.features.feature[feat].HasField("int64_list"): + json_ex[feat] = ex.features.feature[ + feat + ].int64_list.value[0] + elif ex.features.feature[feat].HasField("float_list"): + json_ex[feat] = ex.features.feature[ + feat + ].float_list.value[0] + else: + json_ex[feat] = ensure_str( + ex.features.feature[feat].bytes_list.value[0] + ) + json_exs.append(json_ex) + return json_exs + + def _json_to_tf_examples(self, examples): + def add_single_feature(feat, value, ex): + if isinstance(value, integer_types): + ex.features.feature[feat].int64_list.value.append(value) + elif isinstance(value, Number): + ex.features.feature[feat].float_list.value.append(value) + else: + ex.features.feature[feat].bytes_list.value.append( + value.encode("utf-8") + ) + + tf_examples = [] + for json_ex in examples: + ex = tf.train.Example() + # JSON examples can be lists of values (for xgboost models for instance), + # or dicts of key/value pairs. + if self.config.get("uses_json_list"): + feature_names = self.config.get("feature_names") + for (i, value) in enumerate(json_ex): + # If feature names have been provided, use those feature names instead + # of list indices for feature name when storing as tf.Example. + if feature_names and len(feature_names) > i: + feat = feature_names[i] + else: + feat = str(i) + add_single_feature(feat, value, ex) + tf_examples.append(ex) + else: + for feat in json_ex: + add_single_feature(feat, json_ex[feat], ex) + tf_examples.append(ex) + return tf_examples + + def _predict_aip_model(self, examples): + return self._predict_aip_impl( + examples, + self.config.get("inference_address"), + self.config.get("model_name"), + self.config.get("model_signature"), + self.config.get("force_json_input"), + self.adjust_example_fn, + self.adjust_prediction_fn, + self.adjust_attribution_fn, + self.config.get("aip_service_name"), + self.config.get("aip_service_version"), + self.config.get("get_explanations"), + self.config.get("aip_batch_size"), + self.config.get("aip_api_key"), + ) + + def _predict_aip_compare_model(self, examples): + return self._predict_aip_impl( + examples, + self.config.get("inference_address_2"), + self.config.get("model_name_2"), + self.config.get("model_signature_2"), + self.config.get("compare_force_json_input"), + self.compare_adjust_example_fn, + self.compare_adjust_prediction_fn, + self.compare_adjust_attribution_fn, + self.config.get("compare_aip_service_name"), + self.config.get("compare_aip_service_version"), + self.config.get("compare_get_explanations"), + self.config.get("compare_aip_batch_size"), + self.config.get("compare_aip_api_key"), + ) + + def _predict_aip_impl( + self, + examples, + project, + model, + version, + force_json, + adjust_example, + adjust_prediction, + adjust_attribution, + service_name, + service_version, + get_explanations, + batch_size, + api_key, + ): + """Custom prediction function for running inference through AI + Platform.""" + + # Set up environment for GCP call for specified project. + os.environ["GOOGLE_CLOUD_PROJECT"] = project + + should_explain = get_explanations and not self.running_mutant_infer + + def predict(exs): + """Run prediction on a list of examples and return results.""" + # Properly package the examples to send for prediction. + discovery_url = None + error_during_prediction = False + if api_key is not None: + discovery_url = ( + "https://%s.googleapis.com/$discovery/rest" + "?labels=GOOGLE_INTERNAL&key=%s&version=%s" + ) % (service_name, api_key, "v1") + credentials = GoogleCredentials.get_application_default() + service = googleapiclient.discovery.build( + service_name, + service_version, + cache_discovery=False, + developerKey=api_key, + discoveryServiceUrl=discovery_url, + credentials=credentials, + ) + else: + service = googleapiclient.discovery.build( + service_name, service_version, cache_discovery=False + ) + + name = "projects/{}/models/{}".format(project, model) + if version is not None: + name += "/versions/{}".format(version) + + if self.config.get("uses_json_input") or force_json: + examples_for_predict = self._json_from_tf_examples(exs) + else: + examples_for_predict = [ + { + "b64": base64.b64encode( + example.SerializeToString() + ).decode("utf-8") + } + for example in exs + ] + + # If there is a user-specified input example adjustment to make, make it. + if adjust_example: + examples_for_predict = [ + adjust_example(ex) for ex in examples_for_predict + ] + + # Send request, including custom user-agent for tracking. + request_builder = service.projects().predict( + name=name, body={"instances": examples_for_predict} + ) + user_agent = request_builder.headers.get("user-agent") + request_builder.headers["user-agent"] = ( + USER_AGENT_FOR_CAIP_TRACKING + + ("-" + user_agent if user_agent else "") + ) + try: + response = request_builder.execute() + except Exception as e: + error_during_prediction = True + response = {"error": str(e)} + + # Get the attributions and baseline score if explaination is enabled. + if should_explain and not error_during_prediction: + try: + request_builder = service.projects().explain( + name=name, body={"instances": examples_for_predict} + ) + request_builder.headers["user-agent"] = ( + USER_AGENT_FOR_CAIP_TRACKING + + ("-" + user_agent if user_agent else "") + ) + explain_response = request_builder.execute() + explanations = [ + explain["attributions_by_label"][0]["attributions"] + for explain in explain_response["explanations"] + ] + baseline_scores = [] + for i, explain in enumerate(explanations): + baseline_scores.append( + explain_response["explanations"][i][ + "attributions_by_label" + ][0]["baseline_score"] + ) + response.update( + { + "explanations": explanations, + "baseline_scores": baseline_scores, + } + ) + except Exception as e: + pass + return response + + def chunks(l, n): + """Yield successive n-sized chunks from l.""" + for i in range(0, len(l), n): + yield l[i : i + n] + + # Run prediction in batches in threads. + if batch_size is None: + batch_size = len(examples) + batched_examples = list(chunks(examples, batch_size)) + + pool = multiprocessing.pool.ThreadPool(processes=POOL_SIZE) + responses = pool.map(predict, batched_examples) + pool.close() + pool.join() + + for response in responses: + if "error" in response: + raise RuntimeError(response["error"]) + + # Parse the results from the responses and return them. + all_predictions = [] + all_baseline_scores = [] + all_attributions = [] + + for response in responses: + if "explanations" in response: + # If an attribution adjustment function was provided, use it to adjust + # the attributions. + if adjust_attribution is not None: + all_attributions.extend( + [ + adjust_attribution(attr) + for attr in response["explanations"] + ] + ) + else: + all_attributions.extend(response["explanations"]) + + if "baseline_scores" in response: + all_baseline_scores.extend(response["baseline_scores"]) + + # Use the specified key if one is provided. + key_to_use = self.config.get("predict_output_tensor") + + for pred in response["predictions"]: + # If the prediction contains a key to fetch the prediction, use it. + if isinstance(pred, dict): + if key_to_use is None: + # If the dictionary only contains one key, use it. + returned_keys = list(pred.keys()) + if len(returned_keys) == 1: + key_to_use = returned_keys[0] + # Use a default key if necessary. + elif self.config.get("model_type") == "classification": + key_to_use = "probabilities" + else: + key_to_use = "outputs" + + if key_to_use not in pred: + raise KeyError( + '"%s" not found in model predictions dictionary' + % key_to_use + ) + + pred = pred[key_to_use] + + # If the model is regression and the response is a list, extract the + # score by taking the first element. + if self.config.get("model_type") == "regression" and isinstance( + pred, list + ): + pred = pred[0] + + # If an prediction adjustment function was provided, use it to adjust + # the prediction. + if adjust_prediction: + pred = adjust_prediction(pred) + + # If the model is classification and the response is a single number, + # treat that as the positive class score for a binary classification + # and convert it into a list of those two class scores. WIT only + # accepts lists of class scores as results from classification models. + if self.config.get("model_type") == "classification": + if not isinstance(pred, list): + pred = [pred] + if len(pred) == 1: + pred = [1 - pred[0], pred[0]] + + all_predictions.append(pred) + + results = {"predictions": all_predictions} + if all_attributions: + results.update({"attributions": all_attributions}) + if all_baseline_scores: + results.update({"baseline_score": all_baseline_scores}) + return results + + def create_selection_callback(self, examples, max_examples): + """Returns an example selection callback for use with TFMA. + + The returned function can be provided as an event handler for a TFMA + visualization to dynamically load examples matching a selected slice into + WIT. + + Args: + examples: A list of tf.Examples to filter and use with WIT. + max_examples: The maximum number of examples to create. + """ + + def handle_selection(selected): + def extract_values(feat): + if feat.HasField("bytes_list"): + return [v.decode("utf-8") for v in feat.bytes_list.value] + elif feat.HasField("int64_list"): + return feat.int64_list.value + elif feat.HasField("float_list"): + return feat.float_list.value + return None + + filtered_examples = [] + for ex in examples: + if selected["sliceName"] == "Overall": + filtered_examples.append(ex) + else: + values = extract_values( + ex.features.feature[selected["sliceName"]] + ) + if selected["sliceValue"] in values: + filtered_examples.append(ex) + if len(filtered_examples) == max_examples: + break + + self.set_examples(filtered_examples) + + return handle_selection diff --git a/tensorboard/plugins/interactive_inference/witwidget/notebook/colab/wit.py b/tensorboard/plugins/interactive_inference/witwidget/notebook/colab/wit.py index ae2eb7b468..8a1e0c6243 100644 --- a/tensorboard/plugins/interactive_inference/witwidget/notebook/colab/wit.py +++ b/tensorboard/plugins/interactive_inference/witwidget/notebook/colab/wit.py @@ -21,45 +21,65 @@ # Python functions for requests from javascript. def infer_examples(wit_id): - WitWidget.widgets[wit_id].infer() -output.register_callback('notebook.InferExamples', infer_examples) + WitWidget.widgets[wit_id].infer() + + +output.register_callback("notebook.InferExamples", infer_examples) def delete_example(wit_id, index): - WitWidget.widgets[wit_id].delete_example(index) -output.register_callback('notebook.DeleteExample', delete_example) + WitWidget.widgets[wit_id].delete_example(index) + + +output.register_callback("notebook.DeleteExample", delete_example) def duplicate_example(wit_id, index): - WitWidget.widgets[wit_id].duplicate_example(index) -output.register_callback('notebook.DuplicateExample', duplicate_example) + WitWidget.widgets[wit_id].duplicate_example(index) + + +output.register_callback("notebook.DuplicateExample", duplicate_example) def update_example(wit_id, index, example): - WitWidget.widgets[wit_id].update_example(index, example) -output.register_callback('notebook.UpdateExample', update_example) + WitWidget.widgets[wit_id].update_example(index, example) + + +output.register_callback("notebook.UpdateExample", update_example) def get_eligible_features(wit_id): - WitWidget.widgets[wit_id].get_eligible_features() -output.register_callback('notebook.GetEligibleFeatures', get_eligible_features) + WitWidget.widgets[wit_id].get_eligible_features() + + +output.register_callback("notebook.GetEligibleFeatures", get_eligible_features) def sort_eligible_features(wit_id, details): - WitWidget.widgets[wit_id].sort_eligible_features(details) -output.register_callback('notebook.SortEligibleFeatures', sort_eligible_features) + WitWidget.widgets[wit_id].sort_eligible_features(details) + + +output.register_callback( + "notebook.SortEligibleFeatures", sort_eligible_features +) def infer_mutants(wit_id, details): - WitWidget.widgets[wit_id].infer_mutants(details) -output.register_callback('notebook.InferMutants', infer_mutants) + WitWidget.widgets[wit_id].infer_mutants(details) + + +output.register_callback("notebook.InferMutants", infer_mutants) def compute_custom_distance(wit_id, index, callback_name, params): - WitWidget.widgets[wit_id].compute_custom_distance(index, callback_name, - params) -output.register_callback('notebook.ComputeCustomDistance', - compute_custom_distance) + WitWidget.widgets[wit_id].compute_custom_distance( + index, callback_name, params + ) + + +output.register_callback( + "notebook.ComputeCustomDistance", compute_custom_distance +) # HTML/javascript for the WIT frontend. @@ -211,129 +231,172 @@ def compute_custom_distance(wit_id, index, callback_name, params): class WitWidget(base.WitWidgetBase): - """WIT widget for colab.""" - - # Static instance list of constructed WitWidgets so python global functions - # can call into instances of this object - widgets = [] - - # Static instance index to keep track of ID number of each constructed - # WitWidget. - index = 0 - - def __init__(self, config_builder, height=1000): - """Constructor for colab notebook WitWidget. - - Args: - config_builder: WitConfigBuilder object containing settings for WIT. - height: Optional height in pixels for WIT to occupy. Defaults to 1000. - """ - self._ctor_complete = False - self.id = WitWidget.index - base.WitWidgetBase.__init__(self, config_builder) - # Add this instance to the static instance list. - WitWidget.widgets.append(self) - - # Display WIT Polymer element. - display.display(display.HTML(self._get_element_html())) - display.display(display.HTML( - WIT_HTML.format(height=height, id=self.id))) - - # Increment the static instance WitWidget index counter - WitWidget.index += 1 - - # Send the provided config and examples to JS - output.eval_js("""configCallback({config})""".format( - config=json.dumps(self.config))) - output.eval_js("""updateExamplesCallback({examples})""".format( - examples=json.dumps(self.examples))) - self._generate_sprite() - self._ctor_complete = True - - def _get_element_html(self): - return """ + """WIT widget for colab.""" + + # Static instance list of constructed WitWidgets so python global functions + # can call into instances of this object + widgets = [] + + # Static instance index to keep track of ID number of each constructed + # WitWidget. + index = 0 + + def __init__(self, config_builder, height=1000): + """Constructor for colab notebook WitWidget. + + Args: + config_builder: WitConfigBuilder object containing settings for WIT. + height: Optional height in pixels for WIT to occupy. Defaults to 1000. + """ + self._ctor_complete = False + self.id = WitWidget.index + base.WitWidgetBase.__init__(self, config_builder) + # Add this instance to the static instance list. + WitWidget.widgets.append(self) + + # Display WIT Polymer element. + display.display(display.HTML(self._get_element_html())) + display.display( + display.HTML(WIT_HTML.format(height=height, id=self.id)) + ) + + # Increment the static instance WitWidget index counter + WitWidget.index += 1 + + # Send the provided config and examples to JS + output.eval_js( + """configCallback({config})""".format( + config=json.dumps(self.config) + ) + ) + output.eval_js( + """updateExamplesCallback({examples})""".format( + examples=json.dumps(self.examples) + ) + ) + self._generate_sprite() + self._ctor_complete = True + + def _get_element_html(self): + return """ """ - def set_examples(self, examples): - base.WitWidgetBase.set_examples(self, examples) - # If this is called outside of the ctor, use a BroadcastChannel to send - # the updated examples to the visualization. Inside of the ctor, no action - # is necessary as the ctor handles all communication. - if self._ctor_complete: - # Use BroadcastChannel to allow this call to be made in a separate colab - # cell from the cell that displays WIT. - channel_name = 'updateExamples{}'.format(self.id) - output.eval_js("""(new BroadcastChannel('{channel_name}')).postMessage( + def set_examples(self, examples): + base.WitWidgetBase.set_examples(self, examples) + # If this is called outside of the ctor, use a BroadcastChannel to send + # the updated examples to the visualization. Inside of the ctor, no action + # is necessary as the ctor handles all communication. + if self._ctor_complete: + # Use BroadcastChannel to allow this call to be made in a separate colab + # cell from the cell that displays WIT. + channel_name = "updateExamples{}".format(self.id) + output.eval_js( + """(new BroadcastChannel('{channel_name}')).postMessage( {examples})""".format( - examples=json.dumps(self.examples), channel_name=channel_name)) - self._generate_sprite() - - def infer(self): - try: - inferences = base.WitWidgetBase.infer_impl(self) - output.eval_js("""inferenceCallback({inferences})""".format( - inferences=json.dumps(inferences))) - except Exception as e: - output.eval_js("""backendError({error})""".format( - error=json.dumps({'msg': repr(e)}))) - - def delete_example(self, index): - self.examples.pop(index) - self.updated_example_indices = set([ - i if i < index else i - 1 for i in self.updated_example_indices]) - self._generate_sprite() - - def update_example(self, index, example): - self.updated_example_indices.add(index) - self.examples[index] = example - self._generate_sprite() - - def duplicate_example(self, index): - self.examples.append(self.examples[index]) - self.updated_example_indices.add(len(self.examples) - 1) - self._generate_sprite() - - def compute_custom_distance(self, index, callback_fn, params): - try: - distances = base.WitWidgetBase.compute_custom_distance_impl( - self, index, params['distanceParams']) - callback_dict = { - 'distances': distances, - 'exInd': index, - 'funId': callback_fn, - 'params': params['callbackParams'] - } - output.eval_js("""distanceCallback({callback_dict})""".format( - callback_dict=json.dumps(callback_dict))) - except Exception as e: - output.eval_js( - """backendError({error})""".format( - error=json.dumps({'msg': repr(e)}))) - - def get_eligible_features(self): - features_list = base.WitWidgetBase.get_eligible_features_impl(self) - output.eval_js("""eligibleFeaturesCallback({features_list})""".format( - features_list=json.dumps(features_list))) - - def infer_mutants(self, info): - try: - json_mapping = base.WitWidgetBase.infer_mutants_impl(self, info) - output.eval_js("""inferMutantsCallback({json_mapping})""".format( - json_mapping=json.dumps(json_mapping))) - except Exception as e: - output.eval_js("""backendError({error})""".format( - error=json.dumps({'msg': repr(e)}))) - - def sort_eligible_features(self, info): - try: - features_list = base.WitWidgetBase.sort_eligible_features_impl(self, info) - output.eval_js("""sortEligibleFeaturesCallback({features_list})""".format( - features_list=json.dumps(features_list))) - except Exception as e: - output.eval_js("""backendError({error})""".format( - error=json.dumps({'msg': repr(e)}))) - - def _generate_sprite(self): - sprite = base.WitWidgetBase.create_sprite(self) - if sprite is not None: - output.eval_js("""spriteCallback('{sprite}')""".format(sprite=sprite)) + examples=json.dumps(self.examples), + channel_name=channel_name, + ) + ) + self._generate_sprite() + + def infer(self): + try: + inferences = base.WitWidgetBase.infer_impl(self) + output.eval_js( + """inferenceCallback({inferences})""".format( + inferences=json.dumps(inferences) + ) + ) + except Exception as e: + output.eval_js( + """backendError({error})""".format( + error=json.dumps({"msg": repr(e)}) + ) + ) + + def delete_example(self, index): + self.examples.pop(index) + self.updated_example_indices = set( + [i if i < index else i - 1 for i in self.updated_example_indices] + ) + self._generate_sprite() + + def update_example(self, index, example): + self.updated_example_indices.add(index) + self.examples[index] = example + self._generate_sprite() + + def duplicate_example(self, index): + self.examples.append(self.examples[index]) + self.updated_example_indices.add(len(self.examples) - 1) + self._generate_sprite() + + def compute_custom_distance(self, index, callback_fn, params): + try: + distances = base.WitWidgetBase.compute_custom_distance_impl( + self, index, params["distanceParams"] + ) + callback_dict = { + "distances": distances, + "exInd": index, + "funId": callback_fn, + "params": params["callbackParams"], + } + output.eval_js( + """distanceCallback({callback_dict})""".format( + callback_dict=json.dumps(callback_dict) + ) + ) + except Exception as e: + output.eval_js( + """backendError({error})""".format( + error=json.dumps({"msg": repr(e)}) + ) + ) + + def get_eligible_features(self): + features_list = base.WitWidgetBase.get_eligible_features_impl(self) + output.eval_js( + """eligibleFeaturesCallback({features_list})""".format( + features_list=json.dumps(features_list) + ) + ) + + def infer_mutants(self, info): + try: + json_mapping = base.WitWidgetBase.infer_mutants_impl(self, info) + output.eval_js( + """inferMutantsCallback({json_mapping})""".format( + json_mapping=json.dumps(json_mapping) + ) + ) + except Exception as e: + output.eval_js( + """backendError({error})""".format( + error=json.dumps({"msg": repr(e)}) + ) + ) + + def sort_eligible_features(self, info): + try: + features_list = base.WitWidgetBase.sort_eligible_features_impl( + self, info + ) + output.eval_js( + """sortEligibleFeaturesCallback({features_list})""".format( + features_list=json.dumps(features_list) + ) + ) + except Exception as e: + output.eval_js( + """backendError({error})""".format( + error=json.dumps({"msg": repr(e)}) + ) + ) + + def _generate_sprite(self): + sprite = base.WitWidgetBase.create_sprite(self) + if sprite is not None: + output.eval_js( + """spriteCallback('{sprite}')""".format(sprite=sprite) + ) diff --git a/tensorboard/plugins/interactive_inference/witwidget/notebook/jupyter/wit.py b/tensorboard/plugins/interactive_inference/witwidget/notebook/jupyter/wit.py index 19b1d92532..aef9ddbfd7 100644 --- a/tensorboard/plugins/interactive_inference/witwidget/notebook/jupyter/wit.py +++ b/tensorboard/plugins/interactive_inference/witwidget/notebook/jupyter/wit.py @@ -27,124 +27,128 @@ @widgets.register class WitWidget(widgets.DOMWidget, base.WitWidgetBase): - """WIT widget for Jupyter.""" - _view_name = Unicode('WITView').tag(sync=True) - _view_module = Unicode('wit-widget').tag(sync=True) - _view_module_version = Unicode('^0.1.0').tag(sync=True) - - # Traitlets for communicating between python and javascript. - config = Dict(dict()).tag(sync=True) - examples = List([]).tag(sync=True) - inferences = Dict(dict()).tag(sync=True) - infer = Int(0).tag(sync=True) - update_example = Dict(dict()).tag(sync=True) - delete_example = Dict(dict()).tag(sync=True) - duplicate_example = Dict(dict()).tag(sync=True) - updated_example_indices = Set(set()) - get_eligible_features = Int(0).tag(sync=True) - sort_eligible_features = Dict(dict()).tag(sync=True) - eligible_features = List([]).tag(sync=True) - infer_mutants = Dict(dict()).tag(sync=True) - mutant_charts = Dict([]).tag(sync=True) - mutant_charts_counter = Int(0) - sprite = Unicode('').tag(sync=True) - error = Dict(dict()).tag(sync=True) - compute_custom_distance = Dict(dict()).tag(sync=True) - custom_distance_dict = Dict(dict()).tag(sync=True) - - def __init__(self, config_builder, height=1000): - """Constructor for Jupyter notebook WitWidget. - - Args: - config_builder: WitConfigBuilder object containing settings for WIT. - height: Optional height in pixels for WIT to occupy. Defaults to 1000. - """ - widgets.DOMWidget.__init__(self, layout=Layout(height='%ipx' % height)) - base.WitWidgetBase.__init__(self, config_builder) - self.error_counter = 0 - - # Ensure the visualization takes all available width. - display(HTML("")) - - def set_examples(self, examples): - base.WitWidgetBase.set_examples(self, examples) - self._generate_sprite() - - def _report_error(self, err): - self.error = { - 'msg': repr(err), - 'counter': self.error_counter - } - self.error_counter += 1 - - @observe('infer') - def _infer(self, change): - try: - self.inferences = base.WitWidgetBase.infer_impl(self) - except Exception as e: - self._report_error(e) - - # Observer callbacks for changes from javascript. - @observe('get_eligible_features') - def _get_eligible_features(self, change): - features_list = base.WitWidgetBase.get_eligible_features_impl(self) - self.eligible_features = features_list - - @observe('sort_eligible_features') - def _sort_eligible_features(self, change): - info = self.sort_eligible_features - features_list = base.WitWidgetBase.sort_eligible_features_impl(self, info) - self.eligible_features = features_list - - @observe('infer_mutants') - def _infer_mutants(self, change): - info = self.infer_mutants - try: - json_mapping = base.WitWidgetBase.infer_mutants_impl(self, info) - json_mapping['counter'] = self.mutant_charts_counter - self.mutant_charts_counter += 1 - self.mutant_charts = json_mapping - except Exception as e: - self._report_error(e) - - @observe('update_example') - def _update_example(self, change): - index = self.update_example['index'] - self.updated_example_indices.add(index) - self.examples[index] = self.update_example['example'] - self._generate_sprite() - - @observe('duplicate_example') - def _duplicate_example(self, change): - self.examples.append(self.examples[self.duplicate_example['index']]) - self.updated_example_indices.add(len(self.examples) - 1) - self._generate_sprite() - - @observe('delete_example') - def _delete_example(self, change): - index = self.delete_example['index'] - self.examples.pop(index) - self.updated_example_indices = set([ - i if i < index else i - 1 for i in self.updated_example_indices]) - self._generate_sprite() - - @observe('compute_custom_distance') - def _compute_custom_distance(self, change): - info = self.compute_custom_distance - index = info['index'] - params = info['params'] - callback_fn = info['callback'] - try: - distances = base.WitWidgetBase.compute_custom_distance_impl(self, index, - params['distanceParams']) - self.custom_distance_dict = {'distances': distances, - 'exInd': index, - 'funId': callback_fn, - 'params': params['callbackParams']} - except Exception as e: - self._report_error(e) - - def _generate_sprite(self): - sprite = base.WitWidgetBase.create_sprite(self) - if sprite is not None: - self.sprite = sprite + """WIT widget for Jupyter.""" + + _view_name = Unicode("WITView").tag(sync=True) + _view_module = Unicode("wit-widget").tag(sync=True) + _view_module_version = Unicode("^0.1.0").tag(sync=True) + + # Traitlets for communicating between python and javascript. + config = Dict(dict()).tag(sync=True) + examples = List([]).tag(sync=True) + inferences = Dict(dict()).tag(sync=True) + infer = Int(0).tag(sync=True) + update_example = Dict(dict()).tag(sync=True) + delete_example = Dict(dict()).tag(sync=True) + duplicate_example = Dict(dict()).tag(sync=True) + updated_example_indices = Set(set()) + get_eligible_features = Int(0).tag(sync=True) + sort_eligible_features = Dict(dict()).tag(sync=True) + eligible_features = List([]).tag(sync=True) + infer_mutants = Dict(dict()).tag(sync=True) + mutant_charts = Dict([]).tag(sync=True) + mutant_charts_counter = Int(0) + sprite = Unicode("").tag(sync=True) + error = Dict(dict()).tag(sync=True) + compute_custom_distance = Dict(dict()).tag(sync=True) + custom_distance_dict = Dict(dict()).tag(sync=True) + + def __init__(self, config_builder, height=1000): + """Constructor for Jupyter notebook WitWidget. + + Args: + config_builder: WitConfigBuilder object containing settings for WIT. + height: Optional height in pixels for WIT to occupy. Defaults to 1000. + """ + widgets.DOMWidget.__init__(self, layout=Layout(height="%ipx" % height)) + base.WitWidgetBase.__init__(self, config_builder) + self.error_counter = 0 + + # Ensure the visualization takes all available width. + display(HTML("")) + + def set_examples(self, examples): + base.WitWidgetBase.set_examples(self, examples) + self._generate_sprite() + + def _report_error(self, err): + self.error = {"msg": repr(err), "counter": self.error_counter} + self.error_counter += 1 + + @observe("infer") + def _infer(self, change): + try: + self.inferences = base.WitWidgetBase.infer_impl(self) + except Exception as e: + self._report_error(e) + + # Observer callbacks for changes from javascript. + @observe("get_eligible_features") + def _get_eligible_features(self, change): + features_list = base.WitWidgetBase.get_eligible_features_impl(self) + self.eligible_features = features_list + + @observe("sort_eligible_features") + def _sort_eligible_features(self, change): + info = self.sort_eligible_features + features_list = base.WitWidgetBase.sort_eligible_features_impl( + self, info + ) + self.eligible_features = features_list + + @observe("infer_mutants") + def _infer_mutants(self, change): + info = self.infer_mutants + try: + json_mapping = base.WitWidgetBase.infer_mutants_impl(self, info) + json_mapping["counter"] = self.mutant_charts_counter + self.mutant_charts_counter += 1 + self.mutant_charts = json_mapping + except Exception as e: + self._report_error(e) + + @observe("update_example") + def _update_example(self, change): + index = self.update_example["index"] + self.updated_example_indices.add(index) + self.examples[index] = self.update_example["example"] + self._generate_sprite() + + @observe("duplicate_example") + def _duplicate_example(self, change): + self.examples.append(self.examples[self.duplicate_example["index"]]) + self.updated_example_indices.add(len(self.examples) - 1) + self._generate_sprite() + + @observe("delete_example") + def _delete_example(self, change): + index = self.delete_example["index"] + self.examples.pop(index) + self.updated_example_indices = set( + [i if i < index else i - 1 for i in self.updated_example_indices] + ) + self._generate_sprite() + + @observe("compute_custom_distance") + def _compute_custom_distance(self, change): + info = self.compute_custom_distance + index = info["index"] + params = info["params"] + callback_fn = info["callback"] + try: + distances = base.WitWidgetBase.compute_custom_distance_impl( + self, index, params["distanceParams"] + ) + self.custom_distance_dict = { + "distances": distances, + "exInd": index, + "funId": callback_fn, + "params": params["callbackParams"], + } + except Exception as e: + self._report_error(e) + + def _generate_sprite(self): + sprite = base.WitWidgetBase.create_sprite(self) + if sprite is not None: + self.sprite = sprite diff --git a/tensorboard/plugins/interactive_inference/witwidget/notebook/visualization.py b/tensorboard/plugins/interactive_inference/witwidget/notebook/visualization.py index 98ca6c9de1..e077a3a242 100644 --- a/tensorboard/plugins/interactive_inference/witwidget/notebook/visualization.py +++ b/tensorboard/plugins/interactive_inference/witwidget/notebook/visualization.py @@ -18,673 +18,706 @@ def _is_colab(): - return "google.colab" in sys.modules + return "google.colab" in sys.modules if _is_colab(): - from witwidget.notebook.colab.wit import * # pylint: disable=wildcard-import + from witwidget.notebook.colab.wit import * # pylint: disable=wildcard-import else: - from witwidget.notebook.jupyter.wit import * # pylint: disable=wildcard-import + from witwidget.notebook.jupyter.wit import * # pylint: disable=wildcard-import class WitConfigBuilder(object): - """Configuration builder for WitWidget settings.""" - - def __init__(self, examples, feature_names=None): - """Constructs the WitConfigBuilder object. - - Args: - examples: A list of tf.Example or tf.SequenceExample proto objects, or - raw JSON objects. JSON is allowed only for AI Platform-hosted models (see - 'set_ai_platform_model' and 'set_compare_ai_platform_model methods). - These are the examples that will be displayed in WIT. If no model to - infer these examples with is specified through the methods on this class, - then WIT will display the examples for exploration, but no model inference - will be performed by the tool. - feature_names: Optional, defaults to None. If examples are provided as - JSON lists of numbers (not as feature dictionaries), then this array - maps indices in the feature value lists to human-readable names of those - features, used for display purposes. - """ - self.config = {} - self.set_model_type('classification') - self.set_label_vocab([]) - self.set_examples(examples, feature_names) - - def build(self): - """Returns the configuration set through use of this builder object. - - Used by WitWidget to set the settings on an instance of the What-If Tool. - """ - return self.config - - def store(self, key, value): - self.config[key] = value - - def delete(self, key): - if key in self.config: - del self.config[key] - - def set_examples(self, examples, feature_names=None): - """Sets the examples to be displayed in WIT. - - Args: - examples: List of example protos or JSON objects. - feature_names: Optional, defaults to None. If examples are provided as - JSON lists of numbers (not as feature dictionaries), then this array - maps indices in the feature value lists to human-readable names of those - features, used just for display purposes. - - Returns: - self, in order to enabled method chaining. - """ - self.store('examples', examples) - if feature_names: - self.store('feature_names', feature_names) - if len(examples) > 0 and not ( - isinstance(examples[0], tf.train.Example) or - isinstance(examples[0], tf.train.SequenceExample)): - self._set_uses_json_input(True) - if isinstance(examples[0], list): - self._set_uses_json_list(True) - elif len(examples) > 0: - self.store('are_sequence_examples', - isinstance(examples[0], tf.train.SequenceExample)) - return self - - def set_model_type(self, model): - """Sets the type of the model being used for inference. - - Args: - model: The model type, such as "classification" or "regression". - The model type defaults to "classification". - - Returns: - self, in order to enabled method chaining. - """ - self.store('model_type', model) - return self - - def set_inference_address(self, address): - """Sets the inference address for model inference through TF Serving. - - Args: - address: The address of the served model, including port, such as - "localhost:8888". - - Returns: - self, in order to enabled method chaining. - """ - self.store('inference_address', address) - return self - - def set_model_name(self, name): - """Sets the model name for model inference through TF Serving. - - Setting a model name is required if inferring through a model hosted by - TF Serving. - - Args: - name: The name of the model to be queried through TF Serving at the - address provided by set_inference_address. - - Returns: - self, in order to enabled method chaining. - """ - self.store('model_name', name) - return self - - def has_model_name(self): - return 'model_name' in self.config - - def set_model_version(self, version): - """Sets the optional model version for model inference through TF Serving. - - Args: - version: The string version number of the model to be queried through TF - Serving. This is optional, as TF Serving will use the latest model version - if none is provided. - - Returns: - self, in order to enabled method chaining. - """ - self.store('model_version', version) - return self - - def set_model_signature(self, signature): - """Sets the optional model signature for model inference through TF Serving. - - Args: - signature: The string signature of the model to be queried through TF - Serving. This is optional, as TF Serving will use the default model - signature if none is provided. - - Returns: - self, in order to enabled method chaining. - """ - self.store('model_signature', signature) - return self - - def set_compare_inference_address(self, address): - """Sets the inference address for model inference for a second model hosted - by TF Serving. - - If you wish to compare the results of two models in WIT, use this method - to setup the details of the second model. - - Args: - address: The address of the served model, including port, such as - "localhost:8888". - - Returns: - self, in order to enabled method chaining. - """ - self.store('inference_address_2', address) - return self - - def set_compare_model_name(self, name): - """Sets the model name for a second model hosted by TF Serving. - - If you wish to compare the results of two models in WIT, use this method - to setup the details of the second model. - - Setting a model name is required if inferring through a model hosted by - TF Serving. - - Args: - name: The name of the model to be queried through TF Serving at the - address provided by set_compare_inference_address. - - Returns: - self, in order to enabled method chaining. - """ - self.store('model_name_2', name) - return self - - def has_compare_model_name(self): - return 'model_name_2' in self.config - - def set_compare_model_version(self, version): - """Sets the optional model version for a second model hosted by TF Serving. - - If you wish to compare the results of two models in WIT, use this method - to setup the details of the second model. - - Args: - version: The string version number of the model to be queried through TF - Serving. This is optional, as TF Serving will use the latest model version - if none is provided. - - Returns: - self, in order to enabled method chaining. - """ - self.store('model_version_2', version) - return self - - def set_compare_model_signature(self, signature): - """Sets the optional model signature for a second model hosted by TF - Serving. - - If you wish to compare the results of two models in WIT, use this method - to setup the details of the second model. - - Args: - signature: The string signature of the model to be queried through TF - Serving. This is optional, as TF Serving will use the default model - signature if none is provided. - - Returns: - self, in order to enabled method chaining. - """ - self.store('model_signature_2', signature) - return self - - def set_uses_predict_api(self, predict): - """Indicates that the model uses the Predict API, as opposed to the - Classification or Regression API. - - If the model doesn't use the standard Classification or Regression APIs - provided through TF Serving, but instead uses the more flexible Predict API, - then use this method to indicate that. If this is true, then use the - set_predict_input_tensor and set_predict_output_tensor methods to indicate - the names of the tensors that are used as the input and output for the - models provided in order to perform the appropriate inference request. - - Args: - predict: True if the model or models use the Predict API. - - Returns: - self, in order to enabled method chaining. - """ - self.store('uses_predict_api', predict) - return self - - def set_max_classes_to_display(self, max_classes): - """Sets the maximum number of class results to display for multiclass - classification models. - - When using WIT with a multiclass model with a large number of possible - classes, it can be helpful to restrict WIT to only display some smaller - number of the highest-scoring classes as inference results for any given - example. This method sets that limit. - - Args: - max_classes: The maximum number of classes to display for inference - results for multiclass classification models. - - Returns: - self, in order to enabled method chaining. - """ - self.store('max_classes', max_classes) - return self - - def set_multi_class(self, multiclass): - """Sets if the model(s) to query are mutliclass classification models. - - Args: - multiclass: True if the model or models are multiclass classififcation - models. Defaults to false. - - Returns: - self, in order to enabled method chaining. - """ - self.store('multiclass', multiclass) - return self - - def set_predict_input_tensor(self, tensor): - """Sets the name of the input tensor for models that use the Predict API. - - If using WIT with set_uses_predict_api(True), then call this to specify - the name of the input tensor of the model or models that accepts the - example proto for inference. - - Args: - tensor: The name of the input tensor. - - Returns: - self, in order to enabled method chaining. - """ - self.store('predict_input_tensor', tensor) - return self - - def set_predict_output_tensor(self, tensor): - """Sets the name of the output tensor for models that need output parsing. - - If using WIT with set_uses_predict_api(True), then call this to specify - the name of the output tensor of the model or models that returns the - inference results to be explored by WIT. - - If using an AI Platform model which returns multiple prediction - results in a dictionary, this method specifies the key corresponding to - the inference results to be explored by WIT. - - Args: - tensor: The name of the output tensor. - - Returns: - self, in order to enabled method chaining. - """ - self.store('predict_output_tensor', tensor) - return self - - def set_label_vocab(self, vocab): - """Sets the string value of numeric labels for classification models. - - For classification models, the model returns scores for each class ID - number (classes 0 and 1 for binary classification models). In order for - WIT to visually display the results in a more-readable way, you can specify - string labels for each class ID. - - Args: - vocab: A list of strings, where the string at each index corresponds to - the label for that class ID. For example ['<=50K', '>50K'] for the UCI - census binary classification task. - - Returns: - self, in order to enabled method chaining. - """ - self.store('label_vocab', vocab) - return self - - def set_estimator_and_feature_spec(self, estimator, feature_spec): - """Sets the model for inference as a TF Estimator. - - Instead of using TF Serving to host a model for WIT to query, WIT can - directly use a TF Estimator object as the model to query. In order to - accomplish this, a feature_spec must also be provided to parse the - example protos for input into the estimator. - - Args: - estimator: The TF Estimator which will be used for model inference. - feature_spec: The feature_spec object which will be used for example - parsing. - - Returns: - self, in order to enabled method chaining. - """ - # If custom function is set, remove it before setting estimator - self.delete('custom_predict_fn') - - self.store('estimator_and_spec', { - 'estimator': estimator, 'feature_spec': feature_spec}) - self.set_inference_address('estimator') - # If no model name has been set, give a default - if not self.has_model_name(): - self.set_model_name('1') - return self - - def set_compare_estimator_and_feature_spec(self, estimator, feature_spec): - """Sets a second model for inference as a TF Estimator. - - If you wish to compare the results of two models in WIT, use this method - to setup the details of the second model. - - Instead of using TF Serving to host a model for WIT to query, WIT can - directly use a TF Estimator object as the model to query. In order to - accomplish this, a feature_spec must also be provided to parse the - example protos for input into the estimator. - - Args: - estimator: The TF Estimator which will be used for model inference. - feature_spec: The feature_spec object which will be used for example - parsing. - - Returns: - self, in order to enabled method chaining. - """ - # If custom function is set, remove it before setting estimator - self.delete('compare_custom_predict_fn') - - self.store('compare_estimator_and_spec', { - 'estimator': estimator, 'feature_spec': feature_spec}) - self.set_compare_inference_address('estimator') - # If no model name has been set, give a default - if not self.has_compare_model_name(): - self.set_compare_model_name('2') - return self - - def set_custom_predict_fn(self, predict_fn): - """Sets a custom function for inference. - - Instead of using TF Serving to host a model for WIT to query, WIT can - directly use a custom function as the model to query. In this case, the - provided function should accept example protos and return: - - For classification: A 2D list of numbers. The first dimension is for - each example being predicted. The second dimension are the probabilities - for each class ID in the prediction. - - For regression: A 1D list of numbers, with a regression score for each - example being predicted. - - Optionally, if attributions or other prediction-time information - can be returned by the model with each prediction, then this method - can return a dict with the key 'predictions' containing the predictions - result list described above, and with the key 'attributions' containing - a list of attributions for each example that was predicted. - - For each example, the attributions list should contain a dict mapping - input feature names to attribution values for that feature on that example. - The attribution value can be one of these things: - - A single number representing the attribution for the entire feature - - A list of numbers representing the attribution to each value in the - feature for multivalent features - such as attributions to individual - pixels in an image or numbers in a list of numbers. - - A 2D list for sparse feature attribution. Index 0 contains a list of - feature values that there are attribution scores for. Index 1 contains - a list of attribution values for the corresponding feature values in - the first list. - - This dict can contain any other keys, with their values being a list of - prediction-time strings or numbers for each example being predicted. These - values will be displayed in WIT as extra information for each example, - usable in the same ways by WIT as normal input features (such as for - creating plots and slicing performance data). - - Args: - predict_fn: The custom python function which will be used for model - inference. - - Returns: - self, in order to enabled method chaining. - """ - # If estimator is set, remove it before setting predict_fn - self.delete('estimator_and_spec') - - self.store('custom_predict_fn', predict_fn) - self.set_inference_address('custom_predict_fn') - # If no model name has been set, give a default - if not self.has_model_name(): - self.set_model_name('1') - return self - - def set_compare_custom_predict_fn(self, predict_fn): - """Sets a second custom function for inference. - - If you wish to compare the results of two models in WIT, use this method - to setup the details of the second model. - - Instead of using TF Serving to host a model for WIT to query, WIT can - directly use a custom function as the model to query. In this case, the - provided function should accept example protos and return: - - For classification: A 2D list of numbers. The first dimension is for - each example being predicted. The second dimension are the probabilities - for each class ID in the prediction. - - For regression: A 1D list of numbers, with a regression score for each - example being predicted. - - Optionally, if attributions or other prediction-time information - can be returned by the model with each prediction, then this method - can return a dict with the key 'predictions' containing the predictions - result list described above, and with the key 'attributions' containing - a list of attributions for each example that was predicted. - - For each example, the attributions list should contain a dict mapping - input feature names to attribution values for that feature on that example. - The attribution value can be one of these things: - - A single number representing the attribution for the entire feature - - A list of numbers representing the attribution to each value in the - feature for multivalent features - such as attributions to individual - pixels in an image or numbers in a list of numbers. - - A 2D list for sparse feature attribution. Index 0 contains a list of - feature values that there are attribution scores for. Index 1 contains - a list of attribution values for the corresponding feature values in - the first list. - - This dict can contain any other keys, with their values being a list of - prediction-time strings or numbers for each example being predicted. These - values will be displayed in WIT as extra information for each example, - usable in the same ways by WIT as normal input features (such as for - creating plots and slicing performance data). - - Args: - predict_fn: The custom python function which will be used for model - inference. - - Returns: - self, in order to enabled method chaining. - """ - # If estimator is set, remove it before setting predict_fn - self.delete('compare_estimator_and_spec') - - self.store('compare_custom_predict_fn', predict_fn) - self.set_compare_inference_address('custom_predict_fn') - # If no model name has been set, give a default - if not self.has_compare_model_name(): - self.set_compare_model_name('2') - return self - - def set_custom_distance_fn(self, distance_fn): - """Sets a custom function for distance computation. - - WIT can directly use a custom function for all distance computations within - the tool. In this case, the provided function should accept a query example - proto and a list of example protos to compute the distance against and - return a 1D list of numbers containing the distances. - - Args: - distance_fn: The python function which will be used for distance - computation. - - Returns: - self, in order to enabled method chaining. - """ - if distance_fn is None: - self.delete('custom_distance_fn') - else: - self.store('custom_distance_fn', distance_fn) - return self - - def set_ai_platform_model( - self, project, model, version=None, force_json_input=None, - adjust_prediction=None, adjust_example=None, adjust_attribution=None, - service_name='ml', service_version='v1', get_explanations=True, - batch_size=500, api_key=None): - """Sets the model information for a model served by AI Platform. - - AI Platform Prediction a Google Cloud serving platform. - - Args: - project: The name of the AI Platform Prediction project. - model: The name of the AI Platform Prediction model. - version: Optional, the version of the AI Platform Prediction model. - force_json_input: Optional. If True and examples are provided as - tf.Example protos, convert them to raw JSON objects before sending them - for inference to this model. - adjust_prediction: Optional. If not None then this function takes the - prediction output from the model for a single example and converts it to - the appopriate format - a regression score or a list of class scores. Only - necessary if the model doesn't already abide by this format. - adjust_example: Optional. If not None then this function takes an example - to run prediction on and converts it to the format expected by the model. - Necessary for example if the served model expects a single data value to - run inference on instead of a list or dict of values. - adjust_attribution: Optional. If not None and the model returns attribution - information, then this function takes the attribution information for an - example and converts it to the format expected by the tool, which is a - dictionary of input feature names to attribution scores. Usually necessary - if making use of adjust_example and the model returns attribution results. - service_name: Optional. Name of the AI Platform Prediction service. Defaults - to 'ml'. - service_version: Optional. Version of the AI Platform Prediction service. Defaults - to 'v1'. - get_explanations: Optional. If a model is deployed with explanations, - then this specifies if explainations will be calculated and displayed. - Defaults to True. - batch_size: Optional. Sets the individual batch size to send for - prediction. Defaults to 500. - api_key. Optional. A generated API key to send with the requests to AI - Platform. - - Returns: - self, in order to enabled method chaining. - """ - self.set_inference_address(project) - self.set_model_name(model) - self.store('use_aip', True) - self.store('aip_service_name', service_name) - self.store('aip_service_version', service_version) - self.store('aip_batch_size', batch_size) - self.store('get_explanations', get_explanations) - if version is not None: - self.set_model_signature(version) - if force_json_input: - self.store('force_json_input', True) - if adjust_prediction: - self.store('adjust_prediction', adjust_prediction) - if adjust_example: - self.store('adjust_example', adjust_example) - if adjust_attribution: - self.store('adjust_attribution', adjust_attribution) - if api_key: - self.store('aip_api_key', api_key) - return self - - def set_compare_ai_platform_model( - self, project, model, version=None, force_json_input=None, - adjust_prediction=None, adjust_example=None, adjust_attribution=None, - service_name='ml', service_version='v1', get_explanations=True, - batch_size=500, api_key=None): - """Sets the model information for a second model served by AI Platform. - - AI Platform Prediction a Google Cloud serving platform. - - Args: - project: The name of the AI Platform Prediction project. - model: The name of the AI Platform Prediction model. - version: Optional, the version of the AI Platform Prediction model. - force_json_input: Optional. If True and examples are provided as - tf.Example protos, convert them to raw JSON objects before sending them - for inference to this model. - adjust_prediction: Optional. If not None then this function takes the - prediction output from the model for a single example and converts it to - the appopriate format - a regression score or a list of class scores. Only - necessary if the model doesn't already abide by this format. - adjust_example: Optional. If not None then this function takes an example - to run prediction on and converts it to the format expected by the model. - Necessary for example if the served model expects a single data value to - run inference on instead of a list or dict of values. - adjust_attribution: Optional. If not None and the model returns attribution - information, then this function takes the attribution information for an - example and converts it to the format expected by the tool, which is a - dictionary of input feature names to attribution scores. Usually necessary - if making use of adjust_example and the model returns attribution results. - service_name: Optional. Name of the AI Platform Prediction service. Defaults - to 'ml'. - service_version: Optional. Version of the AI Platform Prediction service. Defaults - to 'v1'. - get_explanations: Optional. If a model is deployed with explanations, - then this specifies if explainations will be calculated and displayed. - Defaults to True. - batch_size: Optional. Sets the individual batch size to send for - prediction. Defaults to 500. - api_key. Optional. A generated API key to send with the requests to AI - Platform. - - Returns: - self, in order to enabled method chaining. - """ - self.set_compare_inference_address(project) - self.set_compare_model_name(model) - self.store('compare_use_aip', True) - self.store('compare_aip_service_name', service_name) - self.store('compare_aip_service_version', service_version) - self.store('compare_aip_batch_size', batch_size) - self.store('compare_get_explanations', get_explanations) - if version is not None: - self.set_compare_model_signature(version) - if force_json_input: - self.store('compare_force_json_input', True) - if adjust_prediction: - self.store('compare_adjust_prediction', adjust_prediction) - if adjust_example: - self.store('compare_adjust_example', adjust_example) - if adjust_attribution: - self.store('compare_adjust_attribution', adjust_attribution) - if api_key: - self.store('compare_aip_api_key', api_key) - return self - - def set_target_feature(self, target): - """Sets the name of the target feature in the provided examples. - - If the provided examples contain a feature that represents the target - that the model is trying to predict, it can be specified by this method. - This is necessary for AI Platform models so that the target feature isn't - sent to the model for prediction, which can cause model inference errors. - - Args: - target: The name of the feature in the examples that represents the value - that the model is trying to predict. - - Returns: - self, in order to enabled method chaining. - """ - self.store('target_feature', target) - return self - - def _set_uses_json_input(self, is_json): - self.store('uses_json_input', is_json) - return self - - def _set_uses_json_list(self, is_list): - self.store('uses_json_list', is_list) - return self + """Configuration builder for WitWidget settings.""" + + def __init__(self, examples, feature_names=None): + """Constructs the WitConfigBuilder object. + + Args: + examples: A list of tf.Example or tf.SequenceExample proto objects, or + raw JSON objects. JSON is allowed only for AI Platform-hosted models (see + 'set_ai_platform_model' and 'set_compare_ai_platform_model methods). + These are the examples that will be displayed in WIT. If no model to + infer these examples with is specified through the methods on this class, + then WIT will display the examples for exploration, but no model inference + will be performed by the tool. + feature_names: Optional, defaults to None. If examples are provided as + JSON lists of numbers (not as feature dictionaries), then this array + maps indices in the feature value lists to human-readable names of those + features, used for display purposes. + """ + self.config = {} + self.set_model_type("classification") + self.set_label_vocab([]) + self.set_examples(examples, feature_names) + + def build(self): + """Returns the configuration set through use of this builder object. + + Used by WitWidget to set the settings on an instance of the + What-If Tool. + """ + return self.config + + def store(self, key, value): + self.config[key] = value + + def delete(self, key): + if key in self.config: + del self.config[key] + + def set_examples(self, examples, feature_names=None): + """Sets the examples to be displayed in WIT. + + Args: + examples: List of example protos or JSON objects. + feature_names: Optional, defaults to None. If examples are provided as + JSON lists of numbers (not as feature dictionaries), then this array + maps indices in the feature value lists to human-readable names of those + features, used just for display purposes. + + Returns: + self, in order to enabled method chaining. + """ + self.store("examples", examples) + if feature_names: + self.store("feature_names", feature_names) + if len(examples) > 0 and not ( + isinstance(examples[0], tf.train.Example) + or isinstance(examples[0], tf.train.SequenceExample) + ): + self._set_uses_json_input(True) + if isinstance(examples[0], list): + self._set_uses_json_list(True) + elif len(examples) > 0: + self.store( + "are_sequence_examples", + isinstance(examples[0], tf.train.SequenceExample), + ) + return self + + def set_model_type(self, model): + """Sets the type of the model being used for inference. + + Args: + model: The model type, such as "classification" or "regression". + The model type defaults to "classification". + + Returns: + self, in order to enabled method chaining. + """ + self.store("model_type", model) + return self + + def set_inference_address(self, address): + """Sets the inference address for model inference through TF Serving. + + Args: + address: The address of the served model, including port, such as + "localhost:8888". + + Returns: + self, in order to enabled method chaining. + """ + self.store("inference_address", address) + return self + + def set_model_name(self, name): + """Sets the model name for model inference through TF Serving. + + Setting a model name is required if inferring through a model hosted by + TF Serving. + + Args: + name: The name of the model to be queried through TF Serving at the + address provided by set_inference_address. + + Returns: + self, in order to enabled method chaining. + """ + self.store("model_name", name) + return self + + def has_model_name(self): + return "model_name" in self.config + + def set_model_version(self, version): + """Sets the optional model version for model inference through TF + Serving. + + Args: + version: The string version number of the model to be queried through TF + Serving. This is optional, as TF Serving will use the latest model version + if none is provided. + + Returns: + self, in order to enabled method chaining. + """ + self.store("model_version", version) + return self + + def set_model_signature(self, signature): + """Sets the optional model signature for model inference through TF + Serving. + + Args: + signature: The string signature of the model to be queried through TF + Serving. This is optional, as TF Serving will use the default model + signature if none is provided. + + Returns: + self, in order to enabled method chaining. + """ + self.store("model_signature", signature) + return self + + def set_compare_inference_address(self, address): + """Sets the inference address for model inference for a second model + hosted by TF Serving. + + If you wish to compare the results of two models in WIT, use this method + to setup the details of the second model. + + Args: + address: The address of the served model, including port, such as + "localhost:8888". + + Returns: + self, in order to enabled method chaining. + """ + self.store("inference_address_2", address) + return self + + def set_compare_model_name(self, name): + """Sets the model name for a second model hosted by TF Serving. + + If you wish to compare the results of two models in WIT, use this method + to setup the details of the second model. + + Setting a model name is required if inferring through a model hosted by + TF Serving. + + Args: + name: The name of the model to be queried through TF Serving at the + address provided by set_compare_inference_address. + + Returns: + self, in order to enabled method chaining. + """ + self.store("model_name_2", name) + return self + + def has_compare_model_name(self): + return "model_name_2" in self.config + + def set_compare_model_version(self, version): + """Sets the optional model version for a second model hosted by TF + Serving. + + If you wish to compare the results of two models in WIT, use this method + to setup the details of the second model. + + Args: + version: The string version number of the model to be queried through TF + Serving. This is optional, as TF Serving will use the latest model version + if none is provided. + + Returns: + self, in order to enabled method chaining. + """ + self.store("model_version_2", version) + return self + + def set_compare_model_signature(self, signature): + """Sets the optional model signature for a second model hosted by TF + Serving. + + If you wish to compare the results of two models in WIT, use this method + to setup the details of the second model. + + Args: + signature: The string signature of the model to be queried through TF + Serving. This is optional, as TF Serving will use the default model + signature if none is provided. + + Returns: + self, in order to enabled method chaining. + """ + self.store("model_signature_2", signature) + return self + + def set_uses_predict_api(self, predict): + """Indicates that the model uses the Predict API, as opposed to the + Classification or Regression API. + + If the model doesn't use the standard Classification or Regression APIs + provided through TF Serving, but instead uses the more flexible Predict API, + then use this method to indicate that. If this is true, then use the + set_predict_input_tensor and set_predict_output_tensor methods to indicate + the names of the tensors that are used as the input and output for the + models provided in order to perform the appropriate inference request. + + Args: + predict: True if the model or models use the Predict API. + + Returns: + self, in order to enabled method chaining. + """ + self.store("uses_predict_api", predict) + return self + + def set_max_classes_to_display(self, max_classes): + """Sets the maximum number of class results to display for multiclass + classification models. + + When using WIT with a multiclass model with a large number of possible + classes, it can be helpful to restrict WIT to only display some smaller + number of the highest-scoring classes as inference results for any given + example. This method sets that limit. + + Args: + max_classes: The maximum number of classes to display for inference + results for multiclass classification models. + + Returns: + self, in order to enabled method chaining. + """ + self.store("max_classes", max_classes) + return self + + def set_multi_class(self, multiclass): + """Sets if the model(s) to query are mutliclass classification models. + + Args: + multiclass: True if the model or models are multiclass classififcation + models. Defaults to false. + + Returns: + self, in order to enabled method chaining. + """ + self.store("multiclass", multiclass) + return self + + def set_predict_input_tensor(self, tensor): + """Sets the name of the input tensor for models that use the Predict + API. + + If using WIT with set_uses_predict_api(True), then call this to specify + the name of the input tensor of the model or models that accepts the + example proto for inference. + + Args: + tensor: The name of the input tensor. + + Returns: + self, in order to enabled method chaining. + """ + self.store("predict_input_tensor", tensor) + return self + + def set_predict_output_tensor(self, tensor): + """Sets the name of the output tensor for models that need output + parsing. + + If using WIT with set_uses_predict_api(True), then call this to specify + the name of the output tensor of the model or models that returns the + inference results to be explored by WIT. + + If using an AI Platform model which returns multiple prediction + results in a dictionary, this method specifies the key corresponding to + the inference results to be explored by WIT. + + Args: + tensor: The name of the output tensor. + + Returns: + self, in order to enabled method chaining. + """ + self.store("predict_output_tensor", tensor) + return self + + def set_label_vocab(self, vocab): + """Sets the string value of numeric labels for classification models. + + For classification models, the model returns scores for each class ID + number (classes 0 and 1 for binary classification models). In order for + WIT to visually display the results in a more-readable way, you can specify + string labels for each class ID. + + Args: + vocab: A list of strings, where the string at each index corresponds to + the label for that class ID. For example ['<=50K', '>50K'] for the UCI + census binary classification task. + + Returns: + self, in order to enabled method chaining. + """ + self.store("label_vocab", vocab) + return self + + def set_estimator_and_feature_spec(self, estimator, feature_spec): + """Sets the model for inference as a TF Estimator. + + Instead of using TF Serving to host a model for WIT to query, WIT can + directly use a TF Estimator object as the model to query. In order to + accomplish this, a feature_spec must also be provided to parse the + example protos for input into the estimator. + + Args: + estimator: The TF Estimator which will be used for model inference. + feature_spec: The feature_spec object which will be used for example + parsing. + + Returns: + self, in order to enabled method chaining. + """ + # If custom function is set, remove it before setting estimator + self.delete("custom_predict_fn") + + self.store( + "estimator_and_spec", + {"estimator": estimator, "feature_spec": feature_spec}, + ) + self.set_inference_address("estimator") + # If no model name has been set, give a default + if not self.has_model_name(): + self.set_model_name("1") + return self + + def set_compare_estimator_and_feature_spec(self, estimator, feature_spec): + """Sets a second model for inference as a TF Estimator. + + If you wish to compare the results of two models in WIT, use this method + to setup the details of the second model. + + Instead of using TF Serving to host a model for WIT to query, WIT can + directly use a TF Estimator object as the model to query. In order to + accomplish this, a feature_spec must also be provided to parse the + example protos for input into the estimator. + + Args: + estimator: The TF Estimator which will be used for model inference. + feature_spec: The feature_spec object which will be used for example + parsing. + + Returns: + self, in order to enabled method chaining. + """ + # If custom function is set, remove it before setting estimator + self.delete("compare_custom_predict_fn") + + self.store( + "compare_estimator_and_spec", + {"estimator": estimator, "feature_spec": feature_spec}, + ) + self.set_compare_inference_address("estimator") + # If no model name has been set, give a default + if not self.has_compare_model_name(): + self.set_compare_model_name("2") + return self + + def set_custom_predict_fn(self, predict_fn): + """Sets a custom function for inference. + + Instead of using TF Serving to host a model for WIT to query, WIT can + directly use a custom function as the model to query. In this case, the + provided function should accept example protos and return: + - For classification: A 2D list of numbers. The first dimension is for + each example being predicted. The second dimension are the probabilities + for each class ID in the prediction. + - For regression: A 1D list of numbers, with a regression score for each + example being predicted. + + Optionally, if attributions or other prediction-time information + can be returned by the model with each prediction, then this method + can return a dict with the key 'predictions' containing the predictions + result list described above, and with the key 'attributions' containing + a list of attributions for each example that was predicted. + + For each example, the attributions list should contain a dict mapping + input feature names to attribution values for that feature on that example. + The attribution value can be one of these things: + - A single number representing the attribution for the entire feature + - A list of numbers representing the attribution to each value in the + feature for multivalent features - such as attributions to individual + pixels in an image or numbers in a list of numbers. + - A 2D list for sparse feature attribution. Index 0 contains a list of + feature values that there are attribution scores for. Index 1 contains + a list of attribution values for the corresponding feature values in + the first list. + + This dict can contain any other keys, with their values being a list of + prediction-time strings or numbers for each example being predicted. These + values will be displayed in WIT as extra information for each example, + usable in the same ways by WIT as normal input features (such as for + creating plots and slicing performance data). + + Args: + predict_fn: The custom python function which will be used for model + inference. + + Returns: + self, in order to enabled method chaining. + """ + # If estimator is set, remove it before setting predict_fn + self.delete("estimator_and_spec") + + self.store("custom_predict_fn", predict_fn) + self.set_inference_address("custom_predict_fn") + # If no model name has been set, give a default + if not self.has_model_name(): + self.set_model_name("1") + return self + + def set_compare_custom_predict_fn(self, predict_fn): + """Sets a second custom function for inference. + + If you wish to compare the results of two models in WIT, use this method + to setup the details of the second model. + + Instead of using TF Serving to host a model for WIT to query, WIT can + directly use a custom function as the model to query. In this case, the + provided function should accept example protos and return: + - For classification: A 2D list of numbers. The first dimension is for + each example being predicted. The second dimension are the probabilities + for each class ID in the prediction. + - For regression: A 1D list of numbers, with a regression score for each + example being predicted. + + Optionally, if attributions or other prediction-time information + can be returned by the model with each prediction, then this method + can return a dict with the key 'predictions' containing the predictions + result list described above, and with the key 'attributions' containing + a list of attributions for each example that was predicted. + + For each example, the attributions list should contain a dict mapping + input feature names to attribution values for that feature on that example. + The attribution value can be one of these things: + - A single number representing the attribution for the entire feature + - A list of numbers representing the attribution to each value in the + feature for multivalent features - such as attributions to individual + pixels in an image or numbers in a list of numbers. + - A 2D list for sparse feature attribution. Index 0 contains a list of + feature values that there are attribution scores for. Index 1 contains + a list of attribution values for the corresponding feature values in + the first list. + + This dict can contain any other keys, with their values being a list of + prediction-time strings or numbers for each example being predicted. These + values will be displayed in WIT as extra information for each example, + usable in the same ways by WIT as normal input features (such as for + creating plots and slicing performance data). + + Args: + predict_fn: The custom python function which will be used for model + inference. + + Returns: + self, in order to enabled method chaining. + """ + # If estimator is set, remove it before setting predict_fn + self.delete("compare_estimator_and_spec") + + self.store("compare_custom_predict_fn", predict_fn) + self.set_compare_inference_address("custom_predict_fn") + # If no model name has been set, give a default + if not self.has_compare_model_name(): + self.set_compare_model_name("2") + return self + + def set_custom_distance_fn(self, distance_fn): + """Sets a custom function for distance computation. + + WIT can directly use a custom function for all distance computations within + the tool. In this case, the provided function should accept a query example + proto and a list of example protos to compute the distance against and + return a 1D list of numbers containing the distances. + + Args: + distance_fn: The python function which will be used for distance + computation. + + Returns: + self, in order to enabled method chaining. + """ + if distance_fn is None: + self.delete("custom_distance_fn") + else: + self.store("custom_distance_fn", distance_fn) + return self + + def set_ai_platform_model( + self, + project, + model, + version=None, + force_json_input=None, + adjust_prediction=None, + adjust_example=None, + adjust_attribution=None, + service_name="ml", + service_version="v1", + get_explanations=True, + batch_size=500, + api_key=None, + ): + """Sets the model information for a model served by AI Platform. + + AI Platform Prediction a Google Cloud serving platform. + + Args: + project: The name of the AI Platform Prediction project. + model: The name of the AI Platform Prediction model. + version: Optional, the version of the AI Platform Prediction model. + force_json_input: Optional. If True and examples are provided as + tf.Example protos, convert them to raw JSON objects before sending them + for inference to this model. + adjust_prediction: Optional. If not None then this function takes the + prediction output from the model for a single example and converts it to + the appopriate format - a regression score or a list of class scores. Only + necessary if the model doesn't already abide by this format. + adjust_example: Optional. If not None then this function takes an example + to run prediction on and converts it to the format expected by the model. + Necessary for example if the served model expects a single data value to + run inference on instead of a list or dict of values. + adjust_attribution: Optional. If not None and the model returns attribution + information, then this function takes the attribution information for an + example and converts it to the format expected by the tool, which is a + dictionary of input feature names to attribution scores. Usually necessary + if making use of adjust_example and the model returns attribution results. + service_name: Optional. Name of the AI Platform Prediction service. Defaults + to 'ml'. + service_version: Optional. Version of the AI Platform Prediction service. Defaults + to 'v1'. + get_explanations: Optional. If a model is deployed with explanations, + then this specifies if explainations will be calculated and displayed. + Defaults to True. + batch_size: Optional. Sets the individual batch size to send for + prediction. Defaults to 500. + api_key. Optional. A generated API key to send with the requests to AI + Platform. + + Returns: + self, in order to enabled method chaining. + """ + self.set_inference_address(project) + self.set_model_name(model) + self.store("use_aip", True) + self.store("aip_service_name", service_name) + self.store("aip_service_version", service_version) + self.store("aip_batch_size", batch_size) + self.store("get_explanations", get_explanations) + if version is not None: + self.set_model_signature(version) + if force_json_input: + self.store("force_json_input", True) + if adjust_prediction: + self.store("adjust_prediction", adjust_prediction) + if adjust_example: + self.store("adjust_example", adjust_example) + if adjust_attribution: + self.store("adjust_attribution", adjust_attribution) + if api_key: + self.store("aip_api_key", api_key) + return self + + def set_compare_ai_platform_model( + self, + project, + model, + version=None, + force_json_input=None, + adjust_prediction=None, + adjust_example=None, + adjust_attribution=None, + service_name="ml", + service_version="v1", + get_explanations=True, + batch_size=500, + api_key=None, + ): + """Sets the model information for a second model served by AI Platform. + + AI Platform Prediction a Google Cloud serving platform. + + Args: + project: The name of the AI Platform Prediction project. + model: The name of the AI Platform Prediction model. + version: Optional, the version of the AI Platform Prediction model. + force_json_input: Optional. If True and examples are provided as + tf.Example protos, convert them to raw JSON objects before sending them + for inference to this model. + adjust_prediction: Optional. If not None then this function takes the + prediction output from the model for a single example and converts it to + the appopriate format - a regression score or a list of class scores. Only + necessary if the model doesn't already abide by this format. + adjust_example: Optional. If not None then this function takes an example + to run prediction on and converts it to the format expected by the model. + Necessary for example if the served model expects a single data value to + run inference on instead of a list or dict of values. + adjust_attribution: Optional. If not None and the model returns attribution + information, then this function takes the attribution information for an + example and converts it to the format expected by the tool, which is a + dictionary of input feature names to attribution scores. Usually necessary + if making use of adjust_example and the model returns attribution results. + service_name: Optional. Name of the AI Platform Prediction service. Defaults + to 'ml'. + service_version: Optional. Version of the AI Platform Prediction service. Defaults + to 'v1'. + get_explanations: Optional. If a model is deployed with explanations, + then this specifies if explainations will be calculated and displayed. + Defaults to True. + batch_size: Optional. Sets the individual batch size to send for + prediction. Defaults to 500. + api_key. Optional. A generated API key to send with the requests to AI + Platform. + + Returns: + self, in order to enabled method chaining. + """ + self.set_compare_inference_address(project) + self.set_compare_model_name(model) + self.store("compare_use_aip", True) + self.store("compare_aip_service_name", service_name) + self.store("compare_aip_service_version", service_version) + self.store("compare_aip_batch_size", batch_size) + self.store("compare_get_explanations", get_explanations) + if version is not None: + self.set_compare_model_signature(version) + if force_json_input: + self.store("compare_force_json_input", True) + if adjust_prediction: + self.store("compare_adjust_prediction", adjust_prediction) + if adjust_example: + self.store("compare_adjust_example", adjust_example) + if adjust_attribution: + self.store("compare_adjust_attribution", adjust_attribution) + if api_key: + self.store("compare_aip_api_key", api_key) + return self + + def set_target_feature(self, target): + """Sets the name of the target feature in the provided examples. + + If the provided examples contain a feature that represents the target + that the model is trying to predict, it can be specified by this method. + This is necessary for AI Platform models so that the target feature isn't + sent to the model for prediction, which can cause model inference errors. + + Args: + target: The name of the feature in the examples that represents the value + that the model is trying to predict. + + Returns: + self, in order to enabled method chaining. + """ + self.store("target_feature", target) + return self + + def _set_uses_json_input(self, is_json): + self.store("uses_json_input", is_json) + return self + + def _set_uses_json_list(self, is_list): + self.store("uses_json_list", is_list) + return self diff --git a/tensorboard/plugins/interactive_inference/witwidget/pip_package/setup.py b/tensorboard/plugins/interactive_inference/witwidget/pip_package/setup.py index 6a2f176b61..6304c6dd91 100644 --- a/tensorboard/plugins/interactive_inference/witwidget/pip_package/setup.py +++ b/tensorboard/plugins/interactive_inference/witwidget/pip_package/setup.py @@ -21,85 +21,81 @@ import sys from setuptools import find_packages, setup -project_name = 'witwidget' +project_name = "witwidget" # Set when building the pip package -if '--project_name' in sys.argv: - project_name_idx = sys.argv.index('--project_name') - project_name = sys.argv[project_name_idx + 1] - sys.argv.remove('--project_name') - sys.argv.pop(project_name_idx) +if "--project_name" in sys.argv: + project_name_idx = sys.argv.index("--project_name") + project_name = sys.argv[project_name_idx + 1] + sys.argv.remove("--project_name") + sys.argv.pop(project_name_idx) -_TF_REQ = [ - 'tensorflow>=1.12.0', - 'tensorflow-serving-api>=1.12.0' -] +_TF_REQ = ["tensorflow>=1.12.0", "tensorflow-serving-api>=1.12.0"] # GPU build (note: the only difference is we depend on tensorflow-gpu and # tensorflow-serving-api-gpu so pip doesn't overwrite them with the CPU builds) -if 'witwidget-gpu' in project_name: - _TF_REQ = [ - 'tensorflow-gpu>=1.12.0', - 'tensorflow-serving-api-gpu>=1.12.0' - ] +if "witwidget-gpu" in project_name: + _TF_REQ = ["tensorflow-gpu>=1.12.0", "tensorflow-serving-api-gpu>=1.12.0"] REQUIRED_PACKAGES = [ - 'absl-py >= 0.4', - 'google-api-python-client>=1.7.8', - 'ipywidgets>=7.0.0', - 'jupyter>=1.0,<2', - 'oauth2client>=4.1.3', - 'six>=1.12.0', + "absl-py >= 0.4", + "google-api-python-client>=1.7.8", + "ipywidgets>=7.0.0", + "jupyter>=1.0,<2", + "oauth2client>=4.1.3", + "six>=1.12.0", ] + _TF_REQ + def get_readme(): - with open('README.rst') as f: - return f.read() + with open("README.rst") as f: + return f.read() + def get_version(): - version_ns = {} - with open(os.path.join('witwidget', 'version.py')) as f: - exec(f.read(), {}, version_ns) - return version_ns['VERSION'].replace('-', '') + version_ns = {} + with open(os.path.join("witwidget", "version.py")) as f: + exec(f.read(), {}, version_ns) + return version_ns["VERSION"].replace("-", "") + setup( - name=project_name, - version=get_version(), - description='What-If Tool jupyter widget', - long_description=get_readme(), - author='Google Inc.', - author_email='packages@tensorflow.org', - url='https://github.com/tensorflow/tensorboard/tree/master/tensorboard/plugins/interactive_inference', - include_package_data=True, - data_files=[ - ('share/jupyter/nbextensions/wit-widget', [ - 'witwidget/static/extension.js', - 'witwidget/static/index.js', - 'witwidget/static/index.js.map', - 'witwidget/static/wit_jupyter.html', - ],), - ('etc/jupyter/nbconfig/notebook.d/', ['wit-widget.json']) - ], - packages=find_packages(), - zip_safe=False, - install_requires=REQUIRED_PACKAGES, - keywords=[ - 'ipython', - 'jupyter', - 'widgets', - ], - license='Apache 2.0', - classifiers=[ - 'Development Status :: 4 - Beta', - 'Framework :: IPython', - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'Topic :: Multimedia :: Graphics', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - ] + name=project_name, + version=get_version(), + description="What-If Tool jupyter widget", + long_description=get_readme(), + author="Google Inc.", + author_email="packages@tensorflow.org", + url="https://github.com/tensorflow/tensorboard/tree/master/tensorboard/plugins/interactive_inference", + include_package_data=True, + data_files=[ + ( + "share/jupyter/nbextensions/wit-widget", + [ + "witwidget/static/extension.js", + "witwidget/static/index.js", + "witwidget/static/index.js.map", + "witwidget/static/wit_jupyter.html", + ], + ), + ("etc/jupyter/nbconfig/notebook.d/", ["wit-widget.json"]), + ], + packages=find_packages(), + zip_safe=False, + install_requires=REQUIRED_PACKAGES, + keywords=["ipython", "jupyter", "widgets",], + license="Apache 2.0", + classifiers=[ + "Development Status :: 4 - Beta", + "Framework :: IPython", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Topic :: Multimedia :: Graphics", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + ], ) diff --git a/tensorboard/plugins/interactive_inference/witwidget/version.py b/tensorboard/plugins/interactive_inference/witwidget/version.py index 0b665b55ed..8a32721f2e 100644 --- a/tensorboard/plugins/interactive_inference/witwidget/version.py +++ b/tensorboard/plugins/interactive_inference/witwidget/version.py @@ -14,4 +14,4 @@ """Contains the version string.""" -VERSION = '1.5.0' +VERSION = "1.5.0" diff --git a/tensorboard/plugins/mesh/demo_utils.py b/tensorboard/plugins/mesh/demo_utils.py index e165745984..1b442f04c6 100644 --- a/tensorboard/plugins/mesh/demo_utils.py +++ b/tensorboard/plugins/mesh/demo_utils.py @@ -23,64 +23,69 @@ def _parse_vertex(vertex_row): - """Parses a line in a PLY file which encodes a vertex coordinates. + """Parses a line in a PLY file which encodes a vertex coordinates. - Args: - vertex_row: string with vertex coordinates and color. + Args: + vertex_row: string with vertex coordinates and color. - Returns: - 2-tuple containing a length-3 array of vertex coordinates (as - floats) and a length-3 array of RGB color values (as ints between 0 - and 255, inclusive). - """ - vertex = vertex_row.strip().split() - # The row must contain coordinates with RGB/RGBA color in addition to that. - if len(vertex) >= 6: - # Supports only RGB colors now, alpha channel will be ignored. - # TODO(b/129298103): add support of RGBA in .ply files. - return ([float(coord) for coord in vertex[:3]], - [int(channel) for channel in vertex[3:6]]) - raise ValueError('PLY file must contain vertices with colors.') + Returns: + 2-tuple containing a length-3 array of vertex coordinates (as + floats) and a length-3 array of RGB color values (as ints between 0 + and 255, inclusive). + """ + vertex = vertex_row.strip().split() + # The row must contain coordinates with RGB/RGBA color in addition to that. + if len(vertex) >= 6: + # Supports only RGB colors now, alpha channel will be ignored. + # TODO(b/129298103): add support of RGBA in .ply files. + return ( + [float(coord) for coord in vertex[:3]], + [int(channel) for channel in vertex[3:6]], + ) + raise ValueError("PLY file must contain vertices with colors.") def _parse_face(face_row): - """Parses a line in a PLY file which encodes a face of the mesh.""" - face = [int(index) for index in face_row.strip().split()] - # Assert that number of vertices in a face is 3, i.e. it is a triangle - if len(face) != 4 or face[0] != 3: - raise ValueError( - 'Only supports face representation as a string with 4 numbers.') + """Parses a line in a PLY file which encodes a face of the mesh.""" + face = [int(index) for index in face_row.strip().split()] + # Assert that number of vertices in a face is 3, i.e. it is a triangle + if len(face) != 4 or face[0] != 3: + raise ValueError( + "Only supports face representation as a string with 4 numbers." + ) - return face[1:] + return face[1:] def read_ascii_ply(filename): - """Reads a PLY file encoded in ASCII format. + """Reads a PLY file encoded in ASCII format. - NOTE: this util method is not intended to be comprehensive PLY reader - and serves as part of demo application. + NOTE: this util method is not intended to be comprehensive PLY reader + and serves as part of demo application. - Args: - filename: path to a PLY file to read. + Args: + filename: path to a PLY file to read. - Returns: - numpy `[dim_1, 3]` array of vertices, `[dim_1, 3]` array of colors and - `[dim_1, 3]` array of faces of the mesh. - """ - with tf.io.gfile.GFile(filename) as ply_file: - for line in ply_file: - if line.startswith('end_header'): - break - elif line.startswith('element vertex'): - vert_count = int(line.split()[-1]) - elif line.startswith('element face'): - face_count = int(line.split()[-1]) - # Read vertices and their colors. - vertex_data = [_parse_vertex(next(ply_file)) for _ in range(vert_count)] - vertices = [datum[0] for datum in vertex_data] - colors = [datum[1] for datum in vertex_data] - # Read faces. - faces = [_parse_face(next(ply_file)) for _ in range(face_count)] - return (np.array(vertices).astype(np.float32), + Returns: + numpy `[dim_1, 3]` array of vertices, `[dim_1, 3]` array of colors and + `[dim_1, 3]` array of faces of the mesh. + """ + with tf.io.gfile.GFile(filename) as ply_file: + for line in ply_file: + if line.startswith("end_header"): + break + elif line.startswith("element vertex"): + vert_count = int(line.split()[-1]) + elif line.startswith("element face"): + face_count = int(line.split()[-1]) + # Read vertices and their colors. + vertex_data = [_parse_vertex(next(ply_file)) for _ in range(vert_count)] + vertices = [datum[0] for datum in vertex_data] + colors = [datum[1] for datum in vertex_data] + # Read faces. + faces = [_parse_face(next(ply_file)) for _ in range(face_count)] + return ( + np.array(vertices).astype(np.float32), np.array(colors).astype(np.uint8), - np.array(faces).astype(np.int32)) + np.array(faces).astype(np.int32), + ) diff --git a/tensorboard/plugins/mesh/demo_utils_test.py b/tensorboard/plugins/mesh/demo_utils_test.py index 47ad137dd6..437c7eaade 100644 --- a/tensorboard/plugins/mesh/demo_utils_test.py +++ b/tensorboard/plugins/mesh/demo_utils_test.py @@ -27,36 +27,41 @@ class TestPLYReader(tf.test.TestCase): - def test_parse_vertex(self): - """Tests vertex coordinate and color parsing.""" - # Vertex 3D coordinates with RGBA color. - vertex_data = [-0.249245, 1.119303, 0.3095566, 60, 253, 32, 255] - coords, colors = demo_utils._parse_vertex(' '.join(map(str, vertex_data))) - self.assertListEqual(coords, vertex_data[:3]) - self.assertListEqual(colors, vertex_data[3:6]) - - def test_prase_vertex_expects_colors(self): - """Tests that method will throw error if color is not poresent.""" - with self.assertRaisesRegexp(ValueError, - 'PLY file must contain vertices with colors'): - demo_utils._parse_vertex('1 2 3') - - def test_parse_face(self): - """Tests face line parsing.""" - face_data = [3, 10, 20, 30] - parsed_face = demo_utils._parse_face(' '.join(map(str, face_data))) - self.assertListEqual(parsed_face, face_data[1:]) - - def test_read_ascii_ply(self): - """Tests end-to-end PLY file reading and parsing.""" - test_ply = os.path.join( - os.path.dirname(os.environ['TEST_BINARY']), - 'test_data', 'icosphere.ply') - vertices, colors, faces = demo_utils.read_ascii_ply(test_ply) - self.assertEqual(len(vertices), 82) - self.assertEqual(len(vertices), len(colors)) - self.assertEqual(len(faces), 80) - - -if __name__ == '__main__': - tf.test.main() + def test_parse_vertex(self): + """Tests vertex coordinate and color parsing.""" + # Vertex 3D coordinates with RGBA color. + vertex_data = [-0.249245, 1.119303, 0.3095566, 60, 253, 32, 255] + coords, colors = demo_utils._parse_vertex( + " ".join(map(str, vertex_data)) + ) + self.assertListEqual(coords, vertex_data[:3]) + self.assertListEqual(colors, vertex_data[3:6]) + + def test_prase_vertex_expects_colors(self): + """Tests that method will throw error if color is not poresent.""" + with self.assertRaisesRegexp( + ValueError, "PLY file must contain vertices with colors" + ): + demo_utils._parse_vertex("1 2 3") + + def test_parse_face(self): + """Tests face line parsing.""" + face_data = [3, 10, 20, 30] + parsed_face = demo_utils._parse_face(" ".join(map(str, face_data))) + self.assertListEqual(parsed_face, face_data[1:]) + + def test_read_ascii_ply(self): + """Tests end-to-end PLY file reading and parsing.""" + test_ply = os.path.join( + os.path.dirname(os.environ["TEST_BINARY"]), + "test_data", + "icosphere.ply", + ) + vertices, colors, faces = demo_utils.read_ascii_ply(test_ply) + self.assertEqual(len(vertices), 82) + self.assertEqual(len(vertices), len(colors)) + self.assertEqual(len(faces), 80) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/mesh/mesh_demo.py b/tensorboard/plugins/mesh/mesh_demo.py index db034ac79e..aab8b7c616 100644 --- a/tensorboard/plugins/mesh/mesh_demo.py +++ b/tensorboard/plugins/mesh/mesh_demo.py @@ -27,9 +27,10 @@ from tensorboard.plugins.mesh import summary as mesh_summary from tensorboard.plugins.mesh import demo_utils -flags.DEFINE_string('logdir', '/tmp/mesh_demo', - 'Directory to write event logs to.') -flags.DEFINE_string('mesh_path', None, 'Path to PLY file to visualize.') +flags.DEFINE_string( + "logdir", "/tmp/mesh_demo", "Directory to write event logs to." +) +flags.DEFINE_string("mesh_path", None, "Path to PLY file to visualize.") FLAGS = flags.FLAGS @@ -40,63 +41,66 @@ def run(): - """Runs session with a mesh summary.""" - # Mesh summaries only work on TensorFlow 1.x. - if int(tf.__version__.split('.')[0]) > 1: - raise ImportError('TensorFlow 1.x is required to run this demo.') - # Flag mesh_path is required. - if FLAGS.mesh_path is None: - raise ValueError( - 'Flag --mesh_path is required and must contain path to PLY file.') - # Camera and scene configuration. - config_dict = { - 'camera': {'cls': 'PerspectiveCamera', 'fov': 75} - } - - # Read sample PLY file. - vertices, colors, faces = demo_utils.read_ascii_ply(FLAGS.mesh_path) - - # Add batch dimension. - vertices = np.expand_dims(vertices, 0) - faces = np.expand_dims(faces, 0) - colors = np.expand_dims(colors, 0) - - # Create placeholders for tensors representing the mesh. - step = tf.placeholder(tf.int32, ()) - vertices_tensor = tf.placeholder( - tf.float32, vertices.shape) - faces_tensor = tf.placeholder( - tf.int32, faces.shape) - colors_tensor = tf.placeholder( - tf.int32, colors.shape) - - # Change colors over time. - t = tf.cast(step, tf.float32) / _MAX_STEPS - transformed_colors = t * (255 - colors) + (1 - t) * colors - - meshes_summary = mesh_summary.op( - 'mesh_color_tensor', vertices=vertices_tensor, faces=faces_tensor, - colors=transformed_colors, config_dict=config_dict) - - # Create summary writer and session. - writer = tf.summary.FileWriter(FLAGS.logdir) - sess = tf.Session() - - for i in range(_MAX_STEPS): - summary = sess.run(meshes_summary, feed_dict={ - vertices_tensor: vertices, - faces_tensor: faces, - colors_tensor: colors, - step: i, - }) - writer.add_summary(summary, global_step=i) + """Runs session with a mesh summary.""" + # Mesh summaries only work on TensorFlow 1.x. + if int(tf.__version__.split(".")[0]) > 1: + raise ImportError("TensorFlow 1.x is required to run this demo.") + # Flag mesh_path is required. + if FLAGS.mesh_path is None: + raise ValueError( + "Flag --mesh_path is required and must contain path to PLY file." + ) + # Camera and scene configuration. + config_dict = {"camera": {"cls": "PerspectiveCamera", "fov": 75}} + + # Read sample PLY file. + vertices, colors, faces = demo_utils.read_ascii_ply(FLAGS.mesh_path) + + # Add batch dimension. + vertices = np.expand_dims(vertices, 0) + faces = np.expand_dims(faces, 0) + colors = np.expand_dims(colors, 0) + + # Create placeholders for tensors representing the mesh. + step = tf.placeholder(tf.int32, ()) + vertices_tensor = tf.placeholder(tf.float32, vertices.shape) + faces_tensor = tf.placeholder(tf.int32, faces.shape) + colors_tensor = tf.placeholder(tf.int32, colors.shape) + + # Change colors over time. + t = tf.cast(step, tf.float32) / _MAX_STEPS + transformed_colors = t * (255 - colors) + (1 - t) * colors + + meshes_summary = mesh_summary.op( + "mesh_color_tensor", + vertices=vertices_tensor, + faces=faces_tensor, + colors=transformed_colors, + config_dict=config_dict, + ) + + # Create summary writer and session. + writer = tf.summary.FileWriter(FLAGS.logdir) + sess = tf.Session() + + for i in range(_MAX_STEPS): + summary = sess.run( + meshes_summary, + feed_dict={ + vertices_tensor: vertices, + faces_tensor: faces, + colors_tensor: colors, + step: i, + }, + ) + writer.add_summary(summary, global_step=i) def main(unused_argv): - print('Saving output to %s.' % FLAGS.logdir) - run() - print('Done. Output saved to %s.' % FLAGS.logdir) + print("Saving output to %s." % FLAGS.logdir) + run() + print("Done. Output saved to %s." % FLAGS.logdir) -if __name__ == '__main__': - app.run(main) +if __name__ == "__main__": + app.run(main) diff --git a/tensorboard/plugins/mesh/mesh_demo_v2.py b/tensorboard/plugins/mesh/mesh_demo_v2.py index ce732dd04b..9e4a5a85da 100644 --- a/tensorboard/plugins/mesh/mesh_demo_v2.py +++ b/tensorboard/plugins/mesh/mesh_demo_v2.py @@ -28,9 +28,10 @@ from tensorboard.plugins.mesh import demo_utils -flags.DEFINE_string('logdir', '/tmp/mesh_demo', - 'Directory to write event logs to.') -flags.DEFINE_string('mesh_path', None, 'Path to PLY file to visualize.') +flags.DEFINE_string( + "logdir", "/tmp/mesh_demo", "Directory to write event logs to." +) +flags.DEFINE_string("mesh_path", None, "Path to PLY file to visualize.") FLAGS = flags.FLAGS @@ -41,50 +42,54 @@ def train_step(vertices, faces, colors, config_dict, step): - """Executes summary as a train step.""" - # Change colors over time. - t = float(step) / _MAX_STEPS - transformed_colors = t * (255 - colors) + (1 - t) * colors - mesh_summary.mesh( - 'mesh_color_tensor', vertices=vertices, faces=faces, - colors=transformed_colors, config_dict=config_dict, step=step) + """Executes summary as a train step.""" + # Change colors over time. + t = float(step) / _MAX_STEPS + transformed_colors = t * (255 - colors) + (1 - t) * colors + mesh_summary.mesh( + "mesh_color_tensor", + vertices=vertices, + faces=faces, + colors=transformed_colors, + config_dict=config_dict, + step=step, + ) def run(): - """Runs training steps with a mesh summary.""" - # Mesh summaries only work on TensorFlow 2.x. - if int(tf.__version__.split('.')[0]) < 1: - raise ImportError('TensorFlow 2.x is required to run this demo.') - # Flag mesh_path is required. - if FLAGS.mesh_path is None: - raise ValueError( - 'Flag --mesh_path is required and must contain path to PLY file.') - # Camera and scene configuration. - config_dict = { - 'camera': {'cls': 'PerspectiveCamera', 'fov': 75} - } - - # Read sample PLY file. - vertices, colors, faces = demo_utils.read_ascii_ply(FLAGS.mesh_path) - - # Add batch dimension. - vertices = np.expand_dims(vertices, 0) - faces = np.expand_dims(faces, 0) - colors = np.expand_dims(colors, 0) - - # Create summary writer. - writer = tf.summary.create_file_writer(FLAGS.logdir) - - with writer.as_default(): - for step in range(_MAX_STEPS): - train_step(vertices, faces, colors, config_dict, step) + """Runs training steps with a mesh summary.""" + # Mesh summaries only work on TensorFlow 2.x. + if int(tf.__version__.split(".")[0]) < 1: + raise ImportError("TensorFlow 2.x is required to run this demo.") + # Flag mesh_path is required. + if FLAGS.mesh_path is None: + raise ValueError( + "Flag --mesh_path is required and must contain path to PLY file." + ) + # Camera and scene configuration. + config_dict = {"camera": {"cls": "PerspectiveCamera", "fov": 75}} + + # Read sample PLY file. + vertices, colors, faces = demo_utils.read_ascii_ply(FLAGS.mesh_path) + + # Add batch dimension. + vertices = np.expand_dims(vertices, 0) + faces = np.expand_dims(faces, 0) + colors = np.expand_dims(colors, 0) + + # Create summary writer. + writer = tf.summary.create_file_writer(FLAGS.logdir) + + with writer.as_default(): + for step in range(_MAX_STEPS): + train_step(vertices, faces, colors, config_dict, step) def main(unused_argv): - print('Saving output to %s.' % FLAGS.logdir) - run() - print('Done. Output saved to %s.' % FLAGS.logdir) + print("Saving output to %s." % FLAGS.logdir) + run() + print("Done. Output saved to %s." % FLAGS.logdir) -if __name__ == '__main__': - app.run(main) +if __name__ == "__main__": + app.run(main) diff --git a/tensorboard/plugins/mesh/mesh_plugin.py b/tensorboard/plugins/mesh/mesh_plugin.py index d1b93debe1..efe4e7330d 100644 --- a/tensorboard/plugins/mesh/mesh_plugin.py +++ b/tensorboard/plugins/mesh/mesh_plugin.py @@ -35,244 +35,258 @@ class MeshPlugin(base_plugin.TBPlugin): - """A plugin that serves 3D visualization of meshes.""" - - plugin_name = metadata.PLUGIN_NAME - - def __init__(self, context): - """Instantiates a MeshPlugin via TensorBoard core. - - Args: - context: A base_plugin.TBContext instance. A magic container that - TensorBoard uses to make objects available to the plugin. - """ - # Retrieve the multiplexer from the context and store a reference to it. - self._multiplexer = context.multiplexer - self._tag_to_instance_tags = collections.defaultdict(list) - self._instance_tag_to_tag = dict() - self._instance_tag_to_metadata = dict() - self.prepare_metadata() - - def prepare_metadata(self): - """Processes all tags and caches metadata for each.""" - if self._tag_to_instance_tags: - return - # This is a dictionary mapping from run to (tag to string content). - # To be clear, the values of the dictionary are dictionaries. - all_runs = self._multiplexer.PluginRunToTagToContent(MeshPlugin.plugin_name) - - # tagToContent is itself a dictionary mapping tag name to string - # SummaryMetadata.plugin_data.content. Retrieve the keys of that dictionary - # to obtain a list of tags associated with each run. For each tag, estimate - # the number of samples. - self._tag_to_instance_tags = collections.defaultdict(list) - self._instance_tag_to_metadata = dict() - for run, tag_to_content in six.iteritems(all_runs): - for tag, content in six.iteritems(tag_to_content): - meta = metadata.parse_plugin_metadata(content) - self._instance_tag_to_metadata[(run, tag)] = meta - # Remember instance_name (instance_tag) for future reference. - self._tag_to_instance_tags[(run, meta.name)].append(tag) - self._instance_tag_to_tag[(run, tag)] = meta.name - - @wrappers.Request.application - def _serve_tags(self, request): - """A route (HTTP handler) that returns a response with tags. - - Args: - request: The werkzeug.Request object. - - Returns: - A response that contains a JSON object. The keys of the object - are all the runs. Each run is mapped to a (potentially empty) - list of all tags that are relevant to this plugin. - """ - # This is a dictionary mapping from run to (tag to string content). - # To be clear, the values of the dictionary are dictionaries. - all_runs = self._multiplexer.PluginRunToTagToContent( - MeshPlugin.plugin_name) - - # Make sure we populate tags mapping structures. - self.prepare_metadata() - - # tagToContent is itself a dictionary mapping tag name to string - # SummaryMetadata.plugin_data.content. Retrieve the keys of that dictionary - # to obtain a list of tags associated with each run. For each tag estimate - # number of samples. - response = dict() - for run, tag_to_content in six.iteritems(all_runs): - response[run] = dict() - for instance_tag, _ in six.iteritems(tag_to_content): - # Make sure we only operate on user-defined tags here. - tag = self._instance_tag_to_tag[(run, instance_tag)] - meta = self._instance_tag_to_metadata[(run, instance_tag)] - # Batch size must be defined, otherwise we don't know how many - # samples were there. - response[run][tag] = {'samples': meta.shape[0]} - return http_util.Respond(request, response, 'application/json') - - def get_plugin_apps(self): - """Gets all routes offered by the plugin. - - This method is called by TensorBoard when retrieving all the - routes offered by the plugin. - - Returns: - A dictionary mapping URL path to route that handles it. - """ - # Note that the methods handling routes are decorated with - # @wrappers.Request.application. - return { - '/tags': self._serve_tags, - '/meshes': self._serve_mesh_metadata, - '/data': self._serve_mesh_data, - } - - def is_active(self): - """Determines whether this plugin is active. - - This plugin is only active if TensorBoard sampled any summaries - relevant to the mesh plugin. - - Returns: - Whether this plugin is active. - """ - all_runs = self._multiplexer.PluginRunToTagToContent( - MeshPlugin.plugin_name) - - # The plugin is active if any of the runs has a tag relevant - # to the plugin. - return bool(self._multiplexer and any(six.itervalues(all_runs))) - - def frontend_metadata(self): - return base_plugin.FrontendMetadata(element_name='mesh-dashboard') - - def _get_sample(self, tensor_event, sample): - """Returns a single sample from a batch of samples.""" - data = tensor_util.make_ndarray(tensor_event.tensor_proto) - return data[sample].tolist() - - def _get_tensor_metadata( - self, event, content_type, components, data_shape, config): - """Converts a TensorEvent into a JSON-compatible response. - - Args: - event: TensorEvent object containing data in proto format. - content_type: enum plugin_data_pb2.MeshPluginData.ContentType value, - representing content type in TensorEvent. - components: Bitmask representing all parts (vertices, colors, etc.) that - belong to the summary. - data_shape: list of dimensions sizes of the tensor. - config: rendering scene configuration as dictionary. - - Returns: - Dictionary of transformed metadata. - """ - return { - 'wall_time': event.wall_time, - 'step': event.step, - 'content_type': content_type, - 'components': components, - 'config': config, - 'data_shape': list(data_shape), - } - - def _get_tensor_data(self, event, sample): - """Convert a TensorEvent into a JSON-compatible response.""" - data = self._get_sample(event, sample) - return data - - def _collect_tensor_events(self, request, step=None): - """Collects list of tensor events based on request.""" - run = request.args.get('run') - tag = request.args.get('tag') - - # TODO(b/128995556): investigate why this additional metadata mapping is - # necessary, it must have something todo with the lifecycle of the request. - # Make sure we populate tags mapping structures. - self.prepare_metadata() - - tensor_events = [] # List of tuples (meta, tensor) that contain tag. - for instance_tag in self._tag_to_instance_tags[(run, tag)]: - tensors = self._multiplexer.Tensors(run, instance_tag) - meta = self._instance_tag_to_metadata[(run, instance_tag)] - tensor_events += [(meta, tensor) for tensor in tensors] - - if step is not None: - tensor_events = [ - event for event in tensor_events if event[1].step == step] - else: - # Make sure tensors sorted by step in ascending order. - tensor_events = sorted( - tensor_events, key=lambda tensor_data: tensor_data[1].step) - - return tensor_events - - @wrappers.Request.application - def _serve_mesh_data(self, request): - """A route that returns data for particular summary of specified type. - - Data can represent vertices coordinates, vertices indices in faces, - vertices colors and so on. Each mesh may have different combination of - abovementioned data and each type/part of mesh summary must be served as - separate roundtrip to the server. - - Args: - request: werkzeug.Request containing content_type as a name of enum - plugin_data_pb2.MeshPluginData.ContentType. - - Returns: - werkzeug.Response either float32 or int32 data in binary format. - """ - step = float(request.args.get('step', 0.0)) - tensor_events = self._collect_tensor_events(request, step) - content_type = request.args.get('content_type') - try: - content_type = plugin_data_pb2.MeshPluginData.ContentType.Value( - content_type) - except ValueError: - return http_util.Respond(request, 'Bad content_type', 'text/plain', 400) - sample = int(request.args.get('sample', 0)) - - response = [ - self._get_tensor_data(tensor, sample) - for meta, tensor in tensor_events - if meta.content_type == content_type - ] - - np_type = { - plugin_data_pb2.MeshPluginData.VERTEX: np.float32, - plugin_data_pb2.MeshPluginData.FACE: np.int32, - plugin_data_pb2.MeshPluginData.COLOR: np.uint8, - }[content_type] - - response = np.array(response, dtype=np_type) - # Looks like reshape can take around 160ms, so why not store it reshaped. - response = response.reshape(-1).tobytes() - - return http_util.Respond(request, response, 'arraybuffer') - - @wrappers.Request.application - def _serve_mesh_metadata(self, request): - """A route that returns the mesh metadata associated with a tag. - - Metadata consists of wall time, type of elements in tensor, scene - configuration and so on. - - Args: - request: The werkzeug.Request object. - - Returns: - A JSON list of mesh data associated with the run and tag - combination. - """ - tensor_events = self._collect_tensor_events(request) - - # We convert the tensor data to text. - response = [ - self._get_tensor_metadata( - tensor, meta.content_type, meta.components, meta.shape, - meta.json_config) - for meta, tensor in tensor_events - ] - return http_util.Respond(request, response, 'application/json') + """A plugin that serves 3D visualization of meshes.""" + + plugin_name = metadata.PLUGIN_NAME + + def __init__(self, context): + """Instantiates a MeshPlugin via TensorBoard core. + + Args: + context: A base_plugin.TBContext instance. A magic container that + TensorBoard uses to make objects available to the plugin. + """ + # Retrieve the multiplexer from the context and store a reference to it. + self._multiplexer = context.multiplexer + self._tag_to_instance_tags = collections.defaultdict(list) + self._instance_tag_to_tag = dict() + self._instance_tag_to_metadata = dict() + self.prepare_metadata() + + def prepare_metadata(self): + """Processes all tags and caches metadata for each.""" + if self._tag_to_instance_tags: + return + # This is a dictionary mapping from run to (tag to string content). + # To be clear, the values of the dictionary are dictionaries. + all_runs = self._multiplexer.PluginRunToTagToContent( + MeshPlugin.plugin_name + ) + + # tagToContent is itself a dictionary mapping tag name to string + # SummaryMetadata.plugin_data.content. Retrieve the keys of that dictionary + # to obtain a list of tags associated with each run. For each tag, estimate + # the number of samples. + self._tag_to_instance_tags = collections.defaultdict(list) + self._instance_tag_to_metadata = dict() + for run, tag_to_content in six.iteritems(all_runs): + for tag, content in six.iteritems(tag_to_content): + meta = metadata.parse_plugin_metadata(content) + self._instance_tag_to_metadata[(run, tag)] = meta + # Remember instance_name (instance_tag) for future reference. + self._tag_to_instance_tags[(run, meta.name)].append(tag) + self._instance_tag_to_tag[(run, tag)] = meta.name + + @wrappers.Request.application + def _serve_tags(self, request): + """A route (HTTP handler) that returns a response with tags. + + Args: + request: The werkzeug.Request object. + + Returns: + A response that contains a JSON object. The keys of the object + are all the runs. Each run is mapped to a (potentially empty) + list of all tags that are relevant to this plugin. + """ + # This is a dictionary mapping from run to (tag to string content). + # To be clear, the values of the dictionary are dictionaries. + all_runs = self._multiplexer.PluginRunToTagToContent( + MeshPlugin.plugin_name + ) + + # Make sure we populate tags mapping structures. + self.prepare_metadata() + + # tagToContent is itself a dictionary mapping tag name to string + # SummaryMetadata.plugin_data.content. Retrieve the keys of that dictionary + # to obtain a list of tags associated with each run. For each tag estimate + # number of samples. + response = dict() + for run, tag_to_content in six.iteritems(all_runs): + response[run] = dict() + for instance_tag, _ in six.iteritems(tag_to_content): + # Make sure we only operate on user-defined tags here. + tag = self._instance_tag_to_tag[(run, instance_tag)] + meta = self._instance_tag_to_metadata[(run, instance_tag)] + # Batch size must be defined, otherwise we don't know how many + # samples were there. + response[run][tag] = {"samples": meta.shape[0]} + return http_util.Respond(request, response, "application/json") + + def get_plugin_apps(self): + """Gets all routes offered by the plugin. + + This method is called by TensorBoard when retrieving all the + routes offered by the plugin. + + Returns: + A dictionary mapping URL path to route that handles it. + """ + # Note that the methods handling routes are decorated with + # @wrappers.Request.application. + return { + "/tags": self._serve_tags, + "/meshes": self._serve_mesh_metadata, + "/data": self._serve_mesh_data, + } + + def is_active(self): + """Determines whether this plugin is active. + + This plugin is only active if TensorBoard sampled any summaries + relevant to the mesh plugin. + + Returns: + Whether this plugin is active. + """ + all_runs = self._multiplexer.PluginRunToTagToContent( + MeshPlugin.plugin_name + ) + + # The plugin is active if any of the runs has a tag relevant + # to the plugin. + return bool(self._multiplexer and any(six.itervalues(all_runs))) + + def frontend_metadata(self): + return base_plugin.FrontendMetadata(element_name="mesh-dashboard") + + def _get_sample(self, tensor_event, sample): + """Returns a single sample from a batch of samples.""" + data = tensor_util.make_ndarray(tensor_event.tensor_proto) + return data[sample].tolist() + + def _get_tensor_metadata( + self, event, content_type, components, data_shape, config + ): + """Converts a TensorEvent into a JSON-compatible response. + + Args: + event: TensorEvent object containing data in proto format. + content_type: enum plugin_data_pb2.MeshPluginData.ContentType value, + representing content type in TensorEvent. + components: Bitmask representing all parts (vertices, colors, etc.) that + belong to the summary. + data_shape: list of dimensions sizes of the tensor. + config: rendering scene configuration as dictionary. + + Returns: + Dictionary of transformed metadata. + """ + return { + "wall_time": event.wall_time, + "step": event.step, + "content_type": content_type, + "components": components, + "config": config, + "data_shape": list(data_shape), + } + + def _get_tensor_data(self, event, sample): + """Convert a TensorEvent into a JSON-compatible response.""" + data = self._get_sample(event, sample) + return data + + def _collect_tensor_events(self, request, step=None): + """Collects list of tensor events based on request.""" + run = request.args.get("run") + tag = request.args.get("tag") + + # TODO(b/128995556): investigate why this additional metadata mapping is + # necessary, it must have something todo with the lifecycle of the request. + # Make sure we populate tags mapping structures. + self.prepare_metadata() + + tensor_events = [] # List of tuples (meta, tensor) that contain tag. + for instance_tag in self._tag_to_instance_tags[(run, tag)]: + tensors = self._multiplexer.Tensors(run, instance_tag) + meta = self._instance_tag_to_metadata[(run, instance_tag)] + tensor_events += [(meta, tensor) for tensor in tensors] + + if step is not None: + tensor_events = [ + event for event in tensor_events if event[1].step == step + ] + else: + # Make sure tensors sorted by step in ascending order. + tensor_events = sorted( + tensor_events, key=lambda tensor_data: tensor_data[1].step + ) + + return tensor_events + + @wrappers.Request.application + def _serve_mesh_data(self, request): + """A route that returns data for particular summary of specified type. + + Data can represent vertices coordinates, vertices indices in faces, + vertices colors and so on. Each mesh may have different combination of + abovementioned data and each type/part of mesh summary must be served as + separate roundtrip to the server. + + Args: + request: werkzeug.Request containing content_type as a name of enum + plugin_data_pb2.MeshPluginData.ContentType. + + Returns: + werkzeug.Response either float32 or int32 data in binary format. + """ + step = float(request.args.get("step", 0.0)) + tensor_events = self._collect_tensor_events(request, step) + content_type = request.args.get("content_type") + try: + content_type = plugin_data_pb2.MeshPluginData.ContentType.Value( + content_type + ) + except ValueError: + return http_util.Respond( + request, "Bad content_type", "text/plain", 400 + ) + sample = int(request.args.get("sample", 0)) + + response = [ + self._get_tensor_data(tensor, sample) + for meta, tensor in tensor_events + if meta.content_type == content_type + ] + + np_type = { + plugin_data_pb2.MeshPluginData.VERTEX: np.float32, + plugin_data_pb2.MeshPluginData.FACE: np.int32, + plugin_data_pb2.MeshPluginData.COLOR: np.uint8, + }[content_type] + + response = np.array(response, dtype=np_type) + # Looks like reshape can take around 160ms, so why not store it reshaped. + response = response.reshape(-1).tobytes() + + return http_util.Respond(request, response, "arraybuffer") + + @wrappers.Request.application + def _serve_mesh_metadata(self, request): + """A route that returns the mesh metadata associated with a tag. + + Metadata consists of wall time, type of elements in tensor, scene + configuration and so on. + + Args: + request: The werkzeug.Request object. + + Returns: + A JSON list of mesh data associated with the run and tag + combination. + """ + tensor_events = self._collect_tensor_events(request) + + # We convert the tensor data to text. + response = [ + self._get_tensor_metadata( + tensor, + meta.content_type, + meta.components, + meta.shape, + meta.json_config, + ) + for meta, tensor in tensor_events + ] + return http_util.Respond(request, response, "application/json") diff --git a/tensorboard/plugins/mesh/mesh_plugin_test.py b/tensorboard/plugins/mesh/mesh_plugin_test.py index 4c602d42f3..21452be7b9 100644 --- a/tensorboard/plugins/mesh/mesh_plugin_test.py +++ b/tensorboard/plugins/mesh/mesh_plugin_test.py @@ -27,7 +27,9 @@ from werkzeug import test as werkzeug_test from werkzeug import wrappers from tensorboard.backend import application -from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer +from tensorboard.backend.event_processing import ( + plugin_event_multiplexer as event_multiplexer, +) from tensorboard.plugins import base_plugin from tensorboard.plugins.mesh import mesh_plugin from tensorboard.plugins.mesh import summary @@ -38,203 +40,250 @@ from mock import patch try: - # python version >= 3.3 - from unittest import mock + # python version >= 3.3 + from unittest import mock except ImportError: - import mock # pylint: disable=unused-import + import mock # pylint: disable=unused-import class MeshPluginTest(tf.test.TestCase): - """Tests for mesh plugin server.""" - - def setUp(self): - # We use numpy.random to generate meshes. We seed to avoid non-determinism - # in this test. - np.random.seed(17) - - # Log dir to save temp events into. - self.log_dir = self.get_temp_dir() - - # Create mesh summary. - with tf.compat.v1.Graph().as_default(): - tf_placeholder = tf.compat.v1.placeholder - sess = tf.compat.v1.Session() - point_cloud = test_utils.get_random_mesh(1000) - point_cloud_vertices = tf_placeholder( - tf.float32, point_cloud.vertices.shape - ) - - mesh_no_color = test_utils.get_random_mesh(2000, add_faces=True) - mesh_no_color_extended = test_utils.get_random_mesh(2500, add_faces=True) - mesh_no_color_vertices = tf_placeholder(tf.float32, [1, None, 3]) - mesh_no_color_faces = tf_placeholder(tf.int32, [1, None, 3]) - - mesh_color = test_utils.get_random_mesh( - 3000, add_faces=True, add_colors=True) - mesh_color_vertices = tf_placeholder(tf.float32, mesh_color.vertices.shape) - mesh_color_faces = tf_placeholder(tf.int32, mesh_color.faces.shape) - mesh_color_colors = tf_placeholder(tf.uint8, mesh_color.colors.shape) - - self.data = [ - point_cloud, mesh_no_color, mesh_no_color_extended, mesh_color] - - # In case when name is present and display_name is not, we will reuse name - # as display_name. Summaries below intended to test both cases. - self.names = ["point_cloud", "mesh_no_color", "mesh_color"] - summary.op( - self.names[0], - point_cloud_vertices, - description="just point cloud") - summary.op( - self.names[1], - mesh_no_color_vertices, - faces=mesh_no_color_faces, - display_name="name_to_display_in_ui", - description="beautiful mesh in grayscale") - summary.op( - self.names[2], - mesh_color_vertices, - faces=mesh_color_faces, - colors=mesh_color_colors, - description="mesh with random colors") - - merged_summary_op = tf.compat.v1.summary.merge_all() - self.runs = ["bar"] - self.steps = 20 - bar_directory = os.path.join(self.log_dir, self.runs[0]) - with tensorboard_test_util.FileWriterCache.get(bar_directory) as writer: - writer.add_graph(sess.graph) - for step in range(self.steps): - # Alternate between two random meshes with different number of - # vertices. - no_color = mesh_no_color if step % 2 == 0 else mesh_no_color_extended - with patch.object(time, 'time', return_value=step): - writer.add_summary( - sess.run( - merged_summary_op, - feed_dict={ - point_cloud_vertices: point_cloud.vertices, - mesh_no_color_vertices: no_color.vertices, - mesh_no_color_faces: no_color.faces, - mesh_color_vertices: mesh_color.vertices, - mesh_color_faces: mesh_color.faces, - mesh_color_colors: mesh_color.colors, - }), - global_step=step) - - # Start a server that will receive requests. - self.multiplexer = event_multiplexer.EventMultiplexer({ - "bar": bar_directory, - }) - self.context = base_plugin.TBContext( - logdir=self.log_dir, multiplexer=self.multiplexer) - self.plugin = mesh_plugin.MeshPlugin(self.context) - # Wait until after plugin construction to reload the multiplexer because the - # plugin caches data from the multiplexer upon construction and this affects - # logic tested later down. - # TODO(https://github.com/tensorflow/tensorboard/issues/2579): Eliminate the - # caching of data at construction time and move this Reload() up to just - # after the multiplexer is created. - self.multiplexer.Reload() - wsgi_app = application.TensorBoardWSGI([self.plugin]) - self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) - self.routes = self.plugin.get_plugin_apps() - - def tearDown(self): - shutil.rmtree(self.log_dir, ignore_errors=True) - - def testRoutes(self): - """Tests that the /tags route offers the correct run to tag mapping.""" - self.assertIsInstance(self.routes["/tags"], collections.Callable) - self.assertIsInstance(self.routes["/meshes"], collections.Callable) - self.assertIsInstance(self.routes["/data"], collections.Callable) - - def testTagsRoute(self): - """Tests that the /tags route offers the correct run to tag mapping.""" - response = self.server.get("/data/plugin/mesh/tags") - self.assertEqual(200, response.status_code) - tags = test_utils.deserialize_json_response(response.get_data()) - self.assertIn(self.runs[0], tags) - for name in self.names: - self.assertIn(name, tags[self.runs[0]]) - - def validate_data_response( - self, run, tag, sample, content_type, dtype, ground_truth_data, - step=0): - """Makes request and checks that response has expected data.""" - response = self.server.get( - "/data/plugin/mesh/data?run=%s&tag=%s&sample=%d&content_type=" - "%s&step=%d" % - (run, tag, sample, content_type, step)) - self.assertEqual(200, response.status_code) - data = test_utils.deserialize_array_buffer_response( - next(response.response), dtype) - self.assertEqual(ground_truth_data.reshape(-1).tolist(), data.tolist()) - - def testDataRoute(self): - """Tests that the /data route returns correct data for meshes.""" - self.validate_data_response( - self.runs[0], self.names[0], 0, "VERTEX", np.float32, - self.data[0].vertices) - - self.validate_data_response( - self.runs[0], self.names[1], 0, "FACE", np.int32, self.data[1].faces) - - # Validate that the same summary has mesh with different number of faces at - # different step=1. - self.validate_data_response( - self.runs[0], self.names[1], 0, "FACE", np.int32, self.data[2].faces, - step=1) - - self.validate_data_response( - self.runs[0], self.names[2], 0, "COLOR", np.uint8, self.data[3].colors) - - def testMetadataRoute(self): - """Tests that the /meshes route returns correct metadata for meshes.""" - response = self.server.get( - "/data/plugin/mesh/meshes?run=%s&tag=%s&sample=%d" % - (self.runs[0], self.names[0], 0)) - self.assertEqual(200, response.status_code) - metadata = test_utils.deserialize_json_response(response.get_data()) - self.assertEqual(len(metadata), self.steps) - self.assertAllEqual(metadata[0]["content_type"], - plugin_data_pb2.MeshPluginData.VERTEX) - self.assertAllEqual(metadata[0]["data_shape"], self.data[0].vertices.shape) - - def testsEventsAlwaysSortedByStep(self): - """Tests that events always sorted by step.""" - response = self.server.get( - "/data/plugin/mesh/meshes?run=%s&tag=%s&sample=%d" % - (self.runs[0], self.names[1], 0)) - self.assertEqual(200, response.status_code) - metadata = test_utils.deserialize_json_response(response.get_data()) - for i in range(1, self.steps): - # Step will be equal when two tensors of different content type - # belong to the same mesh. - self.assertLessEqual(metadata[i - 1]["step"], - metadata[i]["step"]) - - @mock.patch.object( - event_multiplexer.EventMultiplexer, - "PluginRunToTagToContent", - return_value={"bar": {"foo": "".encode("utf-8")}}, - ) - def testMetadataComputedOnce(self, run_to_tag_mock): - """Tests that metadata mapping computed once.""" - self.plugin.prepare_metadata() - self.plugin.prepare_metadata() - self.assertEqual(1, run_to_tag_mock.call_count) - - def testIsActive(self): - self.assertTrue(self.plugin.is_active()) - - @mock.patch.object( - event_multiplexer.EventMultiplexer, - "PluginRunToTagToContent", - return_value={}) - def testIsInactive(self, get_random_mesh_stub): - self.assertFalse(self.plugin.is_active()) + """Tests for mesh plugin server.""" + + def setUp(self): + # We use numpy.random to generate meshes. We seed to avoid non-determinism + # in this test. + np.random.seed(17) + + # Log dir to save temp events into. + self.log_dir = self.get_temp_dir() + + # Create mesh summary. + with tf.compat.v1.Graph().as_default(): + tf_placeholder = tf.compat.v1.placeholder + sess = tf.compat.v1.Session() + point_cloud = test_utils.get_random_mesh(1000) + point_cloud_vertices = tf_placeholder( + tf.float32, point_cloud.vertices.shape + ) + + mesh_no_color = test_utils.get_random_mesh(2000, add_faces=True) + mesh_no_color_extended = test_utils.get_random_mesh( + 2500, add_faces=True + ) + mesh_no_color_vertices = tf_placeholder(tf.float32, [1, None, 3]) + mesh_no_color_faces = tf_placeholder(tf.int32, [1, None, 3]) + + mesh_color = test_utils.get_random_mesh( + 3000, add_faces=True, add_colors=True + ) + mesh_color_vertices = tf_placeholder( + tf.float32, mesh_color.vertices.shape + ) + mesh_color_faces = tf_placeholder(tf.int32, mesh_color.faces.shape) + mesh_color_colors = tf_placeholder( + tf.uint8, mesh_color.colors.shape + ) + + self.data = [ + point_cloud, + mesh_no_color, + mesh_no_color_extended, + mesh_color, + ] + + # In case when name is present and display_name is not, we will reuse name + # as display_name. Summaries below intended to test both cases. + self.names = ["point_cloud", "mesh_no_color", "mesh_color"] + summary.op( + self.names[0], + point_cloud_vertices, + description="just point cloud", + ) + summary.op( + self.names[1], + mesh_no_color_vertices, + faces=mesh_no_color_faces, + display_name="name_to_display_in_ui", + description="beautiful mesh in grayscale", + ) + summary.op( + self.names[2], + mesh_color_vertices, + faces=mesh_color_faces, + colors=mesh_color_colors, + description="mesh with random colors", + ) + + merged_summary_op = tf.compat.v1.summary.merge_all() + self.runs = ["bar"] + self.steps = 20 + bar_directory = os.path.join(self.log_dir, self.runs[0]) + with tensorboard_test_util.FileWriterCache.get( + bar_directory + ) as writer: + writer.add_graph(sess.graph) + for step in range(self.steps): + # Alternate between two random meshes with different number of + # vertices. + no_color = ( + mesh_no_color + if step % 2 == 0 + else mesh_no_color_extended + ) + with patch.object(time, "time", return_value=step): + writer.add_summary( + sess.run( + merged_summary_op, + feed_dict={ + point_cloud_vertices: point_cloud.vertices, + mesh_no_color_vertices: no_color.vertices, + mesh_no_color_faces: no_color.faces, + mesh_color_vertices: mesh_color.vertices, + mesh_color_faces: mesh_color.faces, + mesh_color_colors: mesh_color.colors, + }, + ), + global_step=step, + ) + + # Start a server that will receive requests. + self.multiplexer = event_multiplexer.EventMultiplexer( + {"bar": bar_directory,} + ) + self.context = base_plugin.TBContext( + logdir=self.log_dir, multiplexer=self.multiplexer + ) + self.plugin = mesh_plugin.MeshPlugin(self.context) + # Wait until after plugin construction to reload the multiplexer because the + # plugin caches data from the multiplexer upon construction and this affects + # logic tested later down. + # TODO(https://github.com/tensorflow/tensorboard/issues/2579): Eliminate the + # caching of data at construction time and move this Reload() up to just + # after the multiplexer is created. + self.multiplexer.Reload() + wsgi_app = application.TensorBoardWSGI([self.plugin]) + self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) + self.routes = self.plugin.get_plugin_apps() + + def tearDown(self): + shutil.rmtree(self.log_dir, ignore_errors=True) + + def testRoutes(self): + """Tests that the /tags route offers the correct run to tag mapping.""" + self.assertIsInstance(self.routes["/tags"], collections.Callable) + self.assertIsInstance(self.routes["/meshes"], collections.Callable) + self.assertIsInstance(self.routes["/data"], collections.Callable) + + def testTagsRoute(self): + """Tests that the /tags route offers the correct run to tag mapping.""" + response = self.server.get("/data/plugin/mesh/tags") + self.assertEqual(200, response.status_code) + tags = test_utils.deserialize_json_response(response.get_data()) + self.assertIn(self.runs[0], tags) + for name in self.names: + self.assertIn(name, tags[self.runs[0]]) + + def validate_data_response( + self, run, tag, sample, content_type, dtype, ground_truth_data, step=0 + ): + """Makes request and checks that response has expected data.""" + response = self.server.get( + "/data/plugin/mesh/data?run=%s&tag=%s&sample=%d&content_type=" + "%s&step=%d" % (run, tag, sample, content_type, step) + ) + self.assertEqual(200, response.status_code) + data = test_utils.deserialize_array_buffer_response( + next(response.response), dtype + ) + self.assertEqual(ground_truth_data.reshape(-1).tolist(), data.tolist()) + + def testDataRoute(self): + """Tests that the /data route returns correct data for meshes.""" + self.validate_data_response( + self.runs[0], + self.names[0], + 0, + "VERTEX", + np.float32, + self.data[0].vertices, + ) + + self.validate_data_response( + self.runs[0], self.names[1], 0, "FACE", np.int32, self.data[1].faces + ) + + # Validate that the same summary has mesh with different number of faces at + # different step=1. + self.validate_data_response( + self.runs[0], + self.names[1], + 0, + "FACE", + np.int32, + self.data[2].faces, + step=1, + ) + + self.validate_data_response( + self.runs[0], + self.names[2], + 0, + "COLOR", + np.uint8, + self.data[3].colors, + ) + + def testMetadataRoute(self): + """Tests that the /meshes route returns correct metadata for meshes.""" + response = self.server.get( + "/data/plugin/mesh/meshes?run=%s&tag=%s&sample=%d" + % (self.runs[0], self.names[0], 0) + ) + self.assertEqual(200, response.status_code) + metadata = test_utils.deserialize_json_response(response.get_data()) + self.assertEqual(len(metadata), self.steps) + self.assertAllEqual( + metadata[0]["content_type"], plugin_data_pb2.MeshPluginData.VERTEX + ) + self.assertAllEqual( + metadata[0]["data_shape"], self.data[0].vertices.shape + ) + + def testsEventsAlwaysSortedByStep(self): + """Tests that events always sorted by step.""" + response = self.server.get( + "/data/plugin/mesh/meshes?run=%s&tag=%s&sample=%d" + % (self.runs[0], self.names[1], 0) + ) + self.assertEqual(200, response.status_code) + metadata = test_utils.deserialize_json_response(response.get_data()) + for i in range(1, self.steps): + # Step will be equal when two tensors of different content type + # belong to the same mesh. + self.assertLessEqual(metadata[i - 1]["step"], metadata[i]["step"]) + + @mock.patch.object( + event_multiplexer.EventMultiplexer, + "PluginRunToTagToContent", + return_value={"bar": {"foo": "".encode("utf-8")}}, + ) + def testMetadataComputedOnce(self, run_to_tag_mock): + """Tests that metadata mapping computed once.""" + self.plugin.prepare_metadata() + self.plugin.prepare_metadata() + self.assertEqual(1, run_to_tag_mock.call_count) + + def testIsActive(self): + self.assertTrue(self.plugin.is_active()) + + @mock.patch.object( + event_multiplexer.EventMultiplexer, + "PluginRunToTagToContent", + return_value={}, + ) + def testIsInactive(self, get_random_mesh_stub): + self.assertFalse(self.plugin.is_active()) if __name__ == "__main__": - tf.test.main() + tf.test.main() diff --git a/tensorboard/plugins/mesh/metadata.py b/tensorboard/plugins/mesh/metadata.py index 3ea386c682..b9702f5bc3 100644 --- a/tensorboard/plugins/mesh/metadata.py +++ b/tensorboard/plugins/mesh/metadata.py @@ -24,8 +24,9 @@ MeshTensor = collections.namedtuple( - 'MeshTensor', ('data', 'content_type', 'data_type')) -PLUGIN_NAME = 'mesh' + "MeshTensor", ("data", "content_type", "data_type") +) +PLUGIN_NAME = "mesh" # The most recent value for the `version` field of the # `MeshPluginData` proto. @@ -33,100 +34,111 @@ def get_components_bitmask(content_types): - """Creates bitmask for all existing components of the summary. + """Creates bitmask for all existing components of the summary. - Args: - content_type: list of plugin_data_pb2.MeshPluginData.ContentType, - representing all components related to the summary. - Returns: bitmask based on passed tensors. - """ - components = 0 - for content_type in content_types: - if content_type == plugin_data_pb2.MeshPluginData.UNDEFINED: - raise ValueError('Cannot include UNDEFINED content type in mask.') - components = components | (1 << content_type) - return components + Args: + content_type: list of plugin_data_pb2.MeshPluginData.ContentType, + representing all components related to the summary. + Returns: bitmask based on passed tensors. + """ + components = 0 + for content_type in content_types: + if content_type == plugin_data_pb2.MeshPluginData.UNDEFINED: + raise ValueError("Cannot include UNDEFINED content type in mask.") + components = components | (1 << content_type) + return components def get_current_version(): - """Returns current verions of the proto.""" - return _PROTO_VERSION + """Returns current verions of the proto.""" + return _PROTO_VERSION def get_instance_name(name, content_type): - """Returns a unique instance name for a given summary related to the mesh.""" - return '%s_%s' % ( - name, - plugin_data_pb2.MeshPluginData.ContentType.Name(content_type)) - - -def create_summary_metadata(name, - display_name, - content_type, - components, - shape, - description=None, - json_config=None): - """Creates summary metadata which defined at MeshPluginData proto. - - Arguments: - name: Original merged (summaries of different types) summary name. - display_name: The display name used in TensorBoard. - content_type: Value from MeshPluginData.ContentType enum describing data. - components: Bitmask representing present parts (vertices, colors, etc.) that - belong to the summary. - shape: list of dimensions sizes of the tensor. - description: The description to show in TensorBoard. - json_config: A string, JSON-serialized dictionary of ThreeJS classes - configuration. - - Returns: - A `summary_pb2.SummaryMetadata` protobuf object. - """ - # Shape should be at least BxNx3 where B represents the batch dimensions - # and N - the number of points, each with x,y,z coordinates. - if len(shape) != 3: - raise ValueError( - 'Tensor shape should be of shape BxNx3, but got %s.' % str(shape)) - mesh_plugin_data = plugin_data_pb2.MeshPluginData( - version=get_current_version(), - name=name, - content_type=content_type, - components=components, - shape=shape, - json_config=json_config) - content = mesh_plugin_data.SerializeToString() - return summary_pb2.SummaryMetadata( - display_name=display_name, # Will not be used in TensorBoard UI. - summary_description=description, - plugin_data=summary_pb2.SummaryMetadata.PluginData( - plugin_name=PLUGIN_NAME, - content=content)) + """Returns a unique instance name for a given summary related to the + mesh.""" + return "%s_%s" % ( + name, + plugin_data_pb2.MeshPluginData.ContentType.Name(content_type), + ) + + +def create_summary_metadata( + name, + display_name, + content_type, + components, + shape, + description=None, + json_config=None, +): + """Creates summary metadata which defined at MeshPluginData proto. + + Arguments: + name: Original merged (summaries of different types) summary name. + display_name: The display name used in TensorBoard. + content_type: Value from MeshPluginData.ContentType enum describing data. + components: Bitmask representing present parts (vertices, colors, etc.) that + belong to the summary. + shape: list of dimensions sizes of the tensor. + description: The description to show in TensorBoard. + json_config: A string, JSON-serialized dictionary of ThreeJS classes + configuration. + + Returns: + A `summary_pb2.SummaryMetadata` protobuf object. + """ + # Shape should be at least BxNx3 where B represents the batch dimensions + # and N - the number of points, each with x,y,z coordinates. + if len(shape) != 3: + raise ValueError( + "Tensor shape should be of shape BxNx3, but got %s." % str(shape) + ) + mesh_plugin_data = plugin_data_pb2.MeshPluginData( + version=get_current_version(), + name=name, + content_type=content_type, + components=components, + shape=shape, + json_config=json_config, + ) + content = mesh_plugin_data.SerializeToString() + return summary_pb2.SummaryMetadata( + display_name=display_name, # Will not be used in TensorBoard UI. + summary_description=description, + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME, content=content + ), + ) def parse_plugin_metadata(content): - """Parse summary metadata to a Python object. - - Arguments: - content: The `content` field of a `SummaryMetadata` proto - corresponding to the mesh plugin. - - Returns: - A `MeshPluginData` protobuf object. - Raises: Error if the version of the plugin is not supported. - """ - if not isinstance(content, bytes): - raise TypeError('Content type must be bytes.') - result = plugin_data_pb2.MeshPluginData.FromString(content) - if not 0 <= result.version <= get_current_version(): - raise ValueError('Unknown metadata version: %s. The latest version known to ' - 'this build of TensorBoard is %s; perhaps a newer build is ' - 'available?' % (result.version, get_current_version())) - # Add components field to older version of the proto. - if result.components == 0: - result.components = get_components_bitmask([ - plugin_data_pb2.MeshPluginData.VERTEX, - plugin_data_pb2.MeshPluginData.FACE, - plugin_data_pb2.MeshPluginData.COLOR, - ]) - return result + """Parse summary metadata to a Python object. + + Arguments: + content: The `content` field of a `SummaryMetadata` proto + corresponding to the mesh plugin. + + Returns: + A `MeshPluginData` protobuf object. + Raises: Error if the version of the plugin is not supported. + """ + if not isinstance(content, bytes): + raise TypeError("Content type must be bytes.") + result = plugin_data_pb2.MeshPluginData.FromString(content) + if not 0 <= result.version <= get_current_version(): + raise ValueError( + "Unknown metadata version: %s. The latest version known to " + "this build of TensorBoard is %s; perhaps a newer build is " + "available?" % (result.version, get_current_version()) + ) + # Add components field to older version of the proto. + if result.components == 0: + result.components = get_components_bitmask( + [ + plugin_data_pb2.MeshPluginData.VERTEX, + plugin_data_pb2.MeshPluginData.FACE, + plugin_data_pb2.MeshPluginData.COLOR, + ] + ) + return result diff --git a/tensorboard/plugins/mesh/metadata_test.py b/tensorboard/plugins/mesh/metadata_test.py index a498caf9bc..6bbd0d3681 100644 --- a/tensorboard/plugins/mesh/metadata_test.py +++ b/tensorboard/plugins/mesh/metadata_test.py @@ -27,84 +27,94 @@ class MetadataTest(tf.test.TestCase): + def _create_metadata(self, shape=None): + """Creates metadata with dummy data.""" + self.name = "unique_name" + self.display_name = "my mesh" + self.json_config = "{}" + if shape is None: + shape = [1, 100, 3] + self.shape = shape + self.components = 14 + self.summary_metadata = metadata.create_summary_metadata( + self.name, + self.display_name, + plugin_data_pb2.MeshPluginData.ContentType.Value("VERTEX"), + self.components, + self.shape, + json_config=self.json_config, + ) - def _create_metadata(self, shape=None): - """Creates metadata with dummy data.""" - self.name = 'unique_name' - self.display_name = 'my mesh' - self.json_config = '{}' - if shape is None: - shape = [1, 100, 3] - self.shape = shape - self.components = 14 - self.summary_metadata = metadata.create_summary_metadata( - self.name, - self.display_name, - plugin_data_pb2.MeshPluginData.ContentType.Value('VERTEX'), - self.components, - self.shape, - json_config=self.json_config) + def test_get_instance_name(self): + """Tests proper creation of instance name based on display_name.""" + display_name = "my_mesh" + instance_name = metadata.get_instance_name( + display_name, + plugin_data_pb2.MeshPluginData.ContentType.Value("VERTEX"), + ) + self.assertEqual("%s_VERTEX" % display_name, instance_name) - def test_get_instance_name(self): - """Tests proper creation of instance name based on display_name.""" - display_name = 'my_mesh' - instance_name = metadata.get_instance_name( - display_name, - plugin_data_pb2.MeshPluginData.ContentType.Value('VERTEX')) - self.assertEqual('%s_VERTEX' % display_name, instance_name) + def test_create_summary_metadata(self): + """Tests MeshPlugin metadata creation.""" + self._create_metadata() + self.assertEqual(self.display_name, self.summary_metadata.display_name) + self.assertEqual( + metadata.PLUGIN_NAME, self.summary_metadata.plugin_data.plugin_name + ) - def test_create_summary_metadata(self): - """Tests MeshPlugin metadata creation.""" - self._create_metadata() - self.assertEqual(self.display_name, - self.summary_metadata.display_name) - self.assertEqual(metadata.PLUGIN_NAME, - self.summary_metadata.plugin_data.plugin_name) + def test_parse_plugin_metadata(self): + """Tests parsing of saved plugin metadata.""" + self._create_metadata() + parsed_metadata = metadata.parse_plugin_metadata( + self.summary_metadata.plugin_data.content + ) + self.assertEqual(self.name, parsed_metadata.name) + self.assertEqual( + plugin_data_pb2.MeshPluginData.ContentType.Value("VERTEX"), + parsed_metadata.content_type, + ) + self.assertEqual(self.shape, parsed_metadata.shape) + self.assertEqual(self.json_config, parsed_metadata.json_config) + self.assertEqual(self.components, parsed_metadata.components) - def test_parse_plugin_metadata(self): - """Tests parsing of saved plugin metadata.""" - self._create_metadata() - parsed_metadata = metadata.parse_plugin_metadata( - self.summary_metadata.plugin_data.content) - self.assertEqual(self.name, parsed_metadata.name) - self.assertEqual(plugin_data_pb2.MeshPluginData.ContentType.Value('VERTEX'), - parsed_metadata.content_type) - self.assertEqual(self.shape, parsed_metadata.shape) - self.assertEqual(self.json_config, parsed_metadata.json_config) - self.assertEqual(self.components, parsed_metadata.components) + def test_metadata_version(self): + """Tests that only the latest version of metadata is supported.""" + with patch.object(metadata, "get_current_version", return_value=100): + self._create_metadata() + # Change the version. + with patch.object(metadata, "get_current_version", return_value=1): + # Try to parse metadata from a prior version. + with self.assertRaises(ValueError): + metadata.parse_plugin_metadata( + self.summary_metadata.plugin_data.content + ) - def test_metadata_version(self): - """Tests that only the latest version of metadata is supported.""" - with patch.object(metadata, 'get_current_version', return_value=100): - self._create_metadata() - # Change the version. - with patch.object(metadata, 'get_current_version', return_value=1): - # Try to parse metadata from a prior version. - with self.assertRaises(ValueError): - metadata.parse_plugin_metadata( - self.summary_metadata.plugin_data.content) + def test_tensor_shape(self): + """Tests that target tensor should be of particular shape.""" + with six.assertRaisesRegex( + self, ValueError, r"Tensor shape should be of shape BxNx3.*" + ): + self._create_metadata([1]) - def test_tensor_shape(self): - """Tests that target tensor should be of particular shape.""" - with six.assertRaisesRegex( - self, ValueError, r'Tensor shape should be of shape BxNx3.*'): - self._create_metadata([1]) + def test_metadata_format(self): + """Tests that metadata content must be passed as a serialized + string.""" + with six.assertRaisesRegex( + self, TypeError, r"Content type must be bytes." + ): + metadata.parse_plugin_metadata(123) - def test_metadata_format(self): - """Tests that metadata content must be passed as a serialized string.""" - with six.assertRaisesRegex(self, TypeError, r'Content type must be bytes.'): - metadata.parse_plugin_metadata(123) + def test_default_components(self): + """Tests that defult components are added when necessary.""" + self._create_metadata() + stored_metadata = plugin_data_pb2.MeshPluginData( + version=metadata.get_current_version(), components=0 + ) + parsed_metadata = metadata.parse_plugin_metadata( + stored_metadata.SerializeToString() + ) + self.assertGreater(parsed_metadata.components, 0) - def test_default_components(self): - """Tests that defult components are added when necessary.""" - self._create_metadata() - stored_metadata = plugin_data_pb2.MeshPluginData( - version=metadata.get_current_version(), - components=0) - parsed_metadata = metadata.parse_plugin_metadata( - stored_metadata.SerializeToString()) - self.assertGreater(parsed_metadata.components, 0) - -if __name__ == '__main__': - tf.test.main() +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/mesh/summary.py b/tensorboard/plugins/mesh/summary.py index 1755ad4d4d..b8ea5bd2ca 100644 --- a/tensorboard/plugins/mesh/summary.py +++ b/tensorboard/plugins/mesh/summary.py @@ -30,176 +30,218 @@ def _get_tensor_summary( - name, display_name, description, tensor, content_type, components, - json_config, collections): - """Creates a tensor summary with summary metadata. - - Args: - name: Uniquely identifiable name of the summary op. Could be replaced by - combination of name and type to make it unique even outside of this - summary. - display_name: Will be used as the display name in TensorBoard. - Defaults to `tag`. - description: A longform readable description of the summary data. Markdown - is supported. - tensor: Tensor to display in summary. - content_type: Type of content inside the Tensor. - components: Bitmask representing present parts (vertices, colors, etc.) that - belong to the summary. - json_config: A string, JSON-serialized dictionary of ThreeJS classes - configuration. - collections: List of collections to add this summary to. - - Returns: - Tensor summary with metadata. - """ - tensor = tf.convert_to_tensor(value=tensor) - shape = tensor.shape.as_list() - shape = [dim if dim is not None else -1 for dim in shape] - tensor_metadata = metadata.create_summary_metadata( - name, - display_name, - content_type, - components, - shape, - description, - json_config=json_config) - tensor_summary = tf.compat.v1.summary.tensor_summary( - metadata.get_instance_name(name, content_type), - tensor, - summary_metadata=tensor_metadata, - collections=collections) - return tensor_summary - - -def _get_display_name(name, display_name): - """Returns display_name from display_name and name.""" - if display_name is None: - return name - return display_name - - -def _get_json_config(config_dict): - """Parses and returns JSON string from python dictionary.""" - json_config = '{}' - if config_dict is not None: - json_config = json.dumps(config_dict, sort_keys=True) - return json_config - - -def op(name, vertices, faces=None, colors=None, display_name=None, - description=None, collections=None, config_dict=None): - """Creates a TensorFlow summary op for mesh rendering. - - Args: - name: A name for this summary operation. - vertices: Tensor of shape `[dim_1, ..., dim_n, 3]` representing the 3D - coordinates of vertices. - faces: Tensor of shape `[dim_1, ..., dim_n, 3]` containing indices of - vertices within each triangle. - colors: Tensor of shape `[dim_1, ..., dim_n, 3]` containing colors for each - vertex. - display_name: If set, will be used as the display name in TensorBoard. - Defaults to `name`. - description: A longform readable description of the summary data. Markdown - is supported. - collections: Which TensorFlow graph collections to add the summary op to. - Defaults to `['summaries']`. Can usually be ignored. - config_dict: Dictionary with ThreeJS classes names and configuration. - - Returns: - Merged summary for mesh/point cloud representation. - """ - display_name = _get_display_name(name, display_name) - json_config = _get_json_config(config_dict) - - # All tensors representing a single mesh will be represented as separate - # summaries internally. Those summaries will be regrouped on the client before - # rendering. - summaries = [] - tensors = [ - metadata.MeshTensor( - vertices, plugin_data_pb2.MeshPluginData.VERTEX, tf.float32), - metadata.MeshTensor(faces, plugin_data_pb2.MeshPluginData.FACE, tf.int32), - metadata.MeshTensor( - colors, plugin_data_pb2.MeshPluginData.COLOR, tf.uint8) - ] - tensors = [tensor for tensor in tensors if tensor.data is not None] - - components = metadata.get_components_bitmask([ - tensor.content_type for tensor in tensors]) - - for tensor in tensors: - summaries.append( - _get_tensor_summary(name, display_name, description, tensor.data, - tensor.content_type, components, json_config, - collections)) - - all_summaries = tf.compat.v1.summary.merge( - summaries, collections=collections, name=name) - return all_summaries - - -def pb(name, - vertices, - faces=None, - colors=None, - display_name=None, - description=None, - config_dict=None): - """Create a mesh summary to save in pb format. - - Args: - name: A name for this summary operation. - vertices: numpy array of shape `[dim_1, ..., dim_n, 3]` representing the 3D - coordinates of vertices. - faces: numpy array of shape `[dim_1, ..., dim_n, 3]` containing indices of - vertices within each triangle. - colors: numpy array of shape `[dim_1, ..., dim_n, 3]` containing colors for - each vertex. - display_name: If set, will be used as the display name in TensorBoard. - Defaults to `name`. - description: A longform readable description of the summary data. Markdown - is supported. - config_dict: Dictionary with ThreeJS classes names and configuration. - - Returns: - Instance of tf.Summary class. - """ - display_name = _get_display_name(name, display_name) - json_config = _get_json_config(config_dict) - - summaries = [] - tensors = [ - metadata.MeshTensor( - vertices, plugin_data_pb2.MeshPluginData.VERTEX, tf.float32), - metadata.MeshTensor(faces, plugin_data_pb2.MeshPluginData.FACE, tf.int32), - metadata.MeshTensor( - colors, plugin_data_pb2.MeshPluginData.COLOR, tf.uint8) - ] - tensors = [tensor for tensor in tensors if tensor.data is not None] - components = metadata.get_components_bitmask([ - tensor.content_type for tensor in tensors]) - for tensor in tensors: - shape = tensor.data.shape + name, + display_name, + description, + tensor, + content_type, + components, + json_config, + collections, +): + """Creates a tensor summary with summary metadata. + + Args: + name: Uniquely identifiable name of the summary op. Could be replaced by + combination of name and type to make it unique even outside of this + summary. + display_name: Will be used as the display name in TensorBoard. + Defaults to `tag`. + description: A longform readable description of the summary data. Markdown + is supported. + tensor: Tensor to display in summary. + content_type: Type of content inside the Tensor. + components: Bitmask representing present parts (vertices, colors, etc.) that + belong to the summary. + json_config: A string, JSON-serialized dictionary of ThreeJS classes + configuration. + collections: List of collections to add this summary to. + + Returns: + Tensor summary with metadata. + """ + tensor = tf.convert_to_tensor(value=tensor) + shape = tensor.shape.as_list() shape = [dim if dim is not None else -1 for dim in shape] - tensor_proto = tf.compat.v1.make_tensor_proto( - tensor.data, dtype=tensor.data_type) - summary_metadata = metadata.create_summary_metadata( + tensor_metadata = metadata.create_summary_metadata( name, display_name, - tensor.content_type, + content_type, components, shape, description, - json_config=json_config) - tag = metadata.get_instance_name(name, tensor.content_type) - summaries.append((tag, summary_metadata, tensor_proto)) - - summary = tf.compat.v1.Summary() - for tag, summary_metadata, tensor_proto in summaries: - tf_summary_metadata = tf.compat.v1.SummaryMetadata.FromString( - summary_metadata.SerializeToString()) - summary.value.add( - tag=tag, metadata=tf_summary_metadata, tensor=tensor_proto) - return summary + json_config=json_config, + ) + tensor_summary = tf.compat.v1.summary.tensor_summary( + metadata.get_instance_name(name, content_type), + tensor, + summary_metadata=tensor_metadata, + collections=collections, + ) + return tensor_summary + + +def _get_display_name(name, display_name): + """Returns display_name from display_name and name.""" + if display_name is None: + return name + return display_name + + +def _get_json_config(config_dict): + """Parses and returns JSON string from python dictionary.""" + json_config = "{}" + if config_dict is not None: + json_config = json.dumps(config_dict, sort_keys=True) + return json_config + + +def op( + name, + vertices, + faces=None, + colors=None, + display_name=None, + description=None, + collections=None, + config_dict=None, +): + """Creates a TensorFlow summary op for mesh rendering. + + Args: + name: A name for this summary operation. + vertices: Tensor of shape `[dim_1, ..., dim_n, 3]` representing the 3D + coordinates of vertices. + faces: Tensor of shape `[dim_1, ..., dim_n, 3]` containing indices of + vertices within each triangle. + colors: Tensor of shape `[dim_1, ..., dim_n, 3]` containing colors for each + vertex. + display_name: If set, will be used as the display name in TensorBoard. + Defaults to `name`. + description: A longform readable description of the summary data. Markdown + is supported. + collections: Which TensorFlow graph collections to add the summary op to. + Defaults to `['summaries']`. Can usually be ignored. + config_dict: Dictionary with ThreeJS classes names and configuration. + + Returns: + Merged summary for mesh/point cloud representation. + """ + display_name = _get_display_name(name, display_name) + json_config = _get_json_config(config_dict) + + # All tensors representing a single mesh will be represented as separate + # summaries internally. Those summaries will be regrouped on the client before + # rendering. + summaries = [] + tensors = [ + metadata.MeshTensor( + vertices, plugin_data_pb2.MeshPluginData.VERTEX, tf.float32 + ), + metadata.MeshTensor( + faces, plugin_data_pb2.MeshPluginData.FACE, tf.int32 + ), + metadata.MeshTensor( + colors, plugin_data_pb2.MeshPluginData.COLOR, tf.uint8 + ), + ] + tensors = [tensor for tensor in tensors if tensor.data is not None] + + components = metadata.get_components_bitmask( + [tensor.content_type for tensor in tensors] + ) + + for tensor in tensors: + summaries.append( + _get_tensor_summary( + name, + display_name, + description, + tensor.data, + tensor.content_type, + components, + json_config, + collections, + ) + ) + + all_summaries = tf.compat.v1.summary.merge( + summaries, collections=collections, name=name + ) + return all_summaries + + +def pb( + name, + vertices, + faces=None, + colors=None, + display_name=None, + description=None, + config_dict=None, +): + """Create a mesh summary to save in pb format. + + Args: + name: A name for this summary operation. + vertices: numpy array of shape `[dim_1, ..., dim_n, 3]` representing the 3D + coordinates of vertices. + faces: numpy array of shape `[dim_1, ..., dim_n, 3]` containing indices of + vertices within each triangle. + colors: numpy array of shape `[dim_1, ..., dim_n, 3]` containing colors for + each vertex. + display_name: If set, will be used as the display name in TensorBoard. + Defaults to `name`. + description: A longform readable description of the summary data. Markdown + is supported. + config_dict: Dictionary with ThreeJS classes names and configuration. + + Returns: + Instance of tf.Summary class. + """ + display_name = _get_display_name(name, display_name) + json_config = _get_json_config(config_dict) + + summaries = [] + tensors = [ + metadata.MeshTensor( + vertices, plugin_data_pb2.MeshPluginData.VERTEX, tf.float32 + ), + metadata.MeshTensor( + faces, plugin_data_pb2.MeshPluginData.FACE, tf.int32 + ), + metadata.MeshTensor( + colors, plugin_data_pb2.MeshPluginData.COLOR, tf.uint8 + ), + ] + tensors = [tensor for tensor in tensors if tensor.data is not None] + components = metadata.get_components_bitmask( + [tensor.content_type for tensor in tensors] + ) + for tensor in tensors: + shape = tensor.data.shape + shape = [dim if dim is not None else -1 for dim in shape] + tensor_proto = tf.compat.v1.make_tensor_proto( + tensor.data, dtype=tensor.data_type + ) + summary_metadata = metadata.create_summary_metadata( + name, + display_name, + tensor.content_type, + components, + shape, + description, + json_config=json_config, + ) + tag = metadata.get_instance_name(name, tensor.content_type) + summaries.append((tag, summary_metadata, tensor_proto)) + + summary = tf.compat.v1.Summary() + for tag, summary_metadata, tensor_proto in summaries: + tf_summary_metadata = tf.compat.v1.SummaryMetadata.FromString( + summary_metadata.SerializeToString() + ) + summary.value.add( + tag=tag, metadata=tf_summary_metadata, tensor=tensor_proto + ) + return summary diff --git a/tensorboard/plugins/mesh/summary_test.py b/tensorboard/plugins/mesh/summary_test.py index c044e296b1..20ff4d5688 100644 --- a/tensorboard/plugins/mesh/summary_test.py +++ b/tensorboard/plugins/mesh/summary_test.py @@ -29,86 +29,107 @@ class MeshSummaryTest(tf.test.TestCase): + def pb_via_op(self, summary_op): + """Parses pb proto.""" + actual_pbtxt = summary_op.eval() + actual_proto = summary_pb2.Summary() + actual_proto.ParseFromString(actual_pbtxt) + return actual_proto - def pb_via_op(self, summary_op): - """Parses pb proto.""" - actual_pbtxt = summary_op.eval() - actual_proto = summary_pb2.Summary() - actual_proto.ParseFromString(actual_pbtxt) - return actual_proto + def get_components(self, proto): + return metadata.parse_plugin_metadata( + proto.metadata.plugin_data.content + ).components - def get_components(self, proto): - return metadata.parse_plugin_metadata( - proto.metadata.plugin_data.content).components + def verify_proto(self, proto, name): + """Validates proto.""" + self.assertEqual(3, len(proto.value)) + self.assertEqual("%s_VERTEX" % name, proto.value[0].tag) + self.assertEqual("%s_FACE" % name, proto.value[1].tag) + self.assertEqual("%s_COLOR" % name, proto.value[2].tag) - def verify_proto(self, proto, name): - """Validates proto.""" - self.assertEqual(3, len(proto.value)) - self.assertEqual("%s_VERTEX" % name, proto.value[0].tag) - self.assertEqual("%s_FACE" % name, proto.value[1].tag) - self.assertEqual("%s_COLOR" % name, proto.value[2].tag) + self.assertEqual(14, self.get_components(proto.value[0])) + self.assertEqual(14, self.get_components(proto.value[1])) + self.assertEqual(14, self.get_components(proto.value[2])) - self.assertEqual(14, self.get_components(proto.value[0])) - self.assertEqual(14, self.get_components(proto.value[1])) - self.assertEqual(14, self.get_components(proto.value[2])) + def test_get_tensor_summary(self): + """Tests proper creation of tensor summary with mesh plugin + metadata.""" + name = "my_mesh" + display_name = "my_display_name" + description = "my mesh is the best of meshes" + tensor_data = test_utils.get_random_mesh(100) + components = 14 + with tf.compat.v1.Graph().as_default(): + tensor_summary = summary._get_tensor_summary( + name, + display_name, + description, + tensor_data.vertices, + plugin_data_pb2.MeshPluginData.VERTEX, + components, + "", + None, + ) + with self.test_session(): + proto = self.pb_via_op(tensor_summary) + self.assertEqual("%s_VERTEX" % name, proto.value[0].tag) + self.assertEqual( + metadata.PLUGIN_NAME, + proto.value[0].metadata.plugin_data.plugin_name, + ) + self.assertEqual( + components, self.get_components(proto.value[0]) + ) - def test_get_tensor_summary(self): - """Tests proper creation of tensor summary with mesh plugin metadata.""" - name = "my_mesh" - display_name = "my_display_name" - description = "my mesh is the best of meshes" - tensor_data = test_utils.get_random_mesh(100) - components = 14 - with tf.compat.v1.Graph().as_default(): - tensor_summary = summary._get_tensor_summary( - name, display_name, description, tensor_data.vertices, - plugin_data_pb2.MeshPluginData.VERTEX, components, "", None) - with self.test_session(): - proto = self.pb_via_op(tensor_summary) - self.assertEqual("%s_VERTEX" % name, proto.value[0].tag) - self.assertEqual(metadata.PLUGIN_NAME, - proto.value[0].metadata.plugin_data.plugin_name) - self.assertEqual(components, self.get_components(proto.value[0])) + def test_op(self): + """Tests merged summary with different types of data.""" + name = "my_mesh" + tensor_data = test_utils.get_random_mesh( + 100, add_faces=True, add_colors=True + ) + config_dict = {"foo": 1} + with tf.compat.v1.Graph().as_default(): + tensor_summary = summary.op( + name, + tensor_data.vertices, + faces=tensor_data.faces, + colors=tensor_data.colors, + config_dict=config_dict, + ) + with self.test_session() as sess: + proto = self.pb_via_op(tensor_summary) + self.verify_proto(proto, name) + plugin_metadata = metadata.parse_plugin_metadata( + proto.value[0].metadata.plugin_data.content + ) + self.assertEqual( + json.dumps(config_dict, sort_keys=True), + plugin_metadata.json_config, + ) - def test_op(self): - """Tests merged summary with different types of data.""" - name = "my_mesh" - tensor_data = test_utils.get_random_mesh( - 100, add_faces=True, add_colors=True) - config_dict = {"foo": 1} - with tf.compat.v1.Graph().as_default(): - tensor_summary = summary.op( - name, - tensor_data.vertices, - faces=tensor_data.faces, - colors=tensor_data.colors, - config_dict=config_dict) - with self.test_session() as sess: - proto = self.pb_via_op(tensor_summary) + def test_pb(self): + """Tests merged summary protobuf with different types of data.""" + name = "my_mesh" + tensor_data = test_utils.get_random_mesh( + 100, add_faces=True, add_colors=True + ) + config_dict = {"foo": 1} + proto = summary.pb( + name, + tensor_data.vertices, + faces=tensor_data.faces, + colors=tensor_data.colors, + config_dict=config_dict, + ) self.verify_proto(proto, name) plugin_metadata = metadata.parse_plugin_metadata( - proto.value[0].metadata.plugin_data.content) + proto.value[0].metadata.plugin_data.content + ) self.assertEqual( - json.dumps(config_dict, sort_keys=True), plugin_metadata.json_config) - - def test_pb(self): - """Tests merged summary protobuf with different types of data.""" - name = "my_mesh" - tensor_data = test_utils.get_random_mesh( - 100, add_faces=True, add_colors=True) - config_dict = {"foo": 1} - proto = summary.pb( - name, - tensor_data.vertices, - faces=tensor_data.faces, - colors=tensor_data.colors, - config_dict=config_dict) - self.verify_proto(proto, name) - plugin_metadata = metadata.parse_plugin_metadata( - proto.value[0].metadata.plugin_data.content) - self.assertEqual( - json.dumps(config_dict, sort_keys=True), plugin_metadata.json_config) + json.dumps(config_dict, sort_keys=True), plugin_metadata.json_config + ) if __name__ == "__main__": - tf.test.main() + tf.test.main() diff --git a/tensorboard/plugins/mesh/summary_v2.py b/tensorboard/plugins/mesh/summary_v2.py index b641464b95..2b428d439f 100644 --- a/tensorboard/plugins/mesh/summary_v2.py +++ b/tensorboard/plugins/mesh/summary_v2.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Mesh summaries and TensorFlow operations to create them. V2 versions""" +"""Mesh summaries and TensorFlow operations to create them. + +V2 versions +""" from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -27,158 +30,188 @@ def _write_summary( - name, description, tensor, content_type, components, - json_config, step): - """Creates a tensor summary with summary metadata. - - Args: - name: A name for this summary. The summary tag used for TensorBoard will - be this name prefixed by any active name scopes. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - tensor: Tensor to display in summary. - content_type: Type of content inside the Tensor. - components: Bitmask representing present parts (vertices, colors, etc.) that - belong to the summary. - json_config: A string, JSON-serialized dictionary of ThreeJS classes - configuration. - step: Explicit `int64`-castable monotonic step value for this summary. If - omitted, this defaults to `tf.summary.experimental.get_step()`, which must - not be None. - - Returns: - A boolean indicating if summary was saved successfully or not. - """ - tensor = tf.convert_to_tensor(value=tensor) - shape = tensor.shape.as_list() - shape = [dim if dim is not None else -1 for dim in shape] - tensor_metadata = metadata.create_summary_metadata( - name, - None, # display_name - content_type, - components, - shape, - description, - json_config=json_config) - return tf.summary.write( + name, description, tensor, content_type, components, json_config, step +): + """Creates a tensor summary with summary metadata. + + Args: + name: A name for this summary. The summary tag used for TensorBoard will + be this name prefixed by any active name scopes. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + tensor: Tensor to display in summary. + content_type: Type of content inside the Tensor. + components: Bitmask representing present parts (vertices, colors, etc.) that + belong to the summary. + json_config: A string, JSON-serialized dictionary of ThreeJS classes + configuration. + step: Explicit `int64`-castable monotonic step value for this summary. If + omitted, this defaults to `tf.summary.experimental.get_step()`, which must + not be None. + + Returns: + A boolean indicating if summary was saved successfully or not. + """ + tensor = tf.convert_to_tensor(value=tensor) + shape = tensor.shape.as_list() + shape = [dim if dim is not None else -1 for dim in shape] + tensor_metadata = metadata.create_summary_metadata( + name, + None, # display_name + content_type, + components, + shape, + description, + json_config=json_config, + ) + return tf.summary.write( tag=metadata.get_instance_name(name, content_type), tensor=tensor, step=step, - metadata=tensor_metadata) + metadata=tensor_metadata, + ) def _get_json_config(config_dict): - """Parses and returns JSON string from python dictionary.""" - json_config = '{}' - if config_dict is not None: - json_config = json.dumps(config_dict, sort_keys=True) - return json_config - - -def mesh(name, vertices, faces=None, colors=None, config_dict=None, step=None, - description=None): - """Writes a TensorFlow mesh summary. - - Args: - name: A name for this summary. The summary tag used for TensorBoard will - be this name prefixed by any active name scopes. - vertices: Tensor of shape `[dim_1, ..., dim_n, 3]` representing the 3D - coordinates of vertices. - faces: Tensor of shape `[dim_1, ..., dim_n, 3]` containing indices of - vertices within each triangle. - colors: Tensor of shape `[dim_1, ..., dim_n, 3]` containing colors for each - vertex. - config_dict: Dictionary with ThreeJS classes names and configuration. - step: Explicit `int64`-castable monotonic step value for this summary. If - omitted, this defaults to `tf.summary.experimental.get_step()`, which must - not be None. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - - Returns: - True if all components of the mesh were saved successfully and False - otherwise. - """ - json_config = _get_json_config(config_dict) - - # All tensors representing a single mesh will be represented as separate - # summaries internally. Those summaries will be regrouped on the client before - # rendering. - tensors = [ - metadata.MeshTensor( - vertices, plugin_data_pb2.MeshPluginData.VERTEX, tf.float32), - metadata.MeshTensor(faces, plugin_data_pb2.MeshPluginData.FACE, tf.int32), - metadata.MeshTensor( - colors, plugin_data_pb2.MeshPluginData.COLOR, tf.uint8) - ] - tensors = [tensor for tensor in tensors if tensor.data is not None] - - components = metadata.get_components_bitmask([ - tensor.content_type for tensor in tensors]) - - summary_scope = ( - getattr(tf.summary.experimental, 'summary_scope', None) or - tf.summary.summary_scope) - all_success = True - with summary_scope(name, 'mesh_summary', values=tensors): + """Parses and returns JSON string from python dictionary.""" + json_config = "{}" + if config_dict is not None: + json_config = json.dumps(config_dict, sort_keys=True) + return json_config + + +def mesh( + name, + vertices, + faces=None, + colors=None, + config_dict=None, + step=None, + description=None, +): + """Writes a TensorFlow mesh summary. + + Args: + name: A name for this summary. The summary tag used for TensorBoard will + be this name prefixed by any active name scopes. + vertices: Tensor of shape `[dim_1, ..., dim_n, 3]` representing the 3D + coordinates of vertices. + faces: Tensor of shape `[dim_1, ..., dim_n, 3]` containing indices of + vertices within each triangle. + colors: Tensor of shape `[dim_1, ..., dim_n, 3]` containing colors for each + vertex. + config_dict: Dictionary with ThreeJS classes names and configuration. + step: Explicit `int64`-castable monotonic step value for this summary. If + omitted, this defaults to `tf.summary.experimental.get_step()`, which must + not be None. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + + Returns: + True if all components of the mesh were saved successfully and False + otherwise. + """ + json_config = _get_json_config(config_dict) + + # All tensors representing a single mesh will be represented as separate + # summaries internally. Those summaries will be regrouped on the client before + # rendering. + tensors = [ + metadata.MeshTensor( + vertices, plugin_data_pb2.MeshPluginData.VERTEX, tf.float32 + ), + metadata.MeshTensor( + faces, plugin_data_pb2.MeshPluginData.FACE, tf.int32 + ), + metadata.MeshTensor( + colors, plugin_data_pb2.MeshPluginData.COLOR, tf.uint8 + ), + ] + tensors = [tensor for tensor in tensors if tensor.data is not None] + + components = metadata.get_components_bitmask( + [tensor.content_type for tensor in tensors] + ) + + summary_scope = ( + getattr(tf.summary.experimental, "summary_scope", None) + or tf.summary.summary_scope + ) + all_success = True + with summary_scope(name, "mesh_summary", values=tensors): + for tensor in tensors: + all_success = all_success and _write_summary( + name, + description, + tensor.data, + tensor.content_type, + components, + json_config, + step, + ) + + return all_success + + +def mesh_pb( + tag, vertices, faces=None, colors=None, config_dict=None, description=None +): + """Create a mesh summary to save in pb format. + + Args: + tag: String tag for the summary. + vertices: numpy array of shape `[dim_1, ..., dim_n, 3]` representing the 3D + coordinates of vertices. + faces: numpy array of shape `[dim_1, ..., dim_n, 3]` containing indices of + vertices within each triangle. + colors: numpy array of shape `[dim_1, ..., dim_n, 3]` containing colors for + each vertex. + config_dict: Dictionary with ThreeJS classes names and configuration. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + + Returns: + Instance of tf.Summary class. + """ + json_config = _get_json_config(config_dict) + + summaries = [] + tensors = [ + metadata.MeshTensor( + vertices, plugin_data_pb2.MeshPluginData.VERTEX, tf.float32 + ), + metadata.MeshTensor( + faces, plugin_data_pb2.MeshPluginData.FACE, tf.int32 + ), + metadata.MeshTensor( + colors, plugin_data_pb2.MeshPluginData.COLOR, tf.uint8 + ), + ] + tensors = [tensor for tensor in tensors if tensor.data is not None] + components = metadata.get_components_bitmask( + [tensor.content_type for tensor in tensors] + ) for tensor in tensors: - all_success = all_success and _write_summary( - name, description, tensor.data, tensor.content_type, - components, json_config, step) - - return all_success - - -def mesh_pb(tag, vertices, faces=None, colors=None, config_dict=None, - description=None): - """Create a mesh summary to save in pb format. - - Args: - tag: String tag for the summary. - vertices: numpy array of shape `[dim_1, ..., dim_n, 3]` representing the 3D - coordinates of vertices. - faces: numpy array of shape `[dim_1, ..., dim_n, 3]` containing indices of - vertices within each triangle. - colors: numpy array of shape `[dim_1, ..., dim_n, 3]` containing colors for - each vertex. - config_dict: Dictionary with ThreeJS classes names and configuration. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - - Returns: - Instance of tf.Summary class. - """ - json_config = _get_json_config(config_dict) - - summaries = [] - tensors = [ - metadata.MeshTensor( - vertices, plugin_data_pb2.MeshPluginData.VERTEX, tf.float32), - metadata.MeshTensor(faces, plugin_data_pb2.MeshPluginData.FACE, tf.int32), - metadata.MeshTensor( - colors, plugin_data_pb2.MeshPluginData.COLOR, tf.uint8) - ] - tensors = [tensor for tensor in tensors if tensor.data is not None] - components = metadata.get_components_bitmask([ - tensor.content_type for tensor in tensors]) - for tensor in tensors: - shape = tensor.data.shape - shape = [dim if dim is not None else -1 for dim in shape] - tensor_proto = tensor_util.make_tensor_proto( - tensor.data, dtype=tensor.data_type) - summary_metadata = metadata.create_summary_metadata( - tag, - None, # display_name - tensor.content_type, - components, - shape, - description, - json_config=json_config) - instance_tag = metadata.get_instance_name(tag, tensor.content_type) - summaries.append((instance_tag, summary_metadata, tensor_proto)) - - summary = summary_pb2.Summary() - for instance_tag, summary_metadata, tensor_proto in summaries: - summary.value.add( - tag=instance_tag, metadata=summary_metadata, tensor=tensor_proto) - return summary + shape = tensor.data.shape + shape = [dim if dim is not None else -1 for dim in shape] + tensor_proto = tensor_util.make_tensor_proto( + tensor.data, dtype=tensor.data_type + ) + summary_metadata = metadata.create_summary_metadata( + tag, + None, # display_name + tensor.content_type, + components, + shape, + description, + json_config=json_config, + ) + instance_tag = metadata.get_instance_name(tag, tensor.content_type) + summaries.append((instance_tag, summary_metadata, tensor_proto)) + + summary = summary_pb2.Summary() + for instance_tag, summary_metadata, tensor_proto in summaries: + summary.value.add( + tag=instance_tag, metadata=summary_metadata, tensor=tensor_proto + ) + return summary diff --git a/tensorboard/plugins/mesh/summary_v2_test.py b/tensorboard/plugins/mesh/summary_v2_test.py index b97faf2fc1..e0299e4b9c 100644 --- a/tensorboard/plugins/mesh/summary_v2_test.py +++ b/tensorboard/plugins/mesh/summary_v2_test.py @@ -33,125 +33,144 @@ try: - tf2.__version__ # Force lazy import to resolve + tf2.__version__ # Force lazy import to resolve except ImportError: - tf2 = None + tf2 = None try: - tf.compat.v1.enable_eager_execution() + tf.compat.v1.enable_eager_execution() except AttributeError: - # TF 2.0 doesn't have this symbol because eager is the default. - pass + # TF 2.0 doesn't have this symbol because eager is the default. + pass class MeshSummaryV2Test(tf.test.TestCase): - def setUp(self): - super(MeshSummaryV2Test, self).setUp() - if tf2 is None: - self.skipTest('v2 summary API not available') - - def mesh_events(self, *args, **kwargs): - self.write_mesh_event(*args, **kwargs) - event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), '*'))) - self.assertEqual(len(event_files), 1) - events = list(tf.compat.v1.train.summary_iterator(event_files[0])) - # Expect a boilerplate event for the file_version, then the vertices - # summary one. - num_events = 2 - # All additional tensors (i.e. colors or faces) will be stored as separate - # events, so account for them as well. - num_events += len(frozenset(["colors", "faces"]).intersection(kwargs)) - self.assertEqual(len(events), num_events) - # Delete the event file to reset to an empty directory for later calls. - os.remove(event_files[0]) - return events[1:] - - def write_mesh_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - summary.mesh(*args, **kwargs) - writer.close() - - def get_metadata(self, event): - return metadata.parse_plugin_metadata( - event.summary.value[0].metadata.plugin_data.content) - - def test_step(self): - """Tests that different components of mesh summary share the same step.""" - tensor_data = test_utils.get_random_mesh( - 100, add_faces=True, add_colors=True) - config_dict = {"foo": 1} - events = self.mesh_events( - 'a', - tensor_data.vertices, - faces=tensor_data.faces, - colors=tensor_data.colors, - config_dict=config_dict, - step=333) - self.assertEqual(333, events[0].step) - self.assertEqual(333, events[1].step) - self.assertEqual(333, events[2].step) - - def test_tags(self): - """Tests proper tags for each event/tensor.""" - tensor_data = test_utils.get_random_mesh( - 100, add_faces=True, add_colors=True) - config_dict = {"foo": 1} - name = 'foo' - events = self.mesh_events( - name, - tensor_data.vertices, - faces=tensor_data.faces, - colors=tensor_data.colors, - config_dict=config_dict, - step=333) - expected_names_set = frozenset( - name_tpl % name for name_tpl in ["%s_VERTEX", "%s_FACE", "%s_COLOR"]) - actual_names_set = frozenset([event.summary.value[0].tag for event in events]) - self.assertEqual(expected_names_set, actual_names_set) - expected_bitmask = metadata.get_components_bitmask([ - plugin_data_pb2.MeshPluginData.VERTEX, - plugin_data_pb2.MeshPluginData.FACE, - plugin_data_pb2.MeshPluginData.COLOR, - ]) - for event in events: - self.assertEqual(expected_bitmask, self.get_metadata(event).components) - - def test_pb(self): - """Tests ProtoBuf interface.""" - name = "my_mesh" - tensor_data = test_utils.get_random_mesh( - 100, add_faces=True, add_colors=True) - config_dict = {"foo": 1} - proto = summary.mesh_pb( - name, - tensor_data.vertices, - faces=tensor_data.faces, - colors=tensor_data.colors, - config_dict=config_dict) - plugin_metadata = metadata.parse_plugin_metadata( - proto.value[0].metadata.plugin_data.content) - self.assertEqual( - json.dumps(config_dict, sort_keys=True), plugin_metadata.json_config) + def setUp(self): + super(MeshSummaryV2Test, self).setUp() + if tf2 is None: + self.skipTest("v2 summary API not available") + + def mesh_events(self, *args, **kwargs): + self.write_mesh_event(*args, **kwargs) + event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), "*"))) + self.assertEqual(len(event_files), 1) + events = list(tf.compat.v1.train.summary_iterator(event_files[0])) + # Expect a boilerplate event for the file_version, then the vertices + # summary one. + num_events = 2 + # All additional tensors (i.e. colors or faces) will be stored as separate + # events, so account for them as well. + num_events += len(frozenset(["colors", "faces"]).intersection(kwargs)) + self.assertEqual(len(events), num_events) + # Delete the event file to reset to an empty directory for later calls. + os.remove(event_files[0]) + return events[1:] + + def write_mesh_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + summary.mesh(*args, **kwargs) + writer.close() + + def get_metadata(self, event): + return metadata.parse_plugin_metadata( + event.summary.value[0].metadata.plugin_data.content + ) + + def test_step(self): + """Tests that different components of mesh summary share the same + step.""" + tensor_data = test_utils.get_random_mesh( + 100, add_faces=True, add_colors=True + ) + config_dict = {"foo": 1} + events = self.mesh_events( + "a", + tensor_data.vertices, + faces=tensor_data.faces, + colors=tensor_data.colors, + config_dict=config_dict, + step=333, + ) + self.assertEqual(333, events[0].step) + self.assertEqual(333, events[1].step) + self.assertEqual(333, events[2].step) + + def test_tags(self): + """Tests proper tags for each event/tensor.""" + tensor_data = test_utils.get_random_mesh( + 100, add_faces=True, add_colors=True + ) + config_dict = {"foo": 1} + name = "foo" + events = self.mesh_events( + name, + tensor_data.vertices, + faces=tensor_data.faces, + colors=tensor_data.colors, + config_dict=config_dict, + step=333, + ) + expected_names_set = frozenset( + name_tpl % name for name_tpl in ["%s_VERTEX", "%s_FACE", "%s_COLOR"] + ) + actual_names_set = frozenset( + [event.summary.value[0].tag for event in events] + ) + self.assertEqual(expected_names_set, actual_names_set) + expected_bitmask = metadata.get_components_bitmask( + [ + plugin_data_pb2.MeshPluginData.VERTEX, + plugin_data_pb2.MeshPluginData.FACE, + plugin_data_pb2.MeshPluginData.COLOR, + ] + ) + for event in events: + self.assertEqual( + expected_bitmask, self.get_metadata(event).components + ) + + def test_pb(self): + """Tests ProtoBuf interface.""" + name = "my_mesh" + tensor_data = test_utils.get_random_mesh( + 100, add_faces=True, add_colors=True + ) + config_dict = {"foo": 1} + proto = summary.mesh_pb( + name, + tensor_data.vertices, + faces=tensor_data.faces, + colors=tensor_data.colors, + config_dict=config_dict, + ) + plugin_metadata = metadata.parse_plugin_metadata( + proto.value[0].metadata.plugin_data.content + ) + self.assertEqual( + json.dumps(config_dict, sort_keys=True), plugin_metadata.json_config + ) class MeshSummaryV2GraphTest(MeshSummaryV2Test, tf.test.TestCase): - def write_mesh_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - # Hack to extract current scope since there's no direct API for it. - with tf.name_scope('_') as temp_scope: - scope = temp_scope.rstrip('/_') - @tf2.function - def graph_fn(): - # Recreate the active scope inside the defun since it won't propagate. - with tf.name_scope(scope): - summary.mesh(*args, **kwargs) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - graph_fn() - writer.close() - - -if __name__ == '__main__': - tf.test.main() + def write_mesh_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + # Hack to extract current scope since there's no direct API for it. + with tf.name_scope("_") as temp_scope: + scope = temp_scope.rstrip("/_") + + @tf2.function + def graph_fn(): + # Recreate the active scope inside the defun since it won't propagate. + with tf.name_scope(scope): + summary.mesh(*args, **kwargs) + + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + graph_fn() + writer.close() + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/mesh/test_utils.py b/tensorboard/plugins/mesh/test_utils.py index 572dab95e2..fef0e222d0 100644 --- a/tensorboard/plugins/mesh/test_utils.py +++ b/tensorboard/plugins/mesh/test_utils.py @@ -30,67 +30,69 @@ from tensorboard.compat.proto import summary_pb2 from tensorboard.util import tb_logging -Mesh = collections.namedtuple('Mesh', ('vertices', 'faces', 'colors')) +Mesh = collections.namedtuple("Mesh", ("vertices", "faces", "colors")) logger = tb_logging.get_logger() -def get_random_mesh(num_vertices, - add_faces=False, - add_colors=False, - batch_size=1): - """Returns a random point cloud, optionally with random disconnected faces. - - Args: - num_vertices: Number of vertices in the point cloud or mesh. - add_faces: Random faces will be generated and added to the mesh when True. - add_colors: Random colors will be assigned to each vertex when True. Each - color will be in a range of [0, 255]. - batch_size: Size of batch dimension in output array. - - Returns: - Mesh namedtuple with vertices and optionally with faces and/or colors. - """ - vertices = np.random.random([num_vertices, 3]) * 1000 - # Add batch dimension. - vertices = np.tile(vertices, [batch_size, 1, 1]) - faces = None - colors = None - if add_faces: - arranged_vertices = np.random.permutation(num_vertices) - faces = [] - for i in range(num_vertices - 2): - faces.append([ - arranged_vertices[i], arranged_vertices[i + 1], - arranged_vertices[i + 2] - ]) - faces = np.array(faces) - faces = np.tile(faces, [batch_size, 1, 1]).astype(np.int32) - if add_colors: - colors = np.random.randint(low=0, high=255, size=[num_vertices, 3]) - colors = np.tile(colors, [batch_size, 1, 1]).astype(np.uint8) - return Mesh(vertices.astype(np.float32), faces, colors) +def get_random_mesh( + num_vertices, add_faces=False, add_colors=False, batch_size=1 +): + """Returns a random point cloud, optionally with random disconnected faces. + + Args: + num_vertices: Number of vertices in the point cloud or mesh. + add_faces: Random faces will be generated and added to the mesh when True. + add_colors: Random colors will be assigned to each vertex when True. Each + color will be in a range of [0, 255]. + batch_size: Size of batch dimension in output array. + + Returns: + Mesh namedtuple with vertices and optionally with faces and/or colors. + """ + vertices = np.random.random([num_vertices, 3]) * 1000 + # Add batch dimension. + vertices = np.tile(vertices, [batch_size, 1, 1]) + faces = None + colors = None + if add_faces: + arranged_vertices = np.random.permutation(num_vertices) + faces = [] + for i in range(num_vertices - 2): + faces.append( + [ + arranged_vertices[i], + arranged_vertices[i + 1], + arranged_vertices[i + 2], + ] + ) + faces = np.array(faces) + faces = np.tile(faces, [batch_size, 1, 1]).astype(np.int32) + if add_colors: + colors = np.random.randint(low=0, high=255, size=[num_vertices, 3]) + colors = np.tile(colors, [batch_size, 1, 1]).astype(np.uint8) + return Mesh(vertices.astype(np.float32), faces, colors) def deserialize_json_response(byte_content): - """Deserializes byte content that is a JSON encoding. + """Deserializes byte content that is a JSON encoding. - Args: - byte_content: The byte content of a response. + Args: + byte_content: The byte content of a response. - Returns: - The deserialized python object decoded from JSON. - """ - return json.loads(byte_content.decode('utf-8')) + Returns: + The deserialized python object decoded from JSON. + """ + return json.loads(byte_content.decode("utf-8")) def deserialize_array_buffer_response(byte_content, data_type): - """Deserializes arraybuffer response and optionally tiles the array. + """Deserializes arraybuffer response and optionally tiles the array. - Args: - byte_content: The byte content of a response. - data_type: Numpy type to parse data with. + Args: + byte_content: The byte content of a response. + data_type: Numpy type to parse data with. - Returns: - Flat numpy array with the data. - """ - return np.frombuffer(byte_content, dtype=data_type) + Returns: + Flat numpy array with the data. + """ + return np.frombuffer(byte_content, dtype=data_type) diff --git a/tensorboard/plugins/pr_curve/metadata.py b/tensorboard/plugins/pr_curve/metadata.py index fa1a55b4b7..4ca7961d4f 100644 --- a/tensorboard/plugins/pr_curve/metadata.py +++ b/tensorboard/plugins/pr_curve/metadata.py @@ -24,7 +24,7 @@ logger = tb_logging.get_logger() -PLUGIN_NAME = 'pr_curves' +PLUGIN_NAME = "pr_curves" # Indices for obtaining various values from the tensor stored in a summary. TRUE_POSITIVES_INDEX = 0 @@ -38,46 +38,52 @@ # `PrCurvePluginData` proto. PROTO_VERSION = 0 + def create_summary_metadata(display_name, description, num_thresholds): - """Create a `summary_pb2.SummaryMetadata` proto for pr_curves plugin data. + """Create a `summary_pb2.SummaryMetadata` proto for pr_curves plugin data. - Arguments: - display_name: The display name used in TensorBoard. - description: The description to show in TensorBoard. - num_thresholds: The number of thresholds to use for PR curves. + Arguments: + display_name: The display name used in TensorBoard. + description: The description to show in TensorBoard. + num_thresholds: The number of thresholds to use for PR curves. - Returns: - A `summary_pb2.SummaryMetadata` protobuf object. - """ - pr_curve_plugin_data = plugin_data_pb2.PrCurvePluginData( - version=PROTO_VERSION, num_thresholds=num_thresholds) - content = pr_curve_plugin_data.SerializeToString() - return summary_pb2.SummaryMetadata( - display_name=display_name, - summary_description=description, - plugin_data=summary_pb2.SummaryMetadata.PluginData( - plugin_name=PLUGIN_NAME, - content=content)) + Returns: + A `summary_pb2.SummaryMetadata` protobuf object. + """ + pr_curve_plugin_data = plugin_data_pb2.PrCurvePluginData( + version=PROTO_VERSION, num_thresholds=num_thresholds + ) + content = pr_curve_plugin_data.SerializeToString() + return summary_pb2.SummaryMetadata( + display_name=display_name, + summary_description=description, + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME, content=content + ), + ) def parse_plugin_metadata(content): - """Parse summary metadata to a Python object. + """Parse summary metadata to a Python object. - Arguments: - content: The `content` field of a `SummaryMetadata` proto - corresponding to the pr_curves plugin. + Arguments: + content: The `content` field of a `SummaryMetadata` proto + corresponding to the pr_curves plugin. - Returns: - A `PrCurvesPlugin` protobuf object. - """ - if not isinstance(content, bytes): - raise TypeError('Content type must be bytes') - result = plugin_data_pb2.PrCurvePluginData.FromString(content) - if result.version == 0: - return result - else: - logger.warn( - 'Unknown metadata version: %s. The latest version known to ' - 'this build of TensorBoard is %s; perhaps a newer build is ' - 'available?', result.version, PROTO_VERSION) - return result + Returns: + A `PrCurvesPlugin` protobuf object. + """ + if not isinstance(content, bytes): + raise TypeError("Content type must be bytes") + result = plugin_data_pb2.PrCurvePluginData.FromString(content) + if result.version == 0: + return result + else: + logger.warn( + "Unknown metadata version: %s. The latest version known to " + "this build of TensorBoard is %s; perhaps a newer build is " + "available?", + result.version, + PROTO_VERSION, + ) + return result diff --git a/tensorboard/plugins/pr_curve/pr_curve_demo.py b/tensorboard/plugins/pr_curve/pr_curve_demo.py index 0cf00c3dc3..97643c12ee 100644 --- a/tensorboard/plugins/pr_curve/pr_curve_demo.py +++ b/tensorboard/plugins/pr_curve/pr_curve_demo.py @@ -42,195 +42,238 @@ tf.compat.v1.disable_v2_behavior() FLAGS = flags.FLAGS -flags.DEFINE_string('logdir', '/tmp/pr_curve_demo', - 'Directory into which to write TensorBoard data.') +flags.DEFINE_string( + "logdir", + "/tmp/pr_curve_demo", + "Directory into which to write TensorBoard data.", +) + +flags.DEFINE_integer( + "steps", 10, "Number of steps to generate for each PR curve." +) -flags.DEFINE_integer('steps', 10, - 'Number of steps to generate for each PR curve.') def start_runs( - logdir, - steps, - run_name, - thresholds, - mask_every_other_prediction=False): - """Generate a PR curve with precision and recall evenly weighted. - - Arguments: - logdir: The directory into which to store all the runs' data. - steps: The number of steps to run for. - run_name: The name of the run. - thresholds: The number of thresholds to use for PR curves. - mask_every_other_prediction: Whether to mask every other prediction by - alternating weights between 0 and 1. - """ - tf.compat.v1.reset_default_graph() - tf.compat.v1.set_random_seed(42) - - # Create a normal distribution layer used to generate true color labels. - distribution = tf.compat.v1.distributions.Normal(loc=0., scale=142.) - - # Sample the distribution to generate colors. Lets generate different numbers - # of each color. The first dimension is the count of examples. - - # The calls to sample() are given fixed random seed values that are "magic" - # in that they correspond to the default seeds for those ops when the PR - # curve test (which depends on this code) was written. We've pinned these - # instead of continuing to use the defaults since the defaults are based on - # node IDs from the sequence of nodes added to the graph, which can silently - # change when this code or any TF op implementations it uses are modified. - - # TODO(nickfelt): redo the PR curve test to avoid reliance on random seeds. - - # Generate reds. - number_of_reds = 100 - true_reds = tf.clip_by_value( - tf.concat([ - 255 - tf.abs(distribution.sample([number_of_reds, 1], seed=11)), - tf.abs(distribution.sample([number_of_reds, 2], seed=34)) - ], axis=1), - 0, 255) - - # Generate greens. - number_of_greens = 200 - true_greens = tf.clip_by_value( - tf.concat([ - tf.abs(distribution.sample([number_of_greens, 1], seed=61)), - 255 - tf.abs(distribution.sample([number_of_greens, 1], seed=82)), - tf.abs(distribution.sample([number_of_greens, 1], seed=105)) - ], axis=1), - 0, 255) - - # Generate blues. - number_of_blues = 150 - true_blues = tf.clip_by_value( - tf.concat([ - tf.abs(distribution.sample([number_of_blues, 2], seed=132)), - 255 - tf.abs(distribution.sample([number_of_blues, 1], seed=153)) - ], axis=1), - 0, 255) - - # Assign each color a vector of 3 booleans based on its true label. - labels = tf.concat([ - tf.tile(tf.constant([[True, False, False]]), (number_of_reds, 1)), - tf.tile(tf.constant([[False, True, False]]), (number_of_greens, 1)), - tf.tile(tf.constant([[False, False, True]]), (number_of_blues, 1)), - ], axis=0) - - # We introduce 3 normal distributions. They are used to predict whether a - # color falls under a certain class (based on distances from corners of the - # color triangle). The distributions vary per color. We have the distributions - # narrow over time. - initial_standard_deviations = [v + FLAGS.steps for v in (158, 200, 242)] - iteration = tf.compat.v1.placeholder(tf.int32, shape=[]) - red_predictor = tf.compat.v1.distributions.Normal( - loc=0., - scale=tf.cast( - initial_standard_deviations[0] - iteration, - dtype=tf.float32)) - green_predictor = tf.compat.v1.distributions.Normal( - loc=0., - scale=tf.cast( - initial_standard_deviations[1] - iteration, - dtype=tf.float32)) - blue_predictor = tf.compat.v1.distributions.Normal( - loc=0., - scale=tf.cast( - initial_standard_deviations[2] - iteration, - dtype=tf.float32)) - - # Make predictions (assign 3 probabilities to each color based on each color's - # distance to each of the 3 corners). We seek double the area in the right - # tail of the normal distribution. - examples = tf.concat([true_reds, true_greens, true_blues], axis=0) - probabilities_colors_are_red = (1 - red_predictor.cdf( - tf.norm(tensor=examples - tf.constant([255., 0, 0]), axis=1))) * 2 - probabilities_colors_are_green = (1 - green_predictor.cdf( - tf.norm(tensor=examples - tf.constant([0, 255., 0]), axis=1))) * 2 - probabilities_colors_are_blue = (1 - blue_predictor.cdf( - tf.norm(tensor=examples - tf.constant([0, 0, 255.]), axis=1))) * 2 - - predictions = ( - probabilities_colors_are_red, - probabilities_colors_are_green, - probabilities_colors_are_blue - ) - - # This is the crucial piece. We write data required for generating PR curves. - # We create 1 summary per class because we create 1 PR curve per class. - for i, color in enumerate(('red', 'green', 'blue')): - description = ('The probabilities used to create this PR curve are ' - 'generated from a normal distribution. Its standard ' - 'deviation is initially %0.0f and decreases over time.' % - initial_standard_deviations[i]) - - weights = None - if mask_every_other_prediction: - # Assign a weight of 0 to every even-indexed prediction. Odd-indexed - # predictions are assigned a default weight of 1. - consecutive_indices = tf.reshape( - tf.range(tf.size(input=predictions[i])), tf.shape(input=predictions[i])) - weights = tf.cast(consecutive_indices % 2, dtype=tf.float32) - - summary.op( - name=color, - labels=labels[:, i], - predictions=predictions[i], - num_thresholds=thresholds, - weights=weights, - display_name='classifying %s' % color, - description=description) - merged_summary_op = tf.compat.v1.summary.merge_all() - events_directory = os.path.join(logdir, run_name) - sess = tf.compat.v1.Session() - writer = tf.compat.v1.summary.FileWriter(events_directory, sess.graph) - - for step in xrange(steps): - feed_dict = { - iteration: step, - } - merged_summary = sess.run(merged_summary_op, feed_dict=feed_dict) - writer.add_summary(merged_summary, step) - - writer.close() + logdir, steps, run_name, thresholds, mask_every_other_prediction=False +): + """Generate a PR curve with precision and recall evenly weighted. + + Arguments: + logdir: The directory into which to store all the runs' data. + steps: The number of steps to run for. + run_name: The name of the run. + thresholds: The number of thresholds to use for PR curves. + mask_every_other_prediction: Whether to mask every other prediction by + alternating weights between 0 and 1. + """ + tf.compat.v1.reset_default_graph() + tf.compat.v1.set_random_seed(42) + + # Create a normal distribution layer used to generate true color labels. + distribution = tf.compat.v1.distributions.Normal(loc=0.0, scale=142.0) + + # Sample the distribution to generate colors. Lets generate different numbers + # of each color. The first dimension is the count of examples. + + # The calls to sample() are given fixed random seed values that are "magic" + # in that they correspond to the default seeds for those ops when the PR + # curve test (which depends on this code) was written. We've pinned these + # instead of continuing to use the defaults since the defaults are based on + # node IDs from the sequence of nodes added to the graph, which can silently + # change when this code or any TF op implementations it uses are modified. + + # TODO(nickfelt): redo the PR curve test to avoid reliance on random seeds. + + # Generate reds. + number_of_reds = 100 + true_reds = tf.clip_by_value( + tf.concat( + [ + 255 - tf.abs(distribution.sample([number_of_reds, 1], seed=11)), + tf.abs(distribution.sample([number_of_reds, 2], seed=34)), + ], + axis=1, + ), + 0, + 255, + ) + + # Generate greens. + number_of_greens = 200 + true_greens = tf.clip_by_value( + tf.concat( + [ + tf.abs(distribution.sample([number_of_greens, 1], seed=61)), + 255 + - tf.abs(distribution.sample([number_of_greens, 1], seed=82)), + tf.abs(distribution.sample([number_of_greens, 1], seed=105)), + ], + axis=1, + ), + 0, + 255, + ) + + # Generate blues. + number_of_blues = 150 + true_blues = tf.clip_by_value( + tf.concat( + [ + tf.abs(distribution.sample([number_of_blues, 2], seed=132)), + 255 + - tf.abs(distribution.sample([number_of_blues, 1], seed=153)), + ], + axis=1, + ), + 0, + 255, + ) + + # Assign each color a vector of 3 booleans based on its true label. + labels = tf.concat( + [ + tf.tile(tf.constant([[True, False, False]]), (number_of_reds, 1)), + tf.tile(tf.constant([[False, True, False]]), (number_of_greens, 1)), + tf.tile(tf.constant([[False, False, True]]), (number_of_blues, 1)), + ], + axis=0, + ) + + # We introduce 3 normal distributions. They are used to predict whether a + # color falls under a certain class (based on distances from corners of the + # color triangle). The distributions vary per color. We have the distributions + # narrow over time. + initial_standard_deviations = [v + FLAGS.steps for v in (158, 200, 242)] + iteration = tf.compat.v1.placeholder(tf.int32, shape=[]) + red_predictor = tf.compat.v1.distributions.Normal( + loc=0.0, + scale=tf.cast( + initial_standard_deviations[0] - iteration, dtype=tf.float32 + ), + ) + green_predictor = tf.compat.v1.distributions.Normal( + loc=0.0, + scale=tf.cast( + initial_standard_deviations[1] - iteration, dtype=tf.float32 + ), + ) + blue_predictor = tf.compat.v1.distributions.Normal( + loc=0.0, + scale=tf.cast( + initial_standard_deviations[2] - iteration, dtype=tf.float32 + ), + ) + + # Make predictions (assign 3 probabilities to each color based on each color's + # distance to each of the 3 corners). We seek double the area in the right + # tail of the normal distribution. + examples = tf.concat([true_reds, true_greens, true_blues], axis=0) + probabilities_colors_are_red = ( + 1 + - red_predictor.cdf( + tf.norm(tensor=examples - tf.constant([255.0, 0, 0]), axis=1) + ) + ) * 2 + probabilities_colors_are_green = ( + 1 + - green_predictor.cdf( + tf.norm(tensor=examples - tf.constant([0, 255.0, 0]), axis=1) + ) + ) * 2 + probabilities_colors_are_blue = ( + 1 + - blue_predictor.cdf( + tf.norm(tensor=examples - tf.constant([0, 0, 255.0]), axis=1) + ) + ) * 2 + + predictions = ( + probabilities_colors_are_red, + probabilities_colors_are_green, + probabilities_colors_are_blue, + ) + + # This is the crucial piece. We write data required for generating PR curves. + # We create 1 summary per class because we create 1 PR curve per class. + for i, color in enumerate(("red", "green", "blue")): + description = ( + "The probabilities used to create this PR curve are " + "generated from a normal distribution. Its standard " + "deviation is initially %0.0f and decreases over time." + % initial_standard_deviations[i] + ) + + weights = None + if mask_every_other_prediction: + # Assign a weight of 0 to every even-indexed prediction. Odd-indexed + # predictions are assigned a default weight of 1. + consecutive_indices = tf.reshape( + tf.range(tf.size(input=predictions[i])), + tf.shape(input=predictions[i]), + ) + weights = tf.cast(consecutive_indices % 2, dtype=tf.float32) + + summary.op( + name=color, + labels=labels[:, i], + predictions=predictions[i], + num_thresholds=thresholds, + weights=weights, + display_name="classifying %s" % color, + description=description, + ) + merged_summary_op = tf.compat.v1.summary.merge_all() + events_directory = os.path.join(logdir, run_name) + sess = tf.compat.v1.Session() + writer = tf.compat.v1.summary.FileWriter(events_directory, sess.graph) + + for step in xrange(steps): + feed_dict = { + iteration: step, + } + merged_summary = sess.run(merged_summary_op, feed_dict=feed_dict) + writer.add_summary(merged_summary, step) + + writer.close() + def run_all(logdir, steps, thresholds, verbose=False): - """Generate PR curve summaries. - - Arguments: - logdir: The directory into which to store all the runs' data. - steps: The number of steps to run for. - verbose: Whether to print the names of runs into stdout during execution. - thresholds: The number of thresholds to use for PR curves. - """ - # First, we generate data for a PR curve that assigns even weights for - # predictions of all classes. - run_name = 'colors' - if verbose: - print('--- Running: %s' % run_name) - start_runs( - logdir=logdir, - steps=steps, - run_name=run_name, - thresholds=thresholds) - - # Next, we generate data for a PR curve that assigns arbitrary weights to - # predictions. - run_name = 'mask_every_other_prediction' - if verbose: - print('--- Running: %s' % run_name) - start_runs( - logdir=logdir, - steps=steps, - run_name=run_name, - thresholds=thresholds, - mask_every_other_prediction=True) + """Generate PR curve summaries. + + Arguments: + logdir: The directory into which to store all the runs' data. + steps: The number of steps to run for. + verbose: Whether to print the names of runs into stdout during execution. + thresholds: The number of thresholds to use for PR curves. + """ + # First, we generate data for a PR curve that assigns even weights for + # predictions of all classes. + run_name = "colors" + if verbose: + print("--- Running: %s" % run_name) + start_runs( + logdir=logdir, steps=steps, run_name=run_name, thresholds=thresholds + ) + + # Next, we generate data for a PR curve that assigns arbitrary weights to + # predictions. + run_name = "mask_every_other_prediction" + if verbose: + print("--- Running: %s" % run_name) + start_runs( + logdir=logdir, + steps=steps, + run_name=run_name, + thresholds=thresholds, + mask_every_other_prediction=True, + ) + def main(unused_argv): - print('Saving output to %s.' % FLAGS.logdir) - run_all(FLAGS.logdir, FLAGS.steps, 50, verbose=True) - print('Done. Output saved to %s.' % FLAGS.logdir) + print("Saving output to %s." % FLAGS.logdir) + run_all(FLAGS.logdir, FLAGS.steps, 50, verbose=True) + print("Done. Output saved to %s." % FLAGS.logdir) -if __name__ == '__main__': - app.run(main) +if __name__ == "__main__": + app.run(main) diff --git a/tensorboard/plugins/pr_curve/pr_curves_plugin.py b/tensorboard/plugins/pr_curve/pr_curves_plugin.py index 589196af73..9168ce9f05 100644 --- a/tensorboard/plugins/pr_curve/pr_curves_plugin.py +++ b/tensorboard/plugins/pr_curve/pr_curves_plugin.py @@ -30,67 +30,73 @@ class PrCurvesPlugin(base_plugin.TBPlugin): - """A plugin that serves PR curves for individual classes.""" - - plugin_name = metadata.PLUGIN_NAME - - def __init__(self, context): - """Instantiates a PrCurvesPlugin. - Args: - context: A base_plugin.TBContext instance. A magic container that - TensorBoard uses to make objects available to the plugin. - """ - self._db_connection_provider = context.db_connection_provider - self._multiplexer = context.multiplexer - - @wrappers.Request.application - def pr_curves_route(self, request): - """A route that returns a JSON mapping between runs and PR curve data. - - Returns: - Given a tag and a comma-separated list of runs (both stored within GET - parameters), fetches a JSON object that maps between run name and objects - containing data required for PR curves for that run. Runs that either - cannot be found or that lack tags will be excluded from the response. - """ - runs = request.args.getlist('run') - if not runs: - return http_util.Respond( - request, 'No runs provided when fetching PR curve data', 400) - - tag = request.args.get('tag') - if not tag: - return http_util.Respond( - request, 'No tag provided when fetching PR curve data', 400) - - try: - response = http_util.Respond( - request, self.pr_curves_impl(runs, tag), 'application/json') - except ValueError as e: - return http_util.Respond(request, str(e), 'text/plain', 400) - - return response - - def pr_curves_impl(self, runs, tag): - """Creates the JSON object for the PR curves response for a run-tag combo. - - Arguments: - runs: A list of runs to fetch the curves for. - tag: The tag to fetch the curves for. - - Raises: - ValueError: If no PR curves could be fetched for a run and tag. - - Returns: - The JSON object for the PR curves route response. - """ - if self._db_connection_provider: - # Serve data from the database. - db = self._db_connection_provider() - - # We select for steps greater than -1 because the writer inserts - # placeholder rows en masse. The check for step filters out those rows. - cursor = db.execute(''' + """A plugin that serves PR curves for individual classes.""" + + plugin_name = metadata.PLUGIN_NAME + + def __init__(self, context): + """Instantiates a PrCurvesPlugin. + + Args: + context: A base_plugin.TBContext instance. A magic container that + TensorBoard uses to make objects available to the plugin. + """ + self._db_connection_provider = context.db_connection_provider + self._multiplexer = context.multiplexer + + @wrappers.Request.application + def pr_curves_route(self, request): + """A route that returns a JSON mapping between runs and PR curve data. + + Returns: + Given a tag and a comma-separated list of runs (both stored within GET + parameters), fetches a JSON object that maps between run name and objects + containing data required for PR curves for that run. Runs that either + cannot be found or that lack tags will be excluded from the response. + """ + runs = request.args.getlist("run") + if not runs: + return http_util.Respond( + request, "No runs provided when fetching PR curve data", 400 + ) + + tag = request.args.get("tag") + if not tag: + return http_util.Respond( + request, "No tag provided when fetching PR curve data", 400 + ) + + try: + response = http_util.Respond( + request, self.pr_curves_impl(runs, tag), "application/json" + ) + except ValueError as e: + return http_util.Respond(request, str(e), "text/plain", 400) + + return response + + def pr_curves_impl(self, runs, tag): + """Creates the JSON object for the PR curves response for a run-tag + combo. + + Arguments: + runs: A list of runs to fetch the curves for. + tag: The tag to fetch the curves for. + + Raises: + ValueError: If no PR curves could be fetched for a run and tag. + + Returns: + The JSON object for the PR curves route response. + """ + if self._db_connection_provider: + # Serve data from the database. + db = self._db_connection_provider() + + # We select for steps greater than -1 because the writer inserts + # placeholder rows en masse. The check for step filters out those rows. + cursor = db.execute( + """ SELECT Runs.run_name, Tensors.step, @@ -110,78 +116,102 @@ def pr_curves_impl(self, runs, tag): AND Tags.plugin_name = ? AND Tensors.step > -1 ORDER BY Tensors.step - ''' % ','.join(['?'] * len(runs)), runs + [tag, metadata.PLUGIN_NAME]) - response_mapping = {} - for (run, step, wall_time, data, dtype, shape, plugin_data) in cursor: - if run not in response_mapping: - response_mapping[run] = [] - buf = np.frombuffer(data, dtype=tf.DType(dtype).as_numpy_dtype) - data_array = buf.reshape([int(i) for i in shape.split(',')]) - plugin_data_proto = plugin_data_pb2.PrCurvePluginData() - string_buffer = np.frombuffer(plugin_data, dtype=np.dtype('b')) - plugin_data_proto.ParseFromString(tf.compat.as_bytes( - string_buffer.tostring())) - thresholds = self._compute_thresholds(plugin_data_proto.num_thresholds) - entry = self._make_pr_entry(step, wall_time, data_array, thresholds) - response_mapping[run].append(entry) - else: - # Serve data from events files. - response_mapping = {} - for run in runs: - try: - tensor_events = self._multiplexer.Tensors(run, tag) - except KeyError: - raise ValueError( - 'No PR curves could be found for run %r and tag %r' % (run, tag)) - - content = self._multiplexer.SummaryMetadata( - run, tag).plugin_data.content - pr_curve_data = metadata.parse_plugin_metadata(content) - thresholds = self._compute_thresholds(pr_curve_data.num_thresholds) - response_mapping[run] = [ - self._process_tensor_event(e, thresholds) for e in tensor_events] - return response_mapping - - def _compute_thresholds(self, num_thresholds): - """Computes a list of specific thresholds from the number of thresholds. - - Args: - num_thresholds: The number of thresholds. - - Returns: - A list of specific thresholds (floats). - """ - return [float(v) / num_thresholds for v in range(1, num_thresholds + 1)] - - @wrappers.Request.application - def tags_route(self, request): - """A route (HTTP handler) that returns a response with tags. - - Returns: - A response that contains a JSON object. The keys of the object - are all the runs. Each run is mapped to a (potentially empty) dictionary - whose keys are tags associated with run and whose values are metadata - (dictionaries). - - The metadata dictionaries contain 2 keys: - - displayName: For the display name used atop visualizations in - TensorBoard. - - description: The description that appears near visualizations upon the - user hovering over a certain icon. - """ - return http_util.Respond( - request, self.tags_impl(), 'application/json') - - def tags_impl(self): - """Creates the JSON object for the tags route response. - - Returns: - The JSON object for the tags route response. - """ - if self._db_connection_provider: - # Read tags from the database. - db = self._db_connection_provider() - cursor = db.execute(''' + """ + % ",".join(["?"] * len(runs)), + runs + [tag, metadata.PLUGIN_NAME], + ) + response_mapping = {} + for ( + run, + step, + wall_time, + data, + dtype, + shape, + plugin_data, + ) in cursor: + if run not in response_mapping: + response_mapping[run] = [] + buf = np.frombuffer(data, dtype=tf.DType(dtype).as_numpy_dtype) + data_array = buf.reshape([int(i) for i in shape.split(",")]) + plugin_data_proto = plugin_data_pb2.PrCurvePluginData() + string_buffer = np.frombuffer(plugin_data, dtype=np.dtype("b")) + plugin_data_proto.ParseFromString( + tf.compat.as_bytes(string_buffer.tostring()) + ) + thresholds = self._compute_thresholds( + plugin_data_proto.num_thresholds + ) + entry = self._make_pr_entry( + step, wall_time, data_array, thresholds + ) + response_mapping[run].append(entry) + else: + # Serve data from events files. + response_mapping = {} + for run in runs: + try: + tensor_events = self._multiplexer.Tensors(run, tag) + except KeyError: + raise ValueError( + "No PR curves could be found for run %r and tag %r" + % (run, tag) + ) + + content = self._multiplexer.SummaryMetadata( + run, tag + ).plugin_data.content + pr_curve_data = metadata.parse_plugin_metadata(content) + thresholds = self._compute_thresholds( + pr_curve_data.num_thresholds + ) + response_mapping[run] = [ + self._process_tensor_event(e, thresholds) + for e in tensor_events + ] + return response_mapping + + def _compute_thresholds(self, num_thresholds): + """Computes a list of specific thresholds from the number of + thresholds. + + Args: + num_thresholds: The number of thresholds. + + Returns: + A list of specific thresholds (floats). + """ + return [float(v) / num_thresholds for v in range(1, num_thresholds + 1)] + + @wrappers.Request.application + def tags_route(self, request): + """A route (HTTP handler) that returns a response with tags. + + Returns: + A response that contains a JSON object. The keys of the object + are all the runs. Each run is mapped to a (potentially empty) dictionary + whose keys are tags associated with run and whose values are metadata + (dictionaries). + + The metadata dictionaries contain 2 keys: + - displayName: For the display name used atop visualizations in + TensorBoard. + - description: The description that appears near visualizations upon the + user hovering over a certain icon. + """ + return http_util.Respond(request, self.tags_impl(), "application/json") + + def tags_impl(self): + """Creates the JSON object for the tags route response. + + Returns: + The JSON object for the tags route response. + """ + if self._db_connection_provider: + # Read tags from the database. + db = self._db_connection_provider() + cursor = db.execute( + """ SELECT Tags.tag_name, Tags.display_name, @@ -191,55 +221,67 @@ def tags_impl(self): ON Tags.run_id = Runs.run_id WHERE Tags.plugin_name = ? - ''', (metadata.PLUGIN_NAME,)) - result = {} - for (tag_name, display_name, run_name) in cursor: - if run_name not in result: - result[run_name] = {} - result[run_name][tag_name] = { - 'displayName': display_name, - # TODO(chihuahua): Populate the description. Currently, the tags - # table does not link with the description table. - 'description': '', - } - else: - # Read tags from events files. - runs = self._multiplexer.Runs() - result = {run: {} for run in runs} - - mapping = self._multiplexer.PluginRunToTagToContent(metadata.PLUGIN_NAME) - for (run, tag_to_content) in six.iteritems(mapping): - for (tag, _) in six.iteritems(tag_to_content): - summary_metadata = self._multiplexer.SummaryMetadata(run, tag) - result[run][tag] = {'displayName': summary_metadata.display_name, - 'description': plugin_util.markdown_to_safe_html( - summary_metadata.summary_description)} - - return result - - @wrappers.Request.application - def available_time_entries_route(self, request): - """Gets a dict mapping run to a list of time entries. - Returns: - A dict with string keys (all runs with PR curve data). The values of the - dict are lists of time entries (consisting of the fields below) to be - used in populating values within time sliders. - """ - return http_util.Respond( - request, self.available_time_entries_impl(), 'application/json') - - def available_time_entries_impl(self): - """Creates the JSON object for the available time entries route response. - - Returns: - The JSON object for the available time entries route response. - """ - result = {} - if self._db_connection_provider: - db = self._db_connection_provider() - # For each run, pick a tag. - cursor = db.execute( - ''' + """, + (metadata.PLUGIN_NAME,), + ) + result = {} + for (tag_name, display_name, run_name) in cursor: + if run_name not in result: + result[run_name] = {} + result[run_name][tag_name] = { + "displayName": display_name, + # TODO(chihuahua): Populate the description. Currently, the tags + # table does not link with the description table. + "description": "", + } + else: + # Read tags from events files. + runs = self._multiplexer.Runs() + result = {run: {} for run in runs} + + mapping = self._multiplexer.PluginRunToTagToContent( + metadata.PLUGIN_NAME + ) + for (run, tag_to_content) in six.iteritems(mapping): + for (tag, _) in six.iteritems(tag_to_content): + summary_metadata = self._multiplexer.SummaryMetadata( + run, tag + ) + result[run][tag] = { + "displayName": summary_metadata.display_name, + "description": plugin_util.markdown_to_safe_html( + summary_metadata.summary_description + ), + } + + return result + + @wrappers.Request.application + def available_time_entries_route(self, request): + """Gets a dict mapping run to a list of time entries. + + Returns: + A dict with string keys (all runs with PR curve data). The values of the + dict are lists of time entries (consisting of the fields below) to be + used in populating values within time sliders. + """ + return http_util.Respond( + request, self.available_time_entries_impl(), "application/json" + ) + + def available_time_entries_impl(self): + """Creates the JSON object for the available time entries route + response. + + Returns: + The JSON object for the available time entries route response. + """ + result = {} + if self._db_connection_provider: + db = self._db_connection_provider() + # For each run, pick a tag. + cursor = db.execute( + """ SELECT TagPickingTable.run_name, Tensors.step, @@ -259,147 +301,165 @@ def available_time_entries_impl(self): ON Tensors.series = TagPickingTable.tag_id WHERE Tensors.step IS NOT NULL ORDER BY Tensors.step - ''', (metadata.PLUGIN_NAME,)) - for (run, step, wall_time) in cursor: - if run not in result: - result[run] = [] - result[run].append(self._create_time_entry(step, wall_time)) - else: - # Read data from disk. - all_runs = self._multiplexer.PluginRunToTagToContent( - metadata.PLUGIN_NAME) - for run, tag_to_content in all_runs.items(): - if not tag_to_content: - # This run lacks data for this plugin. - continue - # Just use the list of tensor events for any of the tags to determine - # the steps to list for the run. The steps are often the same across - # tags for each run, albeit the user may elect to sample certain tags - # differently within the same run. If the latter occurs, TensorBoard - # will show the actual step of each tag atop the card for the tag. - tensor_events = self._multiplexer.Tensors( - run, min(six.iterkeys(tag_to_content))) - result[run] = [self._create_time_entry(e.step, e.wall_time) - for e in tensor_events] - return result - - def _create_time_entry(self, step, wall_time): - """Creates a time entry given a tensor event. - - Arguments: - step: The step for the time entry. - wall_time: The wall time for the time entry. - - Returns: - A JSON-able time entry to be passed to the frontend in order to construct - the slider. - """ - return { - 'step': step, - 'wall_time': wall_time, - } - - def get_plugin_apps(self): - """Gets all routes offered by the plugin. - - Returns: - A dictionary mapping URL path to route that handles it. - """ - return { - '/tags': self.tags_route, - '/pr_curves': self.pr_curves_route, - '/available_time_entries': self.available_time_entries_route, - } - - def is_active(self): - """Determines whether this plugin is active. - - This plugin is active only if PR curve summary data is read by TensorBoard. - - Returns: - Whether this plugin is active. - """ - if self._db_connection_provider: - # The plugin is active if one relevant tag can be found in the database. - db = self._db_connection_provider() - cursor = db.execute( - ''' + """, + (metadata.PLUGIN_NAME,), + ) + for (run, step, wall_time) in cursor: + if run not in result: + result[run] = [] + result[run].append(self._create_time_entry(step, wall_time)) + else: + # Read data from disk. + all_runs = self._multiplexer.PluginRunToTagToContent( + metadata.PLUGIN_NAME + ) + for run, tag_to_content in all_runs.items(): + if not tag_to_content: + # This run lacks data for this plugin. + continue + # Just use the list of tensor events for any of the tags to determine + # the steps to list for the run. The steps are often the same across + # tags for each run, albeit the user may elect to sample certain tags + # differently within the same run. If the latter occurs, TensorBoard + # will show the actual step of each tag atop the card for the tag. + tensor_events = self._multiplexer.Tensors( + run, min(six.iterkeys(tag_to_content)) + ) + result[run] = [ + self._create_time_entry(e.step, e.wall_time) + for e in tensor_events + ] + return result + + def _create_time_entry(self, step, wall_time): + """Creates a time entry given a tensor event. + + Arguments: + step: The step for the time entry. + wall_time: The wall time for the time entry. + + Returns: + A JSON-able time entry to be passed to the frontend in order to construct + the slider. + """ + return { + "step": step, + "wall_time": wall_time, + } + + def get_plugin_apps(self): + """Gets all routes offered by the plugin. + + Returns: + A dictionary mapping URL path to route that handles it. + """ + return { + "/tags": self.tags_route, + "/pr_curves": self.pr_curves_route, + "/available_time_entries": self.available_time_entries_route, + } + + def is_active(self): + """Determines whether this plugin is active. + + This plugin is active only if PR curve summary data is read by TensorBoard. + + Returns: + Whether this plugin is active. + """ + if self._db_connection_provider: + # The plugin is active if one relevant tag can be found in the database. + db = self._db_connection_provider() + cursor = db.execute( + """ SELECT 1 FROM Tags WHERE Tags.plugin_name = ? LIMIT 1 - ''', - (metadata.PLUGIN_NAME,)) - return bool(list(cursor)) - - if not self._multiplexer: - return False - - all_runs = self._multiplexer.PluginRunToTagToContent(metadata.PLUGIN_NAME) - - # The plugin is active if any of the runs has a tag relevant to the plugin. - return any(six.itervalues(all_runs)) - - def frontend_metadata(self): - return base_plugin.FrontendMetadata( - element_name='tf-pr-curve-dashboard', - tab_name='PR Curves', - ) - - def _process_tensor_event(self, event, thresholds): - """Converts a TensorEvent into a dict that encapsulates information on it. - - Args: - event: The TensorEvent to convert. - thresholds: An array of floats that ranges from 0 to 1 (in that - direction and inclusive of 0 and 1). - - Returns: - A JSON-able dictionary of PR curve data for 1 step. - """ - return self._make_pr_entry( - event.step, - event.wall_time, - tensor_util.make_ndarray(event.tensor_proto), - thresholds) - - def _make_pr_entry(self, step, wall_time, data_array, thresholds): - """Creates an entry for PR curve data. Each entry corresponds to 1 step. - - Args: - step: The step. - wall_time: The wall time. - data_array: A numpy array of PR curve data stored in the summary format. - thresholds: An array of floating point thresholds. - - Returns: - A PR curve entry. - """ - # Trim entries for which TP + FP = 0 (precision is undefined) at the tail of - # the data. - true_positives = [int(v) for v in data_array[metadata.TRUE_POSITIVES_INDEX]] - false_positives = [ - int(v) for v in data_array[metadata.FALSE_POSITIVES_INDEX]] - tp_index = metadata.TRUE_POSITIVES_INDEX - fp_index = metadata.FALSE_POSITIVES_INDEX - positives = data_array[[tp_index, fp_index], :].astype(int).sum(axis=0) - end_index_inclusive = len(positives) - 1 - while end_index_inclusive > 0 and positives[end_index_inclusive] == 0: - end_index_inclusive -= 1 - end_index = end_index_inclusive + 1 - - return { - 'wall_time': wall_time, - 'step': step, - 'precision': data_array[metadata.PRECISION_INDEX, :end_index].tolist(), - 'recall': data_array[metadata.RECALL_INDEX, :end_index].tolist(), - 'true_positives': true_positives[:end_index], - 'false_positives': false_positives[:end_index], - 'true_negatives': - [int(v) for v in - data_array[metadata.TRUE_NEGATIVES_INDEX][:end_index]], - 'false_negatives': - [int(v) for v in - data_array[metadata.FALSE_NEGATIVES_INDEX][:end_index]], - 'thresholds': thresholds[:end_index], - } + """, + (metadata.PLUGIN_NAME,), + ) + return bool(list(cursor)) + + if not self._multiplexer: + return False + + all_runs = self._multiplexer.PluginRunToTagToContent( + metadata.PLUGIN_NAME + ) + + # The plugin is active if any of the runs has a tag relevant to the plugin. + return any(six.itervalues(all_runs)) + + def frontend_metadata(self): + return base_plugin.FrontendMetadata( + element_name="tf-pr-curve-dashboard", tab_name="PR Curves", + ) + + def _process_tensor_event(self, event, thresholds): + """Converts a TensorEvent into a dict that encapsulates information on + it. + + Args: + event: The TensorEvent to convert. + thresholds: An array of floats that ranges from 0 to 1 (in that + direction and inclusive of 0 and 1). + + Returns: + A JSON-able dictionary of PR curve data for 1 step. + """ + return self._make_pr_entry( + event.step, + event.wall_time, + tensor_util.make_ndarray(event.tensor_proto), + thresholds, + ) + + def _make_pr_entry(self, step, wall_time, data_array, thresholds): + """Creates an entry for PR curve data. Each entry corresponds to 1 + step. + + Args: + step: The step. + wall_time: The wall time. + data_array: A numpy array of PR curve data stored in the summary format. + thresholds: An array of floating point thresholds. + + Returns: + A PR curve entry. + """ + # Trim entries for which TP + FP = 0 (precision is undefined) at the tail of + # the data. + true_positives = [ + int(v) for v in data_array[metadata.TRUE_POSITIVES_INDEX] + ] + false_positives = [ + int(v) for v in data_array[metadata.FALSE_POSITIVES_INDEX] + ] + tp_index = metadata.TRUE_POSITIVES_INDEX + fp_index = metadata.FALSE_POSITIVES_INDEX + positives = data_array[[tp_index, fp_index], :].astype(int).sum(axis=0) + end_index_inclusive = len(positives) - 1 + while end_index_inclusive > 0 and positives[end_index_inclusive] == 0: + end_index_inclusive -= 1 + end_index = end_index_inclusive + 1 + + return { + "wall_time": wall_time, + "step": step, + "precision": data_array[ + metadata.PRECISION_INDEX, :end_index + ].tolist(), + "recall": data_array[metadata.RECALL_INDEX, :end_index].tolist(), + "true_positives": true_positives[:end_index], + "false_positives": false_positives[:end_index], + "true_negatives": [ + int(v) + for v in data_array[metadata.TRUE_NEGATIVES_INDEX][:end_index] + ], + "false_negatives": [ + int(v) + for v in data_array[metadata.FALSE_NEGATIVES_INDEX][:end_index] + ], + "thresholds": thresholds[:end_index], + } diff --git a/tensorboard/plugins/pr_curve/pr_curves_plugin_test.py b/tensorboard/plugins/pr_curve/pr_curves_plugin_test.py index afe1c9a893..077bc0c4cb 100644 --- a/tensorboard/plugins/pr_curve/pr_curves_plugin_test.py +++ b/tensorboard/plugins/pr_curve/pr_curves_plugin_test.py @@ -26,7 +26,9 @@ import six import tensorflow as tf -from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer +from tensorboard.backend.event_processing import ( + plugin_event_multiplexer as event_multiplexer, +) from tensorboard.plugins import base_plugin from tensorboard.plugins.pr_curve import pr_curve_demo from tensorboard.plugins.pr_curve import pr_curves_plugin @@ -35,272 +37,314 @@ # are small. The default relative error (rtol) of 1e-7 yields many undesired # test failures. assert_allclose = functools.partial( - np.testing.assert_allclose, rtol=0, atol=1e-7) + np.testing.assert_allclose, rtol=0, atol=1e-7 +) class PrCurvesPluginTest(tf.test.TestCase): - - def setUp(self): - super(PrCurvesPluginTest, self).setUp() - logdir = os.path.join(self.get_temp_dir(), 'logdir') - - # Generate data. - pr_curve_demo.run_all( - logdir=logdir, - steps=3, - thresholds=5, - verbose=False) - - # Create a multiplexer for reading the data we just wrote. - multiplexer = event_multiplexer.EventMultiplexer() - multiplexer.AddRunsFromDirectory(logdir) - multiplexer.Reload() - - context = base_plugin.TBContext(logdir=logdir, multiplexer=multiplexer) - self.plugin = pr_curves_plugin.PrCurvesPlugin(context) - - def validatePrCurveEntry( - self, - expected_step, - expected_precision, - expected_recall, - expected_true_positives, - expected_false_positives, - expected_true_negatives, - expected_false_negatives, - expected_thresholds, - pr_curve_entry): - """Checks that the values stored within a tensor are correct. - - Args: - expected_step: The expected step. - expected_precision: A list of float values. - expected_recall: A list of float values. - expected_true_positives: A list of int values. - expected_false_positives: A list of int values. - expected_true_negatives: A list of int values. - expected_false_negatives: A list of int values. - expected_thresholds: A list of floats ranging from 0 to 1. - pr_curve_entry: The PR curve entry to evaluate. - """ - self.assertEqual(expected_step, pr_curve_entry['step']) - assert_allclose(expected_precision, pr_curve_entry['precision']) - assert_allclose(expected_recall, pr_curve_entry['recall']) - self.assertListEqual( - expected_true_positives, pr_curve_entry['true_positives']) - self.assertListEqual( - expected_false_positives, pr_curve_entry['false_positives']) - self.assertListEqual( - expected_true_negatives, pr_curve_entry['true_negatives']) - self.assertListEqual( - expected_false_negatives, pr_curve_entry['false_negatives']) - assert_allclose(expected_thresholds, pr_curve_entry['thresholds']) - - def computeCorrectDescription(self, standard_deviation): - """Generates a correct description. - - Arguments: - standard_deviation: An integer standard deviation value. - - Returns: - The correct description given a standard deviation value. - """ - description = ('The probabilities used to create this PR curve are ' - 'generated from a normal distribution. Its standard ' - 'deviation is initially %d and decreases' - ' over time.
') % standard_deviation - return description - - def testRoutesProvided(self): - """Tests that the plugin offers the correct routes.""" - routes = self.plugin.get_plugin_apps() - self.assertIsInstance(routes['/tags'], collections.Callable) - self.assertIsInstance(routes['/pr_curves'], collections.Callable) - self.assertIsInstance( - routes['/available_time_entries'], collections.Callable) - - def testTagsProvided(self): - """Tests that tags are provided.""" - tags_response = self.plugin.tags_impl() - - # Assert that the runs are right. - self.assertItemsEqual( - ['colors', 'mask_every_other_prediction'], list(tags_response.keys())) - - # Assert that the tags for each run are correct. - self.assertItemsEqual( - ['red/pr_curves', 'green/pr_curves', 'blue/pr_curves'], - list(tags_response['colors'].keys())) - self.assertItemsEqual( - ['red/pr_curves', 'green/pr_curves', 'blue/pr_curves'], - list(tags_response['mask_every_other_prediction'].keys())) - - # Verify the data for each run-tag combination. - self.assertDictEqual({ - 'displayName': 'classifying red', - 'description': self.computeCorrectDescription(168), - }, tags_response['colors']['red/pr_curves']) - self.assertDictEqual({ - 'displayName': 'classifying green', - 'description': self.computeCorrectDescription(210), - }, tags_response['colors']['green/pr_curves']) - self.assertDictEqual({ - 'displayName': 'classifying blue', - 'description': self.computeCorrectDescription(252), - }, tags_response['colors']['blue/pr_curves']) - self.assertDictEqual({ - 'displayName': 'classifying red', - 'description': self.computeCorrectDescription(168), - }, tags_response['mask_every_other_prediction']['red/pr_curves']) - self.assertDictEqual({ - 'displayName': 'classifying green', - 'description': self.computeCorrectDescription(210), - }, tags_response['mask_every_other_prediction']['green/pr_curves']) - self.assertDictEqual({ - 'displayName': 'classifying blue', - 'description': self.computeCorrectDescription(252), - }, tags_response['mask_every_other_prediction']['blue/pr_curves']) - - def testAvailableSteps(self): - """Tests that runs are mapped to correct available steps.""" - # Test that all runs are within the keys of the mapping. - response = self.plugin.available_time_entries_impl() - self.assertItemsEqual( - ['colors', 'mask_every_other_prediction'], list(response.keys())) - - # TODO(chizeng): Find a means of testing the wall time and relative time. - # The wall time written to disk is computed within TensorFlow C++. - entries = response['colors'] - entry = entries[0] - self.assertEqual(0, entry['step']) - self.assertIn('wall_time', entry) - entry = entries[1] - self.assertEqual(1, entry['step']) - self.assertIn('wall_time', entry) - entry = entries[2] - self.assertEqual(2, entry['step']) - self.assertIn('wall_time', entry) - - entries = response['mask_every_other_prediction'] - entry = entries[0] - self.assertEqual(0, entry['step']) - self.assertIn('wall_time', entry) - entry = entries[1] - self.assertEqual(1, entry['step']) - self.assertIn('wall_time', entry) - entry = entries[2] - self.assertEqual(2, entry['step']) - self.assertIn('wall_time', entry) - - def testPrCurvesDataCorrect(self): - """Tests that responses for PR curves for run-tag combos are correct.""" - pr_curves_response = self.plugin.pr_curves_impl( - ['colors', 'mask_every_other_prediction'], 'blue/pr_curves') - - # Assert that the runs are correct. - self.assertItemsEqual( - ['colors', 'mask_every_other_prediction'], - list(pr_curves_response.keys())) - - # Assert that PR curve data is correct for the colors run. - entries = pr_curves_response['colors'] - self.assertEqual(3, len(entries)) - self.validatePrCurveEntry( - expected_step=0, - expected_precision=[0.3333333, 0.3853211, 0.5421687, 0.75], - expected_recall=[1.0, 0.84, 0.3, 0.04], - expected_true_positives=[150, 126, 45, 6], - expected_false_positives=[300, 201, 38, 2], - expected_true_negatives=[0, 99, 262, 298], - expected_false_negatives=[0, 24, 105, 144], - expected_thresholds=[0.2, 0.4, 0.6, 0.8], - pr_curve_entry=entries[0]) - self.validatePrCurveEntry( - expected_step=1, - expected_precision=[0.3333333, 0.3855422, 0.5357143, 0.4], - expected_recall=[1.0, 0.8533334, 0.3, 0.0266667], - expected_true_positives=[150, 128, 45, 4], - expected_false_positives=[300, 204, 39, 6], - expected_true_negatives=[0, 96, 261, 294], - expected_false_negatives=[0, 22, 105, 146], - expected_thresholds=[0.2, 0.4, 0.6, 0.8], - pr_curve_entry=entries[1]) - self.validatePrCurveEntry( - expected_step=2, - expected_precision=[0.3333333, 0.3934426, 0.5064935, 0.6666667], - expected_recall=[1.0, 0.8, 0.26, 0.0266667], - expected_true_positives=[150, 120, 39, 4], - expected_false_positives=[300, 185, 38, 2], - expected_true_negatives=[0, 115, 262, 298], - expected_false_negatives=[0, 30, 111, 146], - expected_thresholds=[0.2, 0.4, 0.6, 0.8], - pr_curve_entry=entries[2]) - - # Assert that PR curve data is correct for the mask_every_other_prediction - # run. - entries = pr_curves_response['mask_every_other_prediction'] - self.assertEqual(3, len(entries)) - self.validatePrCurveEntry( - expected_step=0, - expected_precision=[0.3333333, 0.3786982, 0.5384616, 1.0], - expected_recall=[1.0, 0.8533334, 0.28, 0.0666667], - expected_true_positives=[75, 64, 21, 5], - expected_false_positives=[150, 105, 18, 0], - expected_true_negatives=[0, 45, 132, 150], - expected_false_negatives=[0, 11, 54, 70], - expected_thresholds=[0.2, 0.4, 0.6, 0.8], - pr_curve_entry=entries[0]) - self.validatePrCurveEntry( - expected_step=1, - expected_precision=[0.3333333, 0.3850932, 0.5, 0.25], - expected_recall=[1.0, 0.8266667, 0.28, 0.0133333], - expected_true_positives=[75, 62, 21, 1], - expected_false_positives=[150, 99, 21, 3], - expected_true_negatives=[0, 51, 129, 147], - expected_false_negatives=[0, 13, 54, 74], - expected_thresholds=[0.2, 0.4, 0.6, 0.8], - pr_curve_entry=entries[1]) - self.validatePrCurveEntry( - expected_step=2, - expected_precision=[0.3333333, 0.3986928, 0.4444444, 0.6666667], - expected_recall=[1.0, 0.8133333, 0.2133333, 0.0266667], - expected_true_positives=[75, 61, 16, 2], - expected_false_positives=[150, 92, 20, 1], - expected_true_negatives=[0, 58, 130, 149], - expected_false_negatives=[0, 14, 59, 73], - expected_thresholds=[0.2, 0.4, 0.6, 0.8], - pr_curve_entry=entries[2]) - - def testPrCurvesRaisesValueErrorWhenNoData(self): - """Tests that the method for obtaining PR curve data raises a ValueError. - - The handler should raise a ValueError when no PR curve data can be found - for a certain run-tag combination. - """ - with six.assertRaisesRegex( - self, ValueError, r'No PR curves could be found'): - self.plugin.pr_curves_impl(['colors'], 'non_existent_tag') - - with six.assertRaisesRegex( - self, ValueError, r'No PR curves could be found'): - self.plugin.pr_curves_impl(['non_existent_run'], 'blue/pr_curves') - - def testPluginIsNotActive(self): - """Tests that the plugin is inactive when no relevant data exists.""" - empty_logdir = os.path.join(self.get_temp_dir(), 'empty_logdir') - multiplexer = event_multiplexer.EventMultiplexer() - multiplexer.AddRunsFromDirectory(empty_logdir) - multiplexer.Reload() - context = base_plugin.TBContext( - logdir=empty_logdir, multiplexer=multiplexer) - plugin = pr_curves_plugin.PrCurvesPlugin(context) - self.assertFalse(plugin.is_active()) - - def testPluginIsActive(self): - """Tests that the plugin is active when relevant data exists.""" - # The set up for this test generates relevant data. - self.assertTrue(self.plugin.is_active()) - - -if __name__ == '__main__': - tf.test.main() + def setUp(self): + super(PrCurvesPluginTest, self).setUp() + logdir = os.path.join(self.get_temp_dir(), "logdir") + + # Generate data. + pr_curve_demo.run_all( + logdir=logdir, steps=3, thresholds=5, verbose=False + ) + + # Create a multiplexer for reading the data we just wrote. + multiplexer = event_multiplexer.EventMultiplexer() + multiplexer.AddRunsFromDirectory(logdir) + multiplexer.Reload() + + context = base_plugin.TBContext(logdir=logdir, multiplexer=multiplexer) + self.plugin = pr_curves_plugin.PrCurvesPlugin(context) + + def validatePrCurveEntry( + self, + expected_step, + expected_precision, + expected_recall, + expected_true_positives, + expected_false_positives, + expected_true_negatives, + expected_false_negatives, + expected_thresholds, + pr_curve_entry, + ): + """Checks that the values stored within a tensor are correct. + + Args: + expected_step: The expected step. + expected_precision: A list of float values. + expected_recall: A list of float values. + expected_true_positives: A list of int values. + expected_false_positives: A list of int values. + expected_true_negatives: A list of int values. + expected_false_negatives: A list of int values. + expected_thresholds: A list of floats ranging from 0 to 1. + pr_curve_entry: The PR curve entry to evaluate. + """ + self.assertEqual(expected_step, pr_curve_entry["step"]) + assert_allclose(expected_precision, pr_curve_entry["precision"]) + assert_allclose(expected_recall, pr_curve_entry["recall"]) + self.assertListEqual( + expected_true_positives, pr_curve_entry["true_positives"] + ) + self.assertListEqual( + expected_false_positives, pr_curve_entry["false_positives"] + ) + self.assertListEqual( + expected_true_negatives, pr_curve_entry["true_negatives"] + ) + self.assertListEqual( + expected_false_negatives, pr_curve_entry["false_negatives"] + ) + assert_allclose(expected_thresholds, pr_curve_entry["thresholds"]) + + def computeCorrectDescription(self, standard_deviation): + """Generates a correct description. + + Arguments: + standard_deviation: An integer standard deviation value. + + Returns: + The correct description given a standard deviation value. + """ + description = ( + "The probabilities used to create this PR curve are " + "generated from a normal distribution. Its standard " + "deviation is initially %d and decreases" + " over time.
" + ) % standard_deviation + return description + + def testRoutesProvided(self): + """Tests that the plugin offers the correct routes.""" + routes = self.plugin.get_plugin_apps() + self.assertIsInstance(routes["/tags"], collections.Callable) + self.assertIsInstance(routes["/pr_curves"], collections.Callable) + self.assertIsInstance( + routes["/available_time_entries"], collections.Callable + ) + + def testTagsProvided(self): + """Tests that tags are provided.""" + tags_response = self.plugin.tags_impl() + + # Assert that the runs are right. + self.assertItemsEqual( + ["colors", "mask_every_other_prediction"], + list(tags_response.keys()), + ) + + # Assert that the tags for each run are correct. + self.assertItemsEqual( + ["red/pr_curves", "green/pr_curves", "blue/pr_curves"], + list(tags_response["colors"].keys()), + ) + self.assertItemsEqual( + ["red/pr_curves", "green/pr_curves", "blue/pr_curves"], + list(tags_response["mask_every_other_prediction"].keys()), + ) + + # Verify the data for each run-tag combination. + self.assertDictEqual( + { + "displayName": "classifying red", + "description": self.computeCorrectDescription(168), + }, + tags_response["colors"]["red/pr_curves"], + ) + self.assertDictEqual( + { + "displayName": "classifying green", + "description": self.computeCorrectDescription(210), + }, + tags_response["colors"]["green/pr_curves"], + ) + self.assertDictEqual( + { + "displayName": "classifying blue", + "description": self.computeCorrectDescription(252), + }, + tags_response["colors"]["blue/pr_curves"], + ) + self.assertDictEqual( + { + "displayName": "classifying red", + "description": self.computeCorrectDescription(168), + }, + tags_response["mask_every_other_prediction"]["red/pr_curves"], + ) + self.assertDictEqual( + { + "displayName": "classifying green", + "description": self.computeCorrectDescription(210), + }, + tags_response["mask_every_other_prediction"]["green/pr_curves"], + ) + self.assertDictEqual( + { + "displayName": "classifying blue", + "description": self.computeCorrectDescription(252), + }, + tags_response["mask_every_other_prediction"]["blue/pr_curves"], + ) + + def testAvailableSteps(self): + """Tests that runs are mapped to correct available steps.""" + # Test that all runs are within the keys of the mapping. + response = self.plugin.available_time_entries_impl() + self.assertItemsEqual( + ["colors", "mask_every_other_prediction"], list(response.keys()) + ) + + # TODO(chizeng): Find a means of testing the wall time and relative time. + # The wall time written to disk is computed within TensorFlow C++. + entries = response["colors"] + entry = entries[0] + self.assertEqual(0, entry["step"]) + self.assertIn("wall_time", entry) + entry = entries[1] + self.assertEqual(1, entry["step"]) + self.assertIn("wall_time", entry) + entry = entries[2] + self.assertEqual(2, entry["step"]) + self.assertIn("wall_time", entry) + + entries = response["mask_every_other_prediction"] + entry = entries[0] + self.assertEqual(0, entry["step"]) + self.assertIn("wall_time", entry) + entry = entries[1] + self.assertEqual(1, entry["step"]) + self.assertIn("wall_time", entry) + entry = entries[2] + self.assertEqual(2, entry["step"]) + self.assertIn("wall_time", entry) + + def testPrCurvesDataCorrect(self): + """Tests that responses for PR curves for run-tag combos are + correct.""" + pr_curves_response = self.plugin.pr_curves_impl( + ["colors", "mask_every_other_prediction"], "blue/pr_curves" + ) + + # Assert that the runs are correct. + self.assertItemsEqual( + ["colors", "mask_every_other_prediction"], + list(pr_curves_response.keys()), + ) + + # Assert that PR curve data is correct for the colors run. + entries = pr_curves_response["colors"] + self.assertEqual(3, len(entries)) + self.validatePrCurveEntry( + expected_step=0, + expected_precision=[0.3333333, 0.3853211, 0.5421687, 0.75], + expected_recall=[1.0, 0.84, 0.3, 0.04], + expected_true_positives=[150, 126, 45, 6], + expected_false_positives=[300, 201, 38, 2], + expected_true_negatives=[0, 99, 262, 298], + expected_false_negatives=[0, 24, 105, 144], + expected_thresholds=[0.2, 0.4, 0.6, 0.8], + pr_curve_entry=entries[0], + ) + self.validatePrCurveEntry( + expected_step=1, + expected_precision=[0.3333333, 0.3855422, 0.5357143, 0.4], + expected_recall=[1.0, 0.8533334, 0.3, 0.0266667], + expected_true_positives=[150, 128, 45, 4], + expected_false_positives=[300, 204, 39, 6], + expected_true_negatives=[0, 96, 261, 294], + expected_false_negatives=[0, 22, 105, 146], + expected_thresholds=[0.2, 0.4, 0.6, 0.8], + pr_curve_entry=entries[1], + ) + self.validatePrCurveEntry( + expected_step=2, + expected_precision=[0.3333333, 0.3934426, 0.5064935, 0.6666667], + expected_recall=[1.0, 0.8, 0.26, 0.0266667], + expected_true_positives=[150, 120, 39, 4], + expected_false_positives=[300, 185, 38, 2], + expected_true_negatives=[0, 115, 262, 298], + expected_false_negatives=[0, 30, 111, 146], + expected_thresholds=[0.2, 0.4, 0.6, 0.8], + pr_curve_entry=entries[2], + ) + + # Assert that PR curve data is correct for the mask_every_other_prediction + # run. + entries = pr_curves_response["mask_every_other_prediction"] + self.assertEqual(3, len(entries)) + self.validatePrCurveEntry( + expected_step=0, + expected_precision=[0.3333333, 0.3786982, 0.5384616, 1.0], + expected_recall=[1.0, 0.8533334, 0.28, 0.0666667], + expected_true_positives=[75, 64, 21, 5], + expected_false_positives=[150, 105, 18, 0], + expected_true_negatives=[0, 45, 132, 150], + expected_false_negatives=[0, 11, 54, 70], + expected_thresholds=[0.2, 0.4, 0.6, 0.8], + pr_curve_entry=entries[0], + ) + self.validatePrCurveEntry( + expected_step=1, + expected_precision=[0.3333333, 0.3850932, 0.5, 0.25], + expected_recall=[1.0, 0.8266667, 0.28, 0.0133333], + expected_true_positives=[75, 62, 21, 1], + expected_false_positives=[150, 99, 21, 3], + expected_true_negatives=[0, 51, 129, 147], + expected_false_negatives=[0, 13, 54, 74], + expected_thresholds=[0.2, 0.4, 0.6, 0.8], + pr_curve_entry=entries[1], + ) + self.validatePrCurveEntry( + expected_step=2, + expected_precision=[0.3333333, 0.3986928, 0.4444444, 0.6666667], + expected_recall=[1.0, 0.8133333, 0.2133333, 0.0266667], + expected_true_positives=[75, 61, 16, 2], + expected_false_positives=[150, 92, 20, 1], + expected_true_negatives=[0, 58, 130, 149], + expected_false_negatives=[0, 14, 59, 73], + expected_thresholds=[0.2, 0.4, 0.6, 0.8], + pr_curve_entry=entries[2], + ) + + def testPrCurvesRaisesValueErrorWhenNoData(self): + """Tests that the method for obtaining PR curve data raises a + ValueError. + + The handler should raise a ValueError when no PR curve data can + be found for a certain run-tag combination. + """ + with six.assertRaisesRegex( + self, ValueError, r"No PR curves could be found" + ): + self.plugin.pr_curves_impl(["colors"], "non_existent_tag") + + with six.assertRaisesRegex( + self, ValueError, r"No PR curves could be found" + ): + self.plugin.pr_curves_impl(["non_existent_run"], "blue/pr_curves") + + def testPluginIsNotActive(self): + """Tests that the plugin is inactive when no relevant data exists.""" + empty_logdir = os.path.join(self.get_temp_dir(), "empty_logdir") + multiplexer = event_multiplexer.EventMultiplexer() + multiplexer.AddRunsFromDirectory(empty_logdir) + multiplexer.Reload() + context = base_plugin.TBContext( + logdir=empty_logdir, multiplexer=multiplexer + ) + plugin = pr_curves_plugin.PrCurvesPlugin(context) + self.assertFalse(plugin.is_active()) + + def testPluginIsActive(self): + """Tests that the plugin is active when relevant data exists.""" + # The set up for this test generates relevant data. + self.assertTrue(self.plugin.is_active()) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/pr_curve/summary.py b/tensorboard/plugins/pr_curve/summary.py index 76c113f26e..be929bb6e2 100644 --- a/tensorboard/plugins/pr_curve/summary.py +++ b/tensorboard/plugins/pr_curve/summary.py @@ -34,6 +34,7 @@ # The default number of thresholds. _DEFAULT_NUM_THRESHOLDS = 201 + def op( name, labels, @@ -42,307 +43,328 @@ def op( weights=None, display_name=None, description=None, - collections=None): - """Create a PR curve summary op for a single binary classifier. - - Computes true/false positive/negative values for the given `predictions` - against the ground truth `labels`, against a list of evenly distributed - threshold values in `[0, 1]` of length `num_thresholds`. - - Each number in `predictions`, a float in `[0, 1]`, is compared with its - corresponding boolean label in `labels`, and counts as a single tp/fp/tn/fn - value at each threshold. This is then multiplied with `weights` which can be - used to reweight certain values, or more commonly used for masking values. - - Args: - name: A tag attached to the summary. Used by TensorBoard for organization. - labels: The ground truth values. A Tensor of `bool` values with arbitrary - shape. - predictions: A float32 `Tensor` whose values are in the range `[0, 1]`. - Dimensions must match those of `labels`. - num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to - compute PR metrics for. Should be `>= 2`. This value should be a - constant integer value, not a Tensor that stores an integer. - weights: Optional float32 `Tensor`. Individual counts are multiplied by this - value. This tensor must be either the same shape as or broadcastable to - the `labels` tensor. - display_name: Optional name for this summary in TensorBoard, as a - constant `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - collections: Optional list of graph collections keys. The new - summary op is added to these collections. Defaults to - `[Graph Keys.SUMMARIES]`. - - Returns: - A summary operation for use in a TensorFlow graph. The float32 tensor - produced by the summary operation is of dimension (6, num_thresholds). The - first dimension (of length 6) is of the order: true positives, - false positives, true negatives, false negatives, precision, recall. - - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - if num_thresholds is None: - num_thresholds = _DEFAULT_NUM_THRESHOLDS - - if weights is None: - weights = 1.0 - - dtype = predictions.dtype - - with tf.name_scope(name, values=[labels, predictions, weights]): - tf.assert_type(labels, tf.bool) - # We cast to float to ensure we have 0.0 or 1.0. - f_labels = tf.cast(labels, dtype) - # Ensure predictions are all in range [0.0, 1.0]. - predictions = tf.minimum(1.0, tf.maximum(0.0, predictions)) - # Get weighted true/false labels. - true_labels = f_labels * weights - false_labels = (1.0 - f_labels) * weights - - # Before we begin, flatten predictions. - predictions = tf.reshape(predictions, [-1]) - - # Shape the labels so they are broadcast-able for later multiplication. - true_labels = tf.reshape(true_labels, [-1, 1]) - false_labels = tf.reshape(false_labels, [-1, 1]) - - # To compute TP/FP/TN/FN, we are measuring a binary classifier - # C(t) = (predictions >= t) - # at each threshold 't'. So we have - # TP(t) = sum( C(t) * true_labels ) - # FP(t) = sum( C(t) * false_labels ) - # - # But, computing C(t) requires computation for each t. To make it fast, - # observe that C(t) is a cumulative integral, and so if we have - # thresholds = [t_0, ..., t_{n-1}]; t_0 < ... < t_{n-1} - # where n = num_thresholds, and if we can compute the bucket function - # B(i) = Sum( (predictions == t), t_i <= t < t{i+1} ) - # then we get - # C(t_i) = sum( B(j), j >= i ) - # which is the reversed cumulative sum in tf.cumsum(). - # - # We can compute B(i) efficiently by taking advantage of the fact that - # our thresholds are evenly distributed, in that - # width = 1.0 / (num_thresholds - 1) - # thresholds = [0.0, 1*width, 2*width, 3*width, ..., 1.0] - # Given a prediction value p, we can map it to its bucket by - # bucket_index(p) = floor( p * (num_thresholds - 1) ) - # so we can use tf.scatter_add() to update the buckets in one pass. - - # Compute the bucket indices for each prediction value. - bucket_indices = tf.cast( - tf.floor(predictions * (num_thresholds - 1)), tf.int32) - - # Bucket predictions. - tp_buckets = tf.reduce_sum( - input_tensor=tf.one_hot(bucket_indices, depth=num_thresholds) * true_labels, - axis=0) - fp_buckets = tf.reduce_sum( - input_tensor=tf.one_hot(bucket_indices, depth=num_thresholds) * false_labels, - axis=0) - - # Set up the cumulative sums to compute the actual metrics. - tp = tf.cumsum(tp_buckets, reverse=True, name='tp') - fp = tf.cumsum(fp_buckets, reverse=True, name='fp') - # fn = sum(true_labels) - tp - # = sum(tp_buckets) - tp - # = tp[0] - tp - # Similarly, - # tn = fp[0] - fp + collections=None, +): + """Create a PR curve summary op for a single binary classifier. + + Computes true/false positive/negative values for the given `predictions` + against the ground truth `labels`, against a list of evenly distributed + threshold values in `[0, 1]` of length `num_thresholds`. + + Each number in `predictions`, a float in `[0, 1]`, is compared with its + corresponding boolean label in `labels`, and counts as a single tp/fp/tn/fn + value at each threshold. This is then multiplied with `weights` which can be + used to reweight certain values, or more commonly used for masking values. + + Args: + name: A tag attached to the summary. Used by TensorBoard for organization. + labels: The ground truth values. A Tensor of `bool` values with arbitrary + shape. + predictions: A float32 `Tensor` whose values are in the range `[0, 1]`. + Dimensions must match those of `labels`. + num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to + compute PR metrics for. Should be `>= 2`. This value should be a + constant integer value, not a Tensor that stores an integer. + weights: Optional float32 `Tensor`. Individual counts are multiplied by this + value. This tensor must be either the same shape as or broadcastable to + the `labels` tensor. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A summary operation for use in a TensorFlow graph. The float32 tensor + produced by the summary operation is of dimension (6, num_thresholds). The + first dimension (of length 6) is of the order: true positives, + false positives, true negatives, false negatives, precision, recall. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if num_thresholds is None: + num_thresholds = _DEFAULT_NUM_THRESHOLDS + + if weights is None: + weights = 1.0 + + dtype = predictions.dtype + + with tf.name_scope(name, values=[labels, predictions, weights]): + tf.assert_type(labels, tf.bool) + # We cast to float to ensure we have 0.0 or 1.0. + f_labels = tf.cast(labels, dtype) + # Ensure predictions are all in range [0.0, 1.0]. + predictions = tf.minimum(1.0, tf.maximum(0.0, predictions)) + # Get weighted true/false labels. + true_labels = f_labels * weights + false_labels = (1.0 - f_labels) * weights + + # Before we begin, flatten predictions. + predictions = tf.reshape(predictions, [-1]) + + # Shape the labels so they are broadcast-able for later multiplication. + true_labels = tf.reshape(true_labels, [-1, 1]) + false_labels = tf.reshape(false_labels, [-1, 1]) + + # To compute TP/FP/TN/FN, we are measuring a binary classifier + # C(t) = (predictions >= t) + # at each threshold 't'. So we have + # TP(t) = sum( C(t) * true_labels ) + # FP(t) = sum( C(t) * false_labels ) + # + # But, computing C(t) requires computation for each t. To make it fast, + # observe that C(t) is a cumulative integral, and so if we have + # thresholds = [t_0, ..., t_{n-1}]; t_0 < ... < t_{n-1} + # where n = num_thresholds, and if we can compute the bucket function + # B(i) = Sum( (predictions == t), t_i <= t < t{i+1} ) + # then we get + # C(t_i) = sum( B(j), j >= i ) + # which is the reversed cumulative sum in tf.cumsum(). + # + # We can compute B(i) efficiently by taking advantage of the fact that + # our thresholds are evenly distributed, in that + # width = 1.0 / (num_thresholds - 1) + # thresholds = [0.0, 1*width, 2*width, 3*width, ..., 1.0] + # Given a prediction value p, we can map it to its bucket by + # bucket_index(p) = floor( p * (num_thresholds - 1) ) + # so we can use tf.scatter_add() to update the buckets in one pass. + + # Compute the bucket indices for each prediction value. + bucket_indices = tf.cast( + tf.floor(predictions * (num_thresholds - 1)), tf.int32 + ) + + # Bucket predictions. + tp_buckets = tf.reduce_sum( + input_tensor=tf.one_hot(bucket_indices, depth=num_thresholds) + * true_labels, + axis=0, + ) + fp_buckets = tf.reduce_sum( + input_tensor=tf.one_hot(bucket_indices, depth=num_thresholds) + * false_labels, + axis=0, + ) + + # Set up the cumulative sums to compute the actual metrics. + tp = tf.cumsum(tp_buckets, reverse=True, name="tp") + fp = tf.cumsum(fp_buckets, reverse=True, name="fp") + # fn = sum(true_labels) - tp + # = sum(tp_buckets) - tp + # = tp[0] - tp + # Similarly, + # tn = fp[0] - fp + tn = fp[0] - fp + fn = tp[0] - tp + + precision = tp / tf.maximum(_MINIMUM_COUNT, tp + fp) + recall = tp / tf.maximum(_MINIMUM_COUNT, tp + fn) + + return _create_tensor_summary( + name, + tp, + fp, + tn, + fn, + precision, + recall, + num_thresholds, + display_name, + description, + collections, + ) + + +def pb( + name, + labels, + predictions, + num_thresholds=None, + weights=None, + display_name=None, + description=None, +): + """Create a PR curves summary protobuf. + + Arguments: + name: A name for the generated node. Will also serve as a series name in + TensorBoard. + labels: The ground truth values. A bool numpy array. + predictions: A float32 numpy array whose values are in the range `[0, 1]`. + Dimensions must match those of `labels`. + num_thresholds: Optional number of thresholds, evenly distributed in + `[0, 1]`, to compute PR metrics for. When provided, should be an int of + value at least 2. Defaults to 201. + weights: Optional float or float32 numpy array. Individual counts are + multiplied by this value. This tensor must be either the same shape as + or broadcastable to the `labels` numpy array. + display_name: Optional name for this summary in TensorBoard, as a `str`. + Defaults to `name`. + description: Optional long-form description for this summary, as a `str`. + Markdown is supported. Defaults to empty. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if num_thresholds is None: + num_thresholds = _DEFAULT_NUM_THRESHOLDS + + if weights is None: + weights = 1.0 + + # Compute bins of true positives and false positives. + bucket_indices = np.int32(np.floor(predictions * (num_thresholds - 1))) + float_labels = labels.astype(np.float) + histogram_range = (0, num_thresholds - 1) + tp_buckets, _ = np.histogram( + bucket_indices, + bins=num_thresholds, + range=histogram_range, + weights=float_labels * weights, + ) + fp_buckets, _ = np.histogram( + bucket_indices, + bins=num_thresholds, + range=histogram_range, + weights=(1.0 - float_labels) * weights, + ) + + # Obtain the reverse cumulative sum. + tp = np.cumsum(tp_buckets[::-1])[::-1] + fp = np.cumsum(fp_buckets[::-1])[::-1] tn = fp[0] - fp fn = tp[0] - tp + precision = tp / np.maximum(_MINIMUM_COUNT, tp + fp) + recall = tp / np.maximum(_MINIMUM_COUNT, tp + fn) - precision = tp / tf.maximum(_MINIMUM_COUNT, tp + fp) - recall = tp / tf.maximum(_MINIMUM_COUNT, tp + fn) - - return _create_tensor_summary( + return raw_data_pb( name, - tp, - fp, - tn, - fn, - precision, - recall, - num_thresholds, - display_name, - description, - collections) - -def pb(name, - labels, - predictions, - num_thresholds=None, - weights=None, - display_name=None, - description=None): - """Create a PR curves summary protobuf. - - Arguments: - name: A name for the generated node. Will also serve as a series name in - TensorBoard. - labels: The ground truth values. A bool numpy array. - predictions: A float32 numpy array whose values are in the range `[0, 1]`. - Dimensions must match those of `labels`. - num_thresholds: Optional number of thresholds, evenly distributed in - `[0, 1]`, to compute PR metrics for. When provided, should be an int of - value at least 2. Defaults to 201. - weights: Optional float or float32 numpy array. Individual counts are - multiplied by this value. This tensor must be either the same shape as - or broadcastable to the `labels` numpy array. - display_name: Optional name for this summary in TensorBoard, as a `str`. - Defaults to `name`. - description: Optional long-form description for this summary, as a `str`. - Markdown is supported. Defaults to empty. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - if num_thresholds is None: - num_thresholds = _DEFAULT_NUM_THRESHOLDS - - if weights is None: - weights = 1.0 - - # Compute bins of true positives and false positives. - bucket_indices = np.int32(np.floor(predictions * (num_thresholds - 1))) - float_labels = labels.astype(np.float) - histogram_range = (0, num_thresholds - 1) - tp_buckets, _ = np.histogram( - bucket_indices, - bins=num_thresholds, - range=histogram_range, - weights=float_labels * weights) - fp_buckets, _ = np.histogram( - bucket_indices, - bins=num_thresholds, - range=histogram_range, - weights=(1.0 - float_labels) * weights) - - # Obtain the reverse cumulative sum. - tp = np.cumsum(tp_buckets[::-1])[::-1] - fp = np.cumsum(fp_buckets[::-1])[::-1] - tn = fp[0] - fp - fn = tp[0] - tp - precision = tp / np.maximum(_MINIMUM_COUNT, tp + fp) - recall = tp / np.maximum(_MINIMUM_COUNT, tp + fn) - - return raw_data_pb(name, - true_positive_counts=tp, - false_positive_counts=fp, - true_negative_counts=tn, - false_negative_counts=fn, - precision=precision, - recall=recall, - num_thresholds=num_thresholds, - display_name=display_name, - description=description) - -def streaming_op(name, - labels, - predictions, - num_thresholds=None, - weights=None, - metrics_collections=None, - updates_collections=None, - display_name=None, - description=None): - """Computes a precision-recall curve summary across batches of data. - - This function is similar to op() above, but can be used to compute the PR - curve across multiple batches of labels and predictions, in the same style - as the metrics found in tf.metrics. - - This function creates multiple local variables for storing true positives, - true negative, etc. accumulated over each batch of data, and uses these local - variables for computing the final PR curve summary. These variables can be - updated with the returned update_op. - - Args: - name: A tag attached to the summary. Used by TensorBoard for organization. - labels: The ground truth values, a `Tensor` whose dimensions must match - `predictions`. Will be cast to `bool`. - predictions: A floating point `Tensor` of arbitrary shape and whose values - are in the range `[0, 1]`. - num_thresholds: The number of evenly spaced thresholds to generate for - computing the PR curve. Defaults to 201. - weights: Optional `Tensor` whose rank is either 0, or the same rank as - `labels`, and must be broadcastable to `labels` (i.e., all dimensions must - be either `1`, or the same as the corresponding `labels` dimension). - metrics_collections: An optional list of collections that `auc` should be - added to. - updates_collections: An optional list of collections that `update_op` should - be added to. - display_name: Optional name for this summary in TensorBoard, as a - constant `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - - Returns: - pr_curve: A string `Tensor` containing a single value: the - serialized PR curve Tensor summary. The summary contains a - float32 `Tensor` of dimension (6, num_thresholds). The first - dimension (of length 6) is of the order: true positives, false - positives, true negatives, false negatives, precision, recall. - update_op: An operation that updates the summary with the latest data. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - if num_thresholds is None: - num_thresholds = _DEFAULT_NUM_THRESHOLDS - - thresholds = [i / float(num_thresholds - 1) - for i in range(num_thresholds)] - - with tf.name_scope(name, values=[labels, predictions, weights]): - tp, update_tp = tf.metrics.true_positives_at_thresholds( - labels=labels, - predictions=predictions, - thresholds=thresholds, - weights=weights) - fp, update_fp = tf.metrics.false_positives_at_thresholds( - labels=labels, - predictions=predictions, - thresholds=thresholds, - weights=weights) - tn, update_tn = tf.metrics.true_negatives_at_thresholds( - labels=labels, - predictions=predictions, - thresholds=thresholds, - weights=weights) - fn, update_fn = tf.metrics.false_negatives_at_thresholds( - labels=labels, - predictions=predictions, - thresholds=thresholds, - weights=weights) - - def compute_summary(tp, fp, tn, fn, collections): - precision = tp / tf.maximum(_MINIMUM_COUNT, tp + fp) - recall = tp / tf.maximum(_MINIMUM_COUNT, tp + fn) - - return _create_tensor_summary( - name, - tp, - fp, - tn, - fn, - precision, - recall, - num_thresholds, - display_name, - description, - collections) - - pr_curve = compute_summary(tp, fp, tn, fn, metrics_collections) - update_op = tf.group(update_tp, update_fp, update_tn, update_fn) - if updates_collections: - for collection in updates_collections: - tf.add_to_collection(collection, update_op) - - return pr_curve, update_op + true_positive_counts=tp, + false_positive_counts=fp, + true_negative_counts=tn, + false_negative_counts=fn, + precision=precision, + recall=recall, + num_thresholds=num_thresholds, + display_name=display_name, + description=description, + ) + + +def streaming_op( + name, + labels, + predictions, + num_thresholds=None, + weights=None, + metrics_collections=None, + updates_collections=None, + display_name=None, + description=None, +): + """Computes a precision-recall curve summary across batches of data. + + This function is similar to op() above, but can be used to compute the PR + curve across multiple batches of labels and predictions, in the same style + as the metrics found in tf.metrics. + + This function creates multiple local variables for storing true positives, + true negative, etc. accumulated over each batch of data, and uses these local + variables for computing the final PR curve summary. These variables can be + updated with the returned update_op. + + Args: + name: A tag attached to the summary. Used by TensorBoard for organization. + labels: The ground truth values, a `Tensor` whose dimensions must match + `predictions`. Will be cast to `bool`. + predictions: A floating point `Tensor` of arbitrary shape and whose values + are in the range `[0, 1]`. + num_thresholds: The number of evenly spaced thresholds to generate for + computing the PR curve. Defaults to 201. + weights: Optional `Tensor` whose rank is either 0, or the same rank as + `labels`, and must be broadcastable to `labels` (i.e., all dimensions must + be either `1`, or the same as the corresponding `labels` dimension). + metrics_collections: An optional list of collections that `auc` should be + added to. + updates_collections: An optional list of collections that `update_op` should + be added to. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + + Returns: + pr_curve: A string `Tensor` containing a single value: the + serialized PR curve Tensor summary. The summary contains a + float32 `Tensor` of dimension (6, num_thresholds). The first + dimension (of length 6) is of the order: true positives, false + positives, true negatives, false negatives, precision, recall. + update_op: An operation that updates the summary with the latest data. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if num_thresholds is None: + num_thresholds = _DEFAULT_NUM_THRESHOLDS + + thresholds = [i / float(num_thresholds - 1) for i in range(num_thresholds)] + + with tf.name_scope(name, values=[labels, predictions, weights]): + tp, update_tp = tf.metrics.true_positives_at_thresholds( + labels=labels, + predictions=predictions, + thresholds=thresholds, + weights=weights, + ) + fp, update_fp = tf.metrics.false_positives_at_thresholds( + labels=labels, + predictions=predictions, + thresholds=thresholds, + weights=weights, + ) + tn, update_tn = tf.metrics.true_negatives_at_thresholds( + labels=labels, + predictions=predictions, + thresholds=thresholds, + weights=weights, + ) + fn, update_fn = tf.metrics.false_negatives_at_thresholds( + labels=labels, + predictions=predictions, + thresholds=thresholds, + weights=weights, + ) + + def compute_summary(tp, fp, tn, fn, collections): + precision = tp / tf.maximum(_MINIMUM_COUNT, tp + fp) + recall = tp / tf.maximum(_MINIMUM_COUNT, tp + fn) + + return _create_tensor_summary( + name, + tp, + fp, + tn, + fn, + precision, + recall, + num_thresholds, + display_name, + description, + collections, + ) + + pr_curve = compute_summary(tp, fp, tn, fn, metrics_collections) + update_op = tf.group(update_tp, update_fp, update_tn, update_fn) + if updates_collections: + for collection in updates_collections: + tf.add_to_collection(collection, update_op) + + return pr_curve, update_op + def raw_data_op( name, @@ -355,75 +377,81 @@ def raw_data_op( num_thresholds=None, display_name=None, description=None, - collections=None): - """Create an op that collects data for visualizing PR curves. - - Unlike the op above, this one avoids computing precision, recall, and the - intermediate counts. Instead, it accepts those tensors as arguments and - relies on the caller to ensure that the calculations are correct (and the - counts yield the provided precision and recall values). - - This op is useful when a caller seeks to compute precision and recall - differently but still use the PR curves plugin. - - Args: - name: A tag attached to the summary. Used by TensorBoard for organization. - true_positive_counts: A rank-1 tensor of true positive counts. Must contain - `num_thresholds` elements and be castable to float32. Values correspond - to thresholds that increase from left to right (from 0 to 1). - false_positive_counts: A rank-1 tensor of false positive counts. Must - contain `num_thresholds` elements and be castable to float32. Values - correspond to thresholds that increase from left to right (from 0 to 1). - true_negative_counts: A rank-1 tensor of true negative counts. Must contain - `num_thresholds` elements and be castable to float32. Values - correspond to thresholds that increase from left to right (from 0 to 1). - false_negative_counts: A rank-1 tensor of false negative counts. Must - contain `num_thresholds` elements and be castable to float32. Values - correspond to thresholds that increase from left to right (from 0 to 1). - precision: A rank-1 tensor of precision values. Must contain - `num_thresholds` elements and be castable to float32. Values correspond - to thresholds that increase from left to right (from 0 to 1). - recall: A rank-1 tensor of recall values. Must contain `num_thresholds` - elements and be castable to float32. Values correspond to thresholds - that increase from left to right (from 0 to 1). - num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to - compute PR metrics for. Should be `>= 2`. This value should be a - constant integer value, not a Tensor that stores an integer. - display_name: Optional name for this summary in TensorBoard, as a - constant `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - collections: Optional list of graph collections keys. The new - summary op is added to these collections. Defaults to - `[Graph Keys.SUMMARIES]`. - - Returns: - A summary operation for use in a TensorFlow graph. See docs for the `op` - method for details on the float32 tensor produced by this summary. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - with tf.name_scope(name, values=[ - true_positive_counts, - false_positive_counts, - true_negative_counts, - false_negative_counts, - precision, - recall, - ]): - return _create_tensor_summary( + collections=None, +): + """Create an op that collects data for visualizing PR curves. + + Unlike the op above, this one avoids computing precision, recall, and the + intermediate counts. Instead, it accepts those tensors as arguments and + relies on the caller to ensure that the calculations are correct (and the + counts yield the provided precision and recall values). + + This op is useful when a caller seeks to compute precision and recall + differently but still use the PR curves plugin. + + Args: + name: A tag attached to the summary. Used by TensorBoard for organization. + true_positive_counts: A rank-1 tensor of true positive counts. Must contain + `num_thresholds` elements and be castable to float32. Values correspond + to thresholds that increase from left to right (from 0 to 1). + false_positive_counts: A rank-1 tensor of false positive counts. Must + contain `num_thresholds` elements and be castable to float32. Values + correspond to thresholds that increase from left to right (from 0 to 1). + true_negative_counts: A rank-1 tensor of true negative counts. Must contain + `num_thresholds` elements and be castable to float32. Values + correspond to thresholds that increase from left to right (from 0 to 1). + false_negative_counts: A rank-1 tensor of false negative counts. Must + contain `num_thresholds` elements and be castable to float32. Values + correspond to thresholds that increase from left to right (from 0 to 1). + precision: A rank-1 tensor of precision values. Must contain + `num_thresholds` elements and be castable to float32. Values correspond + to thresholds that increase from left to right (from 0 to 1). + recall: A rank-1 tensor of recall values. Must contain `num_thresholds` + elements and be castable to float32. Values correspond to thresholds + that increase from left to right (from 0 to 1). + num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to + compute PR metrics for. Should be `>= 2`. This value should be a + constant integer value, not a Tensor that stores an integer. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A summary operation for use in a TensorFlow graph. See docs for the `op` + method for details on the float32 tensor produced by this summary. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + with tf.name_scope( name, - true_positive_counts, - false_positive_counts, - true_negative_counts, - false_negative_counts, - precision, - recall, - num_thresholds, - display_name, - description, - collections) + values=[ + true_positive_counts, + false_positive_counts, + true_negative_counts, + false_negative_counts, + precision, + recall, + ], + ): + return _create_tensor_summary( + name, + true_positive_counts, + false_positive_counts, + true_negative_counts, + false_negative_counts, + precision, + recall, + num_thresholds, + display_name, + description, + collections, + ) + def raw_data_pb( name, @@ -435,58 +463,65 @@ def raw_data_pb( recall, num_thresholds=None, display_name=None, - description=None): - """Create a PR curves summary protobuf from raw data values. - - Args: - name: A tag attached to the summary. Used by TensorBoard for organization. - true_positive_counts: A rank-1 numpy array of true positive counts. Must - contain `num_thresholds` elements and be castable to float32. - false_positive_counts: A rank-1 numpy array of false positive counts. Must - contain `num_thresholds` elements and be castable to float32. - true_negative_counts: A rank-1 numpy array of true negative counts. Must - contain `num_thresholds` elements and be castable to float32. - false_negative_counts: A rank-1 numpy array of false negative counts. Must - contain `num_thresholds` elements and be castable to float32. - precision: A rank-1 numpy array of precision values. Must contain - `num_thresholds` elements and be castable to float32. - recall: A rank-1 numpy array of recall values. Must contain `num_thresholds` - elements and be castable to float32. - num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to - compute PR metrics for. Should be an int `>= 2`. - display_name: Optional name for this summary in TensorBoard, as a `str`. - Defaults to `name`. - description: Optional long-form description for this summary, as a `str`. - Markdown is supported. Defaults to empty. - - Returns: - A summary operation for use in a TensorFlow graph. See docs for the `op` - method for details on the float32 tensor produced by this summary. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - if display_name is None: - display_name = name - summary_metadata = metadata.create_summary_metadata( - display_name=display_name if display_name is not None else name, - description=description or '', - num_thresholds=num_thresholds) - tf_summary_metadata = tf.SummaryMetadata.FromString( - summary_metadata.SerializeToString()) - summary = tf.Summary() - data = np.stack( - (true_positive_counts, - false_positive_counts, - true_negative_counts, - false_negative_counts, - precision, - recall)) - tensor = tf.make_tensor_proto(np.float32(data), dtype=tf.float32) - summary.value.add(tag='%s/pr_curves' % name, - metadata=tf_summary_metadata, - tensor=tensor) - return summary + description=None, +): + """Create a PR curves summary protobuf from raw data values. + + Args: + name: A tag attached to the summary. Used by TensorBoard for organization. + true_positive_counts: A rank-1 numpy array of true positive counts. Must + contain `num_thresholds` elements and be castable to float32. + false_positive_counts: A rank-1 numpy array of false positive counts. Must + contain `num_thresholds` elements and be castable to float32. + true_negative_counts: A rank-1 numpy array of true negative counts. Must + contain `num_thresholds` elements and be castable to float32. + false_negative_counts: A rank-1 numpy array of false negative counts. Must + contain `num_thresholds` elements and be castable to float32. + precision: A rank-1 numpy array of precision values. Must contain + `num_thresholds` elements and be castable to float32. + recall: A rank-1 numpy array of recall values. Must contain `num_thresholds` + elements and be castable to float32. + num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to + compute PR metrics for. Should be an int `>= 2`. + display_name: Optional name for this summary in TensorBoard, as a `str`. + Defaults to `name`. + description: Optional long-form description for this summary, as a `str`. + Markdown is supported. Defaults to empty. + + Returns: + A summary operation for use in a TensorFlow graph. See docs for the `op` + method for details on the float32 tensor produced by this summary. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name if display_name is not None else name, + description=description or "", + num_thresholds=num_thresholds, + ) + tf_summary_metadata = tf.SummaryMetadata.FromString( + summary_metadata.SerializeToString() + ) + summary = tf.Summary() + data = np.stack( + ( + true_positive_counts, + false_positive_counts, + true_negative_counts, + false_negative_counts, + precision, + recall, + ) + ) + tensor = tf.make_tensor_proto(np.float32(data), dtype=tf.float32) + summary.value.add( + tag="%s/pr_curves" % name, metadata=tf_summary_metadata, tensor=tensor + ) + return summary + def _create_tensor_summary( name, @@ -499,40 +534,46 @@ def _create_tensor_summary( num_thresholds=None, display_name=None, description=None, - collections=None): - """A private helper method for generating a tensor summary. - - We use a helper method instead of having `op` directly call `raw_data_op` - to prevent the scope of `raw_data_op` from being embedded within `op`. - - Arguments are the same as for raw_data_op. - - Returns: - A tensor summary that collects data for PR curves. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - # Store the number of thresholds within the summary metadata because - # that value is constant for all pr curve summaries with the same tag. - summary_metadata = metadata.create_summary_metadata( - display_name=display_name if display_name is not None else name, - description=description or '', - num_thresholds=num_thresholds) - - # Store values within a tensor. We store them in the order: - # true positives, false positives, true negatives, false - # negatives, precision, and recall. - combined_data = tf.stack([ - tf.cast(true_positive_counts, tf.float32), - tf.cast(false_positive_counts, tf.float32), - tf.cast(true_negative_counts, tf.float32), - tf.cast(false_negative_counts, tf.float32), - tf.cast(precision, tf.float32), - tf.cast(recall, tf.float32)]) - - return tf.summary.tensor_summary( - name='pr_curves', - tensor=combined_data, - collections=collections, - summary_metadata=summary_metadata) + collections=None, +): + """A private helper method for generating a tensor summary. + + We use a helper method instead of having `op` directly call `raw_data_op` + to prevent the scope of `raw_data_op` from being embedded within `op`. + + Arguments are the same as for raw_data_op. + + Returns: + A tensor summary that collects data for PR curves. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + # Store the number of thresholds within the summary metadata because + # that value is constant for all pr curve summaries with the same tag. + summary_metadata = metadata.create_summary_metadata( + display_name=display_name if display_name is not None else name, + description=description or "", + num_thresholds=num_thresholds, + ) + + # Store values within a tensor. We store them in the order: + # true positives, false positives, true negatives, false + # negatives, precision, and recall. + combined_data = tf.stack( + [ + tf.cast(true_positive_counts, tf.float32), + tf.cast(false_positive_counts, tf.float32), + tf.cast(true_negative_counts, tf.float32), + tf.cast(false_negative_counts, tf.float32), + tf.cast(precision, tf.float32), + tf.cast(recall, tf.float32), + ] + ) + + return tf.summary.tensor_summary( + name="pr_curves", + tensor=combined_data, + collections=collections, + summary_metadata=summary_metadata, + ) diff --git a/tensorboard/plugins/pr_curve/summary_test.py b/tensorboard/plugins/pr_curve/summary_test.py index 8620ece2c2..ff9c311591 100644 --- a/tensorboard/plugins/pr_curve/summary_test.py +++ b/tensorboard/plugins/pr_curve/summary_test.py @@ -32,408 +32,451 @@ class PrCurveTest(tf.test.TestCase): - - def setUp(self): - super(PrCurveTest, self).setUp() - tf.compat.v1.reset_default_graph() - np.random.seed(42) - - def pb_via_op(self, summary_op, feed_dict=None): - with tf.compat.v1.Session() as sess: - actual_pbtxt = sess.run(summary_op, feed_dict=feed_dict or {}) - actual_proto = summary_pb2.Summary() - actual_proto.ParseFromString(actual_pbtxt) - return actual_proto - - def normalize_summary_pb(self, pb): - """Pass `pb`'s `TensorProto` through a marshalling roundtrip. - `TensorProto`s can be equal in value even if they are not identical - in representation, because data can be stored in either the - `tensor_content` field or the `${dtype}_value` field. This - normalization ensures a canonical form, and should be used before - comparing two `Summary`s for equality. - """ - result = summary_pb2.Summary() - if not isinstance(pb, summary_pb2.Summary): - # pb can come from `pb_via_op` which creates a TB Summary. - pb = test_util.ensure_tb_summary_proto(pb) - result.MergeFrom(pb) - for value in result.value: - if value.HasField('tensor'): - new_tensor = tensor_util.make_tensor_proto( - tensor_util.make_ndarray(value.tensor)) - value.ClearField('tensor') - value.tensor.MergeFrom(new_tensor) - return result - - def compute_and_check_summary_pb(self, - name, - labels, - predictions, - num_thresholds, - weights=None, - display_name=None, - description=None, - feed_dict=None): - """Use both `op` and `pb` to get a summary, asserting equality. - Returns: - a `Summary` protocol buffer - """ - labels_tensor = tf.constant(labels) - predictions_tensor = tf.constant(predictions) - weights_tensor = None if weights is None else tf.constant(weights) - op = summary.op( - name=name, - labels=labels_tensor, - predictions=predictions_tensor, - num_thresholds=num_thresholds, - weights=weights_tensor, - display_name=display_name, - description=description) - pb = self.normalize_summary_pb(summary.pb( - name=name, - labels=labels, - predictions=predictions, - num_thresholds=num_thresholds, - weights=weights, - display_name=display_name, - description=description)) - pb_via_op = self.normalize_summary_pb( - self.pb_via_op(op, feed_dict=feed_dict)) - self.assertProtoEquals(pb, pb_via_op) - return pb - - def verify_float_arrays_are_equal(self, expected, actual): - # We use an absolute error instead of a relative one because the expected - # values are small. The default relative error (trol) of 1e-7 yields many - # undesired test failures. - np.testing.assert_allclose( - expected, actual, rtol=0, atol=1e-7) - - def test_metadata(self): - pb = self.compute_and_check_summary_pb( - name='foo', - labels=np.array([True]), - predictions=np.float32([0.42]), - num_thresholds=3) - summary_metadata = pb.value[0].metadata - plugin_data = summary_metadata.plugin_data - self.assertEqual('foo', summary_metadata.display_name) - self.assertEqual('', summary_metadata.summary_description) - self.assertEqual(metadata.PLUGIN_NAME, plugin_data.plugin_name) - plugin_data = metadata.parse_plugin_metadata( - summary_metadata.plugin_data.content) - self.assertEqual(3, plugin_data.num_thresholds) - - def test_all_true_positives(self): - pb = self.compute_and_check_summary_pb( - name='foo', - labels=np.array([True]), - predictions=np.float32([1]), - num_thresholds=3) - expected = [ - [1.0, 1.0, 1.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [1.0, 1.0, 1.0], - [1.0, 1.0, 1.0], - ] - values = tensor_util.make_ndarray(pb.value[0].tensor) - self.verify_float_arrays_are_equal(expected, values) - - def test_all_true_negatives(self): - pb = self.compute_and_check_summary_pb( - name='foo', - labels=np.array([False]), - predictions=np.float32([0]), - num_thresholds=3) - expected = [ - [0.0, 0.0, 0.0], - [1.0, 0.0, 0.0], - [0.0, 1.0, 1.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - ] - values = tensor_util.make_ndarray(pb.value[0].tensor) - self.verify_float_arrays_are_equal(expected, values) - - def test_all_false_positives(self): - pb = self.compute_and_check_summary_pb( - name='foo', - labels=np.array([False]), - predictions=np.float32([1]), - num_thresholds=3) - expected = [ - [0.0, 0.0, 0.0], - [1.0, 1.0, 1.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - ] - values = tensor_util.make_ndarray(pb.value[0].tensor) - self.verify_float_arrays_are_equal(expected, values) - - def test_all_false_negatives(self): - pb = self.compute_and_check_summary_pb( - name='foo', - labels=np.array([True]), - predictions=np.float32([0]), - num_thresholds=3) - expected = [ - [1.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [0.0, 1.0, 1.0], - [1.0, 0.0, 0.0], - [1.0, 0.0, 0.0], - ] - values = tensor_util.make_ndarray(pb.value[0].tensor) - self.verify_float_arrays_are_equal(expected, values) - - def test_many_values(self): - pb = self.compute_and_check_summary_pb( - name='foo', - labels=np.array([True, False, False, True, True, True]), - predictions=np.float32([0.2, 0.3, 0.4, 0.6, 0.7, 0.8]), - num_thresholds=3) - expected = [ - [4.0, 3.0, 0.0], - [2.0, 0.0, 0.0], - [0.0, 2.0, 2.0], - [0.0, 1.0, 4.0], - [2.0 / 3.0, 1.0, 0.0], - [1.0, 0.75, 0.0], - ] - values = tensor_util.make_ndarray(pb.value[0].tensor) - self.verify_float_arrays_are_equal(expected, values) - - def test_many_values_with_weights(self): - pb = self.compute_and_check_summary_pb( - name='foo', - labels=np.array([True, False, False, True, True, True]), - predictions=np.float32([0.2, 0.3, 0.4, 0.6, 0.7, 0.8]), - num_thresholds=3, - weights=np.float32([0.0, 0.5, 2.0, 0.0, 0.5, 1.0])) - expected = [ - [1.5, 1.5, 0.0], - [2.5, 0.0, 0.0], - [0.0, 2.5, 2.5], - [0.0, 0.0, 1.5], - [0.375, 1.0, 0.0], - [1.0, 1.0, 0.0] - ] - values = tensor_util.make_ndarray(pb.value[0].tensor) - self.verify_float_arrays_are_equal(expected, values) - - def test_exhaustive_random_values(self): - # Most other tests use small and crafted predictions and labels. - # This test exhaustively generates many data points. - data_points = 420 - pb = self.compute_and_check_summary_pb( - name='foo', - labels=np.random.uniform(size=(data_points,)) > 0.5, - predictions=np.float32(np.random.uniform(size=(data_points,))), - num_thresholds=5) - expected = [ - [218.0, 162.0, 111.0, 55.0, 0.0], - [202.0, 148.0, 98.0, 51.0, 0.0], - [0.0, 54.0, 104.0, 151.0, 202.0], - [0.0, 56.0, 107.0, 163.0, 218.0], - [0.5190476, 0.5225806, 0.5311005, 0.5188679, 0.0], - [1.0, 0.7431192, 0.5091743, 0.2522936, 0.0] - ] - values = tensor_util.make_ndarray(pb.value[0].tensor) - self.verify_float_arrays_are_equal(expected, values) - - def test_counts_below_1(self): - """Tests support for counts below 1. - - Certain weights cause TP, FP, TN, FN counts to be below 1. - """ - pb = self.compute_and_check_summary_pb( - name='foo', - labels=np.array([True, False, False, True, True, True]), - predictions=np.float32([0.2, 0.3, 0.4, 0.6, 0.7, 0.8]), - num_thresholds=3, - weights=np.float32([0.0, 0.1, 0.2, 0.1, 0.1, 0.0])) - expected = [ - [0.2, 0.2, 0.0], - [0.3, 0.0, 0.0], - [0.0, 0.3, 0.3], - [0.0, 0.0, 0.2], - [0.4, 1.0, 0.0], - [1.0, 1.0, 0.0] - ] - values = tensor_util.make_ndarray(pb.value[0].tensor) - self.verify_float_arrays_are_equal(expected, values) - - def test_raw_data(self): - # We pass these raw counts and precision/recall values. - name = 'foo' - true_positive_counts = [75, 64, 21, 5, 0] - false_positive_counts = [150, 105, 18, 0, 0] - true_negative_counts = [0, 45, 132, 150, 150] - false_negative_counts = [0, 11, 54, 70, 75] - precision = [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0] - recall = [1.0, 0.8533334, 0.28, 0.0666667, 0.0] - num_thresholds = 5 - display_name = 'some_raw_values' - description = 'We passed raw values into a summary op.' - - op = summary.raw_data_op( - name=name, - true_positive_counts=tf.constant(true_positive_counts), - false_positive_counts=tf.constant(false_positive_counts), - true_negative_counts=tf.constant(true_negative_counts), - false_negative_counts=tf.constant(false_negative_counts), - precision=tf.constant(precision), - recall=tf.constant(recall), - num_thresholds=num_thresholds, - display_name=display_name, - description=description) - pb_via_op = self.normalize_summary_pb(self.pb_via_op(op)) - - # Call the corresponding method that is decoupled from TensorFlow. - pb = self.normalize_summary_pb(summary.raw_data_pb( - name=name, - true_positive_counts=true_positive_counts, - false_positive_counts=false_positive_counts, - true_negative_counts=true_negative_counts, - false_negative_counts=false_negative_counts, - precision=precision, - recall=recall, - num_thresholds=num_thresholds, - display_name=display_name, - description=description)) - - # The 2 methods above should write summaries with the same data. - self.assertProtoEquals(pb, pb_via_op) - - # Test the metadata. - summary_metadata = pb.value[0].metadata - self.assertEqual('some_raw_values', summary_metadata.display_name) - self.assertEqual( - 'We passed raw values into a summary op.', - summary_metadata.summary_description) - self.assertEqual( - metadata.PLUGIN_NAME, summary_metadata.plugin_data.plugin_name) - - plugin_data = metadata.parse_plugin_metadata( - summary_metadata.plugin_data.content) - self.assertEqual(5, plugin_data.num_thresholds) - - # Test the summary contents. - values = tensor_util.make_ndarray(pb.value[0].tensor) - self.verify_float_arrays_are_equal([ - [75.0, 64.0, 21.0, 5.0, 0.0], # True positives. - [150.0, 105.0, 18.0, 0.0, 0.0], # False positives. - [0.0, 45.0, 132.0, 150.0, 150.0], # True negatives. - [0.0, 11.0, 54.0, 70.0, 75.0], # False negatives. - [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0], # Precision. - [1.0, 0.8533334, 0.28, 0.0666667, 0.0], # Recall. - ], values) + def setUp(self): + super(PrCurveTest, self).setUp() + tf.compat.v1.reset_default_graph() + np.random.seed(42) + + def pb_via_op(self, summary_op, feed_dict=None): + with tf.compat.v1.Session() as sess: + actual_pbtxt = sess.run(summary_op, feed_dict=feed_dict or {}) + actual_proto = summary_pb2.Summary() + actual_proto.ParseFromString(actual_pbtxt) + return actual_proto + + def normalize_summary_pb(self, pb): + """Pass `pb`'s `TensorProto` through a marshalling roundtrip. + + `TensorProto`s can be equal in value even if they are not + identical in representation, because data can be stored in + either the `tensor_content` field or the `${dtype}_value` field. + This normalization ensures a canonical form, and should be used + before comparing two `Summary`s for equality. + """ + result = summary_pb2.Summary() + if not isinstance(pb, summary_pb2.Summary): + # pb can come from `pb_via_op` which creates a TB Summary. + pb = test_util.ensure_tb_summary_proto(pb) + result.MergeFrom(pb) + for value in result.value: + if value.HasField("tensor"): + new_tensor = tensor_util.make_tensor_proto( + tensor_util.make_ndarray(value.tensor) + ) + value.ClearField("tensor") + value.tensor.MergeFrom(new_tensor) + return result + + def compute_and_check_summary_pb( + self, + name, + labels, + predictions, + num_thresholds, + weights=None, + display_name=None, + description=None, + feed_dict=None, + ): + """Use both `op` and `pb` to get a summary, asserting equality. + + Returns: + a `Summary` protocol buffer + """ + labels_tensor = tf.constant(labels) + predictions_tensor = tf.constant(predictions) + weights_tensor = None if weights is None else tf.constant(weights) + op = summary.op( + name=name, + labels=labels_tensor, + predictions=predictions_tensor, + num_thresholds=num_thresholds, + weights=weights_tensor, + display_name=display_name, + description=description, + ) + pb = self.normalize_summary_pb( + summary.pb( + name=name, + labels=labels, + predictions=predictions, + num_thresholds=num_thresholds, + weights=weights, + display_name=display_name, + description=description, + ) + ) + pb_via_op = self.normalize_summary_pb( + self.pb_via_op(op, feed_dict=feed_dict) + ) + self.assertProtoEquals(pb, pb_via_op) + return pb + + def verify_float_arrays_are_equal(self, expected, actual): + # We use an absolute error instead of a relative one because the expected + # values are small. The default relative error (trol) of 1e-7 yields many + # undesired test failures. + np.testing.assert_allclose(expected, actual, rtol=0, atol=1e-7) + + def test_metadata(self): + pb = self.compute_and_check_summary_pb( + name="foo", + labels=np.array([True]), + predictions=np.float32([0.42]), + num_thresholds=3, + ) + summary_metadata = pb.value[0].metadata + plugin_data = summary_metadata.plugin_data + self.assertEqual("foo", summary_metadata.display_name) + self.assertEqual("", summary_metadata.summary_description) + self.assertEqual(metadata.PLUGIN_NAME, plugin_data.plugin_name) + plugin_data = metadata.parse_plugin_metadata( + summary_metadata.plugin_data.content + ) + self.assertEqual(3, plugin_data.num_thresholds) + + def test_all_true_positives(self): + pb = self.compute_and_check_summary_pb( + name="foo", + labels=np.array([True]), + predictions=np.float32([1]), + num_thresholds=3, + ) + expected = [ + [1.0, 1.0, 1.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [1.0, 1.0, 1.0], + [1.0, 1.0, 1.0], + ] + values = tensor_util.make_ndarray(pb.value[0].tensor) + self.verify_float_arrays_are_equal(expected, values) + + def test_all_true_negatives(self): + pb = self.compute_and_check_summary_pb( + name="foo", + labels=np.array([False]), + predictions=np.float32([0]), + num_thresholds=3, + ) + expected = [ + [0.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [0.0, 1.0, 1.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + ] + values = tensor_util.make_ndarray(pb.value[0].tensor) + self.verify_float_arrays_are_equal(expected, values) + + def test_all_false_positives(self): + pb = self.compute_and_check_summary_pb( + name="foo", + labels=np.array([False]), + predictions=np.float32([1]), + num_thresholds=3, + ) + expected = [ + [0.0, 0.0, 0.0], + [1.0, 1.0, 1.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + ] + values = tensor_util.make_ndarray(pb.value[0].tensor) + self.verify_float_arrays_are_equal(expected, values) + + def test_all_false_negatives(self): + pb = self.compute_and_check_summary_pb( + name="foo", + labels=np.array([True]), + predictions=np.float32([0]), + num_thresholds=3, + ) + expected = [ + [1.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 1.0, 1.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + ] + values = tensor_util.make_ndarray(pb.value[0].tensor) + self.verify_float_arrays_are_equal(expected, values) + + def test_many_values(self): + pb = self.compute_and_check_summary_pb( + name="foo", + labels=np.array([True, False, False, True, True, True]), + predictions=np.float32([0.2, 0.3, 0.4, 0.6, 0.7, 0.8]), + num_thresholds=3, + ) + expected = [ + [4.0, 3.0, 0.0], + [2.0, 0.0, 0.0], + [0.0, 2.0, 2.0], + [0.0, 1.0, 4.0], + [2.0 / 3.0, 1.0, 0.0], + [1.0, 0.75, 0.0], + ] + values = tensor_util.make_ndarray(pb.value[0].tensor) + self.verify_float_arrays_are_equal(expected, values) + + def test_many_values_with_weights(self): + pb = self.compute_and_check_summary_pb( + name="foo", + labels=np.array([True, False, False, True, True, True]), + predictions=np.float32([0.2, 0.3, 0.4, 0.6, 0.7, 0.8]), + num_thresholds=3, + weights=np.float32([0.0, 0.5, 2.0, 0.0, 0.5, 1.0]), + ) + expected = [ + [1.5, 1.5, 0.0], + [2.5, 0.0, 0.0], + [0.0, 2.5, 2.5], + [0.0, 0.0, 1.5], + [0.375, 1.0, 0.0], + [1.0, 1.0, 0.0], + ] + values = tensor_util.make_ndarray(pb.value[0].tensor) + self.verify_float_arrays_are_equal(expected, values) + + def test_exhaustive_random_values(self): + # Most other tests use small and crafted predictions and labels. + # This test exhaustively generates many data points. + data_points = 420 + pb = self.compute_and_check_summary_pb( + name="foo", + labels=np.random.uniform(size=(data_points,)) > 0.5, + predictions=np.float32(np.random.uniform(size=(data_points,))), + num_thresholds=5, + ) + expected = [ + [218.0, 162.0, 111.0, 55.0, 0.0], + [202.0, 148.0, 98.0, 51.0, 0.0], + [0.0, 54.0, 104.0, 151.0, 202.0], + [0.0, 56.0, 107.0, 163.0, 218.0], + [0.5190476, 0.5225806, 0.5311005, 0.5188679, 0.0], + [1.0, 0.7431192, 0.5091743, 0.2522936, 0.0], + ] + values = tensor_util.make_ndarray(pb.value[0].tensor) + self.verify_float_arrays_are_equal(expected, values) + + def test_counts_below_1(self): + """Tests support for counts below 1. + + Certain weights cause TP, FP, TN, FN counts to be below 1. + """ + pb = self.compute_and_check_summary_pb( + name="foo", + labels=np.array([True, False, False, True, True, True]), + predictions=np.float32([0.2, 0.3, 0.4, 0.6, 0.7, 0.8]), + num_thresholds=3, + weights=np.float32([0.0, 0.1, 0.2, 0.1, 0.1, 0.0]), + ) + expected = [ + [0.2, 0.2, 0.0], + [0.3, 0.0, 0.0], + [0.0, 0.3, 0.3], + [0.0, 0.0, 0.2], + [0.4, 1.0, 0.0], + [1.0, 1.0, 0.0], + ] + values = tensor_util.make_ndarray(pb.value[0].tensor) + self.verify_float_arrays_are_equal(expected, values) + + def test_raw_data(self): + # We pass these raw counts and precision/recall values. + name = "foo" + true_positive_counts = [75, 64, 21, 5, 0] + false_positive_counts = [150, 105, 18, 0, 0] + true_negative_counts = [0, 45, 132, 150, 150] + false_negative_counts = [0, 11, 54, 70, 75] + precision = [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0] + recall = [1.0, 0.8533334, 0.28, 0.0666667, 0.0] + num_thresholds = 5 + display_name = "some_raw_values" + description = "We passed raw values into a summary op." + + op = summary.raw_data_op( + name=name, + true_positive_counts=tf.constant(true_positive_counts), + false_positive_counts=tf.constant(false_positive_counts), + true_negative_counts=tf.constant(true_negative_counts), + false_negative_counts=tf.constant(false_negative_counts), + precision=tf.constant(precision), + recall=tf.constant(recall), + num_thresholds=num_thresholds, + display_name=display_name, + description=description, + ) + pb_via_op = self.normalize_summary_pb(self.pb_via_op(op)) + + # Call the corresponding method that is decoupled from TensorFlow. + pb = self.normalize_summary_pb( + summary.raw_data_pb( + name=name, + true_positive_counts=true_positive_counts, + false_positive_counts=false_positive_counts, + true_negative_counts=true_negative_counts, + false_negative_counts=false_negative_counts, + precision=precision, + recall=recall, + num_thresholds=num_thresholds, + display_name=display_name, + description=description, + ) + ) + + # The 2 methods above should write summaries with the same data. + self.assertProtoEquals(pb, pb_via_op) + + # Test the metadata. + summary_metadata = pb.value[0].metadata + self.assertEqual("some_raw_values", summary_metadata.display_name) + self.assertEqual( + "We passed raw values into a summary op.", + summary_metadata.summary_description, + ) + self.assertEqual( + metadata.PLUGIN_NAME, summary_metadata.plugin_data.plugin_name + ) + + plugin_data = metadata.parse_plugin_metadata( + summary_metadata.plugin_data.content + ) + self.assertEqual(5, plugin_data.num_thresholds) + + # Test the summary contents. + values = tensor_util.make_ndarray(pb.value[0].tensor) + self.verify_float_arrays_are_equal( + [ + [75.0, 64.0, 21.0, 5.0, 0.0], # True positives. + [150.0, 105.0, 18.0, 0.0, 0.0], # False positives. + [0.0, 45.0, 132.0, 150.0, 150.0], # True negatives. + [0.0, 11.0, 54.0, 70.0, 75.0], # False negatives. + [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0], # Precision. + [1.0, 0.8533334, 0.28, 0.0666667, 0.0], # Recall. + ], + values, + ) class StreamingOpTest(tf.test.TestCase): - - def setUp(self): - super(StreamingOpTest, self).setUp() - tf.compat.v1.reset_default_graph() - np.random.seed(1) - - def pb_via_op(self, summary_op): - actual_pbtxt = summary_op.eval() - actual_proto = summary_pb2.Summary() - actual_proto.ParseFromString(actual_pbtxt) - return actual_proto - - def tensor_via_op(self, summary_op): - actual_pbtxt = summary_op.eval() - actual_proto = summary_pb2.Summary() - actual_proto.ParseFromString(actual_pbtxt) - return actual_proto - - def test_matches_op(self): - predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32) - labels = tf.constant([False, True, True, False, True], dtype=tf.bool) - - pr_curve, update_op = summary.streaming_op(name='pr_curve', - predictions=predictions, - labels=labels, - num_thresholds=10) - expected_pr_curve = summary.op(name='pr_curve', - predictions=predictions, - labels=labels, - num_thresholds=10) - with self.test_session() as sess: - sess.run(tf.compat.v1.local_variables_initializer()) - sess.run([update_op]) - - proto = self.pb_via_op(pr_curve) - expected_proto = self.pb_via_op(expected_pr_curve) - - # Need to detect and fix the automatic _1 appended to second namespace. - self.assertEqual(proto.value[0].tag, 'pr_curve/pr_curves') - self.assertEqual(expected_proto.value[0].tag, 'pr_curve_1/pr_curves') - expected_proto.value[0].tag = 'pr_curve/pr_curves' - - self.assertProtoEquals(expected_proto, proto) - - def test_matches_op_with_updates(self): - predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32) - labels = tf.constant([False, True, True, False, True], dtype=tf.bool) - pr_curve, update_op = summary.streaming_op(name='pr_curve', - predictions=predictions, - labels=labels, - num_thresholds=10) - - complete_predictions = tf.tile(predictions, [3]) - complete_labels = tf.tile(labels, [3]) - expected_pr_curve = summary.op(name='pr_curve', - predictions=complete_predictions, - labels=complete_labels, - num_thresholds=10) - with self.test_session() as sess: - sess.run(tf.compat.v1.local_variables_initializer()) - sess.run([update_op]) - sess.run([update_op]) - sess.run([update_op]) - - proto = self.pb_via_op(pr_curve) - expected_proto = self.pb_via_op(expected_pr_curve) - - # Need to detect and fix the automatic _1 appended to second namespace. - self.assertEqual(proto.value[0].tag, 'pr_curve/pr_curves') - self.assertEqual(expected_proto.value[0].tag, 'pr_curve_1/pr_curves') - expected_proto.value[0].tag = 'pr_curve/pr_curves' - - self.assertProtoEquals(expected_proto, proto) - - def test_only_1_summary_generated(self): - """Tests that the streaming op only generates 1 summary for PR curves. - - This test was made in response to a bug in which calling the streaming op - actually introduced 2 tags. - """ - predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32) - labels = tf.constant([False, True, True, False, True], dtype=tf.bool) - _, update_op = summary.streaming_op(name='pr_curve', - predictions=predictions, - labels=labels, - num_thresholds=10) - with self.test_session() as sess: - sess.run(tf.compat.v1.local_variables_initializer()) - sess.run(update_op) - summary_proto = summary_pb2.Summary() - summary_proto.ParseFromString(sess.run(tf.compat.v1.summary.merge_all())) - - tags = [v.tag for v in summary_proto.value] - # Only 1 tag should have been introduced. - self.assertEqual(['pr_curve/pr_curves'], tags) + def setUp(self): + super(StreamingOpTest, self).setUp() + tf.compat.v1.reset_default_graph() + np.random.seed(1) + + def pb_via_op(self, summary_op): + actual_pbtxt = summary_op.eval() + actual_proto = summary_pb2.Summary() + actual_proto.ParseFromString(actual_pbtxt) + return actual_proto + + def tensor_via_op(self, summary_op): + actual_pbtxt = summary_op.eval() + actual_proto = summary_pb2.Summary() + actual_proto.ParseFromString(actual_pbtxt) + return actual_proto + + def test_matches_op(self): + predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32) + labels = tf.constant([False, True, True, False, True], dtype=tf.bool) + + pr_curve, update_op = summary.streaming_op( + name="pr_curve", + predictions=predictions, + labels=labels, + num_thresholds=10, + ) + expected_pr_curve = summary.op( + name="pr_curve", + predictions=predictions, + labels=labels, + num_thresholds=10, + ) + with self.test_session() as sess: + sess.run(tf.compat.v1.local_variables_initializer()) + sess.run([update_op]) + + proto = self.pb_via_op(pr_curve) + expected_proto = self.pb_via_op(expected_pr_curve) + + # Need to detect and fix the automatic _1 appended to second namespace. + self.assertEqual(proto.value[0].tag, "pr_curve/pr_curves") + self.assertEqual( + expected_proto.value[0].tag, "pr_curve_1/pr_curves" + ) + expected_proto.value[0].tag = "pr_curve/pr_curves" + + self.assertProtoEquals(expected_proto, proto) + + def test_matches_op_with_updates(self): + predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32) + labels = tf.constant([False, True, True, False, True], dtype=tf.bool) + pr_curve, update_op = summary.streaming_op( + name="pr_curve", + predictions=predictions, + labels=labels, + num_thresholds=10, + ) + + complete_predictions = tf.tile(predictions, [3]) + complete_labels = tf.tile(labels, [3]) + expected_pr_curve = summary.op( + name="pr_curve", + predictions=complete_predictions, + labels=complete_labels, + num_thresholds=10, + ) + with self.test_session() as sess: + sess.run(tf.compat.v1.local_variables_initializer()) + sess.run([update_op]) + sess.run([update_op]) + sess.run([update_op]) + + proto = self.pb_via_op(pr_curve) + expected_proto = self.pb_via_op(expected_pr_curve) + + # Need to detect and fix the automatic _1 appended to second namespace. + self.assertEqual(proto.value[0].tag, "pr_curve/pr_curves") + self.assertEqual( + expected_proto.value[0].tag, "pr_curve_1/pr_curves" + ) + expected_proto.value[0].tag = "pr_curve/pr_curves" + + self.assertProtoEquals(expected_proto, proto) + + def test_only_1_summary_generated(self): + """Tests that the streaming op only generates 1 summary for PR curves. + + This test was made in response to a bug in which calling the + streaming op actually introduced 2 tags. + """ + predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32) + labels = tf.constant([False, True, True, False, True], dtype=tf.bool) + _, update_op = summary.streaming_op( + name="pr_curve", + predictions=predictions, + labels=labels, + num_thresholds=10, + ) + with self.test_session() as sess: + sess.run(tf.compat.v1.local_variables_initializer()) + sess.run(update_op) + summary_proto = summary_pb2.Summary() + summary_proto.ParseFromString( + sess.run(tf.compat.v1.summary.merge_all()) + ) + + tags = [v.tag for v in summary_proto.value] + # Only 1 tag should have been introduced. + self.assertEqual(["pr_curve/pr_curves"], tags) if __name__ == "__main__": - tf.test.main() + tf.test.main() diff --git a/tensorboard/plugins/profile/profile_demo.py b/tensorboard/plugins/profile/profile_demo.py index 99b13df537..2efcf5c969 100644 --- a/tensorboard/plugins/profile/profile_demo.py +++ b/tensorboard/plugins/profile/profile_demo.py @@ -42,71 +42,78 @@ # Directory into which to write tensorboard data. -LOGDIR = '/tmp/profile_demo' +LOGDIR = "/tmp/profile_demo" # Suffix for the empty eventfile to write. Should be kept in sync with TF # profiler kProfileEmptySuffix constant defined in: # tensorflow/core/profiler/rpc/client/capture_profile.cc. -EVENT_FILE_SUFFIX = '.profile-empty' +EVENT_FILE_SUFFIX = ".profile-empty" def _maybe_create_directory(directory): - try: - os.makedirs(directory) - except OSError: - print('Directory %s already exists.' %directory) + try: + os.makedirs(directory) + except OSError: + print("Directory %s already exists." % directory) def write_empty_event_file(logdir): - w = tf.compat.v2.summary.create_file_writer( - logdir, filename_suffix=EVENT_FILE_SUFFIX) - w.close() + w = tf.compat.v2.summary.create_file_writer( + logdir, filename_suffix=EVENT_FILE_SUFFIX + ) + w.close() def dump_data(logdir): - """Dumps plugin data to the log directory.""" - # Create a tfevents file in the logdir so it is detected as a run. - write_empty_event_file(logdir) - - plugin_logdir = plugin_asset_util.PluginDirectory( - logdir, profile_plugin.ProfilePlugin.plugin_name) - _maybe_create_directory(plugin_logdir) - - for run in profile_demo_data.RUNS: - run_dir = os.path.join(plugin_logdir, run) + """Dumps plugin data to the log directory.""" + # Create a tfevents file in the logdir so it is detected as a run. + write_empty_event_file(logdir) + + plugin_logdir = plugin_asset_util.PluginDirectory( + logdir, profile_plugin.ProfilePlugin.plugin_name + ) + _maybe_create_directory(plugin_logdir) + + for run in profile_demo_data.RUNS: + run_dir = os.path.join(plugin_logdir, run) + _maybe_create_directory(run_dir) + if run in profile_demo_data.TRACES: + with open(os.path.join(run_dir, "trace"), "w") as f: + proto = trace_events_pb2.Trace() + text_format.Merge(profile_demo_data.TRACES[run], proto) + f.write(proto.SerializeToString()) + + if run not in profile_demo_data.TRACE_ONLY: + shutil.copyfile( + "tensorboard/plugins/profile/profile_demo.op_profile.json", + os.path.join(run_dir, "op_profile.json"), + ) + shutil.copyfile( + "tensorboard/plugins/profile/profile_demo.memory_viewer.json", + os.path.join(run_dir, "memory_viewer.json"), + ) + shutil.copyfile( + "tensorboard/plugins/profile/profile_demo.pod_viewer.json", + os.path.join(run_dir, "pod_viewer.json"), + ) + shutil.copyfile( + "tensorboard/plugins/profile/profile_demo.google_chart_demo.json", + os.path.join(run_dir, "google_chart_demo.json"), + ) + + # Unsupported tool data should not be displayed. + run_dir = os.path.join(plugin_logdir, "empty") _maybe_create_directory(run_dir) - if run in profile_demo_data.TRACES: - with open(os.path.join(run_dir, 'trace'), 'w') as f: - proto = trace_events_pb2.Trace() - text_format.Merge(profile_demo_data.TRACES[run], proto) - f.write(proto.SerializeToString()) - - if run not in profile_demo_data.TRACE_ONLY: - shutil.copyfile('tensorboard/plugins/profile/profile_demo.op_profile.json', - os.path.join(run_dir, 'op_profile.json')) - shutil.copyfile( - 'tensorboard/plugins/profile/profile_demo.memory_viewer.json', - os.path.join(run_dir, 'memory_viewer.json')) - shutil.copyfile( - 'tensorboard/plugins/profile/profile_demo.pod_viewer.json', - os.path.join(run_dir, 'pod_viewer.json')) - shutil.copyfile( - 'tensorboard/plugins/profile/profile_demo.google_chart_demo.json', - os.path.join(run_dir, 'google_chart_demo.json')) - - # Unsupported tool data should not be displayed. - run_dir = os.path.join(plugin_logdir, 'empty') - _maybe_create_directory(run_dir) - with open(os.path.join(run_dir, 'unsupported'), 'w') as f: - f.write('unsupported data') + with open(os.path.join(run_dir, "unsupported"), "w") as f: + f.write("unsupported data") def main(unused_argv): - print('Saving output to %s.' % LOGDIR) - dump_data(LOGDIR) - print('Done. Output saved to %s.' % LOGDIR) + print("Saving output to %s." % LOGDIR) + dump_data(LOGDIR) + print("Done. Output saved to %s." % LOGDIR) -if __name__ == '__main__': - app.run(main) +if __name__ == "__main__": + app.run(main) diff --git a/tensorboard/plugins/profile/profile_demo_data.py b/tensorboard/plugins/profile/profile_demo_data.py index 2d8c93adde..d428d8da1a 100644 --- a/tensorboard/plugins/profile/profile_demo_data.py +++ b/tensorboard/plugins/profile/profile_demo_data.py @@ -12,21 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Demo data for the profile dashboard""" +"""Demo data for the profile dashboard.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -RUNS = ['foo', 'bar'] +RUNS = ["foo", "bar"] -TRACE_ONLY = ['foo'] +TRACE_ONLY = ["foo"] TRACES = {} -TRACES['foo'] = """ +TRACES[ + "foo" +] = """ devices { key: 2 value { name: 'Foo2' device_id: 2 @@ -61,7 +63,9 @@ """ -TRACES['bar'] = """ +TRACES[ + "bar" +] = """ devices { key: 2 value { name: 'Bar2' device_id: 2 diff --git a/tensorboard/plugins/profile/profile_plugin.py b/tensorboard/plugins/profile/profile_plugin.py index cf7bab56eb..6258e5eac8 100644 --- a/tensorboard/plugins/profile/profile_plugin.py +++ b/tensorboard/plugins/profile/profile_plugin.py @@ -38,443 +38,504 @@ logger = tb_logging.get_logger() # The prefix of routes provided by this plugin. -PLUGIN_NAME = 'profile' +PLUGIN_NAME = "profile" # HTTP routes -DATA_ROUTE = '/data' -TOOLS_ROUTE = '/tools' -HOSTS_ROUTE = '/hosts' -CAPTURE_ROUTE = '/capture_profile' +DATA_ROUTE = "/data" +TOOLS_ROUTE = "/tools" +HOSTS_ROUTE = "/hosts" +CAPTURE_ROUTE = "/capture_profile" # Available profiling tools -> file name of the tool data. -_FILE_NAME = 'TOOL_FILE_NAME' +_FILE_NAME = "TOOL_FILE_NAME" TOOLS = { - 'trace_viewer': 'trace', - 'trace_viewer@': 'tracetable', #streaming traceviewer - 'op_profile': 'op_profile.json', - 'input_pipeline_analyzer': 'input_pipeline.json', - 'overview_page': 'overview_page.json', - 'memory_viewer': 'memory_viewer.json', - 'pod_viewer': 'pod_viewer.json', - 'google_chart_demo': 'google_chart_demo.json', + "trace_viewer": "trace", + "trace_viewer@": "tracetable", # streaming traceviewer + "op_profile": "op_profile.json", + "input_pipeline_analyzer": "input_pipeline.json", + "overview_page": "overview_page.json", + "memory_viewer": "memory_viewer.json", + "pod_viewer": "pod_viewer.json", + "google_chart_demo": "google_chart_demo.json", } # Tools that consume raw data. -_RAW_DATA_TOOLS = frozenset(['input_pipeline_analyzer', - 'op_profile', - 'overview_page', - 'memory_viewer', - 'pod_viewer', - 'google_chart_demo',]) +_RAW_DATA_TOOLS = frozenset( + [ + "input_pipeline_analyzer", + "op_profile", + "overview_page", + "memory_viewer", + "pod_viewer", + "google_chart_demo", + ] +) + def process_raw_trace(raw_trace): - """Processes raw trace data and returns the UI data.""" - trace = trace_events_pb2.Trace() - trace.ParseFromString(raw_trace) - return ''.join(trace_events_json.TraceEventsJsonStream(trace)) + """Processes raw trace data and returns the UI data.""" + trace = trace_events_pb2.Trace() + trace.ParseFromString(raw_trace) + return "".join(trace_events_json.TraceEventsJsonStream(trace)) + def get_worker_list(cluster_resolver): - """Parses TPU workers list from the cluster resolver.""" - cluster_spec = cluster_resolver.cluster_spec() - task_indices = cluster_spec.task_indices('worker') - worker_list = [ - cluster_spec.task_address('worker', i).split(':')[0] for i in task_indices - ] - return ','.join(worker_list) + """Parses TPU workers list from the cluster resolver.""" + cluster_spec = cluster_resolver.cluster_spec() + task_indices = cluster_spec.task_indices("worker") + worker_list = [ + cluster_spec.task_address("worker", i).split(":")[0] + for i in task_indices + ] + return ",".join(worker_list) + class ProfilePlugin(base_plugin.TBPlugin): - """Profile Plugin for TensorBoard.""" - - plugin_name = PLUGIN_NAME - - def __init__(self, context): - """Constructs a profiler plugin for TensorBoard. - - This plugin adds handlers for performance-related frontends. - - Args: - context: A base_plugin.TBContext instance. - """ - self.logdir = context.logdir - self.multiplexer = context.multiplexer - self.plugin_logdir = plugin_asset_util.PluginDirectory( - self.logdir, PLUGIN_NAME) - self.stub = None - self.master_tpu_unsecure_channel = context.flags.master_tpu_unsecure_channel - - # Whether the plugin is active. This is an expensive computation, so we - # compute this asynchronously and cache positive results indefinitely. - self._is_active = False - # Lock to ensure at most one thread computes _is_active at a time. - self._is_active_lock = threading.Lock() - - def is_active(self): - """Whether this plugin is active and has any profile data to show. - - Detecting profile data is expensive, so this process runs asynchronously - and the value reported by this method is the cached value and may be stale. - - Returns: - Whether any run has profile data. - """ - # If we are already active, we remain active and don't recompute this. - # Otherwise, try to acquire the lock without blocking; if we get it and - # we're still not active, launch a thread to check if we're active and - # release the lock once the computation is finished. Either way, this - # thread returns the current cached value to avoid blocking. - if not self._is_active and self._is_active_lock.acquire(False): - if self._is_active: - self._is_active_lock.release() - else: - def compute_is_active(): - self._is_active = any(self.generate_run_to_tools()) - self._is_active_lock.release() - new_thread = threading.Thread( - target=compute_is_active, - name='ProfilePluginIsActiveThread') - new_thread.start() - return self._is_active - - def frontend_metadata(self): - # TODO(#2338): Keep this in sync with the `registerDashboard` call - # on the frontend until that call is removed. - return base_plugin.FrontendMetadata( - element_name='tf-profile-dashboard', - disable_reload=True, - ) - - def start_grpc_stub_if_necessary(self): - # We will enable streaming trace viewer on two conditions: - # 1. user specify the flags master_tpu_unsecure_channel to the ip address of - # as "master" TPU. grpc will be used to fetch streaming trace data. - # 2. the logdir is on google cloud storage. - if self.master_tpu_unsecure_channel and self.logdir.startswith('gs://'): - if self.stub is None: - import grpc - from tensorflow.python.tpu.profiler import profiler_analysis_pb2_grpc - # Workaround the grpc's 4MB message limitation. - gigabyte = 1024 * 1024 * 1024 - options = [('grpc.max_message_length', gigabyte), - ('grpc.max_send_message_length', gigabyte), - ('grpc.max_receive_message_length', gigabyte)] - tpu_profiler_port = self.master_tpu_unsecure_channel + ':8466' - channel = grpc.insecure_channel(tpu_profiler_port, options) - self.stub = profiler_analysis_pb2_grpc.ProfileAnalysisStub(channel) - - def _run_dir(self, run): - """Helper that maps a frontend run name to a profile "run" directory. - - The frontend run name consists of the TensorBoard run name (aka the relative - path from the logdir root to the directory containing the data) path-joined - to the Profile plugin's "run" concept (which is a subdirectory of the - plugins/profile directory representing an individual run of the tool), with - the special case that TensorBoard run is the logdir root (which is the run - named '.') then only the Profile plugin "run" name is used, for backwards - compatibility. - - To convert back to the actual run directory, we apply the following - transformation: - - If the run name doesn't contain '/', prepend './' - - Split on the rightmost instance of '/' - - Assume the left side is a TensorBoard run name and map it to a directory - path using EventMultiplexer.RunPaths(), then map that to the profile - plugin directory via PluginDirectory() - - Assume the right side is a Profile plugin "run" and path-join it to - the preceding path to get the final directory - - Args: - run: the frontend run name, as described above, e.g. train/run1. - - Returns: - The resolved directory path, e.g. /logdir/train/plugins/profile/run1. - """ - run = run.rstrip('/') - if '/' not in run: - run = './' + run - tb_run_name, _, profile_run_name = run.rpartition('/') - tb_run_directory = self.multiplexer.RunPaths().get(tb_run_name) - if tb_run_directory is None: - # Check if logdir is a directory to handle case where it's actually a - # multipart directory spec, which this plugin does not support. - if tb_run_name == '.' and tf.io.gfile.isdir(self.logdir): - tb_run_directory = self.logdir - else: - raise RuntimeError("No matching run directory for run %s" % run) - plugin_directory = plugin_asset_util.PluginDirectory( - tb_run_directory, PLUGIN_NAME) - return os.path.join(plugin_directory, profile_run_name) - - def generate_run_to_tools(self): - """Generator for pairs of "run name" and a list of tools for that run. - - The "run name" here is a "frontend run name" - see _run_dir() for the - definition of a "frontend run name" and how it maps to a directory of - profile data for a specific profile "run". The profile plugin concept of - "run" is different from the normal TensorBoard run; each run in this case - represents a single instance of profile data collection, more similar to a - "step" of data in typical TensorBoard semantics. These runs reside in - subdirectories of the plugins/profile directory within any regular - TensorBoard run directory (defined as a subdirectory of the logdir that - contains at least one tfevents file) or within the logdir root directory - itself (even if it contains no tfevents file and would thus not be - considered a normal TensorBoard run, for backwards compatibility). - - Within those "profile run directories", there are files in the directory - that correspond to different profiling tools. The file that contains profile - for a specific tool "x" will have a suffix name TOOLS["x"]. - - Example: - logs/ - plugins/ - profile/ + """Profile Plugin for TensorBoard.""" + + plugin_name = PLUGIN_NAME + + def __init__(self, context): + """Constructs a profiler plugin for TensorBoard. + + This plugin adds handlers for performance-related frontends. + + Args: + context: A base_plugin.TBContext instance. + """ + self.logdir = context.logdir + self.multiplexer = context.multiplexer + self.plugin_logdir = plugin_asset_util.PluginDirectory( + self.logdir, PLUGIN_NAME + ) + self.stub = None + self.master_tpu_unsecure_channel = ( + context.flags.master_tpu_unsecure_channel + ) + + # Whether the plugin is active. This is an expensive computation, so we + # compute this asynchronously and cache positive results indefinitely. + self._is_active = False + # Lock to ensure at most one thread computes _is_active at a time. + self._is_active_lock = threading.Lock() + + def is_active(self): + """Whether this plugin is active and has any profile data to show. + + Detecting profile data is expensive, so this process runs asynchronously + and the value reported by this method is the cached value and may be stale. + + Returns: + Whether any run has profile data. + """ + # If we are already active, we remain active and don't recompute this. + # Otherwise, try to acquire the lock without blocking; if we get it and + # we're still not active, launch a thread to check if we're active and + # release the lock once the computation is finished. Either way, this + # thread returns the current cached value to avoid blocking. + if not self._is_active and self._is_active_lock.acquire(False): + if self._is_active: + self._is_active_lock.release() + else: + + def compute_is_active(): + self._is_active = any(self.generate_run_to_tools()) + self._is_active_lock.release() + + new_thread = threading.Thread( + target=compute_is_active, name="ProfilePluginIsActiveThread" + ) + new_thread.start() + return self._is_active + + def frontend_metadata(self): + # TODO(#2338): Keep this in sync with the `registerDashboard` call + # on the frontend until that call is removed. + return base_plugin.FrontendMetadata( + element_name="tf-profile-dashboard", disable_reload=True, + ) + + def start_grpc_stub_if_necessary(self): + # We will enable streaming trace viewer on two conditions: + # 1. user specify the flags master_tpu_unsecure_channel to the ip address of + # as "master" TPU. grpc will be used to fetch streaming trace data. + # 2. the logdir is on google cloud storage. + if self.master_tpu_unsecure_channel and self.logdir.startswith("gs://"): + if self.stub is None: + import grpc + from tensorflow.python.tpu.profiler import ( + profiler_analysis_pb2_grpc, + ) + + # Workaround the grpc's 4MB message limitation. + gigabyte = 1024 * 1024 * 1024 + options = [ + ("grpc.max_message_length", gigabyte), + ("grpc.max_send_message_length", gigabyte), + ("grpc.max_receive_message_length", gigabyte), + ] + tpu_profiler_port = self.master_tpu_unsecure_channel + ":8466" + channel = grpc.insecure_channel(tpu_profiler_port, options) + self.stub = profiler_analysis_pb2_grpc.ProfileAnalysisStub( + channel + ) + + def _run_dir(self, run): + """Helper that maps a frontend run name to a profile "run" directory. + + The frontend run name consists of the TensorBoard run name (aka the relative + path from the logdir root to the directory containing the data) path-joined + to the Profile plugin's "run" concept (which is a subdirectory of the + plugins/profile directory representing an individual run of the tool), with + the special case that TensorBoard run is the logdir root (which is the run + named '.') then only the Profile plugin "run" name is used, for backwards + compatibility. + + To convert back to the actual run directory, we apply the following + transformation: + - If the run name doesn't contain '/', prepend './' + - Split on the rightmost instance of '/' + - Assume the left side is a TensorBoard run name and map it to a directory + path using EventMultiplexer.RunPaths(), then map that to the profile + plugin directory via PluginDirectory() + - Assume the right side is a Profile plugin "run" and path-join it to + the preceding path to get the final directory + + Args: + run: the frontend run name, as described above, e.g. train/run1. + + Returns: + The resolved directory path, e.g. /logdir/train/plugins/profile/run1. + """ + run = run.rstrip("/") + if "/" not in run: + run = "./" + run + tb_run_name, _, profile_run_name = run.rpartition("/") + tb_run_directory = self.multiplexer.RunPaths().get(tb_run_name) + if tb_run_directory is None: + # Check if logdir is a directory to handle case where it's actually a + # multipart directory spec, which this plugin does not support. + if tb_run_name == "." and tf.io.gfile.isdir(self.logdir): + tb_run_directory = self.logdir + else: + raise RuntimeError("No matching run directory for run %s" % run) + plugin_directory = plugin_asset_util.PluginDirectory( + tb_run_directory, PLUGIN_NAME + ) + return os.path.join(plugin_directory, profile_run_name) + + def generate_run_to_tools(self): + """Generator for pairs of "run name" and a list of tools for that run. + + The "run name" here is a "frontend run name" - see _run_dir() for the + definition of a "frontend run name" and how it maps to a directory of + profile data for a specific profile "run". The profile plugin concept of + "run" is different from the normal TensorBoard run; each run in this case + represents a single instance of profile data collection, more similar to a + "step" of data in typical TensorBoard semantics. These runs reside in + subdirectories of the plugins/profile directory within any regular + TensorBoard run directory (defined as a subdirectory of the logdir that + contains at least one tfevents file) or within the logdir root directory + itself (even if it contains no tfevents file and would thus not be + considered a normal TensorBoard run, for backwards compatibility). + + Within those "profile run directories", there are files in the directory + that correspond to different profiling tools. The file that contains profile + for a specific tool "x" will have a suffix name TOOLS["x"]. + + Example: + logs/ + plugins/ + profile/ + run1/ + hostA.trace + train/ + events.out.tfevents.foo + plugins/ + profile/ + run1/ + hostA.trace + hostB.trace + run2/ + hostA.trace + validation/ + events.out.tfevents.foo + plugins/ + profile/ + run1/ + hostA.trace + + Yields: + A sequence of tuples mapping "frontend run names" to lists of tool names + available for those runs. For the above example, this would be: + + ("run1", ["trace_viewer"]) + ("train/run1", ["trace_viewer"]) + ("train/run2", ["trace_viewer"]) + ("validation/run1", ["trace_viewer"]) + """ + self.start_grpc_stub_if_necessary() + + plugin_assets = self.multiplexer.PluginAssets(PLUGIN_NAME) + tb_run_names_to_dirs = self.multiplexer.RunPaths() + + # Ensure that we also check the root logdir, even if it isn't a recognized + # TensorBoard run (i.e. has no tfevents file directly under it), to remain + # backwards compatible with previously profile plugin behavior. Note that we + # check if logdir is a directory to handle case where it's actually a + # multipart directory spec, which this plugin does not support. + if "." not in plugin_assets and tf.io.gfile.isdir(self.logdir): + tb_run_names_to_dirs["."] = self.logdir + plugin_assets["."] = plugin_asset_util.ListAssets( + self.logdir, PLUGIN_NAME + ) + + for tb_run_name, profile_runs in six.iteritems(plugin_assets): + tb_run_dir = tb_run_names_to_dirs[tb_run_name] + tb_plugin_dir = plugin_asset_util.PluginDirectory( + tb_run_dir, PLUGIN_NAME + ) + for profile_run in profile_runs: + # Remove trailing slash; some filesystem implementations emit this. + profile_run = profile_run.rstrip("/") + if tb_run_name == ".": + frontend_run = profile_run + else: + frontend_run = "/".join([tb_run_name, profile_run]) + profile_run_dir = os.path.join(tb_plugin_dir, profile_run) + if tf.io.gfile.isdir(profile_run_dir): + yield frontend_run, self._get_active_tools(profile_run_dir) + + def _get_active_tools(self, profile_run_dir): + tools = [] + for tool in TOOLS: + tool_pattern = "*" + TOOLS[tool] + path = os.path.join(profile_run_dir, tool_pattern) + try: + files = tf.io.gfile.glob(path) + if len(files) >= 1: + tools.append(tool) + except tf.errors.OpError as e: + logger.warn( + "Cannot read asset directory: %s, OpError %s", + profile_run_dir, + e, + ) + if "trace_viewer@" in tools: + # streaming trace viewer always override normal trace viewer. + # the trailing '@' is to inform tf-profile-dashboard.html and + # tf-trace-viewer.html that stream trace viewer should be used. + removed_tool = ( + "trace_viewer@" if self.stub is None else "trace_viewer" + ) + if removed_tool in tools: + tools.remove(removed_tool) + tools.sort() + op = "overview_page" + if op in tools: + # keep overview page at the top of the list + tools.remove(op) + tools.insert(0, op) + return tools + + @wrappers.Request.application + def tools_route(self, request): + run_to_tools = dict(self.generate_run_to_tools()) + return http_util.Respond(request, run_to_tools, "application/json") + + def host_impl(self, run, tool): + """Returns available hosts for the run and tool in the log directory. + + In the plugin log directory, each directory contains profile data for a + single run (identified by the directory name), and files in the run + directory contains data for different tools and hosts. The file that + contains profile for a specific tool "x" will have a prefix name TOOLS["x"]. + + Example: + log/ run1/ - hostA.trace - train/ - events.out.tfevents.foo - plugins/ - profile/ - run1/ - hostA.trace - hostB.trace - run2/ - hostA.trace - validation/ - events.out.tfevents.foo - plugins/ - profile/ - run1/ - hostA.trace - - Yields: - A sequence of tuples mapping "frontend run names" to lists of tool names - available for those runs. For the above example, this would be: - - ("run1", ["trace_viewer"]) - ("train/run1", ["trace_viewer"]) - ("train/run2", ["trace_viewer"]) - ("validation/run1", ["trace_viewer"]) - """ - self.start_grpc_stub_if_necessary() - - plugin_assets = self.multiplexer.PluginAssets(PLUGIN_NAME) - tb_run_names_to_dirs = self.multiplexer.RunPaths() - - # Ensure that we also check the root logdir, even if it isn't a recognized - # TensorBoard run (i.e. has no tfevents file directly under it), to remain - # backwards compatible with previously profile plugin behavior. Note that we - # check if logdir is a directory to handle case where it's actually a - # multipart directory spec, which this plugin does not support. - if '.' not in plugin_assets and tf.io.gfile.isdir(self.logdir): - tb_run_names_to_dirs['.'] = self.logdir - plugin_assets['.'] = plugin_asset_util.ListAssets( - self.logdir, PLUGIN_NAME) - - for tb_run_name, profile_runs in six.iteritems(plugin_assets): - tb_run_dir = tb_run_names_to_dirs[tb_run_name] - tb_plugin_dir = plugin_asset_util.PluginDirectory( - tb_run_dir, PLUGIN_NAME) - for profile_run in profile_runs: - # Remove trailing slash; some filesystem implementations emit this. - profile_run = profile_run.rstrip('/') - if tb_run_name == '.': - frontend_run = profile_run - else: - frontend_run = '/'.join([tb_run_name, profile_run]) - profile_run_dir = os.path.join(tb_plugin_dir, profile_run) - if tf.io.gfile.isdir(profile_run_dir): - yield frontend_run, self._get_active_tools(profile_run_dir) - - def _get_active_tools(self, profile_run_dir): - tools = [] - for tool in TOOLS: - tool_pattern = '*' + TOOLS[tool] - path = os.path.join(profile_run_dir, tool_pattern) - try: - files = tf.io.gfile.glob(path) - if len(files) >= 1: - tools.append(tool) - except tf.errors.OpError as e: - logger.warn("Cannot read asset directory: %s, OpError %s", - profile_run_dir, e) - if 'trace_viewer@' in tools: - # streaming trace viewer always override normal trace viewer. - # the trailing '@' is to inform tf-profile-dashboard.html and - # tf-trace-viewer.html that stream trace viewer should be used. - removed_tool = 'trace_viewer@' if self.stub is None else 'trace_viewer' - if removed_tool in tools: - tools.remove(removed_tool) - tools.sort() - op = 'overview_page' - if op in tools: - # keep overview page at the top of the list - tools.remove(op) - tools.insert(0, op) - return tools - - @wrappers.Request.application - def tools_route(self, request): - run_to_tools = dict(self.generate_run_to_tools()) - return http_util.Respond(request, run_to_tools, 'application/json') - - def host_impl(self, run, tool): - """Returns available hosts for the run and tool in the log directory. - - In the plugin log directory, each directory contains profile data for a - single run (identified by the directory name), and files in the run - directory contains data for different tools and hosts. The file that - contains profile for a specific tool "x" will have a prefix name TOOLS["x"]. - - Example: - log/ - run1/ - plugins/ - profile/ - host1.trace - host2.trace - run2/ - plugins/ - profile/ - host1.trace - host2.trace - - Returns: - A list of host names e.g. - {"host1", "host2", "host3"} for the example. - """ - hosts = {} - run_dir = self._run_dir(run) - if not run_dir: - logger.warn("Cannot find asset directory for: %s", run) - return hosts - tool_pattern = '*' + TOOLS[tool] - try: - files = tf.io.gfile.glob(os.path.join(run_dir, tool_pattern)) - hosts = [os.path.basename(f).replace(TOOLS[tool], '') for f in files] - except tf.errors.OpError as e: - logger.warn("Cannot read asset directory: %s, OpError %s", - run_dir, e) - return hosts - - - @wrappers.Request.application - def hosts_route(self, request): - run = request.args.get('run') - tool = request.args.get('tag') - hosts = self.host_impl(run, tool) - return http_util.Respond(request, hosts, 'application/json') - - def data_impl(self, request): - """Retrieves and processes the tool data for a run and a host. - - Args: - request: XMLHttpRequest - - Returns: - A string that can be served to the frontend tool or None if tool, - run or host is invalid. - """ - run = request.args.get('run') - tool = request.args.get('tag') - host = request.args.get('host') - run_dir = self._run_dir(run) - # Profile plugin "run" is the last component of run dir. - profile_run = os.path.basename(run_dir) - - if tool not in TOOLS: - return None - - self.start_grpc_stub_if_necessary() - if tool == 'trace_viewer@' and self.stub is not None: - from tensorflow.core.profiler import profiler_analysis_pb2 - grpc_request = profiler_analysis_pb2.ProfileSessionDataRequest() - grpc_request.repository_root = os.path.dirname(run_dir) - grpc_request.session_id = profile_run - grpc_request.tool_name = 'trace_viewer' - # Remove the trailing dot if present - grpc_request.host_name = host.rstrip('.') - - grpc_request.parameters['resolution'] = request.args.get('resolution') - if request.args.get('start_time_ms') is not None: - grpc_request.parameters['start_time_ms'] = request.args.get( - 'start_time_ms') - if request.args.get('end_time_ms') is not None: - grpc_request.parameters['end_time_ms'] = request.args.get('end_time_ms') - grpc_response = self.stub.GetSessionToolData(grpc_request) - return grpc_response.output - - if tool not in TOOLS: - return None - tool_name = str(host) + TOOLS[tool] - asset_path = os.path.join(run_dir, tool_name) - raw_data = None - try: - with tf.io.gfile.GFile(asset_path, 'rb') as f: - raw_data = f.read() - except tf.errors.NotFoundError: - logger.warn('Asset path %s not found', asset_path) - except tf.errors.OpError as e: - logger.warn("Couldn't read asset path: %s, OpError %s", asset_path, e) - - if raw_data is None: - return None - if tool == 'trace_viewer': - return process_raw_trace(raw_data) - if tool in _RAW_DATA_TOOLS: - return raw_data - return None - - @wrappers.Request.application - def data_route(self, request): - # params - # request: XMLHTTPRequest. - data = self.data_impl(request) - if data is None: - return http_util.Respond(request, '404 Not Found', 'text/plain', code=404) - return http_util.Respond(request, data, 'application/json') - - @wrappers.Request.application - def capture_route(self, request): - service_addr = request.args.get('service_addr') - duration = int(request.args.get('duration', '1000')) - is_tpu_name = request.args.get('is_tpu_name') == 'true' - worker_list = request.args.get('worker_list') - include_dataset_ops = request.args.get('include_dataset_ops') == 'true' - num_tracing_attempts = int(request.args.get('num_retry', '0')) + 1 - - if is_tpu_name: - try: - tpu_cluster_resolver = ( - tf.distribute.cluster_resolver.TPUClusterResolver(service_addr)) - master_grpc_addr = tpu_cluster_resolver.get_master() - except (ImportError, RuntimeError) as err: - return http_util.Respond(request, {'error': err.message}, - 'application/json', code=200) - except (ValueError, TypeError): - return http_util.Respond(request, - {'error': 'no TPUs with the specified names exist.'}, - 'application/json', code=200) - if not worker_list: - worker_list = get_worker_list(tpu_cluster_resolver) - # TPU cluster resolver always returns port 8470. Replace it with 8466 - # on which profiler service is running. - master_ip = master_grpc_addr.replace('grpc://', '').replace(':8470', '') - service_addr = master_ip + ':8466' - # Set the master TPU for streaming trace viewer. - self.master_tpu_unsecure_channel = master_ip - try: - profiler_client.start_tracing(service_addr, self.logdir, duration, - worker_list, include_dataset_ops, num_tracing_attempts) - return http_util.Respond( - request, {'result': 'Capture profile successfully. Please refresh.'}, - 'application/json') - except tf.errors.UnavailableError: - return http_util.Respond(request, {'error': 'empty trace result.'}, - 'application/json', code=200) - - def get_plugin_apps(self): - return { - TOOLS_ROUTE: self.tools_route, - HOSTS_ROUTE: self.hosts_route, - DATA_ROUTE: self.data_route, - CAPTURE_ROUTE: self.capture_route, - } + plugins/ + profile/ + host1.trace + host2.trace + run2/ + plugins/ + profile/ + host1.trace + host2.trace + + Returns: + A list of host names e.g. + {"host1", "host2", "host3"} for the example. + """ + hosts = {} + run_dir = self._run_dir(run) + if not run_dir: + logger.warn("Cannot find asset directory for: %s", run) + return hosts + tool_pattern = "*" + TOOLS[tool] + try: + files = tf.io.gfile.glob(os.path.join(run_dir, tool_pattern)) + hosts = [ + os.path.basename(f).replace(TOOLS[tool], "") for f in files + ] + except tf.errors.OpError as e: + logger.warn( + "Cannot read asset directory: %s, OpError %s", run_dir, e + ) + return hosts + + @wrappers.Request.application + def hosts_route(self, request): + run = request.args.get("run") + tool = request.args.get("tag") + hosts = self.host_impl(run, tool) + return http_util.Respond(request, hosts, "application/json") + + def data_impl(self, request): + """Retrieves and processes the tool data for a run and a host. + + Args: + request: XMLHttpRequest + + Returns: + A string that can be served to the frontend tool or None if tool, + run or host is invalid. + """ + run = request.args.get("run") + tool = request.args.get("tag") + host = request.args.get("host") + run_dir = self._run_dir(run) + # Profile plugin "run" is the last component of run dir. + profile_run = os.path.basename(run_dir) + + if tool not in TOOLS: + return None + + self.start_grpc_stub_if_necessary() + if tool == "trace_viewer@" and self.stub is not None: + from tensorflow.core.profiler import profiler_analysis_pb2 + + grpc_request = profiler_analysis_pb2.ProfileSessionDataRequest() + grpc_request.repository_root = os.path.dirname(run_dir) + grpc_request.session_id = profile_run + grpc_request.tool_name = "trace_viewer" + # Remove the trailing dot if present + grpc_request.host_name = host.rstrip(".") + + grpc_request.parameters["resolution"] = request.args.get( + "resolution" + ) + if request.args.get("start_time_ms") is not None: + grpc_request.parameters["start_time_ms"] = request.args.get( + "start_time_ms" + ) + if request.args.get("end_time_ms") is not None: + grpc_request.parameters["end_time_ms"] = request.args.get( + "end_time_ms" + ) + grpc_response = self.stub.GetSessionToolData(grpc_request) + return grpc_response.output + + if tool not in TOOLS: + return None + tool_name = str(host) + TOOLS[tool] + asset_path = os.path.join(run_dir, tool_name) + raw_data = None + try: + with tf.io.gfile.GFile(asset_path, "rb") as f: + raw_data = f.read() + except tf.errors.NotFoundError: + logger.warn("Asset path %s not found", asset_path) + except tf.errors.OpError as e: + logger.warn( + "Couldn't read asset path: %s, OpError %s", asset_path, e + ) + + if raw_data is None: + return None + if tool == "trace_viewer": + return process_raw_trace(raw_data) + if tool in _RAW_DATA_TOOLS: + return raw_data + return None + + @wrappers.Request.application + def data_route(self, request): + # params + # request: XMLHTTPRequest. + data = self.data_impl(request) + if data is None: + return http_util.Respond( + request, "404 Not Found", "text/plain", code=404 + ) + return http_util.Respond(request, data, "application/json") + + @wrappers.Request.application + def capture_route(self, request): + service_addr = request.args.get("service_addr") + duration = int(request.args.get("duration", "1000")) + is_tpu_name = request.args.get("is_tpu_name") == "true" + worker_list = request.args.get("worker_list") + include_dataset_ops = request.args.get("include_dataset_ops") == "true" + num_tracing_attempts = int(request.args.get("num_retry", "0")) + 1 + + if is_tpu_name: + try: + tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( + service_addr + ) + master_grpc_addr = tpu_cluster_resolver.get_master() + except (ImportError, RuntimeError) as err: + return http_util.Respond( + request, + {"error": err.message}, + "application/json", + code=200, + ) + except (ValueError, TypeError): + return http_util.Respond( + request, + {"error": "no TPUs with the specified names exist."}, + "application/json", + code=200, + ) + if not worker_list: + worker_list = get_worker_list(tpu_cluster_resolver) + # TPU cluster resolver always returns port 8470. Replace it with 8466 + # on which profiler service is running. + master_ip = master_grpc_addr.replace("grpc://", "").replace( + ":8470", "" + ) + service_addr = master_ip + ":8466" + # Set the master TPU for streaming trace viewer. + self.master_tpu_unsecure_channel = master_ip + try: + profiler_client.start_tracing( + service_addr, + self.logdir, + duration, + worker_list, + include_dataset_ops, + num_tracing_attempts, + ) + return http_util.Respond( + request, + {"result": "Capture profile successfully. Please refresh."}, + "application/json", + ) + except tf.errors.UnavailableError: + return http_util.Respond( + request, + {"error": "empty trace result."}, + "application/json", + code=200, + ) + + def get_plugin_apps(self): + return { + TOOLS_ROUTE: self.tools_route, + HOSTS_ROUTE: self.hosts_route, + DATA_ROUTE: self.data_route, + CAPTURE_ROUTE: self.capture_route, + } diff --git a/tensorboard/plugins/profile/profile_plugin_loader.py b/tensorboard/plugins/profile/profile_plugin_loader.py index cdd4ee856f..6dbeec6625 100644 --- a/tensorboard/plugins/profile/profile_plugin_loader.py +++ b/tensorboard/plugins/profile/profile_plugin_loader.py @@ -22,40 +22,43 @@ class ProfilePluginLoader(base_plugin.TBLoader): - """ProfilePlugin factory. - - This class checks for `tensorflow` install and dependency. - """ - - def define_flags(self, parser): - group = parser.add_argument_group('profile plugin') - group.add_argument( - '--master_tpu_unsecure_channel', - metavar='ADDR', - type=str, - default='', - help='''\ + """ProfilePlugin factory. + + This class checks for `tensorflow` install and dependency. + """ + + def define_flags(self, parser): + group = parser.add_argument_group("profile plugin") + group.add_argument( + "--master_tpu_unsecure_channel", + metavar="ADDR", + type=str, + default="", + help="""\ IP address of "master tpu", used for getting streaming trace data through tpu profiler analysis grpc. The grpc channel is not secured.\ -''') +""", + ) - def load(self, context): - """Returns the plugin, if possible. + def load(self, context): + """Returns the plugin, if possible. - Args: - context: The TBContext flags. + Args: + context: The TBContext flags. - Returns: - A ProfilePlugin instance or None if it couldn't be loaded. - """ - try: - # pylint: disable=unused-import - import tensorflow - # Available in TensorFlow 1.14 or later, so do import check - # pylint: disable=unused-import - from tensorflow.python.eager import profiler_client - except ImportError: - return - - from tensorboard.plugins.profile.profile_plugin import ProfilePlugin - return ProfilePlugin(context) + Returns: + A ProfilePlugin instance or None if it couldn't be loaded. + """ + try: + # pylint: disable=unused-import + import tensorflow + + # Available in TensorFlow 1.14 or later, so do import check + # pylint: disable=unused-import + from tensorflow.python.eager import profiler_client + except ImportError: + return + + from tensorboard.plugins.profile.profile_plugin import ProfilePlugin + + return ProfilePlugin(context) diff --git a/tensorboard/plugins/profile/profile_plugin_test.py b/tensorboard/plugins/profile/profile_plugin_test.py index ce82b58df2..f69951d4a8 100644 --- a/tensorboard/plugins/profile/profile_plugin_test.py +++ b/tensorboard/plugins/profile/profile_plugin_test.py @@ -25,7 +25,9 @@ from werkzeug import Request from tensorboard.backend.event_processing import plugin_asset_util -from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer +from tensorboard.backend.event_processing import ( + plugin_event_multiplexer as event_multiplexer, +) from tensorboard.plugins import base_plugin from tensorboard.plugins.profile import profile_plugin from tensorboard.plugins.profile import trace_events_pb2 @@ -35,44 +37,33 @@ class FakeFlags(object): - def __init__( - self, - logdir, - master_tpu_unsecure_channel=''): - self.logdir = logdir - self.master_tpu_unsecure_channel = master_tpu_unsecure_channel + def __init__(self, logdir, master_tpu_unsecure_channel=""): + self.logdir = logdir + self.master_tpu_unsecure_channel = master_tpu_unsecure_channel RUN_TO_TOOLS = { - 'foo': ['trace_viewer'], - 'bar': ['unsupported'], - 'baz': ['trace_viewer'], - 'empty': [], + "foo": ["trace_viewer"], + "bar": ["unsupported"], + "baz": ["trace_viewer"], + "empty": [], } RUN_TO_HOSTS = { - 'foo': ['host0', 'host1'], - 'bar': ['host1'], - 'baz': ['host2'], - 'empty': [], + "foo": ["host0", "host1"], + "bar": ["host1"], + "baz": ["host2"], + "empty": [], } EXPECTED_TRACE_DATA = dict( - displayTimeUnit='ns', - metadata={'highres-ticks': True}, + displayTimeUnit="ns", + metadata={"highres-ticks": True}, traceEvents=[ - dict( - ph='M', - pid=0, - name='process_name', - args=dict(name='foo')), - dict( - ph='M', - pid=0, - name='process_sort_index', - args=dict(sort_index=0)), + dict(ph="M", pid=0, name="process_name", args=dict(name="foo")), + dict(ph="M", pid=0, name="process_sort_index", args=dict(sort_index=0)), dict(), ], ) @@ -81,173 +72,201 @@ def __init__( # Suffix for the empty eventfile to write. Should be kept in sync with TF # profiler kProfileEmptySuffix constant defined in: # tensorflow/core/profiler/rpc/client/capture_profile.cc. -EVENT_FILE_SUFFIX = '.profile-empty' +EVENT_FILE_SUFFIX = ".profile-empty" def generate_testdata(logdir): - plugin_logdir = plugin_asset_util.PluginDirectory( - logdir, profile_plugin.ProfilePlugin.plugin_name) - os.makedirs(plugin_logdir) - for run in RUN_TO_TOOLS: - run_dir = os.path.join(plugin_logdir, run) - os.mkdir(run_dir) - for tool in RUN_TO_TOOLS[run]: - if tool not in profile_plugin.TOOLS: - continue - for host in RUN_TO_HOSTS[run]: - file_name = host + profile_plugin.TOOLS[tool] - tool_file = os.path.join(run_dir, file_name) - if tool == 'trace_viewer': - trace = trace_events_pb2.Trace() - trace.devices[0].name = run - data = trace.SerializeToString() - else: - data = tool - with open(tool_file, 'wb') as f: - f.write(data) - with open(os.path.join(plugin_logdir, 'noise'), 'w') as f: - f.write('Not a dir, not a run.') + plugin_logdir = plugin_asset_util.PluginDirectory( + logdir, profile_plugin.ProfilePlugin.plugin_name + ) + os.makedirs(plugin_logdir) + for run in RUN_TO_TOOLS: + run_dir = os.path.join(plugin_logdir, run) + os.mkdir(run_dir) + for tool in RUN_TO_TOOLS[run]: + if tool not in profile_plugin.TOOLS: + continue + for host in RUN_TO_HOSTS[run]: + file_name = host + profile_plugin.TOOLS[tool] + tool_file = os.path.join(run_dir, file_name) + if tool == "trace_viewer": + trace = trace_events_pb2.Trace() + trace.devices[0].name = run + data = trace.SerializeToString() + else: + data = tool + with open(tool_file, "wb") as f: + f.write(data) + with open(os.path.join(plugin_logdir, "noise"), "w") as f: + f.write("Not a dir, not a run.") def write_empty_event_file(logdir): - w = tf.compat.v2.summary.create_file_writer( - logdir, filename_suffix=EVENT_FILE_SUFFIX) - w.close() + w = tf.compat.v2.summary.create_file_writer( + logdir, filename_suffix=EVENT_FILE_SUFFIX + ) + w.close() class ProfilePluginTest(tf.test.TestCase): - - def setUp(self): - self.logdir = self.get_temp_dir() - self.multiplexer = event_multiplexer.EventMultiplexer() - self.multiplexer.AddRunsFromDirectory(self.logdir) - context = base_plugin.TBContext( - logdir=self.logdir, - multiplexer=self.multiplexer, - flags=FakeFlags(self.logdir)) - self.plugin = profile_plugin.ProfilePlugin(context) - self.apps = self.plugin.get_plugin_apps() - - def testRuns_logdirWithoutEventFile(self): - generate_testdata(self.logdir) - self.multiplexer.Reload() - runs = dict(self.plugin.generate_run_to_tools()) - self.assertItemsEqual(runs.keys(), RUN_TO_TOOLS.keys()) - self.assertItemsEqual(runs['foo'], RUN_TO_TOOLS['foo']) - self.assertItemsEqual(runs['bar'], []) - self.assertItemsEqual(runs['empty'], []) - - def testRuns_logdirWithEventFIle(self): - write_empty_event_file(self.logdir) - generate_testdata(self.logdir) - self.multiplexer.Reload() - runs = dict(self.plugin.generate_run_to_tools()) - self.assertItemsEqual(runs.keys(), RUN_TO_TOOLS.keys()) - - def testRuns_withSubdirectories(self): - subdir_a = os.path.join(self.logdir, 'a') - subdir_b = os.path.join(self.logdir, 'b') - subdir_b_c = os.path.join(subdir_b, 'c') - generate_testdata(self.logdir) - generate_testdata(subdir_a) - generate_testdata(subdir_b) - generate_testdata(subdir_b_c) - write_empty_event_file(self.logdir) - write_empty_event_file(subdir_a) - # Skip writing an event file for subdir_b - write_empty_event_file(subdir_b_c) - self.multiplexer.AddRunsFromDirectory(self.logdir) - self.multiplexer.Reload() - runs = dict(self.plugin.generate_run_to_tools()) - # Expect runs for the logdir root, 'a', and 'b/c' but not for 'b' - # because it doesn't contain a tfevents file. - expected = list(RUN_TO_TOOLS.keys()) - expected.extend('a/' + run for run in RUN_TO_TOOLS.keys()) - expected.extend('b/c/' + run for run in RUN_TO_TOOLS.keys()) - self.assertItemsEqual(runs.keys(), expected) - - def makeRequest(self, run, tag, host): - req = Request({}) - req.args = {'run': run, 'tag': tag, 'host': host,} - return req - - def testHosts(self): - generate_testdata(self.logdir) - subdir_a = os.path.join(self.logdir, 'a') - generate_testdata(subdir_a) - write_empty_event_file(subdir_a) - self.multiplexer.AddRunsFromDirectory(self.logdir) - self.multiplexer.Reload() - hosts = self.plugin.host_impl('foo', 'trace_viewer') - self.assertItemsEqual(['host0', 'host1'], sorted(hosts)) - hosts_a = self.plugin.host_impl('a/foo', 'trace_viewer') - self.assertItemsEqual(['host0', 'host1'], sorted(hosts_a)) - - def testData(self): - generate_testdata(self.logdir) - subdir_a = os.path.join(self.logdir, 'a') - generate_testdata(subdir_a) - write_empty_event_file(subdir_a) - self.multiplexer.AddRunsFromDirectory(self.logdir) - self.multiplexer.Reload() - trace = json.loads(self.plugin.data_impl( - self.makeRequest('foo', 'trace_viewer', 'host0'))) - self.assertEqual(trace, EXPECTED_TRACE_DATA) - trace_a = json.loads(self.plugin.data_impl( - self.makeRequest('a/foo', 'trace_viewer', 'host0'))) - self.assertEqual(trace_a, EXPECTED_TRACE_DATA) - - # Invalid tool/run. - self.assertEqual(None, self.plugin.data_impl( - self.makeRequest('foo', 'nonono', 'host0'))) - self.assertEqual(None, self.plugin.data_impl( - self.makeRequest('foo', 'trace_viewer', ''))) - self.assertEqual(None, self.plugin.data_impl( - self.makeRequest('bar', 'unsupported', 'host1'))) - self.assertEqual(None, self.plugin.data_impl( - self.makeRequest('empty', 'trace_viewer', ''))) - self.assertEqual(None, self.plugin.data_impl( - self.makeRequest('a', 'trace_viewer', ''))) - - def testActive(self): - def wait_for_thread(): - with self.plugin._is_active_lock: - pass - # Launch thread to check if active. - self.plugin.is_active() - wait_for_thread() - # Should be false since there's no data yet. - self.assertFalse(self.plugin.is_active()) - wait_for_thread() - generate_testdata(self.logdir) - self.multiplexer.Reload() - # Launch a new thread to check if active. - self.plugin.is_active() - wait_for_thread() - # Now that there's data, this should be active. - self.assertTrue(self.plugin.is_active()) - - def testActive_subdirectoryOnly(self): - def wait_for_thread(): - with self.plugin._is_active_lock: - pass - # Launch thread to check if active. - self.plugin.is_active() - wait_for_thread() - # Should be false since there's no data yet. - self.assertFalse(self.plugin.is_active()) - wait_for_thread() - subdir_a = os.path.join(self.logdir, 'a') - generate_testdata(subdir_a) - write_empty_event_file(subdir_a) - self.multiplexer.AddRunsFromDirectory(self.logdir) - self.multiplexer.Reload() - # Launch a new thread to check if active. - self.plugin.is_active() - wait_for_thread() - # Now that there's data, this should be active. - self.assertTrue(self.plugin.is_active()) - - -if __name__ == '__main__': - tf.test.main() + def setUp(self): + self.logdir = self.get_temp_dir() + self.multiplexer = event_multiplexer.EventMultiplexer() + self.multiplexer.AddRunsFromDirectory(self.logdir) + context = base_plugin.TBContext( + logdir=self.logdir, + multiplexer=self.multiplexer, + flags=FakeFlags(self.logdir), + ) + self.plugin = profile_plugin.ProfilePlugin(context) + self.apps = self.plugin.get_plugin_apps() + + def testRuns_logdirWithoutEventFile(self): + generate_testdata(self.logdir) + self.multiplexer.Reload() + runs = dict(self.plugin.generate_run_to_tools()) + self.assertItemsEqual(runs.keys(), RUN_TO_TOOLS.keys()) + self.assertItemsEqual(runs["foo"], RUN_TO_TOOLS["foo"]) + self.assertItemsEqual(runs["bar"], []) + self.assertItemsEqual(runs["empty"], []) + + def testRuns_logdirWithEventFIle(self): + write_empty_event_file(self.logdir) + generate_testdata(self.logdir) + self.multiplexer.Reload() + runs = dict(self.plugin.generate_run_to_tools()) + self.assertItemsEqual(runs.keys(), RUN_TO_TOOLS.keys()) + + def testRuns_withSubdirectories(self): + subdir_a = os.path.join(self.logdir, "a") + subdir_b = os.path.join(self.logdir, "b") + subdir_b_c = os.path.join(subdir_b, "c") + generate_testdata(self.logdir) + generate_testdata(subdir_a) + generate_testdata(subdir_b) + generate_testdata(subdir_b_c) + write_empty_event_file(self.logdir) + write_empty_event_file(subdir_a) + # Skip writing an event file for subdir_b + write_empty_event_file(subdir_b_c) + self.multiplexer.AddRunsFromDirectory(self.logdir) + self.multiplexer.Reload() + runs = dict(self.plugin.generate_run_to_tools()) + # Expect runs for the logdir root, 'a', and 'b/c' but not for 'b' + # because it doesn't contain a tfevents file. + expected = list(RUN_TO_TOOLS.keys()) + expected.extend("a/" + run for run in RUN_TO_TOOLS.keys()) + expected.extend("b/c/" + run for run in RUN_TO_TOOLS.keys()) + self.assertItemsEqual(runs.keys(), expected) + + def makeRequest(self, run, tag, host): + req = Request({}) + req.args = { + "run": run, + "tag": tag, + "host": host, + } + return req + + def testHosts(self): + generate_testdata(self.logdir) + subdir_a = os.path.join(self.logdir, "a") + generate_testdata(subdir_a) + write_empty_event_file(subdir_a) + self.multiplexer.AddRunsFromDirectory(self.logdir) + self.multiplexer.Reload() + hosts = self.plugin.host_impl("foo", "trace_viewer") + self.assertItemsEqual(["host0", "host1"], sorted(hosts)) + hosts_a = self.plugin.host_impl("a/foo", "trace_viewer") + self.assertItemsEqual(["host0", "host1"], sorted(hosts_a)) + + def testData(self): + generate_testdata(self.logdir) + subdir_a = os.path.join(self.logdir, "a") + generate_testdata(subdir_a) + write_empty_event_file(subdir_a) + self.multiplexer.AddRunsFromDirectory(self.logdir) + self.multiplexer.Reload() + trace = json.loads( + self.plugin.data_impl( + self.makeRequest("foo", "trace_viewer", "host0") + ) + ) + self.assertEqual(trace, EXPECTED_TRACE_DATA) + trace_a = json.loads( + self.plugin.data_impl( + self.makeRequest("a/foo", "trace_viewer", "host0") + ) + ) + self.assertEqual(trace_a, EXPECTED_TRACE_DATA) + + # Invalid tool/run. + self.assertEqual( + None, + self.plugin.data_impl(self.makeRequest("foo", "nonono", "host0")), + ) + self.assertEqual( + None, + self.plugin.data_impl(self.makeRequest("foo", "trace_viewer", "")), + ) + self.assertEqual( + None, + self.plugin.data_impl( + self.makeRequest("bar", "unsupported", "host1") + ), + ) + self.assertEqual( + None, + self.plugin.data_impl( + self.makeRequest("empty", "trace_viewer", "") + ), + ) + self.assertEqual( + None, + self.plugin.data_impl(self.makeRequest("a", "trace_viewer", "")), + ) + + def testActive(self): + def wait_for_thread(): + with self.plugin._is_active_lock: + pass + + # Launch thread to check if active. + self.plugin.is_active() + wait_for_thread() + # Should be false since there's no data yet. + self.assertFalse(self.plugin.is_active()) + wait_for_thread() + generate_testdata(self.logdir) + self.multiplexer.Reload() + # Launch a new thread to check if active. + self.plugin.is_active() + wait_for_thread() + # Now that there's data, this should be active. + self.assertTrue(self.plugin.is_active()) + + def testActive_subdirectoryOnly(self): + def wait_for_thread(): + with self.plugin._is_active_lock: + pass + + # Launch thread to check if active. + self.plugin.is_active() + wait_for_thread() + # Should be false since there's no data yet. + self.assertFalse(self.plugin.is_active()) + wait_for_thread() + subdir_a = os.path.join(self.logdir, "a") + generate_testdata(subdir_a) + write_empty_event_file(subdir_a) + self.multiplexer.AddRunsFromDirectory(self.logdir) + self.multiplexer.Reload() + # Launch a new thread to check if active. + self.plugin.is_active() + wait_for_thread() + # Now that there's data, this should be active. + self.assertTrue(self.plugin.is_active()) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/profile/trace_events_json.py b/tensorboard/plugins/profile/trace_events_json.py index 3305da2b84..2ec89fa4b9 100644 --- a/tensorboard/plugins/profile/trace_events_json.py +++ b/tensorboard/plugins/profile/trace_events_json.py @@ -23,84 +23,90 @@ import six # Values for type (ph) and s (scope) parameters in catapult trace format. -_TYPE_METADATA = 'M' -_TYPE_COMPLETE = 'X' -_TYPE_INSTANT = 'i' -_SCOPE_THREAD = 't' +_TYPE_METADATA = "M" +_TYPE_COMPLETE = "X" +_TYPE_INSTANT = "i" +_SCOPE_THREAD = "t" class TraceEventsJsonStream(object): - """A streaming trace file in the format expected by catapult trace viewer. + """A streaming trace file in the format expected by catapult trace viewer. - Iterating over this yields a sequence of string chunks, so it is suitable for - returning in a werkzeug Response. - """ + Iterating over this yields a sequence of string chunks, so it is + suitable for returning in a werkzeug Response. + """ - def __init__(self, proto): - """Create an iterable JSON stream over the supplied Trace. + def __init__(self, proto): + """Create an iterable JSON stream over the supplied Trace. - Args: - proto: a tensorboard.profile.Trace protobuf - """ - self._proto = proto + Args: + proto: a tensorboard.profile.Trace protobuf + """ + self._proto = proto - def _events(self): - """Iterator over all catapult trace events, as python values.""" - for did, device in sorted(six.iteritems(self._proto.devices)): - if device.name: - yield dict( - ph=_TYPE_METADATA, - pid=did, - name='process_name', - args=dict(name=device.name)) - yield dict( - ph=_TYPE_METADATA, - pid=did, - name='process_sort_index', - args=dict(sort_index=did)) - for rid, resource in sorted(six.iteritems(device.resources)): - if resource.name: - yield dict( - ph=_TYPE_METADATA, - pid=did, - tid=rid, - name='thread_name', - args=dict(name=resource.name)) - yield dict( - ph=_TYPE_METADATA, - pid=did, - tid=rid, - name='thread_sort_index', - args=dict(sort_index=rid)) - # TODO(sammccall): filtering and downsampling? - for event in self._proto.trace_events: - yield self._event(event) + def _events(self): + """Iterator over all catapult trace events, as python values.""" + for did, device in sorted(six.iteritems(self._proto.devices)): + if device.name: + yield dict( + ph=_TYPE_METADATA, + pid=did, + name="process_name", + args=dict(name=device.name), + ) + yield dict( + ph=_TYPE_METADATA, + pid=did, + name="process_sort_index", + args=dict(sort_index=did), + ) + for rid, resource in sorted(six.iteritems(device.resources)): + if resource.name: + yield dict( + ph=_TYPE_METADATA, + pid=did, + tid=rid, + name="thread_name", + args=dict(name=resource.name), + ) + yield dict( + ph=_TYPE_METADATA, + pid=did, + tid=rid, + name="thread_sort_index", + args=dict(sort_index=rid), + ) + # TODO(sammccall): filtering and downsampling? + for event in self._proto.trace_events: + yield self._event(event) - def _event(self, event): - """Converts a TraceEvent proto into a catapult trace event python value.""" - result = dict( - pid=event.device_id, - tid=event.resource_id, - name=event.name, - ts=event.timestamp_ps / 1000000.0) - if event.duration_ps: - result['ph'] = _TYPE_COMPLETE - result['dur'] = event.duration_ps / 1000000.0 - else: - result['ph'] = _TYPE_INSTANT - result['s'] = _SCOPE_THREAD - for key in dict(event.args): - if 'args' not in result: - result['args'] = {} - result['args'][key] = event.args[key] - return result + def _event(self, event): + """Converts a TraceEvent proto into a catapult trace event python + value.""" + result = dict( + pid=event.device_id, + tid=event.resource_id, + name=event.name, + ts=event.timestamp_ps / 1000000.0, + ) + if event.duration_ps: + result["ph"] = _TYPE_COMPLETE + result["dur"] = event.duration_ps / 1000000.0 + else: + result["ph"] = _TYPE_INSTANT + result["s"] = _SCOPE_THREAD + for key in dict(event.args): + if "args" not in result: + result["args"] = {} + result["args"][key] = event.args[key] + return result - def __iter__(self): - """Returns an iterator of string chunks of a complete JSON document.""" - yield '{"displayTimeUnit":"ns","metadata":{"highres-ticks":true},\n' - yield '"traceEvents":[\n' - for event in self._events(): - yield json.dumps(event) - yield ',\n' - # Add one fake event to avoid dealing with no-trailing-comma rule. - yield '{}]}\n' + def __iter__(self): + """Returns an iterator of string chunks of a complete JSON document.""" + yield '{"displayTimeUnit":"ns","metadata":{"highres-ticks":true},\n' + yield '"traceEvents":[\n' + for event in self._events(): + yield json.dumps(event) + yield ",\n" + # Add one fake event to avoid dealing with no-trailing-comma rule. + yield "{}]}\n" diff --git a/tensorboard/plugins/profile/trace_events_json_test.py b/tensorboard/plugins/profile/trace_events_json_test.py index ca47e26261..1157a5c3c7 100644 --- a/tensorboard/plugins/profile/trace_events_json_test.py +++ b/tensorboard/plugins/profile/trace_events_json_test.py @@ -28,15 +28,17 @@ class TraceEventsJsonStreamTest(tf.test.TestCase): + def convert(self, proto_text): + proto = trace_events_pb2.Trace() + text_format.Merge(proto_text, proto) + return json.loads( + "".join(trace_events_json.TraceEventsJsonStream(proto)) + ) - def convert(self, proto_text): - proto = trace_events_pb2.Trace() - text_format.Merge(proto_text, proto) - return json.loads(''.join(trace_events_json.TraceEventsJsonStream(proto))) - - def testJsonConversion(self): - self.assertEqual( - self.convert(""" + def testJsonConversion(self): + self.assertEqual( + self.convert( + """ devices { key: 2 value { name: 'D2' device_id: 2 @@ -69,59 +71,73 @@ def testJsonConversion(self): name: "E2.2.1" timestamp_ps: 105000 } - """), - dict( - displayTimeUnit='ns', - metadata={'highres-ticks': True}, - traceEvents=[ - dict(ph='M', pid=1, name='process_name', args=dict(name='D1')), - dict( - ph='M', - pid=1, - name='process_sort_index', - args=dict(sort_index=1)), - dict( - ph='M', - pid=1, - tid=2, - name='thread_name', - args=dict(name='R1.2')), - dict( - ph='M', - pid=1, - tid=2, - name='thread_sort_index', - args=dict(sort_index=2)), - dict(ph='M', pid=2, name='process_name', args=dict(name='D2')), - dict( - ph='M', - pid=2, - name='process_sort_index', - args=dict(sort_index=2)), - dict( - ph='M', - pid=2, - tid=2, - name='thread_name', - args=dict(name='R2.2')), - dict( - ph='M', - pid=2, - tid=2, - name='thread_sort_index', - args=dict(sort_index=2)), - dict( - ph='X', - pid=1, - tid=2, - name='E1.2.1', - ts=0.1, - dur=0.01, - args=dict(label='E1.2.1', extra='extra info')), - dict(ph='i', pid=2, tid=2, name='E2.2.1', ts=0.105, s='t'), - {}, - ])) + """ + ), + dict( + displayTimeUnit="ns", + metadata={"highres-ticks": True}, + traceEvents=[ + dict( + ph="M", pid=1, name="process_name", args=dict(name="D1") + ), + dict( + ph="M", + pid=1, + name="process_sort_index", + args=dict(sort_index=1), + ), + dict( + ph="M", + pid=1, + tid=2, + name="thread_name", + args=dict(name="R1.2"), + ), + dict( + ph="M", + pid=1, + tid=2, + name="thread_sort_index", + args=dict(sort_index=2), + ), + dict( + ph="M", pid=2, name="process_name", args=dict(name="D2") + ), + dict( + ph="M", + pid=2, + name="process_sort_index", + args=dict(sort_index=2), + ), + dict( + ph="M", + pid=2, + tid=2, + name="thread_name", + args=dict(name="R2.2"), + ), + dict( + ph="M", + pid=2, + tid=2, + name="thread_sort_index", + args=dict(sort_index=2), + ), + dict( + ph="X", + pid=1, + tid=2, + name="E1.2.1", + ts=0.1, + dur=0.01, + args=dict(label="E1.2.1", extra="extra info"), + ), + dict(ph="i", pid=2, tid=2, name="E2.2.1", ts=0.105, s="t"), + {}, + ], + ), + ) -if __name__ == '__main__': - tf.test.main() +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/projector/__init__.py b/tensorboard/plugins/projector/__init__.py index 47e0008f56..af0c4e92b0 100644 --- a/tensorboard/plugins/projector/__init__.py +++ b/tensorboard/plugins/projector/__init__.py @@ -36,30 +36,30 @@ def visualize_embeddings(logdir, config): - """Stores a config file used by the embedding projector. + """Stores a config file used by the embedding projector. - Args: - logdir: Directory into which to store the config file, as a `str`. - For compatibility, can also be a `tf.compat.v1.summary.FileWriter` - object open at the desired logdir. - config: `tf.contrib.tensorboard.plugins.projector.ProjectorConfig` - proto that holds the configuration for the projector such as paths to - checkpoint files and metadata files for the embeddings. If - `config.model_checkpoint_path` is none, it defaults to the - `logdir` used by the summary_writer. + Args: + logdir: Directory into which to store the config file, as a `str`. + For compatibility, can also be a `tf.compat.v1.summary.FileWriter` + object open at the desired logdir. + config: `tf.contrib.tensorboard.plugins.projector.ProjectorConfig` + proto that holds the configuration for the projector such as paths to + checkpoint files and metadata files for the embeddings. If + `config.model_checkpoint_path` is none, it defaults to the + `logdir` used by the summary_writer. - Raises: - ValueError: If the summary writer does not have a `logdir`. - """ - # Convert from `tf.compat.v1.summary.FileWriter` if necessary. - logdir = getattr(logdir, 'get_logdir', lambda: logdir)() + Raises: + ValueError: If the summary writer does not have a `logdir`. + """ + # Convert from `tf.compat.v1.summary.FileWriter` if necessary. + logdir = getattr(logdir, "get_logdir", lambda: logdir)() - # Sanity checks. - if logdir is None: - raise ValueError('Expected logdir to be a path, but got None') + # Sanity checks. + if logdir is None: + raise ValueError("Expected logdir to be a path, but got None") - # Saving the config file in the logdir. - config_pbtxt = _text_format.MessageToString(config) - path = os.path.join(logdir, _projector_plugin.PROJECTOR_FILENAME) - with tf.io.gfile.GFile(path, 'w') as f: - f.write(config_pbtxt) + # Saving the config file in the logdir. + config_pbtxt = _text_format.MessageToString(config) + path = os.path.join(logdir, _projector_plugin.PROJECTOR_FILENAME) + with tf.io.gfile.GFile(path, "w") as f: + f.write(config_pbtxt) diff --git a/tensorboard/plugins/projector/projector_api_test.py b/tensorboard/plugins/projector/projector_api_test.py index 417c5839a0..e05f99b915 100644 --- a/tensorboard/plugins/projector/projector_api_test.py +++ b/tensorboard/plugins/projector/projector_api_test.py @@ -28,55 +28,58 @@ from tensorboard.plugins import projector from tensorboard.util import test_util -def create_dummy_config(): - return projector.ProjectorConfig( - model_checkpoint_path='test', - embeddings = [ - projector.EmbeddingInfo( - tensor_name='tensor1', - metadata_path='metadata1', - ), - ], - ) - -class ProjectorApiTest(tf.test.TestCase): - - def test_visualize_embeddings_with_logdir(self): - logdir = self.get_temp_dir() - config = create_dummy_config() - projector.visualize_embeddings(logdir, config) - - # Read the configurations from disk and make sure it matches the original. - with tf.io.gfile.GFile(os.path.join(logdir, 'projector_config.pbtxt')) as f: - config2 = projector.ProjectorConfig() - text_format.Parse(f.read(), config2) - - self.assertEqual(config, config2) - def test_visualize_embeddings_with_file_writer(self): - if tf.__version__ == "stub": - self.skipTest("Requires TensorFlow for FileWriter") - logdir = self.get_temp_dir() - config = create_dummy_config() - - with tf.compat.v1.Graph().as_default(): - with test_util.FileWriterCache.get(logdir) as writer: - projector.visualize_embeddings(writer, config) - - # Read the configurations from disk and make sure it matches the original. - with tf.io.gfile.GFile(os.path.join(logdir, 'projector_config.pbtxt')) as f: - config2 = projector.ProjectorConfig() - text_format.Parse(f.read(), config2) - - self.assertEqual(config, config2) - - def test_visualize_embeddings_no_logdir(self): - with six.assertRaisesRegex( - self, - ValueError, - "Expected logdir to be a path, but got None"): - projector.visualize_embeddings(None, create_dummy_config()) +def create_dummy_config(): + return projector.ProjectorConfig( + model_checkpoint_path="test", + embeddings=[ + projector.EmbeddingInfo( + tensor_name="tensor1", metadata_path="metadata1", + ), + ], + ) -if __name__ == '__main__': - tf.test.main() +class ProjectorApiTest(tf.test.TestCase): + def test_visualize_embeddings_with_logdir(self): + logdir = self.get_temp_dir() + config = create_dummy_config() + projector.visualize_embeddings(logdir, config) + + # Read the configurations from disk and make sure it matches the original. + with tf.io.gfile.GFile( + os.path.join(logdir, "projector_config.pbtxt") + ) as f: + config2 = projector.ProjectorConfig() + text_format.Parse(f.read(), config2) + + self.assertEqual(config, config2) + + def test_visualize_embeddings_with_file_writer(self): + if tf.__version__ == "stub": + self.skipTest("Requires TensorFlow for FileWriter") + logdir = self.get_temp_dir() + config = create_dummy_config() + + with tf.compat.v1.Graph().as_default(): + with test_util.FileWriterCache.get(logdir) as writer: + projector.visualize_embeddings(writer, config) + + # Read the configurations from disk and make sure it matches the original. + with tf.io.gfile.GFile( + os.path.join(logdir, "projector_config.pbtxt") + ) as f: + config2 = projector.ProjectorConfig() + text_format.Parse(f.read(), config2) + + self.assertEqual(config, config2) + + def test_visualize_embeddings_no_logdir(self): + with six.assertRaisesRegex( + self, ValueError, "Expected logdir to be a path, but got None" + ): + projector.visualize_embeddings(None, create_dummy_config()) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/projector/projector_plugin.py b/tensorboard/plugins/projector/projector_plugin.py index 11c36a4b64..a53a538fa4 100644 --- a/tensorboard/plugins/projector/projector_plugin.py +++ b/tensorboard/plugins/projector/projector_plugin.py @@ -42,653 +42,745 @@ logger = tb_logging.get_logger() # The prefix of routes provided by this plugin. -_PLUGIN_PREFIX_ROUTE = 'projector' +_PLUGIN_PREFIX_ROUTE = "projector" # FYI - the PROJECTOR_FILENAME is hardcoded in the visualize_embeddings # method in tf.contrib.tensorboard.plugins.projector module. # TODO(@decentralion): Fix duplication when we find a permanent home for the # projector module. -PROJECTOR_FILENAME = 'projector_config.pbtxt' -_PLUGIN_NAME = 'org_tensorflow_tensorboard_projector' -_PLUGINS_DIR = 'plugins' +PROJECTOR_FILENAME = "projector_config.pbtxt" +_PLUGIN_NAME = "org_tensorflow_tensorboard_projector" +_PLUGINS_DIR = "plugins" # Number of tensors in the LRU cache. _TENSOR_CACHE_CAPACITY = 1 # HTTP routes. -CONFIG_ROUTE = '/info' -TENSOR_ROUTE = '/tensor' -METADATA_ROUTE = '/metadata' -RUNS_ROUTE = '/runs' -BOOKMARKS_ROUTE = '/bookmarks' -SPRITE_IMAGE_ROUTE = '/sprite_image' +CONFIG_ROUTE = "/info" +TENSOR_ROUTE = "/tensor" +METADATA_ROUTE = "/metadata" +RUNS_ROUTE = "/runs" +BOOKMARKS_ROUTE = "/bookmarks" +SPRITE_IMAGE_ROUTE = "/sprite_image" _IMGHDR_TO_MIMETYPE = { - 'bmp': 'image/bmp', - 'gif': 'image/gif', - 'jpeg': 'image/jpeg', - 'png': 'image/png' + "bmp": "image/bmp", + "gif": "image/gif", + "jpeg": "image/jpeg", + "png": "image/png", } -_DEFAULT_IMAGE_MIMETYPE = 'application/octet-stream' +_DEFAULT_IMAGE_MIMETYPE = "application/octet-stream" class LRUCache(object): - """LRU cache. Used for storing the last used tensor.""" + """LRU cache. - def __init__(self, size): - if size < 1: - raise ValueError('The cache size must be >=1') - self._size = size - self._dict = collections.OrderedDict() - - def get(self, key): - try: - value = self._dict.pop(key) - self._dict[key] = value - return value - except KeyError: - return None - - def set(self, key, value): - if value is None: - raise ValueError('value must be != None') - try: - self._dict.pop(key) - except KeyError: - if len(self._dict) >= self._size: - self._dict.popitem(last=False) - self._dict[key] = value - - -class EmbeddingMetadata(object): - """Metadata container for an embedding. - - The metadata holds different columns with values used for visualization - (color by, label by) in the "Embeddings" tab in TensorBoard. - """ + Used for storing the last used tensor. + """ - def __init__(self, num_points): - """Constructs a metadata for an embedding of the specified size. + def __init__(self, size): + if size < 1: + raise ValueError("The cache size must be >=1") + self._size = size + self._dict = collections.OrderedDict() - Args: - num_points: Number of points in the embedding. - """ - self.num_points = num_points - self.column_names = [] - self.name_to_values = {} + def get(self, key): + try: + value = self._dict.pop(key) + self._dict[key] = value + return value + except KeyError: + return None + + def set(self, key, value): + if value is None: + raise ValueError("value must be != None") + try: + self._dict.pop(key) + except KeyError: + if len(self._dict) >= self._size: + self._dict.popitem(last=False) + self._dict[key] = value - def add_column(self, column_name, column_values): - """Adds a named column of metadata values. - Args: - column_name: Name of the column. - column_values: 1D array/list/iterable holding the column values. Must be - of length `num_points`. The i-th value corresponds to the i-th point. +class EmbeddingMetadata(object): + """Metadata container for an embedding. - Raises: - ValueError: If `column_values` is not 1D array, or of length `num_points`, - or the `name` is already used. + The metadata holds different columns with values used for + visualization (color by, label by) in the "Embeddings" tab in + TensorBoard. """ - # Sanity checks. - if isinstance(column_values, list) and isinstance(column_values[0], list): - raise ValueError('"column_values" must be a flat list, but we detected ' - 'that its first entry is a list') - - if isinstance(column_values, np.ndarray) and column_values.ndim != 1: - raise ValueError('"column_values" should be of rank 1, ' - 'but is of rank %d' % column_values.ndim) - if len(column_values) != self.num_points: - raise ValueError('"column_values" should be of length %d, but is of ' - 'length %d' % (self.num_points, len(column_values))) - if column_name in self.name_to_values: - raise ValueError('The column name "%s" is already used' % column_name) - self.column_names.append(column_name) - self.name_to_values[column_name] = column_values + def __init__(self, num_points): + """Constructs a metadata for an embedding of the specified size. + + Args: + num_points: Number of points in the embedding. + """ + self.num_points = num_points + self.column_names = [] + self.name_to_values = {} + + def add_column(self, column_name, column_values): + """Adds a named column of metadata values. + + Args: + column_name: Name of the column. + column_values: 1D array/list/iterable holding the column values. Must be + of length `num_points`. The i-th value corresponds to the i-th point. + + Raises: + ValueError: If `column_values` is not 1D array, or of length `num_points`, + or the `name` is already used. + """ + # Sanity checks. + if isinstance(column_values, list) and isinstance( + column_values[0], list + ): + raise ValueError( + '"column_values" must be a flat list, but we detected ' + "that its first entry is a list" + ) + + if isinstance(column_values, np.ndarray) and column_values.ndim != 1: + raise ValueError( + '"column_values" should be of rank 1, ' + "but is of rank %d" % column_values.ndim + ) + if len(column_values) != self.num_points: + raise ValueError( + '"column_values" should be of length %d, but is of ' + "length %d" % (self.num_points, len(column_values)) + ) + if column_name in self.name_to_values: + raise ValueError( + 'The column name "%s" is already used' % column_name + ) + + self.column_names.append(column_name) + self.name_to_values[column_name] = column_values def _read_tensor_tsv_file(fpath): - with tf.io.gfile.GFile(fpath, 'r') as f: - tensor = [] - for line in f: - line = line.rstrip('\n') - if line: - tensor.append(list(map(float, line.split('\t')))) - return np.array(tensor, dtype='float32') + with tf.io.gfile.GFile(fpath, "r") as f: + tensor = [] + for line in f: + line = line.rstrip("\n") + if line: + tensor.append(list(map(float, line.split("\t")))) + return np.array(tensor, dtype="float32") def _assets_dir_to_logdir(assets_dir): - sub_path = os.path.sep + _PLUGINS_DIR + os.path.sep - if sub_path in assets_dir: - two_parents_up = os.pardir + os.path.sep + os.pardir - return os.path.abspath(os.path.join(assets_dir, two_parents_up)) - return assets_dir + sub_path = os.path.sep + _PLUGINS_DIR + os.path.sep + if sub_path in assets_dir: + two_parents_up = os.pardir + os.path.sep + os.pardir + return os.path.abspath(os.path.join(assets_dir, two_parents_up)) + return assets_dir def _latest_checkpoints_changed(configs, run_path_pairs): - """Returns true if the latest checkpoint has changed in any of the runs.""" - for run_name, assets_dir in run_path_pairs: - if run_name not in configs: - config = ProjectorConfig() - config_fpath = os.path.join(assets_dir, PROJECTOR_FILENAME) - if tf.io.gfile.exists(config_fpath): - with tf.io.gfile.GFile(config_fpath, 'r') as f: - file_content = f.read() - text_format.Merge(file_content, config) - else: - config = configs[run_name] - - # See if you can find a checkpoint file in the logdir. - logdir = _assets_dir_to_logdir(assets_dir) - ckpt_path = _find_latest_checkpoint(logdir) - if not ckpt_path: - continue - if config.model_checkpoint_path != ckpt_path: - return True - return False + """Returns true if the latest checkpoint has changed in any of the runs.""" + for run_name, assets_dir in run_path_pairs: + if run_name not in configs: + config = ProjectorConfig() + config_fpath = os.path.join(assets_dir, PROJECTOR_FILENAME) + if tf.io.gfile.exists(config_fpath): + with tf.io.gfile.GFile(config_fpath, "r") as f: + file_content = f.read() + text_format.Merge(file_content, config) + else: + config = configs[run_name] + + # See if you can find a checkpoint file in the logdir. + logdir = _assets_dir_to_logdir(assets_dir) + ckpt_path = _find_latest_checkpoint(logdir) + if not ckpt_path: + continue + if config.model_checkpoint_path != ckpt_path: + return True + return False def _parse_positive_int_param(request, param_name): - """Parses and asserts a positive (>0) integer query parameter. - - Args: - request: The Werkzeug Request object - param_name: Name of the parameter. - - Returns: - Param, or None, or -1 if parameter is not a positive integer. - """ - param = request.args.get(param_name) - if not param: - return None - try: - param = int(param) - if param <= 0: - raise ValueError() - return param - except ValueError: - return -1 + """Parses and asserts a positive (>0) integer query parameter. + + Args: + request: The Werkzeug Request object + param_name: Name of the parameter. + + Returns: + Param, or None, or -1 if parameter is not a positive integer. + """ + param = request.args.get(param_name) + if not param: + return None + try: + param = int(param) + if param <= 0: + raise ValueError() + return param + except ValueError: + return -1 def _rel_to_abs_asset_path(fpath, config_fpath): - fpath = os.path.expanduser(fpath) - if not os.path.isabs(fpath): - return os.path.join(os.path.dirname(config_fpath), fpath) - return fpath + fpath = os.path.expanduser(fpath) + if not os.path.isabs(fpath): + return os.path.join(os.path.dirname(config_fpath), fpath) + return fpath def _using_tf(): - """Return true if we're not using the fake TF API stub implementation.""" - return tf.__version__ != 'stub' + """Return true if we're not using the fake TF API stub implementation.""" + return tf.__version__ != "stub" class ProjectorPlugin(base_plugin.TBPlugin): - """Embedding projector.""" - - plugin_name = _PLUGIN_PREFIX_ROUTE - - def __init__(self, context): - """Instantiates ProjectorPlugin via TensorBoard core. - - Args: - context: A base_plugin.TBContext instance. - """ - self.multiplexer = context.multiplexer - self.logdir = context.logdir - self._handlers = None - self.readers = {} - self.run_paths = None - self._configs = {} - self.old_num_run_paths = None - self.config_fpaths = None - self.tensor_cache = LRUCache(_TENSOR_CACHE_CAPACITY) - - # Whether the plugin is active (has meaningful data to process and serve). - # Once the plugin is deemed active, we no longer re-compute the value - # because doing so is potentially expensive. - self._is_active = False - - # The running thread that is currently determining whether the plugin is - # active. If such a thread exists, do not start a duplicate thread. - self._thread_for_determining_is_active = None - - if self.multiplexer: - self.run_paths = self.multiplexer.RunPaths() - - def get_plugin_apps(self): - self._handlers = { - RUNS_ROUTE: self._serve_runs, - CONFIG_ROUTE: self._serve_config, - TENSOR_ROUTE: self._serve_tensor, - METADATA_ROUTE: self._serve_metadata, - BOOKMARKS_ROUTE: self._serve_bookmarks, - SPRITE_IMAGE_ROUTE: self._serve_sprite_image, - '/index.js': - functools.partial( + """Embedding projector.""" + + plugin_name = _PLUGIN_PREFIX_ROUTE + + def __init__(self, context): + """Instantiates ProjectorPlugin via TensorBoard core. + + Args: + context: A base_plugin.TBContext instance. + """ + self.multiplexer = context.multiplexer + self.logdir = context.logdir + self._handlers = None + self.readers = {} + self.run_paths = None + self._configs = {} + self.old_num_run_paths = None + self.config_fpaths = None + self.tensor_cache = LRUCache(_TENSOR_CACHE_CAPACITY) + + # Whether the plugin is active (has meaningful data to process and serve). + # Once the plugin is deemed active, we no longer re-compute the value + # because doing so is potentially expensive. + self._is_active = False + + # The running thread that is currently determining whether the plugin is + # active. If such a thread exists, do not start a duplicate thread. + self._thread_for_determining_is_active = None + + if self.multiplexer: + self.run_paths = self.multiplexer.RunPaths() + + def get_plugin_apps(self): + self._handlers = { + RUNS_ROUTE: self._serve_runs, + CONFIG_ROUTE: self._serve_config, + TENSOR_ROUTE: self._serve_tensor, + METADATA_ROUTE: self._serve_metadata, + BOOKMARKS_ROUTE: self._serve_bookmarks, + SPRITE_IMAGE_ROUTE: self._serve_sprite_image, + "/index.js": functools.partial( self._serve_file, - os.path.join('tf_projector_plugin', 'index.js')), - '/projector_binary.html': - functools.partial( + os.path.join("tf_projector_plugin", "index.js"), + ), + "/projector_binary.html": functools.partial( self._serve_file, - os.path.join('tf_projector_plugin', 'projector_binary.html')), - '/projector_binary.js': - functools.partial( + os.path.join("tf_projector_plugin", "projector_binary.html"), + ), + "/projector_binary.js": functools.partial( self._serve_file, - os.path.join('tf_projector_plugin', 'projector_binary.js')), - } - return self._handlers - - def is_active(self): - """Determines whether this plugin is active. - - This plugin is only active if any run has an embedding. - - Returns: - Whether any run has embedding data to show in the projector. - """ - if not self.multiplexer: - return False - - if self._is_active: - # We have already determined that the projector plugin should be active. - # Do not re-compute that. We have no reason to later set this plugin to be - # inactive. - return True - - if self._thread_for_determining_is_active: - # We are currently determining whether the plugin is active. Do not start - # a separate thread. - return self._is_active - - # The plugin is currently not active. The frontend might check again later. - # For now, spin off a separate thread to determine whether the plugin is - # active. - new_thread = threading.Thread( - target=self._determine_is_active, - name='ProjectorPluginIsActiveThread') - self._thread_for_determining_is_active = new_thread - new_thread.start() - return False - - def frontend_metadata(self): - return base_plugin.FrontendMetadata( - es_module_path='/index.js', - disable_reload=True, - ) - - def _determine_is_active(self): - """Determines whether the plugin is active. - - This method is run in a separate thread so that the plugin can offer an - immediate response to whether it is active and determine whether it should - be active in a separate thread. - """ - if self.configs: - self._is_active = True - self._thread_for_determining_is_active = None - - @property - def configs(self): - """Returns a map of run paths to `ProjectorConfig` protos.""" - run_path_pairs = list(self.run_paths.items()) - self._append_plugin_asset_directories(run_path_pairs) - # If there are no summary event files, the projector should still work, - # treating the `logdir` as the model checkpoint directory. - if not run_path_pairs: - run_path_pairs.append(('.', self.logdir)) - if (self._run_paths_changed() or - _latest_checkpoints_changed(self._configs, run_path_pairs)): - self.readers = {} - self._configs, self.config_fpaths = self._read_latest_config_files( - run_path_pairs) - self._augment_configs_with_checkpoint_info() - return self._configs - - def _run_paths_changed(self): - num_run_paths = len(list(self.run_paths.keys())) - if num_run_paths != self.old_num_run_paths: - self.old_num_run_paths = num_run_paths - return True - return False - - def _augment_configs_with_checkpoint_info(self): - for run, config in self._configs.items(): - for embedding in config.embeddings: - # Normalize the name of the embeddings. - if embedding.tensor_name.endswith(':0'): - embedding.tensor_name = embedding.tensor_name[:-2] - # Find the size of embeddings associated with a tensors file. - if embedding.tensor_path and not embedding.tensor_shape: - fpath = _rel_to_abs_asset_path(embedding.tensor_path, - self.config_fpaths[run]) - tensor = self.tensor_cache.get((run, embedding.tensor_name)) - if tensor is None: - tensor = _read_tensor_tsv_file(fpath) - self.tensor_cache.set((run, embedding.tensor_name), tensor) - embedding.tensor_shape.extend([len(tensor), len(tensor[0])]) - - reader = self._get_reader_for_run(run) - if not reader: - continue - # Augment the configuration with the tensors in the checkpoint file. - special_embedding = None - if config.embeddings and not config.embeddings[0].tensor_name: - special_embedding = config.embeddings[0] - config.embeddings.remove(special_embedding) - var_map = reader.get_variable_to_shape_map() - for tensor_name, tensor_shape in var_map.items(): - if len(tensor_shape) != 2: - continue - embedding = self._get_embedding(tensor_name, config) - if not embedding: - embedding = config.embeddings.add() - embedding.tensor_name = tensor_name - if special_embedding: - embedding.metadata_path = special_embedding.metadata_path - embedding.bookmarks_path = special_embedding.bookmarks_path - if not embedding.tensor_shape: - embedding.tensor_shape.extend(tensor_shape) - - # Remove configs that do not have any valid (2D) tensors. - runs_to_remove = [] - for run, config in self._configs.items(): - if not config.embeddings: - runs_to_remove.append(run) - for run in runs_to_remove: - del self._configs[run] - del self.config_fpaths[run] - - def _read_latest_config_files(self, run_path_pairs): - """Reads and returns the projector config files in every run directory.""" - configs = {} - config_fpaths = {} - for run_name, assets_dir in run_path_pairs: - config = ProjectorConfig() - config_fpath = os.path.join(assets_dir, PROJECTOR_FILENAME) - if tf.io.gfile.exists(config_fpath): - with tf.io.gfile.GFile(config_fpath, 'r') as f: - file_content = f.read() - text_format.Merge(file_content, config) - has_tensor_files = False - for embedding in config.embeddings: - if embedding.tensor_path: - if not embedding.tensor_name: - embedding.tensor_name = os.path.basename(embedding.tensor_path) - has_tensor_files = True - break - - if not config.model_checkpoint_path: - # See if you can find a checkpoint file in the logdir. - logdir = _assets_dir_to_logdir(assets_dir) - ckpt_path = _find_latest_checkpoint(logdir) - if not ckpt_path and not has_tensor_files: - continue - if ckpt_path: - config.model_checkpoint_path = ckpt_path - - # Sanity check for the checkpoint file existing. - if (config.model_checkpoint_path and _using_tf() and - not tf.io.gfile.glob(config.model_checkpoint_path + '*')): - logger.warn('Checkpoint file "%s" not found', - config.model_checkpoint_path) - continue - configs[run_name] = config - config_fpaths[run_name] = config_fpath - return configs, config_fpaths - - def _get_reader_for_run(self, run): - if run in self.readers: - return self.readers[run] - - config = self._configs[run] - reader = None - if config.model_checkpoint_path and _using_tf(): - try: - reader = tf.train.load_checkpoint(config.model_checkpoint_path) - except Exception: # pylint: disable=broad-except - logger.warn('Failed reading "%s"', config.model_checkpoint_path) - self.readers[run] = reader - return reader - - def _get_metadata_file_for_tensor(self, tensor_name, config): - embedding_info = self._get_embedding(tensor_name, config) - if embedding_info: - return embedding_info.metadata_path - return None - - def _get_bookmarks_file_for_tensor(self, tensor_name, config): - embedding_info = self._get_embedding(tensor_name, config) - if embedding_info: - return embedding_info.bookmarks_path - return None - - def _canonical_tensor_name(self, tensor_name): - if ':' not in tensor_name: - return tensor_name + ':0' - else: - return tensor_name - - def _get_embedding(self, tensor_name, config): - if not config.embeddings: - return None - for info in config.embeddings: - if (self._canonical_tensor_name(info.tensor_name) == - self._canonical_tensor_name(tensor_name)): - return info - return None - - def _append_plugin_asset_directories(self, run_path_pairs): - for run, assets in self.multiplexer.PluginAssets(_PLUGIN_NAME).items(): - if PROJECTOR_FILENAME not in assets: - continue - assets_dir = os.path.join(self.run_paths[run], _PLUGINS_DIR, _PLUGIN_NAME) - assets_path_pair = (run, os.path.abspath(assets_dir)) - run_path_pairs.append(assets_path_pair) - - @wrappers.Request.application - def _serve_file(self, file_path, request): - """Returns a resource file.""" - res_path = os.path.join(os.path.dirname(__file__), file_path) - with open(res_path, 'rb') as read_file: - mimetype = mimetypes.guess_type(file_path)[0] - return Respond(request, read_file.read(), content_type=mimetype) - - @wrappers.Request.application - def _serve_runs(self, request): - """Returns a list of runs that have embeddings.""" - return Respond(request, list(self.configs.keys()), 'application/json') - - @wrappers.Request.application - def _serve_config(self, request): - run = request.args.get('run') - if run is None: - return Respond(request, 'query parameter "run" is required', 'text/plain', - 400) - if run not in self.configs: - return Respond(request, 'Unknown run: "%s"' % run, 'text/plain', 400) - - config = self.configs[run] - return Respond(request, - json_format.MessageToJson(config), 'application/json') - - @wrappers.Request.application - def _serve_metadata(self, request): - run = request.args.get('run') - if run is None: - return Respond(request, 'query parameter "run" is required', 'text/plain', - 400) - - name = request.args.get('name') - if name is None: - return Respond(request, 'query parameter "name" is required', - 'text/plain', 400) - - num_rows = _parse_positive_int_param(request, 'num_rows') - if num_rows == -1: - return Respond(request, 'query parameter num_rows must be integer > 0', - 'text/plain', 400) - - if run not in self.configs: - return Respond(request, 'Unknown run: "%s"' % run, 'text/plain', 400) - - config = self.configs[run] - fpath = self._get_metadata_file_for_tensor(name, config) - if not fpath: - return Respond( - request, - 'No metadata file found for tensor "%s" in the config file "%s"' % - (name, self.config_fpaths[run]), 'text/plain', 400) - fpath = _rel_to_abs_asset_path(fpath, self.config_fpaths[run]) - if not tf.io.gfile.exists(fpath) or tf.io.gfile.isdir(fpath): - return Respond(request, '"%s" not found, or is not a file' % fpath, - 'text/plain', 400) - - num_header_rows = 0 - with tf.io.gfile.GFile(fpath, 'r') as f: - lines = [] - # Stream reading the file with early break in case the file doesn't fit in - # memory. - for line in f: - lines.append(line) - if len(lines) == 1 and '\t' in lines[0]: - num_header_rows = 1 - if num_rows and len(lines) >= num_rows + num_header_rows: - break - return Respond(request, ''.join(lines), 'text/plain') - - @wrappers.Request.application - def _serve_tensor(self, request): - run = request.args.get('run') - if run is None: - return Respond(request, 'query parameter "run" is required', 'text/plain', - 400) - - name = request.args.get('name') - if name is None: - return Respond(request, 'query parameter "name" is required', - 'text/plain', 400) - - num_rows = _parse_positive_int_param(request, 'num_rows') - if num_rows == -1: - return Respond(request, 'query parameter num_rows must be integer > 0', - 'text/plain', 400) - - if run not in self.configs: - return Respond(request, 'Unknown run: "%s"' % run, 'text/plain', 400) - - config = self.configs[run] - - tensor = self.tensor_cache.get((run, name)) - if tensor is None: - # See if there is a tensor file in the config. - embedding = self._get_embedding(name, config) - - if embedding and embedding.tensor_path: - fpath = _rel_to_abs_asset_path(embedding.tensor_path, - self.config_fpaths[run]) - if not tf.io.gfile.exists(fpath): - return Respond(request, - 'Tensor file "%s" does not exist' % fpath, - 'text/plain', 400) - tensor = _read_tensor_tsv_file(fpath) - else: - reader = self._get_reader_for_run(run) - if not reader or not reader.has_tensor(name): - return Respond(request, - 'Tensor "%s" not found in checkpoint dir "%s"' % - (name, config.model_checkpoint_path), 'text/plain', - 400) - try: - tensor = reader.get_tensor(name) - except tf.errors.InvalidArgumentError as e: - return Respond(request, str(e), 'text/plain', 400) - - self.tensor_cache.set((run, name), tensor) - - if num_rows: - tensor = tensor[:num_rows] - if tensor.dtype != 'float32': - tensor = tensor.astype(dtype='float32', copy=False) - data_bytes = tensor.tobytes() - return Respond(request, data_bytes, 'application/octet-stream') - - @wrappers.Request.application - def _serve_bookmarks(self, request): - run = request.args.get('run') - if not run: - return Respond(request, 'query parameter "run" is required', 'text/plain', - 400) - - name = request.args.get('name') - if name is None: - return Respond(request, 'query parameter "name" is required', - 'text/plain', 400) - - if run not in self.configs: - return Respond(request, 'Unknown run: "%s"' % run, 'text/plain', 400) - - config = self.configs[run] - fpath = self._get_bookmarks_file_for_tensor(name, config) - if not fpath: - return Respond( - request, - 'No bookmarks file found for tensor "%s" in the config file "%s"' % - (name, self.config_fpaths[run]), 'text/plain', 400) - fpath = _rel_to_abs_asset_path(fpath, self.config_fpaths[run]) - if not tf.io.gfile.exists(fpath) or tf.io.gfile.isdir(fpath): - return Respond(request, '"%s" not found, or is not a file' % fpath, - 'text/plain', 400) - - bookmarks_json = None - with tf.io.gfile.GFile(fpath, 'rb') as f: - bookmarks_json = f.read() - return Respond(request, bookmarks_json, 'application/json') - - @wrappers.Request.application - def _serve_sprite_image(self, request): - run = request.args.get('run') - if not run: - return Respond(request, 'query parameter "run" is required', 'text/plain', - 400) - - name = request.args.get('name') - if name is None: - return Respond(request, 'query parameter "name" is required', - 'text/plain', 400) - - if run not in self.configs: - return Respond(request, 'Unknown run: "%s"' % run, 'text/plain', 400) - - config = self.configs[run] - embedding_info = self._get_embedding(name, config) - - if not embedding_info or not embedding_info.sprite.image_path: - return Respond( - request, - 'No sprite image file found for tensor "%s" in the config file "%s"' % - (name, self.config_fpaths[run]), 'text/plain', 400) - - fpath = os.path.expanduser(embedding_info.sprite.image_path) - fpath = _rel_to_abs_asset_path(fpath, self.config_fpaths[run]) - if not tf.io.gfile.exists(fpath) or tf.io.gfile.isdir(fpath): - return Respond(request, '"%s" does not exist or is directory' % fpath, - 'text/plain', 400) - f = tf.io.gfile.GFile(fpath, 'rb') - encoded_image_string = f.read() - f.close() - image_type = imghdr.what(None, encoded_image_string) - mime_type = _IMGHDR_TO_MIMETYPE.get(image_type, _DEFAULT_IMAGE_MIMETYPE) - return Respond(request, encoded_image_string, mime_type) + os.path.join("tf_projector_plugin", "projector_binary.js"), + ), + } + return self._handlers + + def is_active(self): + """Determines whether this plugin is active. + + This plugin is only active if any run has an embedding. + + Returns: + Whether any run has embedding data to show in the projector. + """ + if not self.multiplexer: + return False + + if self._is_active: + # We have already determined that the projector plugin should be active. + # Do not re-compute that. We have no reason to later set this plugin to be + # inactive. + return True + + if self._thread_for_determining_is_active: + # We are currently determining whether the plugin is active. Do not start + # a separate thread. + return self._is_active + + # The plugin is currently not active. The frontend might check again later. + # For now, spin off a separate thread to determine whether the plugin is + # active. + new_thread = threading.Thread( + target=self._determine_is_active, + name="ProjectorPluginIsActiveThread", + ) + self._thread_for_determining_is_active = new_thread + new_thread.start() + return False + + def frontend_metadata(self): + return base_plugin.FrontendMetadata( + es_module_path="/index.js", disable_reload=True, + ) + + def _determine_is_active(self): + """Determines whether the plugin is active. + + This method is run in a separate thread so that the plugin can + offer an immediate response to whether it is active and + determine whether it should be active in a separate thread. + """ + if self.configs: + self._is_active = True + self._thread_for_determining_is_active = None + + @property + def configs(self): + """Returns a map of run paths to `ProjectorConfig` protos.""" + run_path_pairs = list(self.run_paths.items()) + self._append_plugin_asset_directories(run_path_pairs) + # If there are no summary event files, the projector should still work, + # treating the `logdir` as the model checkpoint directory. + if not run_path_pairs: + run_path_pairs.append((".", self.logdir)) + if self._run_paths_changed() or _latest_checkpoints_changed( + self._configs, run_path_pairs + ): + self.readers = {} + self._configs, self.config_fpaths = self._read_latest_config_files( + run_path_pairs + ) + self._augment_configs_with_checkpoint_info() + return self._configs + + def _run_paths_changed(self): + num_run_paths = len(list(self.run_paths.keys())) + if num_run_paths != self.old_num_run_paths: + self.old_num_run_paths = num_run_paths + return True + return False + + def _augment_configs_with_checkpoint_info(self): + for run, config in self._configs.items(): + for embedding in config.embeddings: + # Normalize the name of the embeddings. + if embedding.tensor_name.endswith(":0"): + embedding.tensor_name = embedding.tensor_name[:-2] + # Find the size of embeddings associated with a tensors file. + if embedding.tensor_path and not embedding.tensor_shape: + fpath = _rel_to_abs_asset_path( + embedding.tensor_path, self.config_fpaths[run] + ) + tensor = self.tensor_cache.get((run, embedding.tensor_name)) + if tensor is None: + tensor = _read_tensor_tsv_file(fpath) + self.tensor_cache.set( + (run, embedding.tensor_name), tensor + ) + embedding.tensor_shape.extend([len(tensor), len(tensor[0])]) + + reader = self._get_reader_for_run(run) + if not reader: + continue + # Augment the configuration with the tensors in the checkpoint file. + special_embedding = None + if config.embeddings and not config.embeddings[0].tensor_name: + special_embedding = config.embeddings[0] + config.embeddings.remove(special_embedding) + var_map = reader.get_variable_to_shape_map() + for tensor_name, tensor_shape in var_map.items(): + if len(tensor_shape) != 2: + continue + embedding = self._get_embedding(tensor_name, config) + if not embedding: + embedding = config.embeddings.add() + embedding.tensor_name = tensor_name + if special_embedding: + embedding.metadata_path = ( + special_embedding.metadata_path + ) + embedding.bookmarks_path = ( + special_embedding.bookmarks_path + ) + if not embedding.tensor_shape: + embedding.tensor_shape.extend(tensor_shape) + + # Remove configs that do not have any valid (2D) tensors. + runs_to_remove = [] + for run, config in self._configs.items(): + if not config.embeddings: + runs_to_remove.append(run) + for run in runs_to_remove: + del self._configs[run] + del self.config_fpaths[run] + + def _read_latest_config_files(self, run_path_pairs): + """Reads and returns the projector config files in every run + directory.""" + configs = {} + config_fpaths = {} + for run_name, assets_dir in run_path_pairs: + config = ProjectorConfig() + config_fpath = os.path.join(assets_dir, PROJECTOR_FILENAME) + if tf.io.gfile.exists(config_fpath): + with tf.io.gfile.GFile(config_fpath, "r") as f: + file_content = f.read() + text_format.Merge(file_content, config) + has_tensor_files = False + for embedding in config.embeddings: + if embedding.tensor_path: + if not embedding.tensor_name: + embedding.tensor_name = os.path.basename( + embedding.tensor_path + ) + has_tensor_files = True + break + + if not config.model_checkpoint_path: + # See if you can find a checkpoint file in the logdir. + logdir = _assets_dir_to_logdir(assets_dir) + ckpt_path = _find_latest_checkpoint(logdir) + if not ckpt_path and not has_tensor_files: + continue + if ckpt_path: + config.model_checkpoint_path = ckpt_path + + # Sanity check for the checkpoint file existing. + if ( + config.model_checkpoint_path + and _using_tf() + and not tf.io.gfile.glob(config.model_checkpoint_path + "*") + ): + logger.warn( + 'Checkpoint file "%s" not found', + config.model_checkpoint_path, + ) + continue + configs[run_name] = config + config_fpaths[run_name] = config_fpath + return configs, config_fpaths + + def _get_reader_for_run(self, run): + if run in self.readers: + return self.readers[run] + + config = self._configs[run] + reader = None + if config.model_checkpoint_path and _using_tf(): + try: + reader = tf.train.load_checkpoint(config.model_checkpoint_path) + except Exception: # pylint: disable=broad-except + logger.warn('Failed reading "%s"', config.model_checkpoint_path) + self.readers[run] = reader + return reader + + def _get_metadata_file_for_tensor(self, tensor_name, config): + embedding_info = self._get_embedding(tensor_name, config) + if embedding_info: + return embedding_info.metadata_path + return None + + def _get_bookmarks_file_for_tensor(self, tensor_name, config): + embedding_info = self._get_embedding(tensor_name, config) + if embedding_info: + return embedding_info.bookmarks_path + return None + + def _canonical_tensor_name(self, tensor_name): + if ":" not in tensor_name: + return tensor_name + ":0" + else: + return tensor_name + + def _get_embedding(self, tensor_name, config): + if not config.embeddings: + return None + for info in config.embeddings: + if self._canonical_tensor_name( + info.tensor_name + ) == self._canonical_tensor_name(tensor_name): + return info + return None + + def _append_plugin_asset_directories(self, run_path_pairs): + for run, assets in self.multiplexer.PluginAssets(_PLUGIN_NAME).items(): + if PROJECTOR_FILENAME not in assets: + continue + assets_dir = os.path.join( + self.run_paths[run], _PLUGINS_DIR, _PLUGIN_NAME + ) + assets_path_pair = (run, os.path.abspath(assets_dir)) + run_path_pairs.append(assets_path_pair) + + @wrappers.Request.application + def _serve_file(self, file_path, request): + """Returns a resource file.""" + res_path = os.path.join(os.path.dirname(__file__), file_path) + with open(res_path, "rb") as read_file: + mimetype = mimetypes.guess_type(file_path)[0] + return Respond(request, read_file.read(), content_type=mimetype) + + @wrappers.Request.application + def _serve_runs(self, request): + """Returns a list of runs that have embeddings.""" + return Respond(request, list(self.configs.keys()), "application/json") + + @wrappers.Request.application + def _serve_config(self, request): + run = request.args.get("run") + if run is None: + return Respond( + request, 'query parameter "run" is required', "text/plain", 400 + ) + if run not in self.configs: + return Respond( + request, 'Unknown run: "%s"' % run, "text/plain", 400 + ) + + config = self.configs[run] + return Respond( + request, json_format.MessageToJson(config), "application/json" + ) + + @wrappers.Request.application + def _serve_metadata(self, request): + run = request.args.get("run") + if run is None: + return Respond( + request, 'query parameter "run" is required', "text/plain", 400 + ) + + name = request.args.get("name") + if name is None: + return Respond( + request, 'query parameter "name" is required', "text/plain", 400 + ) + + num_rows = _parse_positive_int_param(request, "num_rows") + if num_rows == -1: + return Respond( + request, + "query parameter num_rows must be integer > 0", + "text/plain", + 400, + ) + + if run not in self.configs: + return Respond( + request, 'Unknown run: "%s"' % run, "text/plain", 400 + ) + + config = self.configs[run] + fpath = self._get_metadata_file_for_tensor(name, config) + if not fpath: + return Respond( + request, + 'No metadata file found for tensor "%s" in the config file "%s"' + % (name, self.config_fpaths[run]), + "text/plain", + 400, + ) + fpath = _rel_to_abs_asset_path(fpath, self.config_fpaths[run]) + if not tf.io.gfile.exists(fpath) or tf.io.gfile.isdir(fpath): + return Respond( + request, + '"%s" not found, or is not a file' % fpath, + "text/plain", + 400, + ) + + num_header_rows = 0 + with tf.io.gfile.GFile(fpath, "r") as f: + lines = [] + # Stream reading the file with early break in case the file doesn't fit in + # memory. + for line in f: + lines.append(line) + if len(lines) == 1 and "\t" in lines[0]: + num_header_rows = 1 + if num_rows and len(lines) >= num_rows + num_header_rows: + break + return Respond(request, "".join(lines), "text/plain") + + @wrappers.Request.application + def _serve_tensor(self, request): + run = request.args.get("run") + if run is None: + return Respond( + request, 'query parameter "run" is required', "text/plain", 400 + ) + + name = request.args.get("name") + if name is None: + return Respond( + request, 'query parameter "name" is required', "text/plain", 400 + ) + + num_rows = _parse_positive_int_param(request, "num_rows") + if num_rows == -1: + return Respond( + request, + "query parameter num_rows must be integer > 0", + "text/plain", + 400, + ) + + if run not in self.configs: + return Respond( + request, 'Unknown run: "%s"' % run, "text/plain", 400 + ) + + config = self.configs[run] + + tensor = self.tensor_cache.get((run, name)) + if tensor is None: + # See if there is a tensor file in the config. + embedding = self._get_embedding(name, config) + + if embedding and embedding.tensor_path: + fpath = _rel_to_abs_asset_path( + embedding.tensor_path, self.config_fpaths[run] + ) + if not tf.io.gfile.exists(fpath): + return Respond( + request, + 'Tensor file "%s" does not exist' % fpath, + "text/plain", + 400, + ) + tensor = _read_tensor_tsv_file(fpath) + else: + reader = self._get_reader_for_run(run) + if not reader or not reader.has_tensor(name): + return Respond( + request, + 'Tensor "%s" not found in checkpoint dir "%s"' + % (name, config.model_checkpoint_path), + "text/plain", + 400, + ) + try: + tensor = reader.get_tensor(name) + except tf.errors.InvalidArgumentError as e: + return Respond(request, str(e), "text/plain", 400) + + self.tensor_cache.set((run, name), tensor) + + if num_rows: + tensor = tensor[:num_rows] + if tensor.dtype != "float32": + tensor = tensor.astype(dtype="float32", copy=False) + data_bytes = tensor.tobytes() + return Respond(request, data_bytes, "application/octet-stream") + + @wrappers.Request.application + def _serve_bookmarks(self, request): + run = request.args.get("run") + if not run: + return Respond( + request, 'query parameter "run" is required', "text/plain", 400 + ) + + name = request.args.get("name") + if name is None: + return Respond( + request, 'query parameter "name" is required', "text/plain", 400 + ) + + if run not in self.configs: + return Respond( + request, 'Unknown run: "%s"' % run, "text/plain", 400 + ) + + config = self.configs[run] + fpath = self._get_bookmarks_file_for_tensor(name, config) + if not fpath: + return Respond( + request, + 'No bookmarks file found for tensor "%s" in the config file "%s"' + % (name, self.config_fpaths[run]), + "text/plain", + 400, + ) + fpath = _rel_to_abs_asset_path(fpath, self.config_fpaths[run]) + if not tf.io.gfile.exists(fpath) or tf.io.gfile.isdir(fpath): + return Respond( + request, + '"%s" not found, or is not a file' % fpath, + "text/plain", + 400, + ) + + bookmarks_json = None + with tf.io.gfile.GFile(fpath, "rb") as f: + bookmarks_json = f.read() + return Respond(request, bookmarks_json, "application/json") + + @wrappers.Request.application + def _serve_sprite_image(self, request): + run = request.args.get("run") + if not run: + return Respond( + request, 'query parameter "run" is required', "text/plain", 400 + ) + + name = request.args.get("name") + if name is None: + return Respond( + request, 'query parameter "name" is required', "text/plain", 400 + ) + + if run not in self.configs: + return Respond( + request, 'Unknown run: "%s"' % run, "text/plain", 400 + ) + + config = self.configs[run] + embedding_info = self._get_embedding(name, config) + + if not embedding_info or not embedding_info.sprite.image_path: + return Respond( + request, + 'No sprite image file found for tensor "%s" in the config file "%s"' + % (name, self.config_fpaths[run]), + "text/plain", + 400, + ) + + fpath = os.path.expanduser(embedding_info.sprite.image_path) + fpath = _rel_to_abs_asset_path(fpath, self.config_fpaths[run]) + if not tf.io.gfile.exists(fpath) or tf.io.gfile.isdir(fpath): + return Respond( + request, + '"%s" does not exist or is directory' % fpath, + "text/plain", + 400, + ) + f = tf.io.gfile.GFile(fpath, "rb") + encoded_image_string = f.read() + f.close() + image_type = imghdr.what(None, encoded_image_string) + mime_type = _IMGHDR_TO_MIMETYPE.get(image_type, _DEFAULT_IMAGE_MIMETYPE) + return Respond(request, encoded_image_string, mime_type) def _find_latest_checkpoint(dir_path): - if not _using_tf(): - return None - try: - ckpt_path = tf.train.latest_checkpoint(dir_path) - if not ckpt_path: - # Check the parent directory. - ckpt_path = tf.train.latest_checkpoint(os.path.join(dir_path, os.pardir)) - return ckpt_path - except tf.errors.NotFoundError: - return None + if not _using_tf(): + return None + try: + ckpt_path = tf.train.latest_checkpoint(dir_path) + if not ckpt_path: + # Check the parent directory. + ckpt_path = tf.train.latest_checkpoint( + os.path.join(dir_path, os.pardir) + ) + return ckpt_path + except tf.errors.NotFoundError: + return None diff --git a/tensorboard/plugins/projector/projector_plugin_test.py b/tensorboard/plugins/projector/projector_plugin_test.py index 4907bd6144..a6b6932353 100644 --- a/tensorboard/plugins/projector/projector_plugin_test.py +++ b/tensorboard/plugins/projector/projector_plugin_test.py @@ -33,7 +33,9 @@ from google.protobuf import text_format from tensorboard.backend import application -from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer +from tensorboard.backend.event_processing import ( + plugin_event_multiplexer as event_multiplexer, +) from tensorboard.compat.proto import event_pb2 from tensorboard.compat.proto import summary_pb2 from tensorboard.compat import tf as tf_compat @@ -44,378 +46,393 @@ tf.compat.v1.disable_v2_behavior() -USING_REAL_TF = tf_compat.__version__ != 'stub' +USING_REAL_TF = tf_compat.__version__ != "stub" class ProjectorAppTest(tf.test.TestCase): - - def __init__(self, *args, **kwargs): - super(ProjectorAppTest, self).__init__(*args, **kwargs) - self.logdir = None - self.plugin = None - self.server = None - - def setUp(self): - self.log_dir = self.get_temp_dir() - - def testRunsWithValidCheckpoint(self): - self._GenerateProjectorTestData() - self._SetupWSGIApp() - run_json = self._GetJson('/data/plugin/projector/runs') - if USING_REAL_TF: + def __init__(self, *args, **kwargs): + super(ProjectorAppTest, self).__init__(*args, **kwargs) + self.logdir = None + self.plugin = None + self.server = None + + def setUp(self): + self.log_dir = self.get_temp_dir() + + def testRunsWithValidCheckpoint(self): + self._GenerateProjectorTestData() + self._SetupWSGIApp() + run_json = self._GetJson("/data/plugin/projector/runs") + if USING_REAL_TF: + self.assertTrue(run_json) + else: + self.assertFalse(run_json) + + def testRunsWithNoCheckpoint(self): + self._SetupWSGIApp() + run_json = self._GetJson("/data/plugin/projector/runs") + self.assertEqual(run_json, []) + + def testRunsWithInvalidModelCheckpointPath(self): + checkpoint_file = os.path.join(self.log_dir, "checkpoint") + f = open(checkpoint_file, "w") + f.write('model_checkpoint_path: "does_not_exist"\n') + f.write('all_model_checkpoint_paths: "does_not_exist"\n') + f.close() + self._SetupWSGIApp() + + run_json = self._GetJson("/data/plugin/projector/runs") + self.assertEqual(run_json, []) + + # TODO(#2007): Cleanly separate out projector tests that require real TF + @unittest.skipUnless(USING_REAL_TF, "Test only passes when using real TF") + def testRunsWithInvalidModelCheckpointPathInConfig(self): + config_path = os.path.join(self.log_dir, "projector_config.pbtxt") + config = projector_config_pb2.ProjectorConfig() + config.model_checkpoint_path = "does_not_exist" + embedding = config.embeddings.add() + embedding.tensor_name = "var1" + with tf.io.gfile.GFile(config_path, "w") as f: + f.write(text_format.MessageToString(config)) + self._SetupWSGIApp() + + run_json = self._GetJson("/data/plugin/projector/runs") + self.assertEqual(run_json, []) + + # TODO(#2007): Cleanly separate out projector tests that require real TF + @unittest.skipUnless(USING_REAL_TF, "Test only passes when using real TF") + def testInfoWithValidCheckpointNoEventsData(self): + self._GenerateProjectorTestData() + self._SetupWSGIApp() + + info_json = self._GetJson("/data/plugin/projector/info?run=.") + self.assertItemsEqual( + info_json["embeddings"], + [ + { + "tensorShape": [1, 2], + "tensorName": "var1", + "bookmarksPath": "bookmarks.json", + }, + {"tensorShape": [10, 10], "tensorName": "var2"}, + {"tensorShape": [100, 100], "tensorName": "var3"}, + ], + ) + + # TODO(#2007): Cleanly separate out projector tests that require real TF + @unittest.skipUnless(USING_REAL_TF, "Test only passes when using real TF") + def testInfoWithValidCheckpointAndEventsData(self): + self._GenerateProjectorTestData() + self._GenerateEventsData() + self._SetupWSGIApp() + + run_json = self._GetJson("/data/plugin/projector/runs") self.assertTrue(run_json) - else: - self.assertFalse(run_json) - - def testRunsWithNoCheckpoint(self): - self._SetupWSGIApp() - run_json = self._GetJson('/data/plugin/projector/runs') - self.assertEqual(run_json, []) - - def testRunsWithInvalidModelCheckpointPath(self): - checkpoint_file = os.path.join(self.log_dir, 'checkpoint') - f = open(checkpoint_file, 'w') - f.write('model_checkpoint_path: "does_not_exist"\n') - f.write('all_model_checkpoint_paths: "does_not_exist"\n') - f.close() - self._SetupWSGIApp() - - run_json = self._GetJson('/data/plugin/projector/runs') - self.assertEqual(run_json, []) - - # TODO(#2007): Cleanly separate out projector tests that require real TF - @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF') - def testRunsWithInvalidModelCheckpointPathInConfig(self): - config_path = os.path.join(self.log_dir, 'projector_config.pbtxt') - config = projector_config_pb2.ProjectorConfig() - config.model_checkpoint_path = 'does_not_exist' - embedding = config.embeddings.add() - embedding.tensor_name = 'var1' - with tf.io.gfile.GFile(config_path, 'w') as f: - f.write(text_format.MessageToString(config)) - self._SetupWSGIApp() - - run_json = self._GetJson('/data/plugin/projector/runs') - self.assertEqual(run_json, []) - - # TODO(#2007): Cleanly separate out projector tests that require real TF - @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF') - def testInfoWithValidCheckpointNoEventsData(self): - self._GenerateProjectorTestData() - self._SetupWSGIApp() - - info_json = self._GetJson('/data/plugin/projector/info?run=.') - self.assertItemsEqual(info_json['embeddings'], [{ - 'tensorShape': [1, 2], - 'tensorName': 'var1', - 'bookmarksPath': 'bookmarks.json' - }, { - 'tensorShape': [10, 10], - 'tensorName': 'var2' - }, { - 'tensorShape': [100, 100], - 'tensorName': 'var3' - }]) - - # TODO(#2007): Cleanly separate out projector tests that require real TF - @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF') - def testInfoWithValidCheckpointAndEventsData(self): - self._GenerateProjectorTestData() - self._GenerateEventsData() - self._SetupWSGIApp() - - run_json = self._GetJson('/data/plugin/projector/runs') - self.assertTrue(run_json) - run = run_json[0] - info_json = self._GetJson('/data/plugin/projector/info?run=%s' % run) - self.assertItemsEqual(info_json['embeddings'], [{ - 'tensorShape': [1, 2], - 'tensorName': 'var1', - 'bookmarksPath': 'bookmarks.json' - }, { - 'tensorShape': [10, 10], - 'tensorName': 'var2' - }, { - 'tensorShape': [100, 100], - 'tensorName': 'var3' - }]) - - # TODO(#2007): Cleanly separate out projector tests that require real TF - @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF') - def testTensorWithValidCheckpoint(self): - self._GenerateProjectorTestData() - self._SetupWSGIApp() - - url = '/data/plugin/projector/tensor?run=.&name=var1' - tensor_bytes = self._Get(url).data - expected_tensor = np.array([[6, 6]], dtype=np.float32) - self._AssertTensorResponse(tensor_bytes, expected_tensor) - - def testBookmarksRequestMissingRunAndName(self): - self._GenerateProjectorTestData() - self._SetupWSGIApp() - - url = '/data/plugin/projector/bookmarks' - self.assertEqual(self._Get(url).status_code, 400) - - def testBookmarksRequestMissingName(self): - self._GenerateProjectorTestData() - self._SetupWSGIApp() - - url = '/data/plugin/projector/bookmarks?run=.' - self.assertEqual(self._Get(url).status_code, 400) - - def testBookmarksRequestMissingRun(self): - self._GenerateProjectorTestData() - self._SetupWSGIApp() - - url = '/data/plugin/projector/bookmarks?name=var1' - self.assertEqual(self._Get(url).status_code, 400) - - def testBookmarksUnknownRun(self): - self._GenerateProjectorTestData() - self._SetupWSGIApp() - - url = '/data/plugin/projector/bookmarks?run=unknown&name=var1' - self.assertEqual(self._Get(url).status_code, 400) - - def testBookmarksUnknownName(self): - self._GenerateProjectorTestData() - self._SetupWSGIApp() - - url = '/data/plugin/projector/bookmarks?run=.&name=unknown' - self.assertEqual(self._Get(url).status_code, 400) - - # TODO(#2007): Cleanly separate out projector tests that require real TF - @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF') - def testBookmarks(self): - self._GenerateProjectorTestData() - self._SetupWSGIApp() - - url = '/data/plugin/projector/bookmarks?run=.&name=var1' - bookmark = self._GetJson(url) - self.assertEqual(bookmark, {'a': 'b'}) - - def testEndpointsNoAssets(self): - g = tf.Graph() - - with test_util.FileWriterCache.get(self.log_dir) as writer: - writer.add_graph(g) - - self._SetupWSGIApp() - run_json = self._GetJson('/data/plugin/projector/runs') - self.assertEqual(run_json, []) - - def _AssertTensorResponse(self, tensor_bytes, expected_tensor): - tensor = np.reshape(np.fromstring(tensor_bytes, dtype=np.float32), - expected_tensor.shape) - self.assertTrue(np.array_equal(tensor, expected_tensor)) - - # TODO(#2007): Cleanly separate out projector tests that require real TF - @unittest.skipUnless(USING_REAL_TF, 'Test only passes when using real TF') - def testPluginIsActive(self): - self._GenerateProjectorTestData() - self._SetupWSGIApp() - - patcher = tf.compat.v1.test.mock.patch('threading.Thread.start', autospec=True) - mock = patcher.start() - self.addCleanup(patcher.stop) - - # The projector plugin has not yet determined whether it is active, but it - # should now start a thread to determine that. - self.assertFalse(self.plugin.is_active()) - thread = self.plugin._thread_for_determining_is_active - mock.assert_called_once_with(thread) - - # The logic has not finished running yet, so the plugin should still not - # have deemed itself to be active. - self.assertFalse(self.plugin.is_active()) - mock.assert_called_once_with(thread) - - self.plugin._thread_for_determining_is_active.run() - - # The plugin later finds that embedding data is available. - self.assertTrue(self.plugin.is_active()) - - # Subsequent calls to is_active should not start a new thread. The mock - # should only have been called once throughout this test. - self.assertTrue(self.plugin.is_active()) - mock.assert_called_once_with(thread) - - def testPluginIsNotActive(self): - self._SetupWSGIApp() - - # The is_active method makes use of a separate thread, so we mock threading - # behavior to make this test deterministic. - patcher = tf.compat.v1.test.mock.patch('threading.Thread.start', autospec=True) - mock = patcher.start() - self.addCleanup(patcher.stop) - - # The projector plugin has not yet determined whether it is active, but it - # should now start a thread to determine that. - self.assertFalse(self.plugin.is_active()) - mock.assert_called_once_with(self.plugin._thread_for_determining_is_active) - - self.plugin._thread_for_determining_is_active.run() - - # The plugin later finds that embedding data is not available. - self.assertFalse(self.plugin.is_active()) - - # Furthermore, the plugin should have spawned a new thread to check whether - # it is active (because it might now be active even though it had not been - # beforehand), so the mock should now be called twice. - self.assertEqual(2, mock.call_count) - - def _SetupWSGIApp(self): - multiplexer = event_multiplexer.EventMultiplexer( - size_guidance=application.DEFAULT_SIZE_GUIDANCE, - purge_orphaned_data=True) - context = base_plugin.TBContext( - logdir=self.log_dir, multiplexer=multiplexer) - self.plugin = projector_plugin.ProjectorPlugin(context) - wsgi_app = application.TensorBoardWSGI([self.plugin]) - self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) - - def _Get(self, path): - return self.server.get(path) - - def _GetJson(self, path): - response = self.server.get(path) - data = response.data - if response.headers.get('Content-Encoding') == 'gzip': - data = gzip.GzipFile('', 'rb', 9, io.BytesIO(data)).read() - return json.loads(data.decode('utf-8')) - - def _GenerateEventsData(self): - with test_util.FileWriterCache.get(self.log_dir) as fw: - event = event_pb2.Event( - wall_time=1, - step=1, - summary=summary_pb2.Summary( - value=[summary_pb2.Summary.Value(tag='s1', simple_value=0)])) - fw.add_event(event) - - def _GenerateProjectorTestData(self): - config_path = os.path.join(self.log_dir, 'projector_config.pbtxt') - config = projector_config_pb2.ProjectorConfig() - embedding = config.embeddings.add() - # Add an embedding by its canonical tensor name. - embedding.tensor_name = 'var1:0' - - with tf.io.gfile.GFile(os.path.join(self.log_dir, 'bookmarks.json'), 'w') as f: - f.write('{"a": "b"}') - embedding.bookmarks_path = 'bookmarks.json' - - config_pbtxt = text_format.MessageToString(config) - with tf.io.gfile.GFile(config_path, 'w') as f: - f.write(config_pbtxt) - - # Write a checkpoint with some dummy variables. - with tf.Graph().as_default(): - sess = tf.compat.v1.Session() - checkpoint_path = os.path.join(self.log_dir, 'model') - tf.compat.v1.get_variable('var1', - initializer=tf.constant(np.full([1, 2], 6.0))) - tf.compat.v1.get_variable('var2', [10, 10]) - tf.compat.v1.get_variable('var3', [100, 100]) - sess.run(tf.compat.v1.global_variables_initializer()) - saver = tf.compat.v1.train.Saver(write_version=tf.compat.v1.train.SaverDef.V1) - saver.save(sess, checkpoint_path) + run = run_json[0] + info_json = self._GetJson("/data/plugin/projector/info?run=%s" % run) + self.assertItemsEqual( + info_json["embeddings"], + [ + { + "tensorShape": [1, 2], + "tensorName": "var1", + "bookmarksPath": "bookmarks.json", + }, + {"tensorShape": [10, 10], "tensorName": "var2"}, + {"tensorShape": [100, 100], "tensorName": "var3"}, + ], + ) + + # TODO(#2007): Cleanly separate out projector tests that require real TF + @unittest.skipUnless(USING_REAL_TF, "Test only passes when using real TF") + def testTensorWithValidCheckpoint(self): + self._GenerateProjectorTestData() + self._SetupWSGIApp() + + url = "/data/plugin/projector/tensor?run=.&name=var1" + tensor_bytes = self._Get(url).data + expected_tensor = np.array([[6, 6]], dtype=np.float32) + self._AssertTensorResponse(tensor_bytes, expected_tensor) + + def testBookmarksRequestMissingRunAndName(self): + self._GenerateProjectorTestData() + self._SetupWSGIApp() + + url = "/data/plugin/projector/bookmarks" + self.assertEqual(self._Get(url).status_code, 400) + + def testBookmarksRequestMissingName(self): + self._GenerateProjectorTestData() + self._SetupWSGIApp() + + url = "/data/plugin/projector/bookmarks?run=." + self.assertEqual(self._Get(url).status_code, 400) + + def testBookmarksRequestMissingRun(self): + self._GenerateProjectorTestData() + self._SetupWSGIApp() + + url = "/data/plugin/projector/bookmarks?name=var1" + self.assertEqual(self._Get(url).status_code, 400) + + def testBookmarksUnknownRun(self): + self._GenerateProjectorTestData() + self._SetupWSGIApp() + + url = "/data/plugin/projector/bookmarks?run=unknown&name=var1" + self.assertEqual(self._Get(url).status_code, 400) + + def testBookmarksUnknownName(self): + self._GenerateProjectorTestData() + self._SetupWSGIApp() + + url = "/data/plugin/projector/bookmarks?run=.&name=unknown" + self.assertEqual(self._Get(url).status_code, 400) + + # TODO(#2007): Cleanly separate out projector tests that require real TF + @unittest.skipUnless(USING_REAL_TF, "Test only passes when using real TF") + def testBookmarks(self): + self._GenerateProjectorTestData() + self._SetupWSGIApp() + + url = "/data/plugin/projector/bookmarks?run=.&name=var1" + bookmark = self._GetJson(url) + self.assertEqual(bookmark, {"a": "b"}) + + def testEndpointsNoAssets(self): + g = tf.Graph() + + with test_util.FileWriterCache.get(self.log_dir) as writer: + writer.add_graph(g) + + self._SetupWSGIApp() + run_json = self._GetJson("/data/plugin/projector/runs") + self.assertEqual(run_json, []) + + def _AssertTensorResponse(self, tensor_bytes, expected_tensor): + tensor = np.reshape( + np.fromstring(tensor_bytes, dtype=np.float32), expected_tensor.shape + ) + self.assertTrue(np.array_equal(tensor, expected_tensor)) + + # TODO(#2007): Cleanly separate out projector tests that require real TF + @unittest.skipUnless(USING_REAL_TF, "Test only passes when using real TF") + def testPluginIsActive(self): + self._GenerateProjectorTestData() + self._SetupWSGIApp() + + patcher = tf.compat.v1.test.mock.patch( + "threading.Thread.start", autospec=True + ) + mock = patcher.start() + self.addCleanup(patcher.stop) + + # The projector plugin has not yet determined whether it is active, but it + # should now start a thread to determine that. + self.assertFalse(self.plugin.is_active()) + thread = self.plugin._thread_for_determining_is_active + mock.assert_called_once_with(thread) + + # The logic has not finished running yet, so the plugin should still not + # have deemed itself to be active. + self.assertFalse(self.plugin.is_active()) + mock.assert_called_once_with(thread) + + self.plugin._thread_for_determining_is_active.run() + + # The plugin later finds that embedding data is available. + self.assertTrue(self.plugin.is_active()) + + # Subsequent calls to is_active should not start a new thread. The mock + # should only have been called once throughout this test. + self.assertTrue(self.plugin.is_active()) + mock.assert_called_once_with(thread) + + def testPluginIsNotActive(self): + self._SetupWSGIApp() + + # The is_active method makes use of a separate thread, so we mock threading + # behavior to make this test deterministic. + patcher = tf.compat.v1.test.mock.patch( + "threading.Thread.start", autospec=True + ) + mock = patcher.start() + self.addCleanup(patcher.stop) + + # The projector plugin has not yet determined whether it is active, but it + # should now start a thread to determine that. + self.assertFalse(self.plugin.is_active()) + mock.assert_called_once_with( + self.plugin._thread_for_determining_is_active + ) + + self.plugin._thread_for_determining_is_active.run() + + # The plugin later finds that embedding data is not available. + self.assertFalse(self.plugin.is_active()) + + # Furthermore, the plugin should have spawned a new thread to check whether + # it is active (because it might now be active even though it had not been + # beforehand), so the mock should now be called twice. + self.assertEqual(2, mock.call_count) + + def _SetupWSGIApp(self): + multiplexer = event_multiplexer.EventMultiplexer( + size_guidance=application.DEFAULT_SIZE_GUIDANCE, + purge_orphaned_data=True, + ) + context = base_plugin.TBContext( + logdir=self.log_dir, multiplexer=multiplexer + ) + self.plugin = projector_plugin.ProjectorPlugin(context) + wsgi_app = application.TensorBoardWSGI([self.plugin]) + self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) + + def _Get(self, path): + return self.server.get(path) + + def _GetJson(self, path): + response = self.server.get(path) + data = response.data + if response.headers.get("Content-Encoding") == "gzip": + data = gzip.GzipFile("", "rb", 9, io.BytesIO(data)).read() + return json.loads(data.decode("utf-8")) + + def _GenerateEventsData(self): + with test_util.FileWriterCache.get(self.log_dir) as fw: + event = event_pb2.Event( + wall_time=1, + step=1, + summary=summary_pb2.Summary( + value=[summary_pb2.Summary.Value(tag="s1", simple_value=0)] + ), + ) + fw.add_event(event) + + def _GenerateProjectorTestData(self): + config_path = os.path.join(self.log_dir, "projector_config.pbtxt") + config = projector_config_pb2.ProjectorConfig() + embedding = config.embeddings.add() + # Add an embedding by its canonical tensor name. + embedding.tensor_name = "var1:0" + + with tf.io.gfile.GFile( + os.path.join(self.log_dir, "bookmarks.json"), "w" + ) as f: + f.write('{"a": "b"}') + embedding.bookmarks_path = "bookmarks.json" + + config_pbtxt = text_format.MessageToString(config) + with tf.io.gfile.GFile(config_path, "w") as f: + f.write(config_pbtxt) + + # Write a checkpoint with some dummy variables. + with tf.Graph().as_default(): + sess = tf.compat.v1.Session() + checkpoint_path = os.path.join(self.log_dir, "model") + tf.compat.v1.get_variable( + "var1", initializer=tf.constant(np.full([1, 2], 6.0)) + ) + tf.compat.v1.get_variable("var2", [10, 10]) + tf.compat.v1.get_variable("var3", [100, 100]) + sess.run(tf.compat.v1.global_variables_initializer()) + saver = tf.compat.v1.train.Saver( + write_version=tf.compat.v1.train.SaverDef.V1 + ) + saver.save(sess, checkpoint_path) class MetadataColumnsTest(tf.test.TestCase): - - def testLengthDoesNotMatch(self): - metadata = projector_plugin.EmbeddingMetadata(10) - - with self.assertRaises(ValueError): - metadata.add_column('Labels', [''] * 11) - - def testValuesNot1D(self): - metadata = projector_plugin.EmbeddingMetadata(3) - values = np.array([[1, 2, 3]]) - - with self.assertRaises(ValueError): - metadata.add_column('Labels', values) - - def testMultipleColumnsRetrieval(self): - metadata = projector_plugin.EmbeddingMetadata(3) - metadata.add_column('Sizes', [1, 2, 3]) - metadata.add_column('Labels', ['a', 'b', 'c']) - self.assertEqual(metadata.column_names, ['Sizes', 'Labels']) - self.assertEqual(metadata.name_to_values['Labels'], ['a', 'b', 'c']) - self.assertEqual(metadata.name_to_values['Sizes'], [1, 2, 3]) - - def testValuesAreListofLists(self): - metadata = projector_plugin.EmbeddingMetadata(3) - values = [[1, 2, 3], [4, 5, 6]] - with self.assertRaises(ValueError): - metadata.add_column('Labels', values) - - def testStringListRetrieval(self): - metadata = projector_plugin.EmbeddingMetadata(3) - metadata.add_column('Labels', ['a', 'B', 'c']) - self.assertEqual(metadata.name_to_values['Labels'], ['a', 'B', 'c']) - self.assertEqual(metadata.column_names, ['Labels']) - - def testNumericListRetrieval(self): - metadata = projector_plugin.EmbeddingMetadata(3) - metadata.add_column('Labels', [1, 2, 3]) - self.assertEqual(metadata.name_to_values['Labels'], [1, 2, 3]) - - def testNumericNdArrayRetrieval(self): - metadata = projector_plugin.EmbeddingMetadata(3) - metadata.add_column('Labels', np.array([1, 2, 3])) - self.assertEqual(metadata.name_to_values['Labels'].tolist(), [1, 2, 3]) - - def testStringNdArrayRetrieval(self): - metadata = projector_plugin.EmbeddingMetadata(2) - metadata.add_column('Labels', np.array(['a', 'b'])) - self.assertEqual(metadata.name_to_values['Labels'].tolist(), ['a', 'b']) - - def testDuplicateColumnName(self): - metadata = projector_plugin.EmbeddingMetadata(2) - metadata.add_column('Labels', np.array(['a', 'b'])) - with self.assertRaises(ValueError): - metadata.add_column('Labels', np.array(['a', 'b'])) + def testLengthDoesNotMatch(self): + metadata = projector_plugin.EmbeddingMetadata(10) + + with self.assertRaises(ValueError): + metadata.add_column("Labels", [""] * 11) + + def testValuesNot1D(self): + metadata = projector_plugin.EmbeddingMetadata(3) + values = np.array([[1, 2, 3]]) + + with self.assertRaises(ValueError): + metadata.add_column("Labels", values) + + def testMultipleColumnsRetrieval(self): + metadata = projector_plugin.EmbeddingMetadata(3) + metadata.add_column("Sizes", [1, 2, 3]) + metadata.add_column("Labels", ["a", "b", "c"]) + self.assertEqual(metadata.column_names, ["Sizes", "Labels"]) + self.assertEqual(metadata.name_to_values["Labels"], ["a", "b", "c"]) + self.assertEqual(metadata.name_to_values["Sizes"], [1, 2, 3]) + + def testValuesAreListofLists(self): + metadata = projector_plugin.EmbeddingMetadata(3) + values = [[1, 2, 3], [4, 5, 6]] + with self.assertRaises(ValueError): + metadata.add_column("Labels", values) + + def testStringListRetrieval(self): + metadata = projector_plugin.EmbeddingMetadata(3) + metadata.add_column("Labels", ["a", "B", "c"]) + self.assertEqual(metadata.name_to_values["Labels"], ["a", "B", "c"]) + self.assertEqual(metadata.column_names, ["Labels"]) + + def testNumericListRetrieval(self): + metadata = projector_plugin.EmbeddingMetadata(3) + metadata.add_column("Labels", [1, 2, 3]) + self.assertEqual(metadata.name_to_values["Labels"], [1, 2, 3]) + + def testNumericNdArrayRetrieval(self): + metadata = projector_plugin.EmbeddingMetadata(3) + metadata.add_column("Labels", np.array([1, 2, 3])) + self.assertEqual(metadata.name_to_values["Labels"].tolist(), [1, 2, 3]) + + def testStringNdArrayRetrieval(self): + metadata = projector_plugin.EmbeddingMetadata(2) + metadata.add_column("Labels", np.array(["a", "b"])) + self.assertEqual(metadata.name_to_values["Labels"].tolist(), ["a", "b"]) + + def testDuplicateColumnName(self): + metadata = projector_plugin.EmbeddingMetadata(2) + metadata.add_column("Labels", np.array(["a", "b"])) + with self.assertRaises(ValueError): + metadata.add_column("Labels", np.array(["a", "b"])) class LRUCacheTest(tf.test.TestCase): - - def testInvalidSize(self): - with self.assertRaises(ValueError): - projector_plugin.LRUCache(0) - - def testSimpleGetAndSet(self): - cache = projector_plugin.LRUCache(1) - value = cache.get('a') - self.assertIsNone(value) - cache.set('a', 10) - self.assertEqual(cache.get('a'), 10) - - def testErrorsWhenSettingNoneAsValue(self): - cache = projector_plugin.LRUCache(1) - with self.assertRaises(ValueError): - cache.set('a', None) - - def testLRUReplacementPolicy(self): - cache = projector_plugin.LRUCache(2) - cache.set('a', 1) - cache.set('b', 2) - cache.set('c', 3) - self.assertIsNone(cache.get('a')) - self.assertEqual(cache.get('b'), 2) - self.assertEqual(cache.get('c'), 3) - - # Make 'b' the most recently used. - cache.get('b') - cache.set('d', 4) - - # Make sure 'c' got replaced with 'd'. - self.assertIsNone(cache.get('c')) - self.assertEqual(cache.get('b'), 2) - self.assertEqual(cache.get('d'), 4) - - -if __name__ == '__main__': - tf.test.main() + def testInvalidSize(self): + with self.assertRaises(ValueError): + projector_plugin.LRUCache(0) + + def testSimpleGetAndSet(self): + cache = projector_plugin.LRUCache(1) + value = cache.get("a") + self.assertIsNone(value) + cache.set("a", 10) + self.assertEqual(cache.get("a"), 10) + + def testErrorsWhenSettingNoneAsValue(self): + cache = projector_plugin.LRUCache(1) + with self.assertRaises(ValueError): + cache.set("a", None) + + def testLRUReplacementPolicy(self): + cache = projector_plugin.LRUCache(2) + cache.set("a", 1) + cache.set("b", 2) + cache.set("c", 3) + self.assertIsNone(cache.get("a")) + self.assertEqual(cache.get("b"), 2) + self.assertEqual(cache.get("c"), 3) + + # Make 'b' the most recently used. + cache.get("b") + cache.set("d", 4) + + # Make sure 'c' got replaced with 'd'. + self.assertIsNone(cache.get("c")) + self.assertEqual(cache.get("b"), 2) + self.assertEqual(cache.get("d"), 4) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/scalar/metadata.py b/tensorboard/plugins/scalar/metadata.py index 2d1e273568..1b3903807d 100644 --- a/tensorboard/plugins/scalar/metadata.py +++ b/tensorboard/plugins/scalar/metadata.py @@ -24,7 +24,7 @@ logger = tb_logging.get_logger() -PLUGIN_NAME = 'scalars' +PLUGIN_NAME = "scalars" # The most recent value for the `version` field of the # `ScalarPluginData` proto. @@ -32,39 +32,43 @@ def create_summary_metadata(display_name, description): - """Create a `summary_pb2.SummaryMetadata` proto for scalar plugin data. + """Create a `summary_pb2.SummaryMetadata` proto for scalar plugin data. - Returns: - A `summary_pb2.SummaryMetadata` protobuf object. - """ - content = plugin_data_pb2.ScalarPluginData(version=PROTO_VERSION) - metadata = summary_pb2.SummaryMetadata( - display_name=display_name, - summary_description=description, - plugin_data=summary_pb2.SummaryMetadata.PluginData( - plugin_name=PLUGIN_NAME, - content=content.SerializeToString())) - return metadata + Returns: + A `summary_pb2.SummaryMetadata` protobuf object. + """ + content = plugin_data_pb2.ScalarPluginData(version=PROTO_VERSION) + metadata = summary_pb2.SummaryMetadata( + display_name=display_name, + summary_description=description, + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME, content=content.SerializeToString() + ), + ) + return metadata def parse_plugin_metadata(content): - """Parse summary metadata to a Python object. + """Parse summary metadata to a Python object. - Arguments: - content: The `content` field of a `SummaryMetadata` proto - corresponding to the scalar plugin. + Arguments: + content: The `content` field of a `SummaryMetadata` proto + corresponding to the scalar plugin. - Returns: - A `ScalarPluginData` protobuf object. - """ - if not isinstance(content, bytes): - raise TypeError('Content type must be bytes') - result = plugin_data_pb2.ScalarPluginData.FromString(content) - if result.version == 0: - return result - else: - logger.warn( - 'Unknown metadata version: %s. The latest version known to ' - 'this build of TensorBoard is %s; perhaps a newer build is ' - 'available?', result.version, PROTO_VERSION) - return result + Returns: + A `ScalarPluginData` protobuf object. + """ + if not isinstance(content, bytes): + raise TypeError("Content type must be bytes") + result = plugin_data_pb2.ScalarPluginData.FromString(content) + if result.version == 0: + return result + else: + logger.warn( + "Unknown metadata version: %s. The latest version known to " + "this build of TensorBoard is %s; perhaps a newer build is " + "available?", + result.version, + PROTO_VERSION, + ) + return result diff --git a/tensorboard/plugins/scalar/scalars_demo.py b/tensorboard/plugins/scalar/scalars_demo.py index e16316075f..b561f1e8a3 100644 --- a/tensorboard/plugins/scalar/scalars_demo.py +++ b/tensorboard/plugins/scalar/scalars_demo.py @@ -26,119 +26,138 @@ from tensorboard.plugins.scalar import summary # Directory into which to write tensorboard data. -LOGDIR = '/tmp/scalars_demo' +LOGDIR = "/tmp/scalars_demo" # Duration of the simulation. STEPS = 1000 -def run(logdir, run_name, - initial_temperature, ambient_temperature, heat_coefficient): - """Run a temperature simulation. - - This will simulate an object at temperature `initial_temperature` - sitting at rest in a large room at temperature `ambient_temperature`. - The object has some intrinsic `heat_coefficient`, which indicates - how much thermal conductivity it has: for instance, metals have high - thermal conductivity, while the thermal conductivity of water is low. - - Over time, the object's temperature will adjust to match the - temperature of its environment. We'll track the object's temperature, - how far it is from the room's temperature, and how much it changes at - each time step. - - Arguments: - logdir: the top-level directory into which to write summary data - run_name: the name of this run; will be created as a subdirectory - under logdir - initial_temperature: float; the object's initial temperature - ambient_temperature: float; the temperature of the enclosing room - heat_coefficient: float; a measure of the object's thermal - conductivity - """ - tf.compat.v1.reset_default_graph() - tf.compat.v1.set_random_seed(0) - - with tf.name_scope('temperature'): - # Create a mutable variable to hold the object's temperature, and - # create a scalar summary to track its value over time. The name of - # the summary will appear as "temperature/current" due to the - # name-scope above. - temperature = tf.Variable(tf.constant(initial_temperature), - name='temperature') - summary.op('current', temperature, - display_name='Temperature', - description='The temperature of the object under ' - 'simulation, in Kelvins.') - - # Compute how much the object's temperature differs from that of its - # environment, and track this, too: likewise, as - # "temperature/difference_to_ambient". - ambient_difference = temperature - ambient_temperature - summary.op('difference_to_ambient', ambient_difference, - display_name='Difference to ambient temperature', - description='The difference between the ambient ' - 'temperature and the temperature of the ' - 'object under simulation, in Kelvins.') - - # Newton suggested that the rate of change of the temperature of an - # object is directly proportional to this `ambient_difference` above, - # where the proportionality constant is what we called the heat - # coefficient. But in real life, not everything is quite so clean, so - # we'll add in some noise. (The value of 50 is arbitrary, chosen to - # make the data look somewhat interesting. :-) ) - noise = 50 * tf.random.normal([]) - delta = -heat_coefficient * (ambient_difference + noise) - summary.op('delta', delta, - description='The change in temperature from the previous ' - 'step, in Kelvins.') - - # Collect all the scalars that we want to keep track of. - summ = tf.compat.v1.summary.merge_all() - - # Now, augment the current temperature by this delta that we computed, - # blocking the assignment on summary collection to avoid race conditions - # and ensure that the summary always reports the pre-update value. - with tf.control_dependencies([summ]): - update_step = temperature.assign_add(delta) - - sess = tf.compat.v1.Session() - writer = tf.summary.FileWriter(os.path.join(logdir, run_name)) - writer.add_graph(sess.graph) - sess.run(tf.compat.v1.global_variables_initializer()) - for step in xrange(STEPS): - # By asking TensorFlow to compute the update step, we force it to - # change the value of the temperature variable. We don't actually - # care about this value, so we discard it; instead, we grab the - # summary data computed along the way. - (s, _) = sess.run([summ, update_step]) - writer.add_summary(s, global_step=step) - writer.close() +def run( + logdir, run_name, initial_temperature, ambient_temperature, heat_coefficient +): + """Run a temperature simulation. + + This will simulate an object at temperature `initial_temperature` + sitting at rest in a large room at temperature `ambient_temperature`. + The object has some intrinsic `heat_coefficient`, which indicates + how much thermal conductivity it has: for instance, metals have high + thermal conductivity, while the thermal conductivity of water is low. + + Over time, the object's temperature will adjust to match the + temperature of its environment. We'll track the object's temperature, + how far it is from the room's temperature, and how much it changes at + each time step. + + Arguments: + logdir: the top-level directory into which to write summary data + run_name: the name of this run; will be created as a subdirectory + under logdir + initial_temperature: float; the object's initial temperature + ambient_temperature: float; the temperature of the enclosing room + heat_coefficient: float; a measure of the object's thermal + conductivity + """ + tf.compat.v1.reset_default_graph() + tf.compat.v1.set_random_seed(0) + + with tf.name_scope("temperature"): + # Create a mutable variable to hold the object's temperature, and + # create a scalar summary to track its value over time. The name of + # the summary will appear as "temperature/current" due to the + # name-scope above. + temperature = tf.Variable( + tf.constant(initial_temperature), name="temperature" + ) + summary.op( + "current", + temperature, + display_name="Temperature", + description="The temperature of the object under " + "simulation, in Kelvins.", + ) + + # Compute how much the object's temperature differs from that of its + # environment, and track this, too: likewise, as + # "temperature/difference_to_ambient". + ambient_difference = temperature - ambient_temperature + summary.op( + "difference_to_ambient", + ambient_difference, + display_name="Difference to ambient temperature", + description="The difference between the ambient " + "temperature and the temperature of the " + "object under simulation, in Kelvins.", + ) + + # Newton suggested that the rate of change of the temperature of an + # object is directly proportional to this `ambient_difference` above, + # where the proportionality constant is what we called the heat + # coefficient. But in real life, not everything is quite so clean, so + # we'll add in some noise. (The value of 50 is arbitrary, chosen to + # make the data look somewhat interesting. :-) ) + noise = 50 * tf.random.normal([]) + delta = -heat_coefficient * (ambient_difference + noise) + summary.op( + "delta", + delta, + description="The change in temperature from the previous " + "step, in Kelvins.", + ) + + # Collect all the scalars that we want to keep track of. + summ = tf.compat.v1.summary.merge_all() + + # Now, augment the current temperature by this delta that we computed, + # blocking the assignment on summary collection to avoid race conditions + # and ensure that the summary always reports the pre-update value. + with tf.control_dependencies([summ]): + update_step = temperature.assign_add(delta) + + sess = tf.compat.v1.Session() + writer = tf.summary.FileWriter(os.path.join(logdir, run_name)) + writer.add_graph(sess.graph) + sess.run(tf.compat.v1.global_variables_initializer()) + for step in xrange(STEPS): + # By asking TensorFlow to compute the update step, we force it to + # change the value of the temperature variable. We don't actually + # care about this value, so we discard it; instead, we grab the + # summary data computed along the way. + (s, _) = sess.run([summ, update_step]) + writer.add_summary(s, global_step=step) + writer.close() def run_all(logdir, verbose=False): - """Run simulations on a reasonable set of parameters. - - Arguments: - logdir: the directory into which to store all the runs' data - verbose: if true, print out each run's name as it begins - """ - for initial_temperature in [270.0, 310.0, 350.0]: - for final_temperature in [270.0, 310.0, 350.0]: - for heat_coefficient in [0.001, 0.005]: - run_name = 'temperature:t0=%g,tA=%g,kH=%g' % ( - initial_temperature, final_temperature, heat_coefficient) - if verbose: - print('--- Running: %s' % run_name) - run(logdir, run_name, - initial_temperature, final_temperature, heat_coefficient) + """Run simulations on a reasonable set of parameters. + + Arguments: + logdir: the directory into which to store all the runs' data + verbose: if true, print out each run's name as it begins + """ + for initial_temperature in [270.0, 310.0, 350.0]: + for final_temperature in [270.0, 310.0, 350.0]: + for heat_coefficient in [0.001, 0.005]: + run_name = "temperature:t0=%g,tA=%g,kH=%g" % ( + initial_temperature, + final_temperature, + heat_coefficient, + ) + if verbose: + print("--- Running: %s" % run_name) + run( + logdir, + run_name, + initial_temperature, + final_temperature, + heat_coefficient, + ) def main(unused_argv): - print('Saving output to %s.' % LOGDIR) - run_all(LOGDIR, verbose=True) - print('Done. Output saved to %s.' % LOGDIR) + print("Saving output to %s." % LOGDIR) + run_all(LOGDIR, verbose=True) + print("Done. Output saved to %s." % LOGDIR) -if __name__ == '__main__': - app.run(main) +if __name__ == "__main__": + app.run(main) diff --git a/tensorboard/plugins/scalar/scalars_plugin.py b/tensorboard/plugins/scalar/scalars_plugin.py index 4314b65b61..4004abbaed 100644 --- a/tensorboard/plugins/scalar/scalars_plugin.py +++ b/tensorboard/plugins/scalar/scalars_plugin.py @@ -41,83 +41,93 @@ class OutputFormat(object): - """An enum used to list the valid output formats for API calls.""" - JSON = 'json' - CSV = 'csv' + """An enum used to list the valid output formats for API calls.""" + + JSON = "json" + CSV = "csv" class ScalarsPlugin(base_plugin.TBPlugin): - """Scalars Plugin for TensorBoard.""" - - plugin_name = metadata.PLUGIN_NAME - - def __init__(self, context): - """Instantiates ScalarsPlugin via TensorBoard core. - - Args: - context: A base_plugin.TBContext instance. - """ - self._multiplexer = context.multiplexer - self._db_connection_provider = context.db_connection_provider - if context.flags and context.flags.generic_data == 'true': - self._data_provider = context.data_provider - else: - self._data_provider = None - - def get_plugin_apps(self): - return { - '/scalars': self.scalars_route, - '/tags': self.tags_route, - } - - def is_active(self): - """The scalars plugin is active iff any run has at least one scalar tag.""" - if self._data_provider: - # We don't have an experiment ID, and modifying the backend core - # to provide one would break backward compatibility. Hack for now. - return True - - if self._db_connection_provider: - # The plugin is active if one relevant tag can be found in the database. - db = self._db_connection_provider() - cursor = db.execute(''' + """Scalars Plugin for TensorBoard.""" + + plugin_name = metadata.PLUGIN_NAME + + def __init__(self, context): + """Instantiates ScalarsPlugin via TensorBoard core. + + Args: + context: A base_plugin.TBContext instance. + """ + self._multiplexer = context.multiplexer + self._db_connection_provider = context.db_connection_provider + if context.flags and context.flags.generic_data == "true": + self._data_provider = context.data_provider + else: + self._data_provider = None + + def get_plugin_apps(self): + return { + "/scalars": self.scalars_route, + "/tags": self.tags_route, + } + + def is_active(self): + """The scalars plugin is active iff any run has at least one scalar + tag.""" + if self._data_provider: + # We don't have an experiment ID, and modifying the backend core + # to provide one would break backward compatibility. Hack for now. + return True + + if self._db_connection_provider: + # The plugin is active if one relevant tag can be found in the database. + db = self._db_connection_provider() + cursor = db.execute( + """ SELECT 1 FROM Tags WHERE Tags.plugin_name = ? LIMIT 1 - ''', (metadata.PLUGIN_NAME,)) - return bool(list(cursor)) - - if not self._multiplexer: - return False - - return bool(self._multiplexer.PluginRunToTagToContent(metadata.PLUGIN_NAME)) - - def frontend_metadata(self): - return base_plugin.FrontendMetadata(element_name='tf-scalar-dashboard') - - def index_impl(self, experiment=None): - """Return {runName: {tagName: {displayName: ..., description: ...}}}.""" - if self._data_provider: - mapping = self._data_provider.list_scalars( - experiment_id=experiment, - plugin_name=metadata.PLUGIN_NAME, - ) - result = {run: {} for run in mapping} - for (run, tag_to_content) in six.iteritems(mapping): - for (tag, metadatum) in six.iteritems(tag_to_content): - description = plugin_util.markdown_to_safe_html(metadatum.description) - result[run][tag] = { - 'displayName': metadatum.display_name, - 'description': description, - } - return result - - if self._db_connection_provider: - # Read tags from the database. - db = self._db_connection_provider() - cursor = db.execute(''' + """, + (metadata.PLUGIN_NAME,), + ) + return bool(list(cursor)) + + if not self._multiplexer: + return False + + return bool( + self._multiplexer.PluginRunToTagToContent(metadata.PLUGIN_NAME) + ) + + def frontend_metadata(self): + return base_plugin.FrontendMetadata(element_name="tf-scalar-dashboard") + + def index_impl(self, experiment=None): + """Return {runName: {tagName: {displayName: ..., description: + ...}}}.""" + if self._data_provider: + mapping = self._data_provider.list_scalars( + experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME, + ) + result = {run: {} for run in mapping} + for (run, tag_to_content) in six.iteritems(mapping): + for (tag, metadatum) in six.iteritems(tag_to_content): + description = plugin_util.markdown_to_safe_html( + metadatum.description + ) + result[run][tag] = { + "displayName": metadatum.display_name, + "description": description, + } + return result + + if self._db_connection_provider: + # Read tags from the database. + db = self._db_connection_provider() + cursor = db.execute( + """ SELECT Tags.tag_name, Tags.display_name, @@ -127,54 +137,62 @@ def index_impl(self, experiment=None): ON Tags.run_id = Runs.run_id WHERE Tags.plugin_name = ? - ''', (metadata.PLUGIN_NAME,)) - result = collections.defaultdict(dict) - for row in cursor: - tag_name, display_name, run_name = row - result[run_name][tag_name] = { - 'displayName': display_name, - # TODO(chihuahua): Populate the description. Currently, the tags - # table does not link with the description table. - 'description': '', - } - return result - - result = collections.defaultdict(lambda: {}) - mapping = self._multiplexer.PluginRunToTagToContent(metadata.PLUGIN_NAME) - for (run, tag_to_content) in six.iteritems(mapping): - for (tag, content) in six.iteritems(tag_to_content): - content = metadata.parse_plugin_metadata(content) - summary_metadata = self._multiplexer.SummaryMetadata(run, tag) - result[run][tag] = {'displayName': summary_metadata.display_name, - 'description': plugin_util.markdown_to_safe_html( - summary_metadata.summary_description)} - - return result - - def scalars_impl(self, tag, run, experiment, output_format): - """Result of the form `(body, mime_type)`.""" - if self._data_provider: - # Downsample reads to 1000 scalars per time series, which is the - # default size guidance for scalars under the multiplexer loading - # logic. - SAMPLE_COUNT = 1000 - all_scalars = self._data_provider.read_scalars( - experiment_id=experiment, - plugin_name=metadata.PLUGIN_NAME, - downsample=SAMPLE_COUNT, - run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), - ) - scalars = all_scalars.get(run, {}).get(tag, None) - if scalars is None: - raise errors.NotFoundError( - 'No scalar data for run=%r, tag=%r' % (run, tag) + """, + (metadata.PLUGIN_NAME,), + ) + result = collections.defaultdict(dict) + for row in cursor: + tag_name, display_name, run_name = row + result[run_name][tag_name] = { + "displayName": display_name, + # TODO(chihuahua): Populate the description. Currently, the tags + # table does not link with the description table. + "description": "", + } + return result + + result = collections.defaultdict(lambda: {}) + mapping = self._multiplexer.PluginRunToTagToContent( + metadata.PLUGIN_NAME ) - values = [(x.wall_time, x.step, x.value) for x in scalars] - elif self._db_connection_provider: - db = self._db_connection_provider() - # We select for steps greater than -1 because the writer inserts - # placeholder rows en masse. The check for step filters out those rows. - cursor = db.execute(''' + for (run, tag_to_content) in six.iteritems(mapping): + for (tag, content) in six.iteritems(tag_to_content): + content = metadata.parse_plugin_metadata(content) + summary_metadata = self._multiplexer.SummaryMetadata(run, tag) + result[run][tag] = { + "displayName": summary_metadata.display_name, + "description": plugin_util.markdown_to_safe_html( + summary_metadata.summary_description + ), + } + + return result + + def scalars_impl(self, tag, run, experiment, output_format): + """Result of the form `(body, mime_type)`.""" + if self._data_provider: + # Downsample reads to 1000 scalars per time series, which is the + # default size guidance for scalars under the multiplexer loading + # logic. + SAMPLE_COUNT = 1000 + all_scalars = self._data_provider.read_scalars( + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + downsample=SAMPLE_COUNT, + run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), + ) + scalars = all_scalars.get(run, {}).get(tag, None) + if scalars is None: + raise errors.NotFoundError( + "No scalar data for run=%r, tag=%r" % (run, tag) + ) + values = [(x.wall_time, x.step, x.value) for x in scalars] + elif self._db_connection_provider: + db = self._db_connection_provider() + # We select for steps greater than -1 because the writer inserts + # placeholder rows en masse. The check for step filters out those rows. + cursor = db.execute( + """ SELECT Tensors.step, Tensors.computed_time, @@ -195,56 +213,73 @@ def scalars_impl(self, tag, run, experiment, output_format): AND Tensors.shape = '' AND Tensors.step > -1 ORDER BY Tensors.step - ''', dict(exp=experiment, run=run, tag=tag, plugin=metadata.PLUGIN_NAME)) - values = [(wall_time, step, self._get_value(data, dtype_enum)) - for (step, wall_time, data, dtype_enum) in cursor] - else: - try: - tensor_events = self._multiplexer.Tensors(run, tag) - except KeyError: - raise errors.NotFoundError( - 'No scalar data for run=%r, tag=%r' % (run, tag) + """, + dict( + exp=experiment, + run=run, + tag=tag, + plugin=metadata.PLUGIN_NAME, + ), + ) + values = [ + (wall_time, step, self._get_value(data, dtype_enum)) + for (step, wall_time, data, dtype_enum) in cursor + ] + else: + try: + tensor_events = self._multiplexer.Tensors(run, tag) + except KeyError: + raise errors.NotFoundError( + "No scalar data for run=%r, tag=%r" % (run, tag) + ) + values = [ + ( + tensor_event.wall_time, + tensor_event.step, + tensor_util.make_ndarray(tensor_event.tensor_proto).item(), + ) + for tensor_event in tensor_events + ] + + if output_format == OutputFormat.CSV: + string_io = StringIO() + writer = csv.writer(string_io) + writer.writerow(["Wall time", "Step", "Value"]) + writer.writerows(values) + return (string_io.getvalue(), "text/csv") + else: + return (values, "application/json") + + def _get_value(self, scalar_data_blob, dtype_enum): + """Obtains value for scalar event given blob and dtype enum. + + Args: + scalar_data_blob: The blob obtained from the database. + dtype_enum: The enum representing the dtype. + + Returns: + The scalar value. + """ + tensorflow_dtype = tf.DType(dtype_enum) + buf = np.frombuffer( + scalar_data_blob, dtype=tensorflow_dtype.as_numpy_dtype + ) + return np.asscalar(buf) + + @wrappers.Request.application + def tags_route(self, request): + experiment = plugin_util.experiment_id(request.environ) + index = self.index_impl(experiment=experiment) + return http_util.Respond(request, index, "application/json") + + @wrappers.Request.application + def scalars_route(self, request): + """Given a tag and single run, return array of ScalarEvents.""" + tag = request.args.get("tag") + run = request.args.get("run") + experiment = plugin_util.experiment_id(request.environ) + output_format = request.args.get("format") + (body, mime_type) = self.scalars_impl( + tag, run, experiment, output_format ) - values = [(tensor_event.wall_time, - tensor_event.step, - tensor_util.make_ndarray(tensor_event.tensor_proto).item()) - for tensor_event in tensor_events] - - if output_format == OutputFormat.CSV: - string_io = StringIO() - writer = csv.writer(string_io) - writer.writerow(['Wall time', 'Step', 'Value']) - writer.writerows(values) - return (string_io.getvalue(), 'text/csv') - else: - return (values, 'application/json') - - def _get_value(self, scalar_data_blob, dtype_enum): - """Obtains value for scalar event given blob and dtype enum. - - Args: - scalar_data_blob: The blob obtained from the database. - dtype_enum: The enum representing the dtype. - - Returns: - The scalar value. - """ - tensorflow_dtype = tf.DType(dtype_enum) - buf = np.frombuffer(scalar_data_blob, dtype=tensorflow_dtype.as_numpy_dtype) - return np.asscalar(buf) - - @wrappers.Request.application - def tags_route(self, request): - experiment = plugin_util.experiment_id(request.environ) - index = self.index_impl(experiment=experiment) - return http_util.Respond(request, index, 'application/json') - - @wrappers.Request.application - def scalars_route(self, request): - """Given a tag and single run, return array of ScalarEvents.""" - tag = request.args.get('tag') - run = request.args.get('run') - experiment = plugin_util.experiment_id(request.environ) - output_format = request.args.get('format') - (body, mime_type) = self.scalars_impl(tag, run, experiment, output_format) - return http_util.Respond(request, body, mime_type) + return http_util.Respond(request, body, mime_type) diff --git a/tensorboard/plugins/scalar/scalars_plugin_test.py b/tensorboard/plugins/scalar/scalars_plugin_test.py index 60840c2b11..30bd7f1574 100644 --- a/tensorboard/plugins/scalar/scalars_plugin_test.py +++ b/tensorboard/plugins/scalar/scalars_plugin_test.py @@ -32,8 +32,12 @@ from tensorboard import errors from tensorboard.backend import application from tensorboard.backend.event_processing import data_provider -from tensorboard.backend.event_processing import plugin_event_accumulator as event_accumulator -from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer +from tensorboard.backend.event_processing import ( + plugin_event_accumulator as event_accumulator, +) +from tensorboard.backend.event_processing import ( + plugin_event_multiplexer as event_multiplexer, +) from tensorboard.plugins import base_plugin from tensorboard.plugins.core import core_plugin from tensorboard.plugins.scalar import metadata @@ -47,255 +51,304 @@ class ScalarsPluginTest(tf.test.TestCase): - _STEPS = 9 - - _LEGACY_SCALAR_TAG = 'ancient-values' - _SCALAR_TAG = 'simple-values' - _HISTOGRAM_TAG = 'complicated-values' - - _DISPLAY_NAME = 'Walrus population' - _DESCRIPTION = 'the *most* valuable statistic' - _HTML_DESCRIPTION = 'the most valuable statistic
' - - _RUN_WITH_LEGACY_SCALARS = '_RUN_WITH_LEGACY_SCALARS' - _RUN_WITH_SCALARS = '_RUN_WITH_SCALARS' - _RUN_WITH_HISTOGRAM = '_RUN_WITH_HISTOGRAM' - - def __init__(self, *args, **kwargs): - super(ScalarsPluginTest, self).__init__(*args, **kwargs) - self.plugin = None # used by DB tests only - - def load_runs(self, run_names): - logdir = self.get_temp_dir() - for run_name in run_names: - self.generate_run(logdir, run_name) - multiplexer = event_multiplexer.EventMultiplexer(size_guidance={ - # don't truncate my test data, please - event_accumulator.TENSORS: self._STEPS, - }) - multiplexer.AddRunsFromDirectory(logdir) - multiplexer.Reload() - return (logdir, multiplexer) - - def set_up_db(self): - self.db_path = os.path.join(self.get_temp_dir(), 'db.db') - self.db_uri = 'sqlite:' + self.db_path - db_connection_provider = application.create_sqlite_connection_provider( - self.db_uri) - context = base_plugin.TBContext( - db_connection_provider=db_connection_provider, - db_uri=self.db_uri) - self.core_plugin = core_plugin.CorePlugin(context) - self.plugin = scalars_plugin.ScalarsPlugin(context) - - def generate_run_to_db(self, experiment_name, run_name): - # This method uses `tf.contrib.summary`, and so must only be invoked - # when TensorFlow 1.x is installed. - tf.compat.v1.reset_default_graph() - with tf.compat.v1.Graph().as_default(): - global_step = tf.compat.v1.placeholder(tf.int64) - db_writer = tf.contrib.summary.create_db_writer( - db_uri=self.db_path, - experiment_name=experiment_name, - run_name=run_name, - user_name='user') - with db_writer.as_default(), tf.contrib.summary.always_record_summaries(): - tf.contrib.summary.scalar(self._SCALAR_TAG, 42, step=global_step) - flush_op = tf.contrib.summary.flush(db_writer._resource) - with tf.compat.v1.Session() as sess: - sess.run(tf.contrib.summary.summary_writer_initializer_op()) - summaries = tf.contrib.summary.all_summary_ops() - for step in xrange(self._STEPS): - feed_dict = {global_step: step} - sess.run(summaries, feed_dict=feed_dict) - sess.run(flush_op) - - def with_runs(run_names): - """Run a test with a bare multiplexer and with a `data_provider`. - - The decorated function will receive an initialized `ScalarsPlugin` - object as its first positional argument. - """ - def decorator(fn): - @functools.wraps(fn) - def wrapper(self, *args, **kwargs): - (logdir, multiplexer) = self.load_runs(run_names) - with self.subTest('bare multiplexer'): - ctx = base_plugin.TBContext(logdir=logdir, multiplexer=multiplexer) - fn(self, scalars_plugin.ScalarsPlugin(ctx), *args, **kwargs) - with self.subTest('generic data provider'): - flags = argparse.Namespace(generic_data='true') - provider = data_provider.MultiplexerDataProvider(multiplexer, logdir) - ctx = base_plugin.TBContext( - flags=flags, - logdir=logdir, - multiplexer=multiplexer, - data_provider=provider, - ) - fn(self, scalars_plugin.ScalarsPlugin(ctx), *args, **kwargs) - return wrapper - return decorator - - def generate_run(self, logdir, run_name): - subdir = os.path.join(logdir, run_name) - with test_util.FileWriterCache.get(subdir) as writer: - for step in xrange(self._STEPS): - data = [1 + step, 2 + step, 3 + step] - if run_name == self._RUN_WITH_LEGACY_SCALARS: - summ = tf.compat.v1.summary.scalar( - self._LEGACY_SCALAR_TAG, tf.reduce_mean(data), - ).numpy() - elif run_name == self._RUN_WITH_SCALARS: - summ = summary.op( - self._SCALAR_TAG, - tf.reduce_sum(data), - display_name=self._DISPLAY_NAME, - description=self._DESCRIPTION, - ).numpy() - elif run_name == self._RUN_WITH_HISTOGRAM: - summ = tf.compat.v1.summary.histogram( - self._HISTOGRAM_TAG, data - ).numpy() - else: - assert False, 'Invalid run name: %r' % run_name - writer.add_summary(summ, global_step=step) - - @with_runs([]) - def testRoutesProvided(self, plugin): - """Tests that the plugin offers the correct routes.""" - routes = plugin.get_plugin_apps() - self.assertIsInstance(routes['/scalars'], collections.Callable) - self.assertIsInstance(routes['/tags'], collections.Callable) - - @with_runs([_RUN_WITH_LEGACY_SCALARS, _RUN_WITH_SCALARS, _RUN_WITH_HISTOGRAM]) - def test_index(self, plugin): - self.assertEqual({ - self._RUN_WITH_LEGACY_SCALARS: { - self._LEGACY_SCALAR_TAG: { - 'displayName': self._LEGACY_SCALAR_TAG, - 'description': '', - }, - }, - self._RUN_WITH_SCALARS: { - '%s/scalar_summary' % self._SCALAR_TAG: { - 'displayName': self._DISPLAY_NAME, - 'description': self._HTML_DESCRIPTION, + _STEPS = 9 + + _LEGACY_SCALAR_TAG = "ancient-values" + _SCALAR_TAG = "simple-values" + _HISTOGRAM_TAG = "complicated-values" + + _DISPLAY_NAME = "Walrus population" + _DESCRIPTION = "the *most* valuable statistic" + _HTML_DESCRIPTION = "the most valuable statistic
" + + _RUN_WITH_LEGACY_SCALARS = "_RUN_WITH_LEGACY_SCALARS" + _RUN_WITH_SCALARS = "_RUN_WITH_SCALARS" + _RUN_WITH_HISTOGRAM = "_RUN_WITH_HISTOGRAM" + + def __init__(self, *args, **kwargs): + super(ScalarsPluginTest, self).__init__(*args, **kwargs) + self.plugin = None # used by DB tests only + + def load_runs(self, run_names): + logdir = self.get_temp_dir() + for run_name in run_names: + self.generate_run(logdir, run_name) + multiplexer = event_multiplexer.EventMultiplexer( + size_guidance={ + # don't truncate my test data, please + event_accumulator.TENSORS: self._STEPS, + } + ) + multiplexer.AddRunsFromDirectory(logdir) + multiplexer.Reload() + return (logdir, multiplexer) + + def set_up_db(self): + self.db_path = os.path.join(self.get_temp_dir(), "db.db") + self.db_uri = "sqlite:" + self.db_path + db_connection_provider = application.create_sqlite_connection_provider( + self.db_uri + ) + context = base_plugin.TBContext( + db_connection_provider=db_connection_provider, db_uri=self.db_uri + ) + self.core_plugin = core_plugin.CorePlugin(context) + self.plugin = scalars_plugin.ScalarsPlugin(context) + + def generate_run_to_db(self, experiment_name, run_name): + # This method uses `tf.contrib.summary`, and so must only be invoked + # when TensorFlow 1.x is installed. + tf.compat.v1.reset_default_graph() + with tf.compat.v1.Graph().as_default(): + global_step = tf.compat.v1.placeholder(tf.int64) + db_writer = tf.contrib.summary.create_db_writer( + db_uri=self.db_path, + experiment_name=experiment_name, + run_name=run_name, + user_name="user", + ) + with db_writer.as_default(), tf.contrib.summary.always_record_summaries(): + tf.contrib.summary.scalar( + self._SCALAR_TAG, 42, step=global_step + ) + flush_op = tf.contrib.summary.flush(db_writer._resource) + with tf.compat.v1.Session() as sess: + sess.run(tf.contrib.summary.summary_writer_initializer_op()) + summaries = tf.contrib.summary.all_summary_ops() + for step in xrange(self._STEPS): + feed_dict = {global_step: step} + sess.run(summaries, feed_dict=feed_dict) + sess.run(flush_op) + + def with_runs(run_names): + """Run a test with a bare multiplexer and with a `data_provider`. + + The decorated function will receive an initialized + `ScalarsPlugin` object as its first positional argument. + """ + + def decorator(fn): + @functools.wraps(fn) + def wrapper(self, *args, **kwargs): + (logdir, multiplexer) = self.load_runs(run_names) + with self.subTest("bare multiplexer"): + ctx = base_plugin.TBContext( + logdir=logdir, multiplexer=multiplexer + ) + fn(self, scalars_plugin.ScalarsPlugin(ctx), *args, **kwargs) + with self.subTest("generic data provider"): + flags = argparse.Namespace(generic_data="true") + provider = data_provider.MultiplexerDataProvider( + multiplexer, logdir + ) + ctx = base_plugin.TBContext( + flags=flags, + logdir=logdir, + multiplexer=multiplexer, + data_provider=provider, + ) + fn(self, scalars_plugin.ScalarsPlugin(ctx), *args, **kwargs) + + return wrapper + + return decorator + + def generate_run(self, logdir, run_name): + subdir = os.path.join(logdir, run_name) + with test_util.FileWriterCache.get(subdir) as writer: + for step in xrange(self._STEPS): + data = [1 + step, 2 + step, 3 + step] + if run_name == self._RUN_WITH_LEGACY_SCALARS: + summ = tf.compat.v1.summary.scalar( + self._LEGACY_SCALAR_TAG, tf.reduce_mean(data), + ).numpy() + elif run_name == self._RUN_WITH_SCALARS: + summ = summary.op( + self._SCALAR_TAG, + tf.reduce_sum(data), + display_name=self._DISPLAY_NAME, + description=self._DESCRIPTION, + ).numpy() + elif run_name == self._RUN_WITH_HISTOGRAM: + summ = tf.compat.v1.summary.histogram( + self._HISTOGRAM_TAG, data + ).numpy() + else: + assert False, "Invalid run name: %r" % run_name + writer.add_summary(summ, global_step=step) + + @with_runs([]) + def testRoutesProvided(self, plugin): + """Tests that the plugin offers the correct routes.""" + routes = plugin.get_plugin_apps() + self.assertIsInstance(routes["/scalars"], collections.Callable) + self.assertIsInstance(routes["/tags"], collections.Callable) + + @with_runs( + [_RUN_WITH_LEGACY_SCALARS, _RUN_WITH_SCALARS, _RUN_WITH_HISTOGRAM] + ) + def test_index(self, plugin): + self.assertEqual( + { + self._RUN_WITH_LEGACY_SCALARS: { + self._LEGACY_SCALAR_TAG: { + "displayName": self._LEGACY_SCALAR_TAG, + "description": "", + }, + }, + self._RUN_WITH_SCALARS: { + "%s/scalar_summary" + % self._SCALAR_TAG: { + "displayName": self._DISPLAY_NAME, + "description": self._HTML_DESCRIPTION, + }, + }, + # _RUN_WITH_HISTOGRAM omitted: No scalar data. }, - }, - # _RUN_WITH_HISTOGRAM omitted: No scalar data. - }, plugin.index_impl('eid')) - - @with_runs([_RUN_WITH_LEGACY_SCALARS, _RUN_WITH_SCALARS, _RUN_WITH_HISTOGRAM]) - def _test_scalars_json(self, plugin, run_name, tag_name, should_work=True): - if should_work: - (data, mime_type) = plugin.scalars_impl( - tag_name, run_name, 'eid', scalars_plugin.OutputFormat.JSON) - self.assertEqual('application/json', mime_type) - self.assertEqual(len(data), self._STEPS) - else: - with self.assertRaises(errors.NotFoundError): - plugin.scalars_impl( - self._SCALAR_TAG, run_name, 'eid', scalars_plugin.OutputFormat.JSON + plugin.index_impl("eid"), + ) + + @with_runs( + [_RUN_WITH_LEGACY_SCALARS, _RUN_WITH_SCALARS, _RUN_WITH_HISTOGRAM] + ) + def _test_scalars_json(self, plugin, run_name, tag_name, should_work=True): + if should_work: + (data, mime_type) = plugin.scalars_impl( + tag_name, run_name, "eid", scalars_plugin.OutputFormat.JSON + ) + self.assertEqual("application/json", mime_type) + self.assertEqual(len(data), self._STEPS) + else: + with self.assertRaises(errors.NotFoundError): + plugin.scalars_impl( + self._SCALAR_TAG, + run_name, + "eid", + scalars_plugin.OutputFormat.JSON, + ) + + @with_runs( + [_RUN_WITH_LEGACY_SCALARS, _RUN_WITH_SCALARS, _RUN_WITH_HISTOGRAM] + ) + def _test_scalars_csv(self, plugin, run_name, tag_name, should_work=True): + if should_work: + (data, mime_type) = plugin.scalars_impl( + tag_name, run_name, "eid", scalars_plugin.OutputFormat.CSV + ) + self.assertEqual("text/csv", mime_type) + s = StringIO(data) + reader = csv.reader(s) + self.assertEqual(["Wall time", "Step", "Value"], next(reader)) + self.assertEqual(len(list(reader)), self._STEPS) + else: + with self.assertRaises(errors.NotFoundError): + plugin.scalars_impl( + self._SCALAR_TAG, + run_name, + "eid", + scalars_plugin.OutputFormat.CSV, + ) + + def test_scalars_json_with_legacy_scalars(self): + self._test_scalars_json( + self._RUN_WITH_LEGACY_SCALARS, self._LEGACY_SCALAR_TAG + ) + + def test_scalars_json_with_scalars(self): + self._test_scalars_json( + self._RUN_WITH_SCALARS, "%s/scalar_summary" % self._SCALAR_TAG + ) + + def test_scalars_json_with_histogram(self): + self._test_scalars_json( + self._RUN_WITH_HISTOGRAM, self._HISTOGRAM_TAG, should_work=False ) - @with_runs([_RUN_WITH_LEGACY_SCALARS, _RUN_WITH_SCALARS, _RUN_WITH_HISTOGRAM]) - def _test_scalars_csv(self, plugin, run_name, tag_name, should_work=True): - if should_work: - (data, mime_type) = plugin.scalars_impl( - tag_name, run_name, 'eid', scalars_plugin.OutputFormat.CSV) - self.assertEqual('text/csv', mime_type) - s = StringIO(data) - reader = csv.reader(s) - self.assertEqual(['Wall time', 'Step', 'Value'], next(reader)) - self.assertEqual(len(list(reader)), self._STEPS) - else: - with self.assertRaises(errors.NotFoundError): - plugin.scalars_impl( - self._SCALAR_TAG, run_name, 'eid', scalars_plugin.OutputFormat.CSV + def test_scalars_csv_with_legacy_scalars(self): + self._test_scalars_csv( + self._RUN_WITH_LEGACY_SCALARS, self._LEGACY_SCALAR_TAG ) - def test_scalars_json_with_legacy_scalars(self): - self._test_scalars_json(self._RUN_WITH_LEGACY_SCALARS, - self._LEGACY_SCALAR_TAG) - - def test_scalars_json_with_scalars(self): - self._test_scalars_json(self._RUN_WITH_SCALARS, - '%s/scalar_summary' % self._SCALAR_TAG) - - def test_scalars_json_with_histogram(self): - self._test_scalars_json(self._RUN_WITH_HISTOGRAM, self._HISTOGRAM_TAG, - should_work=False) - - def test_scalars_csv_with_legacy_scalars(self): - self._test_scalars_csv(self._RUN_WITH_LEGACY_SCALARS, - self._LEGACY_SCALAR_TAG) - - def test_scalars_csv_with_scalars(self): - self._test_scalars_csv(self._RUN_WITH_SCALARS, - '%s/scalar_summary' % self._SCALAR_TAG) - - def test_scalars_csv_with_histogram(self): - self._test_scalars_csv(self._RUN_WITH_HISTOGRAM, self._HISTOGRAM_TAG, - should_work=False) - - @with_runs([_RUN_WITH_LEGACY_SCALARS]) - def test_active_with_legacy_scalars(self, plugin): - self.assertTrue(plugin.is_active()) - - @with_runs([_RUN_WITH_SCALARS]) - def test_active_with_scalars(self, plugin): - self.assertTrue(plugin.is_active()) - - @with_runs([_RUN_WITH_HISTOGRAM]) - def test_active_with_histogram(self, plugin): - if plugin._data_provider: - # Hack, for now. - self.assertTrue(plugin.is_active()) - else: - self.assertFalse(plugin.is_active()) - - @with_runs([_RUN_WITH_LEGACY_SCALARS, _RUN_WITH_SCALARS, _RUN_WITH_HISTOGRAM]) - def test_active_with_all(self, plugin): - self.assertTrue(plugin.is_active()) - - @test_util.run_v1_only('Requires contrib for db writer') - def test_scalars_db_without_exp(self): - self.set_up_db() - self.generate_run_to_db('exp1', self._RUN_WITH_SCALARS) - - (data, mime_type) = self.plugin.scalars_impl( - self._SCALAR_TAG, self._RUN_WITH_SCALARS, 'eid', - scalars_plugin.OutputFormat.JSON) - self.assertEqual('application/json', mime_type) - # When querying DB-based backend without an experiment id, it returns all - # scalars without an experiment id. Such scalar can only be generated using - # raw SQL queries though. - self.assertEqual(len(data), 0) - - @test_util.run_v1_only('Requires contrib for db writer') - def test_scalars_db_filter_by_experiment(self): - self.set_up_db() - self.generate_run_to_db('exp1', self._RUN_WITH_SCALARS) - all_exps = self.core_plugin.list_experiments_impl() - exp1 = next((x for x in all_exps if x.get('name') == 'exp1'), {}) - - (data, mime_type) = self.plugin.scalars_impl( - self._SCALAR_TAG, self._RUN_WITH_SCALARS, exp1.get('id'), - scalars_plugin.OutputFormat.JSON) - self.assertEqual('application/json', mime_type) - self.assertEqual(len(data), self._STEPS) - - @test_util.run_v1_only('Requires contrib for db writer') - def test_scalars_db_no_match(self): - self.set_up_db() - self.generate_run_to_db('exp1', self._RUN_WITH_SCALARS) - - # experiment_id is a number but we passed a string here. - (data, mime_type) = self.plugin.scalars_impl( - self._SCALAR_TAG, self._RUN_WITH_SCALARS, 'random_exp_id', - scalars_plugin.OutputFormat.JSON) - self.assertEqual('application/json', mime_type) - self.assertEqual(len(data), 0) - -if __name__ == '__main__': - tf.test.main() + def test_scalars_csv_with_scalars(self): + self._test_scalars_csv( + self._RUN_WITH_SCALARS, "%s/scalar_summary" % self._SCALAR_TAG + ) + + def test_scalars_csv_with_histogram(self): + self._test_scalars_csv( + self._RUN_WITH_HISTOGRAM, self._HISTOGRAM_TAG, should_work=False + ) + + @with_runs([_RUN_WITH_LEGACY_SCALARS]) + def test_active_with_legacy_scalars(self, plugin): + self.assertTrue(plugin.is_active()) + + @with_runs([_RUN_WITH_SCALARS]) + def test_active_with_scalars(self, plugin): + self.assertTrue(plugin.is_active()) + + @with_runs([_RUN_WITH_HISTOGRAM]) + def test_active_with_histogram(self, plugin): + if plugin._data_provider: + # Hack, for now. + self.assertTrue(plugin.is_active()) + else: + self.assertFalse(plugin.is_active()) + + @with_runs( + [_RUN_WITH_LEGACY_SCALARS, _RUN_WITH_SCALARS, _RUN_WITH_HISTOGRAM] + ) + def test_active_with_all(self, plugin): + self.assertTrue(plugin.is_active()) + + @test_util.run_v1_only("Requires contrib for db writer") + def test_scalars_db_without_exp(self): + self.set_up_db() + self.generate_run_to_db("exp1", self._RUN_WITH_SCALARS) + + (data, mime_type) = self.plugin.scalars_impl( + self._SCALAR_TAG, + self._RUN_WITH_SCALARS, + "eid", + scalars_plugin.OutputFormat.JSON, + ) + self.assertEqual("application/json", mime_type) + # When querying DB-based backend without an experiment id, it returns all + # scalars without an experiment id. Such scalar can only be generated using + # raw SQL queries though. + self.assertEqual(len(data), 0) + + @test_util.run_v1_only("Requires contrib for db writer") + def test_scalars_db_filter_by_experiment(self): + self.set_up_db() + self.generate_run_to_db("exp1", self._RUN_WITH_SCALARS) + all_exps = self.core_plugin.list_experiments_impl() + exp1 = next((x for x in all_exps if x.get("name") == "exp1"), {}) + + (data, mime_type) = self.plugin.scalars_impl( + self._SCALAR_TAG, + self._RUN_WITH_SCALARS, + exp1.get("id"), + scalars_plugin.OutputFormat.JSON, + ) + self.assertEqual("application/json", mime_type) + self.assertEqual(len(data), self._STEPS) + + @test_util.run_v1_only("Requires contrib for db writer") + def test_scalars_db_no_match(self): + self.set_up_db() + self.generate_run_to_db("exp1", self._RUN_WITH_SCALARS) + + # experiment_id is a number but we passed a string here. + (data, mime_type) = self.plugin.scalars_impl( + self._SCALAR_TAG, + self._RUN_WITH_SCALARS, + "random_exp_id", + scalars_plugin.OutputFormat.JSON, + ) + self.assertEqual("application/json", mime_type) + self.assertEqual(len(data), 0) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/scalar/summary.py b/tensorboard/plugins/scalar/summary.py index e391971896..b1de180302 100644 --- a/tensorboard/plugins/scalar/summary.py +++ b/tensorboard/plugins/scalar/summary.py @@ -14,7 +14,8 @@ # ============================================================================== """Scalar summaries and TensorFlow operations to create them. -A scalar summary stores a single floating-point value, as a rank-0 tensor. +A scalar summary stores a single floating-point value, as a rank-0 +tensor. """ from __future__ import absolute_import @@ -32,78 +33,82 @@ scalar_pb = summary_v2.scalar_pb -def op(name, - data, - display_name=None, - description=None, - collections=None): - """Create a legacy scalar summary op. - - Arguments: - name: A unique name for the generated summary node. - data: A real numeric rank-0 `Tensor`. Must have `dtype` castable - to `float32`. - display_name: Optional name for this summary in TensorBoard, as a - constant `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - collections: Optional list of graph collections keys. The new - summary op is added to these collections. Defaults to - `[Graph Keys.SUMMARIES]`. - - Returns: - A TensorFlow summary op. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - if display_name is None: - display_name = name - summary_metadata = metadata.create_summary_metadata( - display_name=display_name, description=description) - with tf.name_scope(name): - with tf.control_dependencies([tf.assert_scalar(data)]): - return tf.summary.tensor_summary(name='scalar_summary', - tensor=tf.cast(data, tf.float32), - collections=collections, - summary_metadata=summary_metadata) +def op(name, data, display_name=None, description=None, collections=None): + """Create a legacy scalar summary op. + + Arguments: + name: A unique name for the generated summary node. + data: A real numeric rank-0 `Tensor`. Must have `dtype` castable + to `float32`. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A TensorFlow summary op. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description + ) + with tf.name_scope(name): + with tf.control_dependencies([tf.assert_scalar(data)]): + return tf.summary.tensor_summary( + name="scalar_summary", + tensor=tf.cast(data, tf.float32), + collections=collections, + summary_metadata=summary_metadata, + ) def pb(name, data, display_name=None, description=None): - """Create a legacy scalar summary protobuf. - - Arguments: - name: A unique name for the generated summary, including any desired - name scopes. - data: A rank-0 `np.array` or array-like form (so raw `int`s and - `float`s are fine, too). - display_name: Optional name for this summary in TensorBoard, as a - `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - `str`. Markdown is supported. Defaults to empty. - - Returns: - A `tf.Summary` protobuf object. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - data = np.array(data) - if data.shape != (): - raise ValueError('Expected scalar shape for data, saw shape: %s.' - % data.shape) - if data.dtype.kind not in ('b', 'i', 'u', 'f'): # bool, int, uint, float - raise ValueError('Cast %s to float is not supported' % data.dtype.name) - tensor = tf.make_tensor_proto(data.astype(np.float32)) - - if display_name is None: - display_name = name - summary_metadata = metadata.create_summary_metadata( - display_name=display_name, description=description) - tf_summary_metadata = tf.SummaryMetadata.FromString( - summary_metadata.SerializeToString()) - summary = tf.Summary() - summary.value.add(tag='%s/scalar_summary' % name, - metadata=tf_summary_metadata, - tensor=tensor) - return summary + """Create a legacy scalar summary protobuf. + + Arguments: + name: A unique name for the generated summary, including any desired + name scopes. + data: A rank-0 `np.array` or array-like form (so raw `int`s and + `float`s are fine, too). + display_name: Optional name for this summary in TensorBoard, as a + `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + `str`. Markdown is supported. Defaults to empty. + + Returns: + A `tf.Summary` protobuf object. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + data = np.array(data) + if data.shape != (): + raise ValueError( + "Expected scalar shape for data, saw shape: %s." % data.shape + ) + if data.dtype.kind not in ("b", "i", "u", "f"): # bool, int, uint, float + raise ValueError("Cast %s to float is not supported" % data.dtype.name) + tensor = tf.make_tensor_proto(data.astype(np.float32)) + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description + ) + tf_summary_metadata = tf.SummaryMetadata.FromString( + summary_metadata.SerializeToString() + ) + summary = tf.Summary() + summary.value.add( + tag="%s/scalar_summary" % name, + metadata=tf_summary_metadata, + tensor=tensor, + ) + return summary diff --git a/tensorboard/plugins/scalar/summary_test.py b/tensorboard/plugins/scalar/summary_test.py index 3be3a4190f..754591209e 100644 --- a/tensorboard/plugins/scalar/summary_test.py +++ b/tensorboard/plugins/scalar/summary_test.py @@ -34,171 +34,179 @@ from tensorboard.util import tensor_util try: - tf2.__version__ # Force lazy import to resolve + tf2.__version__ # Force lazy import to resolve except ImportError: - tf2 = None + tf2 = None try: - tf.compat.v1.enable_eager_execution() + tf.compat.v1.enable_eager_execution() except AttributeError: - # TF 2.0 doesn't have this symbol because eager is the default. - pass + # TF 2.0 doesn't have this symbol because eager is the default. + pass class SummaryBaseTest(object): - - def scalar(self, *args, **kwargs): - raise NotImplementedError() - - def test_tag(self): - self.assertEqual('a', self.scalar('a', 1).value[0].tag) - self.assertEqual('a/b', self.scalar('a/b', 1).value[0].tag) - - def test_metadata(self): - pb = self.scalar('a', 1.13) - summary_metadata = pb.value[0].metadata - plugin_data = summary_metadata.plugin_data - self.assertEqual(summary_metadata.summary_description, '') - self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) - content = summary_metadata.plugin_data.content - # There's no content, so successfully parsing is fine. - metadata.parse_plugin_metadata(content) - - def test_explicit_description(self): - description = 'The first letter of the alphabet.' - pb = self.scalar('a', 1.13, description=description) - summary_metadata = pb.value[0].metadata - self.assertEqual(summary_metadata.summary_description, description) - plugin_data = summary_metadata.plugin_data - self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) - content = summary_metadata.plugin_data.content - # There's no content, so successfully parsing is fine. - metadata.parse_plugin_metadata(content) - - def test_float_value(self): - pb = self.scalar('a', 1.13) - value = tensor_util.make_ndarray(pb.value[0].tensor).item() - self.assertEqual(float, type(value)) - self.assertNear(1.13, value, 1e-6) - - def test_int_value(self): - # ints should be valid, but converted to floats. - pb = self.scalar('a', 113) - value = tensor_util.make_ndarray(pb.value[0].tensor).item() - self.assertEqual(float, type(value)) - self.assertNear(113.0, value, 1e-6) - - def test_bool_value(self): - # bools should be valid, but converted to floats. - pb = self.scalar('a', True) - value = tensor_util.make_ndarray(pb.value[0].tensor).item() - self.assertEqual(float, type(value)) - self.assertEqual(1.0, value) - - def test_string_value(self): - # Use str.* in regex because PY3 numpy refers to string arrays using - # length-dependent type names in the format "str%d" % (32 * len(str)). - with six.assertRaisesRegex(self, (ValueError, tf.errors.UnimplementedError), - r'Cast str.*float'): - self.scalar('a', np.array('113')) - - def test_requires_rank_0(self): - with six.assertRaisesRegex(self, ValueError, r'Expected scalar shape'): - self.scalar('a', np.array([1, 1, 3])) + def scalar(self, *args, **kwargs): + raise NotImplementedError() + + def test_tag(self): + self.assertEqual("a", self.scalar("a", 1).value[0].tag) + self.assertEqual("a/b", self.scalar("a/b", 1).value[0].tag) + + def test_metadata(self): + pb = self.scalar("a", 1.13) + summary_metadata = pb.value[0].metadata + plugin_data = summary_metadata.plugin_data + self.assertEqual(summary_metadata.summary_description, "") + self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) + content = summary_metadata.plugin_data.content + # There's no content, so successfully parsing is fine. + metadata.parse_plugin_metadata(content) + + def test_explicit_description(self): + description = "The first letter of the alphabet." + pb = self.scalar("a", 1.13, description=description) + summary_metadata = pb.value[0].metadata + self.assertEqual(summary_metadata.summary_description, description) + plugin_data = summary_metadata.plugin_data + self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) + content = summary_metadata.plugin_data.content + # There's no content, so successfully parsing is fine. + metadata.parse_plugin_metadata(content) + + def test_float_value(self): + pb = self.scalar("a", 1.13) + value = tensor_util.make_ndarray(pb.value[0].tensor).item() + self.assertEqual(float, type(value)) + self.assertNear(1.13, value, 1e-6) + + def test_int_value(self): + # ints should be valid, but converted to floats. + pb = self.scalar("a", 113) + value = tensor_util.make_ndarray(pb.value[0].tensor).item() + self.assertEqual(float, type(value)) + self.assertNear(113.0, value, 1e-6) + + def test_bool_value(self): + # bools should be valid, but converted to floats. + pb = self.scalar("a", True) + value = tensor_util.make_ndarray(pb.value[0].tensor).item() + self.assertEqual(float, type(value)) + self.assertEqual(1.0, value) + + def test_string_value(self): + # Use str.* in regex because PY3 numpy refers to string arrays using + # length-dependent type names in the format "str%d" % (32 * len(str)). + with six.assertRaisesRegex( + self, (ValueError, tf.errors.UnimplementedError), r"Cast str.*float" + ): + self.scalar("a", np.array("113")) + + def test_requires_rank_0(self): + with six.assertRaisesRegex(self, ValueError, r"Expected scalar shape"): + self.scalar("a", np.array([1, 1, 3])) class SummaryV1PbTest(SummaryBaseTest, tf.test.TestCase): - def scalar(self, *args, **kwargs): - return summary.pb(*args, **kwargs) + def scalar(self, *args, **kwargs): + return summary.pb(*args, **kwargs) - def test_tag(self): - self.assertEqual('a/scalar_summary', self.scalar('a', 1).value[0].tag) - self.assertEqual('a/b/scalar_summary', self.scalar('a/b', 1).value[0].tag) + def test_tag(self): + self.assertEqual("a/scalar_summary", self.scalar("a", 1).value[0].tag) + self.assertEqual( + "a/b/scalar_summary", self.scalar("a/b", 1).value[0].tag + ) class SummaryV1OpTest(SummaryBaseTest, tf.test.TestCase): - def scalar(self, *args, **kwargs): - return summary_pb2.Summary.FromString(summary.op(*args, **kwargs).numpy()) + def scalar(self, *args, **kwargs): + return summary_pb2.Summary.FromString( + summary.op(*args, **kwargs).numpy() + ) - def test_tag(self): - self.assertEqual('a/scalar_summary', self.scalar('a', 1).value[0].tag) - self.assertEqual('a/b/scalar_summary', self.scalar('a/b', 1).value[0].tag) + def test_tag(self): + self.assertEqual("a/scalar_summary", self.scalar("a", 1).value[0].tag) + self.assertEqual( + "a/b/scalar_summary", self.scalar("a/b", 1).value[0].tag + ) - def test_scoped_tag(self): - with tf.name_scope('scope'): - self.assertEqual('scope/a/scalar_summary', - self.scalar('a', 1).value[0].tag) + def test_scoped_tag(self): + with tf.name_scope("scope"): + self.assertEqual( + "scope/a/scalar_summary", self.scalar("a", 1).value[0].tag + ) class SummaryV2PbTest(SummaryBaseTest, tf.test.TestCase): - def scalar(self, *args, **kwargs): - return summary.scalar_pb(*args, **kwargs) + def scalar(self, *args, **kwargs): + return summary.scalar_pb(*args, **kwargs) class SummaryV2OpTest(SummaryBaseTest, tf.test.TestCase): - - def setUp(self): - super(SummaryV2OpTest, self).setUp() - if tf2 is None: - self.skipTest('v2 summary API not available') - - def scalar(self, *args, **kwargs): - return self.scalar_event(*args, **kwargs).summary - - def scalar_event(self, *args, **kwargs): - self.write_scalar_event(*args, **kwargs) - event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), '*'))) - self.assertEqual(len(event_files), 1) - events = list(tf.compat.v1.train.summary_iterator(event_files[0])) - # Expect a boilerplate event for the file_version, then the summary one. - self.assertEqual(len(events), 2) - # Delete the event file to reset to an empty directory for later calls. - # TODO(nickfelt): use a unique subdirectory per writer instead. - os.remove(event_files[0]) - return events[1] - - def write_scalar_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - summary.scalar(*args, **kwargs) - writer.close() - - def test_scoped_tag(self): - with tf.name_scope('scope'): - self.assertEqual('scope/a', self.scalar('a', 1).value[0].tag) - - def test_step(self): - event = self.scalar_event('a', 1.0, step=333) - self.assertEqual(333, event.step) - - def test_default_step(self): - try: - tf2.summary.experimental.set_step(333) - # TODO(nickfelt): change test logic so we can just omit `step` entirely. - event = self.scalar_event('a', 1.0, step=None) - self.assertEqual(333, event.step) - finally: - # Reset to default state for other tests. - tf2.summary.experimental.set_step(None) + def setUp(self): + super(SummaryV2OpTest, self).setUp() + if tf2 is None: + self.skipTest("v2 summary API not available") + + def scalar(self, *args, **kwargs): + return self.scalar_event(*args, **kwargs).summary + + def scalar_event(self, *args, **kwargs): + self.write_scalar_event(*args, **kwargs) + event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), "*"))) + self.assertEqual(len(event_files), 1) + events = list(tf.compat.v1.train.summary_iterator(event_files[0])) + # Expect a boilerplate event for the file_version, then the summary one. + self.assertEqual(len(events), 2) + # Delete the event file to reset to an empty directory for later calls. + # TODO(nickfelt): use a unique subdirectory per writer instead. + os.remove(event_files[0]) + return events[1] + + def write_scalar_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + summary.scalar(*args, **kwargs) + writer.close() + + def test_scoped_tag(self): + with tf.name_scope("scope"): + self.assertEqual("scope/a", self.scalar("a", 1).value[0].tag) + + def test_step(self): + event = self.scalar_event("a", 1.0, step=333) + self.assertEqual(333, event.step) + + def test_default_step(self): + try: + tf2.summary.experimental.set_step(333) + # TODO(nickfelt): change test logic so we can just omit `step` entirely. + event = self.scalar_event("a", 1.0, step=None) + self.assertEqual(333, event.step) + finally: + # Reset to default state for other tests. + tf2.summary.experimental.set_step(None) class SummaryV2OpGraphTest(SummaryV2OpTest, tf.test.TestCase): - def write_scalar_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - # Hack to extract current scope since there's no direct API for it. - with tf.name_scope('_') as temp_scope: - scope = temp_scope.rstrip('/_') - @tf2.function - def graph_fn(): - # Recreate the active scope inside the defun since it won't propagate. - with tf.name_scope(scope): - summary.scalar(*args, **kwargs) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - graph_fn() - writer.close() - - -if __name__ == '__main__': - tf.test.main() + def write_scalar_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + # Hack to extract current scope since there's no direct API for it. + with tf.name_scope("_") as temp_scope: + scope = temp_scope.rstrip("/_") + + @tf2.function + def graph_fn(): + # Recreate the active scope inside the defun since it won't propagate. + with tf.name_scope(scope): + summary.scalar(*args, **kwargs) + + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + graph_fn() + writer.close() + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/scalar/summary_v2.py b/tensorboard/plugins/scalar/summary_v2.py index be5fc3b214..17aae4543a 100644 --- a/tensorboard/plugins/scalar/summary_v2.py +++ b/tensorboard/plugins/scalar/summary_v2.py @@ -14,7 +14,8 @@ # ============================================================================== """Scalar summaries and TensorFlow operations to create them, V2 versions. -A scalar summary stores a single floating-point value, as a rank-0 tensor. +A scalar summary stores a single floating-point value, as a rank-0 +tensor. """ from __future__ import absolute_import @@ -30,67 +31,70 @@ def scalar(name, data, step=None, description=None): - """Write a scalar summary. + """Write a scalar summary. - Arguments: - name: A name for this summary. The summary tag used for TensorBoard will - be this name prefixed by any active name scopes. - data: A real numeric scalar value, convertible to a `float32` Tensor. - step: Explicit `int64`-castable monotonic step value for this summary. If - omitted, this defaults to `tf.summary.experimental.get_step()`, which must - not be None. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. + Arguments: + name: A name for this summary. The summary tag used for TensorBoard will + be this name prefixed by any active name scopes. + data: A real numeric scalar value, convertible to a `float32` Tensor. + step: Explicit `int64`-castable monotonic step value for this summary. If + omitted, this defaults to `tf.summary.experimental.get_step()`, which must + not be None. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. - Returns: - True on success, or false if no summary was written because no default - summary writer was available. + Returns: + True on success, or false if no summary was written because no default + summary writer was available. - Raises: - ValueError: if a default writer exists, but no step was provided and - `tf.summary.experimental.get_step()` is None. - """ - summary_metadata = metadata.create_summary_metadata( - display_name=None, description=description) - # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback - summary_scope = ( - getattr(tf.summary.experimental, 'summary_scope', None) or - tf.summary.summary_scope) - with summary_scope( - name, 'scalar_summary', values=[data, step]) as (tag, _): - tf.debugging.assert_scalar(data) - return tf.summary.write(tag=tag, - tensor=tf.cast(data, tf.float32), - step=step, - metadata=summary_metadata) + Raises: + ValueError: if a default writer exists, but no step was provided and + `tf.summary.experimental.get_step()` is None. + """ + summary_metadata = metadata.create_summary_metadata( + display_name=None, description=description + ) + # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback + summary_scope = ( + getattr(tf.summary.experimental, "summary_scope", None) + or tf.summary.summary_scope + ) + with summary_scope(name, "scalar_summary", values=[data, step]) as (tag, _): + tf.debugging.assert_scalar(data) + return tf.summary.write( + tag=tag, + tensor=tf.cast(data, tf.float32), + step=step, + metadata=summary_metadata, + ) def scalar_pb(tag, data, description=None): - """Create a scalar summary_pb2.Summary protobuf. + """Create a scalar summary_pb2.Summary protobuf. - Arguments: - tag: String tag for the summary. - data: A 0-dimensional `np.array` or a compatible python number type. - description: Optional long-form description for this summary, as a - `str`. Markdown is supported. Defaults to empty. + Arguments: + tag: String tag for the summary. + data: A 0-dimensional `np.array` or a compatible python number type. + description: Optional long-form description for this summary, as a + `str`. Markdown is supported. Defaults to empty. - Raises: - ValueError: If the type or shape of the data is unsupported. + Raises: + ValueError: If the type or shape of the data is unsupported. - Returns: - A `summary_pb2.Summary` protobuf object. - """ - arr = np.array(data) - if arr.shape != (): - raise ValueError('Expected scalar shape for tensor, got shape: %s.' - % arr.shape) - if arr.dtype.kind not in ('b', 'i', 'u', 'f'): # bool, int, uint, float - raise ValueError('Cast %s to float is not supported' % arr.dtype.name) - tensor_proto = tensor_util.make_tensor_proto(arr.astype(np.float32)) - summary_metadata = metadata.create_summary_metadata( - display_name=None, description=description) - summary = summary_pb2.Summary() - summary.value.add(tag=tag, - metadata=summary_metadata, - tensor=tensor_proto) - return summary + Returns: + A `summary_pb2.Summary` protobuf object. + """ + arr = np.array(data) + if arr.shape != (): + raise ValueError( + "Expected scalar shape for tensor, got shape: %s." % arr.shape + ) + if arr.dtype.kind not in ("b", "i", "u", "f"): # bool, int, uint, float + raise ValueError("Cast %s to float is not supported" % arr.dtype.name) + tensor_proto = tensor_util.make_tensor_proto(arr.astype(np.float32)) + summary_metadata = metadata.create_summary_metadata( + display_name=None, description=description + ) + summary = summary_pb2.Summary() + summary.value.add(tag=tag, metadata=summary_metadata, tensor=tensor_proto) + return summary diff --git a/tensorboard/plugins/text/metadata.py b/tensorboard/plugins/text/metadata.py index 7d7225890d..4236fb07e2 100644 --- a/tensorboard/plugins/text/metadata.py +++ b/tensorboard/plugins/text/metadata.py @@ -24,7 +24,7 @@ logger = tb_logging.get_logger() -PLUGIN_NAME = 'text' +PLUGIN_NAME = "text" # The most recent value for the `version` field of the # `TextPluginData` proto. @@ -32,36 +32,42 @@ def create_summary_metadata(display_name, description): - """Create a `summary_pb2.SummaryMetadata` proto for text plugin data. - Returns: - A `summary_pb2.SummaryMetadata` protobuf object. - """ - content = plugin_data_pb2.TextPluginData(version=PROTO_VERSION) - metadata = summary_pb2.SummaryMetadata( - display_name=display_name, - summary_description=description, - plugin_data=summary_pb2.SummaryMetadata.PluginData( - plugin_name=PLUGIN_NAME, - content=content.SerializeToString())) - return metadata + """Create a `summary_pb2.SummaryMetadata` proto for text plugin data. + + Returns: + A `summary_pb2.SummaryMetadata` protobuf object. + """ + content = plugin_data_pb2.TextPluginData(version=PROTO_VERSION) + metadata = summary_pb2.SummaryMetadata( + display_name=display_name, + summary_description=description, + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME, content=content.SerializeToString() + ), + ) + return metadata def parse_plugin_metadata(content): - """Parse summary metadata to a Python object. - Arguments: - content: The `content` field of a `SummaryMetadata` proto corresponding to - the text plugin. - Returns: - A `TextPluginData` protobuf object. - """ - if not isinstance(content, bytes): - raise TypeError('Content type must be bytes') - result = plugin_data_pb2.TextPluginData.FromString(content) - if result.version == 0: - return result - else: - logger.warn( - 'Unknown metadata version: %s. The latest version known to ' - 'this build of TensorBoard is %s; perhaps a newer build is ' - 'available?', result.version, PROTO_VERSION) - return result + """Parse summary metadata to a Python object. + + Arguments: + content: The `content` field of a `SummaryMetadata` proto corresponding to + the text plugin. + Returns: + A `TextPluginData` protobuf object. + """ + if not isinstance(content, bytes): + raise TypeError("Content type must be bytes") + result = plugin_data_pb2.TextPluginData.FromString(content) + if result.version == 0: + return result + else: + logger.warn( + "Unknown metadata version: %s. The latest version known to " + "this build of TensorBoard is %s; perhaps a newer build is " + "available?", + result.version, + PROTO_VERSION, + ) + return result diff --git a/tensorboard/plugins/text/summary.py b/tensorboard/plugins/text/summary.py index 4248c8fcb4..76f78be157 100644 --- a/tensorboard/plugins/text/summary.py +++ b/tensorboard/plugins/text/summary.py @@ -27,90 +27,93 @@ text_pb = summary_v2.text_pb -def op(name, - data, - display_name=None, - description=None, - collections=None): - """Create a legacy text summary op. - - Text data summarized via this plugin will be visible in the Text Dashboard - in TensorBoard. The standard TensorBoard Text Dashboard will render markdown - in the strings, and will automatically organize 1D and 2D tensors into tables. - If a tensor with more than 2 dimensions is provided, a 2D subarray will be - displayed along with a warning message. (Note that this behavior is not - intrinsic to the text summary API, but rather to the default TensorBoard text - plugin.) - - Args: - name: A name for the generated node. Will also serve as a series name in - TensorBoard. - data: A string-type Tensor to summarize. The text must be encoded in UTF-8. - display_name: Optional name for this summary in TensorBoard, as a - constant `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. - collections: Optional list of ops.GraphKeys. The collections to which to add - the summary. Defaults to [Graph Keys.SUMMARIES]. - - Returns: - A TensorSummary op that is configured so that TensorBoard will recognize - that it contains textual data. The TensorSummary is a scalar `Tensor` of - type `string` which contains `Summary` protobufs. - - Raises: - ValueError: If tensor has the wrong type. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - if display_name is None: - display_name = name - summary_metadata = metadata.create_summary_metadata( - display_name=display_name, description=description) - with tf.name_scope(name): - with tf.control_dependencies([tf.assert_type(data, tf.string)]): - return tf.summary.tensor_summary(name='text_summary', - tensor=data, - collections=collections, - summary_metadata=summary_metadata) +def op(name, data, display_name=None, description=None, collections=None): + """Create a legacy text summary op. + + Text data summarized via this plugin will be visible in the Text Dashboard + in TensorBoard. The standard TensorBoard Text Dashboard will render markdown + in the strings, and will automatically organize 1D and 2D tensors into tables. + If a tensor with more than 2 dimensions is provided, a 2D subarray will be + displayed along with a warning message. (Note that this behavior is not + intrinsic to the text summary API, but rather to the default TensorBoard text + plugin.) + + Args: + name: A name for the generated node. Will also serve as a series name in + TensorBoard. + data: A string-type Tensor to summarize. The text must be encoded in UTF-8. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of ops.GraphKeys. The collections to which to add + the summary. Defaults to [Graph Keys.SUMMARIES]. + + Returns: + A TensorSummary op that is configured so that TensorBoard will recognize + that it contains textual data. The TensorSummary is a scalar `Tensor` of + type `string` which contains `Summary` protobufs. + + Raises: + ValueError: If tensor has the wrong type. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description + ) + with tf.name_scope(name): + with tf.control_dependencies([tf.assert_type(data, tf.string)]): + return tf.summary.tensor_summary( + name="text_summary", + tensor=data, + collections=collections, + summary_metadata=summary_metadata, + ) def pb(name, data, display_name=None, description=None): - """Create a legacy text summary protobuf. - - Arguments: - name: A name for the generated node. Will also serve as a series name in - TensorBoard. - data: A Python bytestring (of type bytes), or Unicode string. Or a numpy - data array of those types. - display_name: Optional name for this summary in TensorBoard, as a - `str`. Defaults to `name`. - description: Optional long-form description for this summary, as a - `str`. Markdown is supported. Defaults to empty. - - Raises: - ValueError: If the type of the data is unsupported. - - Returns: - A `tf.Summary` protobuf object. - """ - # TODO(nickfelt): remove on-demand imports once dep situation is fixed. - import tensorflow.compat.v1 as tf - - try: - tensor = tf.make_tensor_proto(data, dtype=tf.string) - except TypeError as e: - raise ValueError(e) - - if display_name is None: - display_name = name - summary_metadata = metadata.create_summary_metadata( - display_name=display_name, description=description) - tf_summary_metadata = tf.SummaryMetadata.FromString( - summary_metadata.SerializeToString()) - summary = tf.Summary() - summary.value.add(tag='%s/text_summary' % name, - metadata=tf_summary_metadata, - tensor=tensor) - return summary + """Create a legacy text summary protobuf. + + Arguments: + name: A name for the generated node. Will also serve as a series name in + TensorBoard. + data: A Python bytestring (of type bytes), or Unicode string. Or a numpy + data array of those types. + display_name: Optional name for this summary in TensorBoard, as a + `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + `str`. Markdown is supported. Defaults to empty. + + Raises: + ValueError: If the type of the data is unsupported. + + Returns: + A `tf.Summary` protobuf object. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + try: + tensor = tf.make_tensor_proto(data, dtype=tf.string) + except TypeError as e: + raise ValueError(e) + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description + ) + tf_summary_metadata = tf.SummaryMetadata.FromString( + summary_metadata.SerializeToString() + ) + summary = tf.Summary() + summary.value.add( + tag="%s/text_summary" % name, + metadata=tf_summary_metadata, + tensor=tensor, + ) + return summary diff --git a/tensorboard/plugins/text/summary_test.py b/tensorboard/plugins/text/summary_test.py index 2de1b572df..ee9d5d4eb4 100644 --- a/tensorboard/plugins/text/summary_test.py +++ b/tensorboard/plugins/text/summary_test.py @@ -34,188 +34,203 @@ from tensorboard.util import tensor_util try: - tf2.__version__ # Force lazy import to resolve + tf2.__version__ # Force lazy import to resolve except ImportError: - tf2 = None + tf2 = None try: - tf.compat.v1.enable_eager_execution() + tf.compat.v1.enable_eager_execution() except AttributeError: - # TF 2.0 doesn't have this symbol because eager is the default. - pass + # TF 2.0 doesn't have this symbol because eager is the default. + pass class SummaryBaseTest(object): - - def text(self, *args, **kwargs): - raise NotImplementedError() - - def test_tag(self): - self.assertEqual('a', self.text('a', 'foo').value[0].tag) - self.assertEqual('a/b', self.text('a/b', 'foo').value[0].tag) - - def test_metadata(self): - pb = self.text('do', 'A deer. A female deer.') - summary_metadata = pb.value[0].metadata - plugin_data = summary_metadata.plugin_data - self.assertEqual(summary_metadata.summary_description, '') - self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) - content = summary_metadata.plugin_data.content - # There's no content, so successfully parsing is fine. - metadata.parse_plugin_metadata(content) - - def test_explicit_description(self): - description = 'A whole step above do.' - pb = self.text('re', 'A drop of golden sun.', description=description) - summary_metadata = pb.value[0].metadata - self.assertEqual(summary_metadata.summary_description, description) - plugin_data = summary_metadata.plugin_data - self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) - content = summary_metadata.plugin_data.content - # There's no content, so successfully parsing is fine. - metadata.parse_plugin_metadata(content) - - def test_bytes_value(self): - pb = self.text('mi', b'A name\xe2\x80\xa6I call myself') - value = tensor_util.make_ndarray(pb.value[0].tensor).item() - self.assertIsInstance(value, six.binary_type) - self.assertEqual(b'A name\xe2\x80\xa6I call myself', value) - - def test_unicode_value(self): - pb = self.text('mi', u'A name\u2026I call myself') - value = tensor_util.make_ndarray(pb.value[0].tensor).item() - self.assertIsInstance(value, six.binary_type) - self.assertEqual(b'A name\xe2\x80\xa6I call myself', value) - - def test_np_array_bytes_value(self): - pb = self.text( - 'fa', - np.array( - [[b'A', b'long', b'long'], [b'way', b'to', b'run \xe2\x80\xbc']])) - values = tensor_util.make_ndarray(pb.value[0].tensor).tolist() - self.assertEqual( - [[b'A', b'long', b'long'], [b'way', b'to', b'run \xe2\x80\xbc']], - values) - # Check that all entries are byte strings. - for vectors in values: - for value in vectors: + def text(self, *args, **kwargs): + raise NotImplementedError() + + def test_tag(self): + self.assertEqual("a", self.text("a", "foo").value[0].tag) + self.assertEqual("a/b", self.text("a/b", "foo").value[0].tag) + + def test_metadata(self): + pb = self.text("do", "A deer. A female deer.") + summary_metadata = pb.value[0].metadata + plugin_data = summary_metadata.plugin_data + self.assertEqual(summary_metadata.summary_description, "") + self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) + content = summary_metadata.plugin_data.content + # There's no content, so successfully parsing is fine. + metadata.parse_plugin_metadata(content) + + def test_explicit_description(self): + description = "A whole step above do." + pb = self.text("re", "A drop of golden sun.", description=description) + summary_metadata = pb.value[0].metadata + self.assertEqual(summary_metadata.summary_description, description) + plugin_data = summary_metadata.plugin_data + self.assertEqual(plugin_data.plugin_name, metadata.PLUGIN_NAME) + content = summary_metadata.plugin_data.content + # There's no content, so successfully parsing is fine. + metadata.parse_plugin_metadata(content) + + def test_bytes_value(self): + pb = self.text("mi", b"A name\xe2\x80\xa6I call myself") + value = tensor_util.make_ndarray(pb.value[0].tensor).item() self.assertIsInstance(value, six.binary_type) + self.assertEqual(b"A name\xe2\x80\xa6I call myself", value) - def test_np_array_unicode_value(self): - pb = self.text( - 'fa', - np.array( - [[u'A', u'long', u'long'], [u'way', u'to', u'run \u203C']])) - values = tensor_util.make_ndarray (pb.value[0].tensor).tolist() - self.assertEqual( - [[b'A', b'long', b'long'], [b'way', b'to', b'run \xe2\x80\xbc']], - values) - # Check that all entries are byte strings. - for vectors in values: - for value in vectors: + def test_unicode_value(self): + pb = self.text("mi", u"A name\u2026I call myself") + value = tensor_util.make_ndarray(pb.value[0].tensor).item() self.assertIsInstance(value, six.binary_type) - - def test_non_string_value(self): - with six.assertRaisesRegex(self, TypeError, r'must be of type.*string'): - self.text('la', np.array(range(42))) + self.assertEqual(b"A name\xe2\x80\xa6I call myself", value) + + def test_np_array_bytes_value(self): + pb = self.text( + "fa", + np.array( + [[b"A", b"long", b"long"], [b"way", b"to", b"run \xe2\x80\xbc"]] + ), + ) + values = tensor_util.make_ndarray(pb.value[0].tensor).tolist() + self.assertEqual( + [[b"A", b"long", b"long"], [b"way", b"to", b"run \xe2\x80\xbc"]], + values, + ) + # Check that all entries are byte strings. + for vectors in values: + for value in vectors: + self.assertIsInstance(value, six.binary_type) + + def test_np_array_unicode_value(self): + pb = self.text( + "fa", + np.array( + [[u"A", u"long", u"long"], [u"way", u"to", u"run \u203C"]] + ), + ) + values = tensor_util.make_ndarray(pb.value[0].tensor).tolist() + self.assertEqual( + [[b"A", b"long", b"long"], [b"way", b"to", b"run \xe2\x80\xbc"]], + values, + ) + # Check that all entries are byte strings. + for vectors in values: + for value in vectors: + self.assertIsInstance(value, six.binary_type) + + def test_non_string_value(self): + with six.assertRaisesRegex(self, TypeError, r"must be of type.*string"): + self.text("la", np.array(range(42))) class SummaryV1PbTest(SummaryBaseTest, tf.test.TestCase): - def text(self, *args, **kwargs): - return summary.pb(*args, **kwargs) + def text(self, *args, **kwargs): + return summary.pb(*args, **kwargs) - def test_tag(self): - self.assertEqual('a/text_summary', self.text('a', 'foo').value[0].tag) - self.assertEqual('a/b/text_summary', self.text('a/b', 'foo').value[0].tag) + def test_tag(self): + self.assertEqual("a/text_summary", self.text("a", "foo").value[0].tag) + self.assertEqual( + "a/b/text_summary", self.text("a/b", "foo").value[0].tag + ) - def test_non_string_value(self): - with six.assertRaisesRegex(self, ValueError, - r'Expected binary or unicode string, got 0'): - self.text('la', np.array(range(42))) + def test_non_string_value(self): + with six.assertRaisesRegex( + self, ValueError, r"Expected binary or unicode string, got 0" + ): + self.text("la", np.array(range(42))) class SummaryV1OpTest(SummaryBaseTest, tf.test.TestCase): - def text(self, *args, **kwargs): - return summary_pb2.Summary.FromString(summary.op(*args, **kwargs).numpy()) + def text(self, *args, **kwargs): + return summary_pb2.Summary.FromString( + summary.op(*args, **kwargs).numpy() + ) - def test_tag(self): - self.assertEqual('a/text_summary', self.text('a', 'foo').value[0].tag) - self.assertEqual('a/b/text_summary', self.text('a/b', 'foo').value[0].tag) + def test_tag(self): + self.assertEqual("a/text_summary", self.text("a", "foo").value[0].tag) + self.assertEqual( + "a/b/text_summary", self.text("a/b", "foo").value[0].tag + ) - def test_scoped_tag(self): - with tf.name_scope('scope'): - self.assertEqual('scope/a/text_summary', - self.text('a', 'foo').value[0].tag) + def test_scoped_tag(self): + with tf.name_scope("scope"): + self.assertEqual( + "scope/a/text_summary", self.text("a", "foo").value[0].tag + ) class SummaryV2PbTest(SummaryBaseTest, tf.test.TestCase): - def text(self, *args, **kwargs): - return summary.text_pb(*args, **kwargs) + def text(self, *args, **kwargs): + return summary.text_pb(*args, **kwargs) class SummaryV2OpTest(SummaryBaseTest, tf.test.TestCase): - def setUp(self): - super(SummaryV2OpTest, self).setUp() - if tf2 is None: - self.skipTest('TF v2 summary API not available') - - def text(self, *args, **kwargs): - return self.text_event(*args, **kwargs).summary - - def text_event(self, *args, **kwargs): - self.write_text_event(*args, **kwargs) - event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), '*'))) - self.assertEqual(len(event_files), 1) - events = list(tf.compat.v1.train.summary_iterator(event_files[0])) - # Expect a boilerplate event for the file_version, then the summary one. - self.assertEqual(len(events), 2) - # Delete the event file to reset to an empty directory for later calls. - # TODO(nickfelt): use a unique subdirectory per writer instead. - os.remove(event_files[0]) - return events[1] - - def write_text_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - summary.text(*args, **kwargs) - writer.close() - - def test_scoped_tag(self): - with tf.name_scope('scope'): - self.assertEqual('scope/a', self.text('a', 'foo').value[0].tag) - - def test_step(self): - event = self.text_event('a', 'foo', step=333) - self.assertEqual(333, event.step) - - def test_default_step(self): - try: - tf2.summary.experimental.set_step(333) - # TODO(nickfelt): change test logic so we can just omit `step` entirely. - event = self.text_event('a', 'foo', step=None) - self.assertEqual(333, event.step) - finally: - # Reset to default state for other tests. - tf2.summary.experimental.set_step(None) + def setUp(self): + super(SummaryV2OpTest, self).setUp() + if tf2 is None: + self.skipTest("TF v2 summary API not available") + + def text(self, *args, **kwargs): + return self.text_event(*args, **kwargs).summary + + def text_event(self, *args, **kwargs): + self.write_text_event(*args, **kwargs) + event_files = sorted(glob.glob(os.path.join(self.get_temp_dir(), "*"))) + self.assertEqual(len(event_files), 1) + events = list(tf.compat.v1.train.summary_iterator(event_files[0])) + # Expect a boilerplate event for the file_version, then the summary one. + self.assertEqual(len(events), 2) + # Delete the event file to reset to an empty directory for later calls. + # TODO(nickfelt): use a unique subdirectory per writer instead. + os.remove(event_files[0]) + return events[1] + + def write_text_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + summary.text(*args, **kwargs) + writer.close() + + def test_scoped_tag(self): + with tf.name_scope("scope"): + self.assertEqual("scope/a", self.text("a", "foo").value[0].tag) + + def test_step(self): + event = self.text_event("a", "foo", step=333) + self.assertEqual(333, event.step) + + def test_default_step(self): + try: + tf2.summary.experimental.set_step(333) + # TODO(nickfelt): change test logic so we can just omit `step` entirely. + event = self.text_event("a", "foo", step=None) + self.assertEqual(333, event.step) + finally: + # Reset to default state for other tests. + tf2.summary.experimental.set_step(None) class SummaryV2OpGraphTest(SummaryV2OpTest, tf.test.TestCase): - def write_text_event(self, *args, **kwargs): - kwargs.setdefault('step', 1) - # Hack to extract current scope since there's no direct API for it. - with tf.name_scope('_') as temp_scope: - scope = temp_scope.rstrip('/_') - @tf2.function - def graph_fn(): - # Recreate the active scope inside the defun since it won't propagate. - with tf.name_scope(scope): - summary.text(*args, **kwargs) - writer = tf2.summary.create_file_writer(self.get_temp_dir()) - with writer.as_default(): - graph_fn() - writer.close() - - -if __name__ == '__main__': - tf.test.main() + def write_text_event(self, *args, **kwargs): + kwargs.setdefault("step", 1) + # Hack to extract current scope since there's no direct API for it. + with tf.name_scope("_") as temp_scope: + scope = temp_scope.rstrip("/_") + + @tf2.function + def graph_fn(): + # Recreate the active scope inside the defun since it won't propagate. + with tf.name_scope(scope): + summary.text(*args, **kwargs) + + writer = tf2.summary.create_file_writer(self.get_temp_dir()) + with writer.as_default(): + graph_fn() + writer.close() + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorboard/plugins/text/summary_v2.py b/tensorboard/plugins/text/summary_v2.py index a5b076196c..5d14c84b24 100644 --- a/tensorboard/plugins/text/summary_v2.py +++ b/tensorboard/plugins/text/summary_v2.py @@ -27,63 +27,64 @@ def text(name, data, step=None, description=None): - """Write a text summary. + """Write a text summary. - Arguments: - name: A name for this summary. The summary tag used for TensorBoard will - be this name prefixed by any active name scopes. - data: A UTF-8 string tensor value. - step: Explicit `int64`-castable monotonic step value for this summary. If - omitted, this defaults to `tf.summary.experimental.get_step()`, which must - not be None. - description: Optional long-form description for this summary, as a - constant `str`. Markdown is supported. Defaults to empty. + Arguments: + name: A name for this summary. The summary tag used for TensorBoard will + be this name prefixed by any active name scopes. + data: A UTF-8 string tensor value. + step: Explicit `int64`-castable monotonic step value for this summary. If + omitted, this defaults to `tf.summary.experimental.get_step()`, which must + not be None. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. - Returns: - True on success, or false if no summary was emitted because no default - summary writer was available. + Returns: + True on success, or false if no summary was emitted because no default + summary writer was available. - Raises: - ValueError: if a default writer exists, but no step was provided and - `tf.summary.experimental.get_step()` is None. - """ - summary_metadata = metadata.create_summary_metadata( - display_name=None, description=description) - # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback - summary_scope = ( - getattr(tf.summary.experimental, 'summary_scope', None) or - tf.summary.summary_scope) - with summary_scope( - name, 'text_summary', values=[data, step]) as (tag, _): - tf.debugging.assert_type(data, tf.string) - return tf.summary.write( - tag=tag, tensor=data, step=step, metadata=summary_metadata) + Raises: + ValueError: if a default writer exists, but no step was provided and + `tf.summary.experimental.get_step()` is None. + """ + summary_metadata = metadata.create_summary_metadata( + display_name=None, description=description + ) + # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback + summary_scope = ( + getattr(tf.summary.experimental, "summary_scope", None) + or tf.summary.summary_scope + ) + with summary_scope(name, "text_summary", values=[data, step]) as (tag, _): + tf.debugging.assert_type(data, tf.string) + return tf.summary.write( + tag=tag, tensor=data, step=step, metadata=summary_metadata + ) def text_pb(tag, data, description=None): - """Create a text tf.Summary protobuf. + """Create a text tf.Summary protobuf. - Arguments: - tag: String tag for the summary. - data: A Python bytestring (of type bytes), a Unicode string, or a numpy data - array of those types. - description: Optional long-form description for this summary, as a `str`. - Markdown is supported. Defaults to empty. + Arguments: + tag: String tag for the summary. + data: A Python bytestring (of type bytes), a Unicode string, or a numpy data + array of those types. + description: Optional long-form description for this summary, as a `str`. + Markdown is supported. Defaults to empty. - Raises: - TypeError: If the type of the data is unsupported. + Raises: + TypeError: If the type of the data is unsupported. - Returns: - A `tf.Summary` protobuf object. - """ - try: - tensor = tensor_util.make_tensor_proto(data, dtype=np.object) - except TypeError as e: - raise TypeError('tensor must be of type string', e) - summary_metadata = metadata.create_summary_metadata( - display_name=None, description=description) - summary = summary_pb2.Summary() - summary.value.add(tag=tag, - metadata=summary_metadata, - tensor=tensor) - return summary + Returns: + A `tf.Summary` protobuf object. + """ + try: + tensor = tensor_util.make_tensor_proto(data, dtype=np.object) + except TypeError as e: + raise TypeError("tensor must be of type string", e) + summary_metadata = metadata.create_summary_metadata( + display_name=None, description=description + ) + summary = summary_pb2.Summary() + summary.value.add(tag=tag, metadata=summary_metadata, tensor=tensor) + return summary diff --git a/tensorboard/plugins/text/text_demo.py b/tensorboard/plugins/text/text_demo.py index c2e47f1641..eea016c7eb 100644 --- a/tensorboard/plugins/text/text_demo.py +++ b/tensorboard/plugins/text/text_demo.py @@ -28,108 +28,108 @@ logger = tb_logging.get_logger() # Directory into which to write tensorboard data. -LOGDIR = '/tmp/text_demo' +LOGDIR = "/tmp/text_demo" # Number of steps for which to write data. STEPS = 16 def simple_example(step): - # Text summaries log arbitrary text. This can be encoded with ASCII or - # UTF-8. Here's a simple example, wherein we greet the user on each - # step: - step_string = tf.as_string(step) - greeting = tf.strings.join(['Hello from step ', step_string, '!']) - tf.compat.v1.summary.text('greeting', greeting) + # Text summaries log arbitrary text. This can be encoded with ASCII or + # UTF-8. Here's a simple example, wherein we greet the user on each + # step: + step_string = tf.as_string(step) + greeting = tf.strings.join(["Hello from step ", step_string, "!"]) + tf.compat.v1.summary.text("greeting", greeting) def markdown_table(step): - # The text summary can also contain Markdown, including Markdown - # tables. Markdown tables look like this: - # - # | hello | there | - # |-------|-------| - # | this | is | - # | a | table | - # - # The leading and trailing pipes in each row are optional, and the text - # doesn't actually have to be neatly aligned, so we can create these - # pretty easily. Let's do so. - header_row = 'Pounds of chocolate | Happiness' - chocolate = tf.range(step) - happiness = tf.square(chocolate + 1) - chocolate_column = tf.as_string(chocolate) - happiness_column = tf.as_string(happiness) - table_rows = tf.strings.join([chocolate_column, " | ", happiness_column]) - table_body = tf.strings.reduce_join(inputs=table_rows, separator='\n') - table = tf.strings.join([header_row, "---|---", table_body], separator='\n') - preamble = 'We conducted an experiment and found the following data:\n\n' - result = tf.strings.join([preamble, table]) - tf.compat.v1.summary.text('chocolate_study', result) + # The text summary can also contain Markdown, including Markdown + # tables. Markdown tables look like this: + # + # | hello | there | + # |-------|-------| + # | this | is | + # | a | table | + # + # The leading and trailing pipes in each row are optional, and the text + # doesn't actually have to be neatly aligned, so we can create these + # pretty easily. Let's do so. + header_row = "Pounds of chocolate | Happiness" + chocolate = tf.range(step) + happiness = tf.square(chocolate + 1) + chocolate_column = tf.as_string(chocolate) + happiness_column = tf.as_string(happiness) + table_rows = tf.strings.join([chocolate_column, " | ", happiness_column]) + table_body = tf.strings.reduce_join(inputs=table_rows, separator="\n") + table = tf.strings.join([header_row, "---|---", table_body], separator="\n") + preamble = "We conducted an experiment and found the following data:\n\n" + result = tf.strings.join([preamble, table]) + tf.compat.v1.summary.text("chocolate_study", result) def higher_order_tensors(step): - # We're not limited to passing scalar tensors to the summary - # operation. If we pass a rank-1 or rank-2 tensor, it'll be visualized - # as a table in TensorBoard. (For higher-ranked tensors, you'll see - # just a 2D slice of the data.) - # - # To demonstrate this, let's create a multiplication table. - - # First, we'll create the table body, a `step`-by-`step` array of - # strings. - numbers = tf.range(step) - numbers_row = tf.expand_dims(numbers, 0) # shape: [1, step] - numbers_column = tf.expand_dims(numbers, 1) # shape: [step, 1] - products = tf.matmul(numbers_column, numbers_row) # shape: [step, step] - table_body = tf.as_string(products) - - # Next, we'll create a header row and column, and a little - # multiplication sign to put in the corner. - bold_numbers = tf.strings.join(['**', tf.as_string(numbers), '**']) - bold_row = tf.expand_dims(bold_numbers, 0) - bold_column = tf.expand_dims(bold_numbers, 1) - corner_cell = tf.constant(u'\u00d7'.encode('utf-8')) # MULTIPLICATION SIGN - - # Now, we have to put the pieces together. Using `axis=0` stacks - # vertically; using `axis=1` juxtaposes horizontally. - table_body_and_top_row = tf.concat([bold_row, table_body], axis=0) - table_left_column = tf.concat([[[corner_cell]], bold_column], axis=0) - table_full = tf.concat([table_left_column, table_body_and_top_row], axis=1) - - # The result, `table_full`, is a rank-2 string tensor of shape - # `[step + 1, step + 1]`. We can pass it directly to the summary, and - # we'll get a nicely formatted table in TensorBoard. - tf.compat.v1.summary.text('multiplication_table', table_full) + # We're not limited to passing scalar tensors to the summary + # operation. If we pass a rank-1 or rank-2 tensor, it'll be visualized + # as a table in TensorBoard. (For higher-ranked tensors, you'll see + # just a 2D slice of the data.) + # + # To demonstrate this, let's create a multiplication table. + + # First, we'll create the table body, a `step`-by-`step` array of + # strings. + numbers = tf.range(step) + numbers_row = tf.expand_dims(numbers, 0) # shape: [1, step] + numbers_column = tf.expand_dims(numbers, 1) # shape: [step, 1] + products = tf.matmul(numbers_column, numbers_row) # shape: [step, step] + table_body = tf.as_string(products) + + # Next, we'll create a header row and column, and a little + # multiplication sign to put in the corner. + bold_numbers = tf.strings.join(["**", tf.as_string(numbers), "**"]) + bold_row = tf.expand_dims(bold_numbers, 0) + bold_column = tf.expand_dims(bold_numbers, 1) + corner_cell = tf.constant(u"\u00d7".encode("utf-8")) # MULTIPLICATION SIGN + + # Now, we have to put the pieces together. Using `axis=0` stacks + # vertically; using `axis=1` juxtaposes horizontally. + table_body_and_top_row = tf.concat([bold_row, table_body], axis=0) + table_left_column = tf.concat([[[corner_cell]], bold_column], axis=0) + table_full = tf.concat([table_left_column, table_body_and_top_row], axis=1) + + # The result, `table_full`, is a rank-2 string tensor of shape + # `[step + 1, step + 1]`. We can pass it directly to the summary, and + # we'll get a nicely formatted table in TensorBoard. + tf.compat.v1.summary.text("multiplication_table", table_full) def run_all(logdir): - tf.compat.v1.reset_default_graph() - step_placeholder = tf.compat.v1.placeholder(tf.int32) - - with tf.name_scope('simple_example'): - simple_example(step_placeholder) - with tf.name_scope('markdown_table'): - markdown_table(step_placeholder) - with tf.name_scope('higher_order_tensors'): - higher_order_tensors(step_placeholder) - all_summaries = tf.compat.v1.summary.merge_all() - - with tf.compat.v1.Session() as sess: - writer = tf.summary.FileWriter(logdir) - writer.add_graph(sess.graph) - for step in xrange(STEPS): - s = sess.run(all_summaries, feed_dict={step_placeholder: step}) - writer.add_summary(s, global_step=step) - writer.close() + tf.compat.v1.reset_default_graph() + step_placeholder = tf.compat.v1.placeholder(tf.int32) + + with tf.name_scope("simple_example"): + simple_example(step_placeholder) + with tf.name_scope("markdown_table"): + markdown_table(step_placeholder) + with tf.name_scope("higher_order_tensors"): + higher_order_tensors(step_placeholder) + all_summaries = tf.compat.v1.summary.merge_all() + + with tf.compat.v1.Session() as sess: + writer = tf.summary.FileWriter(logdir) + writer.add_graph(sess.graph) + for step in xrange(STEPS): + s = sess.run(all_summaries, feed_dict={step_placeholder: step}) + writer.add_summary(s, global_step=step) + writer.close() def main(unused_argv): - logging.set_verbosity(logging.INFO) - logger.info('Saving output to %s.' % LOGDIR) - run_all(LOGDIR) - logger.info('Done. Output saved to %s.' % LOGDIR) + logging.set_verbosity(logging.INFO) + logger.info("Saving output to %s." % LOGDIR) + run_all(LOGDIR) + logger.info("Done. Output saved to %s." % LOGDIR) -if __name__ == '__main__': - app.run(main) +if __name__ == "__main__": + app.run(main) diff --git a/tensorboard/plugins/text/text_plugin.py b/tensorboard/plugins/text/text_plugin.py index b97a5196c9..bdf4b8ad8d 100644 --- a/tensorboard/plugins/text/text_plugin.py +++ b/tensorboard/plugins/text/text_plugin.py @@ -24,6 +24,7 @@ # pylint: disable=g-bad-import-order # Necessary for an internal test with special behavior for numpy. import numpy as np + # pylint: enable=g-bad-import-order import six @@ -36,217 +37,230 @@ from tensorboard.util import tensor_util # HTTP routes -TAGS_ROUTE = '/tags' -TEXT_ROUTE = '/text' +TAGS_ROUTE = "/tags" +TEXT_ROUTE = "/text" -WARNING_TEMPLATE = textwrap.dedent("""\ +WARNING_TEMPLATE = textwrap.dedent( + """\ **Warning:** This text summary contained data of dimensionality %d, but only \ - 2d tables are supported. Showing a 2d slice of the data instead.""") - + 2d tables are supported. Showing a 2d slice of the data instead.""" +) -def make_table_row(contents, tag='td'): - """Given an iterable of string contents, make a table row. - Args: - contents: An iterable yielding strings. - tag: The tag to place contents in. Defaults to 'td', you might want 'th'. +def make_table_row(contents, tag="td"): + """Given an iterable of string contents, make a table row. - Returns: - A string containing the content strings, organized into a table row. + Args: + contents: An iterable yielding strings. + tag: The tag to place contents in. Defaults to 'td', you might want 'th'. - Example: make_table_row(['one', 'two', 'three']) == ''' -four |
| four |
| four |
| five |
| five |
foo
' - self.assertEqual(convert(scalar), scalar_expected) - - # Check that underscores are preserved correctly; this detects erroneous - # use of UTF-16 or UTF-32 encoding when calling markdown_to_safe_html(), - # which would introduce spurious null bytes and cause undesired tags - # around the underscores. - scalar_underscores = np.array('word_with_underscores') - scalar_underscores_expected = 'word_with_underscores
' - self.assertEqual(convert(scalar_underscores), scalar_underscores_expected) - - vector = np.array(['foo', 'bar']) - vector_expected = textwrap.dedent("""\ + """ + ) + actual_1d_with_headers = text_plugin.make_table(array_1d, headers=["X"]) + self.assertEqual(actual_1d_with_headers, expected_1d_with_headers) + + def testMakeTableExceptions(self): + # Verify that contents is being type-checked and shape-checked. + with self.assertRaises(ValueError): + text_plugin.make_table([]) + + with self.assertRaises(ValueError): + text_plugin.make_table("foo") + + with self.assertRaises(ValueError): + invalid_shape = np.full((3, 3, 3), "nope", dtype=np.dtype("S3")) + text_plugin.make_table(invalid_shape) + + # Test headers exceptions in 2d array case. + test_array = np.full((3, 3), "foo", dtype=np.dtype("S3")) + with self.assertRaises(ValueError): + # Headers is wrong type. + text_plugin.make_table(test_array, headers="foo") + with self.assertRaises(ValueError): + # Too many headers. + text_plugin.make_table( + test_array, headers=["foo", "bar", "zod", "zoink"] + ) + with self.assertRaises(ValueError): + # headers is 2d + text_plugin.make_table(test_array, headers=test_array) + + # Also make sure the column counting logic works in the 1d array case. + test_array = np.array(["foo", "bar", "zod"]) + with self.assertRaises(ValueError): + # Too many headers. + text_plugin.make_table(test_array, headers=test_array) + + def test_reduce_to_2d(self): + def make_range_array(dim): + """Produce an incrementally increasing multidimensional array. + + Args: + dim: the number of dimensions for the array + + Returns: + An array of increasing integer elements, with dim dimensions and size + two in each dimension. + + Example: rangeArray(2) results in [[0,1],[2,3]]. + """ + return np.array(range(2 ** dim)).reshape([2] * dim) + + for i in range(2, 5): + actual = text_plugin.reduce_to_2d(make_range_array(i)) + expected = make_range_array(2) + np.testing.assert_array_equal(actual, expected) + + def test_text_array_to_html(self): + convert = text_plugin.text_array_to_html + scalar = np.array("foo") + scalar_expected = "foo
" + self.assertEqual(convert(scalar), scalar_expected) + + # Check that underscores are preserved correctly; this detects erroneous + # use of UTF-16 or UTF-32 encoding when calling markdown_to_safe_html(), + # which would introduce spurious null bytes and cause undesired tags + # around the underscores. + scalar_underscores = np.array("word_with_underscores") + scalar_underscores_expected = "word_with_underscores
" + self.assertEqual( + convert(scalar_underscores), scalar_underscores_expected + ) + + vector = np.array(["foo", "bar"]) + vector_expected = textwrap.dedent( + """\bar |
zod |
zod |
")
- print("
")
-
- for suggestion in suggestions:
+ set_up_logging()
+
+ print("### Diagnostics")
print()
- print("### Suggestion: %s" % suggestion.headline)
+
+ print("Diagnostics output
") - print() - - markdown_code_fence = "``````" # seems likely to be sufficient - print(markdown_code_fence) - suggestions = [] - for (i, check) in enumerate(CHECKS): - if i > 0: - print() - print("--- check: %s" % check.__name__) - try: - suggestions.extend(check()) - except Exception: - traceback.print_exc(file=sys.stdout) - pass - print(markdown_code_fence) - print() - print("")
+ print("
")
+
+ for suggestion in suggestions:
+ print()
+ print("### Suggestion: %s" % suggestion.headline)
+ print()
+ print(suggestion.description)
+
+ print()
+ print("### Next steps")
+ print()
+ if suggestions:
+ print(
+ reflow(
+ """
Please try each suggestion enumerated above to determine whether
it solves your problem. If none of these suggestions works,
please copy ALL of the above output, including the lines
containing only backticks, into your GitHub issue or comment. Be
sure to redact any sensitive information.
"""
- ))
- else:
- print(reflow(
- """
+ )
+ )
+ else:
+ print(
+ reflow(
+ """
No action items identified. Please copy ALL of the above output,
including the lines containing only backticks, into your GitHub
issue or comment. Be sure to redact any sensitive information.
"""
- ))
+ )
+ )
if __name__ == "__main__":
- main()
+ main()
diff --git a/tensorboard/tools/import_google_fonts.py b/tensorboard/tools/import_google_fonts.py
index c2cc739874..f6de26cb19 100644
--- a/tensorboard/tools/import_google_fonts.py
+++ b/tensorboard/tools/import_google_fonts.py
@@ -37,11 +37,11 @@
import tensorflow as tf
ROBOTO_URLS = [
- 'https://fonts.googleapis.com/css?family=Roboto:400,300,300italic,400italic,500,500italic,700,700italic',
- 'https://fonts.googleapis.com/css?family=Roboto+Mono:400,700',
+ "https://fonts.googleapis.com/css?family=Roboto:400,300,300italic,400italic,500,500italic,700,700italic",
+ "https://fonts.googleapis.com/css?family=Roboto+Mono:400,700",
]
-GOOGLE_LICENSE_HTML = '''\
+GOOGLE_LICENSE_HTML = """\
-'''
-
-flags.DEFINE_string('urls', ';'.join(ROBOTO_URLS),
- 'Google Fonts stylesheet URLs (semicolons delimited)')
-flags.DEFINE_string('path', '/font-roboto/roboto.html', 'Path of HTML file')
-flags.DEFINE_string('repo', 'com_google_fonts_roboto', 'Name of repository')
-flags.DEFINE_string('license', 'notice', 'Bazel category of license')
-flags.DEFINE_string('license_summary', 'Apache 2.0', 'License description')
-flags.DEFINE_string('license_html', GOOGLE_LICENSE_HTML,
- 'HTML @license comment')
-flags.DEFINE_string('user_agent',
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) '
- 'AppleWebKit/537.36 (KHTML, like Gecko) '
- 'Chrome/62.0.3202.94 '
- 'Safari/537.36',
- 'HTTP User-Agent header to send to Google Fonts')
-flags.DEFINE_string('mirror', 'http://mirror.tensorflow.org/',
- 'Mirror URL prefix')
+"""
+
+flags.DEFINE_string(
+ "urls",
+ ";".join(ROBOTO_URLS),
+ "Google Fonts stylesheet URLs (semicolons delimited)",
+)
+flags.DEFINE_string("path", "/font-roboto/roboto.html", "Path of HTML file")
+flags.DEFINE_string("repo", "com_google_fonts_roboto", "Name of repository")
+flags.DEFINE_string("license", "notice", "Bazel category of license")
+flags.DEFINE_string("license_summary", "Apache 2.0", "License description")
+flags.DEFINE_string(
+ "license_html", GOOGLE_LICENSE_HTML, "HTML @license comment"
+)
+flags.DEFINE_string(
+ "user_agent",
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) "
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/62.0.3202.94 "
+ "Safari/537.36",
+ "HTTP User-Agent header to send to Google Fonts",
+)
+flags.DEFINE_string(
+ "mirror", "http://mirror.tensorflow.org/", "Mirror URL prefix"
+)
FLAGS = flags.FLAGS
-BAR = '/%s/' % ('*' * 78)
-NON_REPO_PATTERN = re.compile(r'[^_a-z0-9]')
-SCHEME_PATTERN = re.compile(r'https?://')
-CSS_PATTERN = re.compile(r'(?:/\* (?P