diff --git a/tensorboard/uploader/BUILD b/tensorboard/uploader/BUILD index 0b17a7e0dc..cc8a34c88f 100644 --- a/tensorboard/uploader/BUILD +++ b/tensorboard/uploader/BUILD @@ -20,6 +20,7 @@ py_library( "//tensorboard:expect_grpc_installed", "//tensorboard/uploader/proto:protos_all_py_pb2", "//tensorboard/util:grpc_util", + "@org_pythonhosted_six", ], ) @@ -58,6 +59,7 @@ py_library( ":exporter_lib", ":server_info", ":uploader_lib", + ":util", "//tensorboard:expect_absl_app_installed", "//tensorboard:expect_absl_flags_argparse_flags_installed", "//tensorboard:expect_absl_flags_installed", diff --git a/tensorboard/uploader/exporter.py b/tensorboard/uploader/exporter.py index a5d7172bfd..ef6d6d7205 100644 --- a/tensorboard/uploader/exporter.py +++ b/tensorboard/uploader/exporter.py @@ -26,6 +26,8 @@ import string import time +import six + from tensorboard.uploader.proto import export_service_pb2 from tensorboard.uploader import util from tensorboard.util import grpc_util @@ -126,7 +128,13 @@ def export(self, read_time=None): def _request_experiment_ids(self, read_time): """Yields all of the calling user's experiment IDs, as strings.""" - return list_experiments(self._api, read_time=read_time) + for experiment in list_experiments(self._api, read_time=read_time): + if isinstance(experiment, export_service_pb2.Experiment): + yield experiment.experiment_id + elif isinstance(experiment, six.string_types): + yield experiment + else: + raise AssertionError("Unexpected experiment type: %r" % (experiment,)) def _request_scalar_data(self, experiment_id, read_time): """Yields JSON-serializable blocks of scalar data.""" @@ -157,31 +165,36 @@ def _request_scalar_data(self, experiment_id, read_time): } -def list_experiments(api_client, read_time=None): - """Yields all of the calling user's experiment IDs. +def list_experiments(api_client, fieldmask=None, read_time=None): + """Yields all of the calling user's experiments. Args: api_client: A TensorBoardExporterService stub instance. + fieldmask: An optional `export_service_pb2.ExperimentMask` value. read_time: A fixed timestamp from which to export data, as float seconds since epoch (like `time.time()`). Optional; defaults to the current time. Yields: - One string for each experiment owned by the calling user, in arbitrary - order. + For each experiment owned by the user, an `export_service_pb2.Experiment` + value, or a simple string experiment ID for older servers. """ if read_time is None: read_time = time.time() request = export_service_pb2.StreamExperimentsRequest(limit=_MAX_INT64) util.set_timestamp(request.read_timestamp, read_time) + if fieldmask: + request.experiments_mask.CopyFrom(fieldmask) stream = api_client.StreamExperiments( request, metadata=grpc_util.version_metadata()) for response in stream: - if not response.experiments: + if response.experiments: + for experiment in response.experiments: + yield experiment + else: + # Old servers. for experiment_id in response.experiment_ids: yield experiment_id - for experiment in response.experiments: - yield experiment.experiment_id class OutputDirectoryExistsError(ValueError): diff --git a/tensorboard/uploader/exporter_test.py b/tensorboard/uploader/exporter_test.py index a6f889b66a..aef6947ee3 100644 --- a/tensorboard/uploader/exporter_test.py +++ b/tensorboard/uploader/exporter_test.py @@ -360,7 +360,15 @@ def stream_experiments(request, **kwargs): mock_api_client.StreamExperiments = mock.Mock(wraps=stream_experiments) gen = exporter_lib.list_experiments(mock_api_client) mock_api_client.StreamExperiments.assert_not_called() - self.assertEqual(list(gen), ["123", "456", "789", "012", "345", "678"]) + expected = [ + "123", + "456", + export_service_pb2.Experiment(experiment_id="789"), + export_service_pb2.Experiment(experiment_id="012"), + export_service_pb2.Experiment(experiment_id="345"), + export_service_pb2.Experiment(experiment_id="678"), + ] + self.assertEqual(list(gen), expected) class MkdirPTest(tb_test.TestCase): diff --git a/tensorboard/uploader/uploader_main.py b/tensorboard/uploader/uploader_main.py index b75fd7fab5..87ac1b6276 100644 --- a/tensorboard/uploader/uploader_main.py +++ b/tensorboard/uploader/uploader_main.py @@ -31,12 +31,14 @@ import six from tensorboard.uploader import dev_creds +from tensorboard.uploader.proto import export_service_pb2 from tensorboard.uploader.proto import export_service_pb2_grpc from tensorboard.uploader.proto import write_service_pb2_grpc from tensorboard.uploader import auth from tensorboard.uploader import exporter as exporter_lib from tensorboard.uploader import server_info as server_info_lib from tensorboard.uploader import uploader as uploader_lib +from tensorboard.uploader import util from tensorboard.uploader.proto import server_info_pb2 from tensorboard import program from tensorboard.plugins import base_plugin @@ -356,12 +358,34 @@ def get_ack_message_body(self): def execute(self, server_info, channel): api_client = export_service_pb2_grpc.TensorBoardExporterServiceStub(channel) - gen = exporter_lib.list_experiments(api_client) + fieldmask = export_service_pb2.ExperimentMask( + create_time=True, + update_time=True, + num_scalars=True, + num_runs=True, + num_tags=True, + ) + gen = exporter_lib.list_experiments(api_client, fieldmask=fieldmask) count = 0 - for experiment_id in gen: + for experiment in gen: count += 1 + if not isinstance(experiment, export_service_pb2.Experiment): + url = server_info_lib.experiment_url(server_info, experiment) + print(url) + continue + experiment_id = experiment.experiment_id url = server_info_lib.experiment_url(server_info, experiment_id) print(url) + data = [ + ('Id', experiment.experiment_id), + ('Created', util.format_time(experiment.create_time)), + ('Updated', util.format_time(experiment.update_time)), + ('Scalars', str(experiment.num_scalars)), + ('Runs', str(experiment.num_runs)), + ('Tags', str(experiment.num_tags)), + ] + for (name, value) in data: + print('\t%s %s' % (name.ljust(10), value)) sys.stdout.flush() if not count: sys.stderr.write( diff --git a/tensorboard/uploader/util.py b/tensorboard/uploader/util.py index 795758c4c1..16ad493bc5 100644 --- a/tensorboard/uploader/util.py +++ b/tensorboard/uploader/util.py @@ -18,6 +18,7 @@ from __future__ import division from __future__ import print_function +import datetime import errno import os import os.path @@ -112,3 +113,51 @@ def set_timestamp(pb, seconds_since_epoch): """ pb.seconds = int(seconds_since_epoch) pb.nanos = int(round((seconds_since_epoch % 1) * 10**9)) + + +def format_time(timestamp_pb, now=None): + """Converts a `timestamp_pb2.Timestamp` to human-readable string. + + This always includes the absolute date and time, and for recent dates + may include a relative time like "(just now)" or "(2 hours ago)". + + Args: + timestamp_pb: A `google.protobuf.timestamp_pb2.Timestamp` value to + convert to string. The input will not be modified. + now: A `datetime.datetime` object representing the current time, + used for determining relative times like "just now". Optional; + defaults to `datetime.datetime.now()`. + + Returns: + A string suitable for human consumption. + """ + + # Add and subtract a day for , + # which breaks early datetime conversions on Windows for small + # timestamps. + dt = datetime.datetime.fromtimestamp(timestamp_pb.seconds + 86400) + dt = dt - datetime.timedelta(seconds=86400) + + if now is None: + now = datetime.datetime.now() + ago = now.replace(microsecond=0) - dt + + def ago_text(n, singular, plural): + return "%d %s ago" % (n, singular if n == 1 else plural) + + relative = None + if ago < datetime.timedelta(seconds=5): + relative = "just now" + elif ago < datetime.timedelta(minutes=1): + relative = ago_text(int(ago.total_seconds()), "second", "seconds") + elif ago < datetime.timedelta(hours=1): + relative = ago_text(int(ago.total_seconds()) // 60, "minute", "minutes") + elif ago < datetime.timedelta(days=1): + relative = ago_text(int(ago.total_seconds()) // 3600, "hour", "hours") + + relative_part = " (%s)" % relative if relative is not None else "" + return str(dt) + relative_part + + +def _ngettext(n, singular, plural): + return "%d %s ago" % (n, singular if n == 1 else plural) diff --git a/tensorboard/uploader/util_test.py b/tensorboard/uploader/util_test.py index b0670d5315..090547f70c 100644 --- a/tensorboard/uploader/util_test.py +++ b/tensorboard/uploader/util_test.py @@ -18,8 +18,10 @@ from __future__ import division from __future__ import print_function +import datetime import os import unittest +import mock try: @@ -195,5 +197,50 @@ def test_set_timestamp(self): self.assertEqual(pb.nanos, 7812500) +class FormatTimeTest(tb_test.TestCase): + + def _run(self, t=None, now=None): + timestamp_pb = timestamp_pb2.Timestamp() + util.set_timestamp(timestamp_pb, t) + now = datetime.datetime.fromtimestamp(now) + with mock.patch.dict(os.environ, {"TZ": "UTC"}): + return util.format_time(timestamp_pb, now=now) + + def test_just_now(self): + base = 1546398245 + actual = self._run(t=base, now=base + 1) + self.assertEqual(actual, "2019-01-02 03:04:05 (just now)") + + def test_seconds_ago(self): + base = 1546398245 + actual = self._run(t=base, now=base + 10) + self.assertEqual(actual, "2019-01-02 03:04:05 (10 seconds ago)") + + def test_minute_ago(self): + base = 1546398245 + actual = self._run(t=base, now=base + 66) + self.assertEqual(actual, "2019-01-02 03:04:05 (1 minute ago)") + + def test_minutes_ago(self): + base = 1546398245 + actual = self._run(t=base, now=base + 222) + self.assertEqual(actual, "2019-01-02 03:04:05 (3 minutes ago)") + + def test_hour_ago(self): + base = 1546398245 + actual = self._run(t=base, now=base + 3601) + self.assertEqual(actual, "2019-01-02 03:04:05 (1 hour ago)") + + def test_hours_ago(self): + base = 1546398245 + actual = self._run(t=base, now=base + 9999) + self.assertEqual(actual, "2019-01-02 03:04:05 (2 hours ago)") + + def test_long_ago(self): + base = 1546398245 + actual = self._run(t=base, now=base + 7 * 86400) + self.assertEqual(actual, "2019-01-02 03:04:05") + + if __name__ == "__main__": tb_test.main()