uploader: include metadata in list output

tensorboard/uploader/BUILD

@@ -20,6 +20,7 @@ py_library(
         "//tensorboard:expect_grpc_installed",
         "//tensorboard/uploader/proto:protos_all_py_pb2",
         "//tensorboard/util:grpc_util",
+        "@org_pythonhosted_six",
     ],
 )
 
@@ -58,6 +59,7 @@ py_library(
         ":exporter_lib",
         ":server_info",
         ":uploader_lib",
+        ":util",
         "//tensorboard:expect_absl_app_installed",
         "//tensorboard:expect_absl_flags_argparse_flags_installed",
         "//tensorboard:expect_absl_flags_installed",

tensorboard/uploader/exporter.py

@@ -26,6 +26,8 @@
 import string
 import time
 
+import six
+
 from tensorboard.uploader.proto import export_service_pb2
 from tensorboard.uploader import util
 from tensorboard.util import grpc_util
@@ -126,7 +128,13 @@ def export(self, read_time=None):
 
   def _request_experiment_ids(self, read_time):
     """Yields all of the calling user's experiment IDs, as strings."""
-    return list_experiments(self._api, read_time=read_time)
+    for experiment in list_experiments(self._api, read_time=read_time):
+      if isinstance(experiment, export_service_pb2.Experiment):
+        yield experiment.experiment_id
+      elif isinstance(experiment, six.string_types):
+        yield experiment
+      else:
+        raise AssertionError("Unexpected experiment type: %r" % (experiment,))
 
   def _request_scalar_data(self, experiment_id, read_time):
     """Yields JSON-serializable blocks of scalar data."""
@@ -157,31 +165,36 @@ def _request_scalar_data(self, experiment_id, read_time):
       }
 
 
-def list_experiments(api_client, read_time=None):
-  """Yields all of the calling user's experiment IDs.
+def list_experiments(api_client, fieldmask=None, read_time=None):
+  """Yields all of the calling user's experiments.
 
   Args:
     api_client: A TensorBoardExporterService stub instance.
+    fieldmask: An optional `export_service_pb2.ExperimentMask` value.
     read_time: A fixed timestamp from which to export data, as float seconds
       since epoch (like `time.time()`). Optional; defaults to the current
       time.
 
   Yields:
-    One string for each experiment owned by the calling user, in arbitrary
-    order.
+    For each experiment owned by the user, an `export_service_pb2.Experiment`
+    value, or a simple string experiment ID for older servers.
   """
   if read_time is None:
     read_time = time.time()
   request = export_service_pb2.StreamExperimentsRequest(limit=_MAX_INT64)
   util.set_timestamp(request.read_timestamp, read_time)
+  if fieldmask:
+    request.experiments_mask.CopyFrom(fieldmask)
   stream = api_client.StreamExperiments(
       request, metadata=grpc_util.version_metadata())
   for response in stream:
-    if not response.experiments:
+    if response.experiments:
+      for experiment in response.experiments:
+        yield experiment
+    else:
+      # Old servers.
       for experiment_id in response.experiment_ids:
         yield experiment_id
-    for experiment in response.experiments:
-      yield experiment.experiment_id
 
 
 class OutputDirectoryExistsError(ValueError):

tensorboard/uploader/exporter_test.py

@@ -360,7 +360,15 @@ def stream_experiments(request, **kwargs):
     mock_api_client.StreamExperiments = mock.Mock(wraps=stream_experiments)
     gen = exporter_lib.list_experiments(mock_api_client)
     mock_api_client.StreamExperiments.assert_not_called()
-    self.assertEqual(list(gen), ["123", "456", "789", "012", "345", "678"])
+    expected = [
+        "123",
+        "456",
+        export_service_pb2.Experiment(experiment_id="789"),
+        export_service_pb2.Experiment(experiment_id="012"),
+        export_service_pb2.Experiment(experiment_id="345"),
+        export_service_pb2.Experiment(experiment_id="678"),
+    ]
+    self.assertEqual(list(gen), expected)
 
 
 class MkdirPTest(tb_test.TestCase):

tensorboard/uploader/uploader_main.py

@@ -31,12 +31,14 @@
 import six
 
 from tensorboard.uploader import dev_creds
+from tensorboard.uploader.proto import export_service_pb2
 from tensorboard.uploader.proto import export_service_pb2_grpc
 from tensorboard.uploader.proto import write_service_pb2_grpc
 from tensorboard.uploader import auth
 from tensorboard.uploader import exporter as exporter_lib
 from tensorboard.uploader import server_info as server_info_lib
 from tensorboard.uploader import uploader as uploader_lib
+from tensorboard.uploader import util
 from tensorboard.uploader.proto import server_info_pb2
 from tensorboard import program
 from tensorboard.plugins import base_plugin
@@ -356,12 +358,34 @@ def get_ack_message_body(self):
 
   def execute(self, server_info, channel):
     api_client = export_service_pb2_grpc.TensorBoardExporterServiceStub(channel)
-    gen = exporter_lib.list_experiments(api_client)
+    fieldmask = export_service_pb2.ExperimentMask(
+        create_time=True,
+        update_time=True,
+        num_scalars=True,
+        num_runs=True,
+        num_tags=True,
+    )
+    gen = exporter_lib.list_experiments(api_client, fieldmask=fieldmask)
     count = 0
-    for experiment_id in gen:
+    for experiment in gen:
       count += 1
+      if not isinstance(experiment, export_service_pb2.Experiment):
+        url = server_info_lib.experiment_url(server_info, experiment)
+        print(url)
+        continue
+      experiment_id = experiment.experiment_id
       url = server_info_lib.experiment_url(server_info, experiment_id)
       print(url)
+      data = [
+          ('Id', experiment.experiment_id),
+          ('Created', util.format_time(experiment.create_time)),
+          ('Updated', util.format_time(experiment.update_time)),
+          ('Scalars', str(experiment.num_scalars)),
+          ('Runs', str(experiment.num_runs)),
+          ('Tags', str(experiment.num_tags)),
+      ]
+      for (name, value) in data:
+        print('\t%s %s' % (name.ljust(10), value))
     sys.stdout.flush()
     if not count:
       sys.stderr.write(

tensorboard/uploader/util.py

@@ -18,6 +18,7 @@
 from __future__ import division
 from __future__ import print_function
 
+import datetime
 import errno
 import os
 import os.path
@@ -112,3 +113,51 @@ def set_timestamp(pb, seconds_since_epoch):
   """
   pb.seconds = int(seconds_since_epoch)
   pb.nanos = int(round((seconds_since_epoch % 1) * 10**9))
+
+
+def format_time(timestamp_pb, now=None):
+  """Converts a `timestamp_pb2.Timestamp` to human-readable string.
+
+  This always includes the absolute date and time, and for recent dates
+  may include a relative time like "(just now)" or "(2 hours ago)".
+
+  Args:
+    timestamp_pb: A `google.protobuf.timestamp_pb2.Timestamp` value to
+      convert to string. The input will not be modified.
+    now: A `datetime.datetime` object representing the current time,
+      used for determining relative times like "just now". Optional;
+      defaults to `datetime.datetime.now()`.
+
+  Returns:
+    A string suitable for human consumption.
+  """
+
+  # Add and subtract a day for <https://bugs.python.org/issue29097>,
+  # which breaks early datetime conversions on Windows for small
+  # timestamps.
+  dt = datetime.datetime.fromtimestamp(timestamp_pb.seconds + 86400)
+  dt = dt - datetime.timedelta(seconds=86400)
+
+  if now is None:
+    now = datetime.datetime.now()
+  ago = now.replace(microsecond=0) - dt
+
+  def ago_text(n, singular, plural):
+    return "%d %s ago" % (n, singular if n == 1 else plural)
+
+  relative = None
+  if ago < datetime.timedelta(seconds=5):
+    relative = "just now"
+  elif ago < datetime.timedelta(minutes=1):
+    relative = ago_text(int(ago.total_seconds()), "second", "seconds")
+  elif ago < datetime.timedelta(hours=1):
+    relative = ago_text(int(ago.total_seconds()) // 60, "minute", "minutes")
+  elif ago < datetime.timedelta(days=1):
+    relative = ago_text(int(ago.total_seconds()) // 3600, "hour", "hours")
+
+  relative_part = " (%s)" % relative if relative is not None else ""
+  return str(dt) + relative_part
+
+
+def _ngettext(n, singular, plural):
+  return "%d %s ago" % (n, singular if n == 1 else plural)

tensorboard/uploader/util_test.py

@@ -18,8 +18,10 @@
 from __future__ import division
 from __future__ import print_function
 
+import datetime
 import os
 import unittest
+import mock
 
 
 try:
@@ -195,5 +197,50 @@ def test_set_timestamp(self):
     self.assertEqual(pb.nanos, 7812500)
 
 
+class FormatTimeTest(tb_test.TestCase):
+
+  def _run(self, t=None, now=None):
+    timestamp_pb = timestamp_pb2.Timestamp()
+    util.set_timestamp(timestamp_pb, t)
+    now = datetime.datetime.fromtimestamp(now)
+    with mock.patch.dict(os.environ, {"TZ": "UTC"}):
+      return util.format_time(timestamp_pb, now=now)
+
+  def test_just_now(self):
+    base = 1546398245
+    actual = self._run(t=base, now=base + 1)
+    self.assertEqual(actual, "2019-01-02 03:04:05 (just now)")
+
+  def test_seconds_ago(self):
+    base = 1546398245
+    actual = self._run(t=base, now=base + 10)
+    self.assertEqual(actual, "2019-01-02 03:04:05 (10 seconds ago)")
+
+  def test_minute_ago(self):
+    base = 1546398245
+    actual = self._run(t=base, now=base + 66)
+    self.assertEqual(actual, "2019-01-02 03:04:05 (1 minute ago)")
+
+  def test_minutes_ago(self):
+    base = 1546398245
+    actual = self._run(t=base, now=base + 222)
+    self.assertEqual(actual, "2019-01-02 03:04:05 (3 minutes ago)")
+
+  def test_hour_ago(self):
+    base = 1546398245
+    actual = self._run(t=base, now=base + 3601)
+    self.assertEqual(actual, "2019-01-02 03:04:05 (1 hour ago)")
+
+  def test_hours_ago(self):
+    base = 1546398245
+    actual = self._run(t=base, now=base + 9999)
+    self.assertEqual(actual, "2019-01-02 03:04:05 (2 hours ago)")
+
+  def test_long_ago(self):
+    base = 1546398245
+    actual = self._run(t=base, now=base + 7 * 86400)
+    self.assertEqual(actual, "2019-01-02 03:04:05")
+
+
 if __name__ == "__main__":
   tb_test.main()