Update export/write services for eventual tensor storage

tensorboard/uploader/proto/BUILD

@@ -13,6 +13,7 @@ tb_proto_library(
         "export_service.proto",
         "scalar.proto",
         "server_info.proto",
+        "tensor.proto",
         "write_service.proto",
     ],
     has_services = True,

tensorboard/uploader/proto/export_service.proto

@@ -4,6 +4,7 @@ package tensorboard.service;
 
 import "google/protobuf/timestamp.proto";
 import "tensorboard/compat/proto/summary.proto";
+import "tensorboard/compat/proto/tensor.proto";
 
 // Service for exporting data from TensorBoard.dev.
 service TensorBoardExporterService {
@@ -96,6 +97,9 @@ message Experiment {
   string name = 7;
   // User provided description of the experiment, in markdown source format.
   string description = 8;
+  // The number of bytes used for storage of tensors in this experiment,
+  // across all time series, including estimated overhead.
+  int64 total_tensor_bytes = 9;
 }
 
 // Field mask for `Experiment`. The `experiment_id` field is always implicitly
@@ -110,6 +114,7 @@ message ExperimentMask {
   bool num_tags = 6;
   bool name = 7;
   bool description = 8;
+  bool total_tensor_bytes = 9;
 }
 
 // Request to stream scalars from all the runs and tags in an experiment.
@@ -123,15 +128,15 @@ message StreamExperimentDataRequest {
   google.protobuf.Timestamp read_timestamp = 2;
 }
 
-// Streams scalars from all the runs and tags in an experiment. Each stream
+// Streams data from all the runs and tags in an experiment. Each stream
 // result only contains data for a single tag from a single run. For example if
 // there are five runs and each run had two tags, the RPC will return a stream
 // of at least ten `StreamExperimentDataResponse`s, each one having the
-// scalars for one tag. The values from a single tag may be split among multiple
-// responses.  Users need to aggregate information from entire stream to get
-// data for the entire experiment. Empty experiments will have zero stream
-// results. Empty runs that doesn't have any tags need not be supported by a
-// hosted service.
+// either scalars or tensors for one tag. The values from a single tag may be
+// split among multiple responses.  Users need to aggregate information from
+// entire stream to get data for the entire experiment. Empty experiments will
+// have zero stream results. Empty runs that doesn't have any tags need not
+// be supported by a hosted service.
 message StreamExperimentDataResponse {
   // Name of the tag whose data is contained in this response.
   string tag_name = 1;
@@ -141,6 +146,8 @@ message StreamExperimentDataResponse {
   .tensorboard.SummaryMetadata tag_metadata = 3;
   // Data to store for the tag `tag_name.
   ScalarPoints points = 4;
+  // Tensor data to store.
+  TensorPoints tensors = 5;
 
   // Data for the scalars are stored in a columnar fashion to optimize it for
   // exporting the data into textual formats like JSON.
@@ -154,4 +161,17 @@ message StreamExperimentDataResponse {
     // Value of the point at this step / timestamp.
     repeated double values = 3;
   }
+
+  // Data for the Tensors are stored in a columnar fashion to optimize it for
+  // exporting the data into textual formats like JSON.
+  // The data for the ith tensor is { steps[i], wall_times[i], values[i] }.
+  // The data here is sorted by step values in ascending order.
+  message TensorPoints {
+    // Step index within the run.
+    repeated int64 steps = 1;
+    // Timestamp of the creation of this point.
+    repeated google.protobuf.Timestamp wall_times = 2;
+    // Value of the point at this step / timestamp.
+    repeated .tensorboard.TensorProto values = 3;
+  }
 }

tensorboard/uploader/proto/tensor.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+package tensorboard.service;
+
+import "google/protobuf/timestamp.proto";
+import "tensorboard/compat/proto/summary.proto";
+import "tensorboard/compat/proto/tensor.proto";
+
+// One point viewable on a tensor metric plot.
+message TensorPoint {
+  // Step index within the run.
+  int64 step = 1;
+  // Timestamp of the creation of this point.
+  google.protobuf.Timestamp wall_time = 2;
+  // Value of the point at this step / timestamp.
+  .tensorboard.TensorProto value = 3;
+}
+
+// Metadata for the TensorPoints stored for one (Experiment, Run, Tag).
+message TensorPointMetadata {
+  // Maximum step recorded for the tag.
+  int64 max_step = 1;
+  // Timestamp corresponding to the max step.
+  google.protobuf.Timestamp max_wall_time = 2;
+  // Information about the plugin which created this tensor data.
+  // Note: The period is required part of the type here due to the
+  // package name resolution logic.
+  .tensorboard.SummaryMetadata summary_metadata = 3;
+}

tensorboard/uploader/proto/write_service.proto

@@ -7,6 +7,7 @@ package tensorboard.service;
 // those into one file that both services depend on.
 import "tensorboard/uploader/proto/export_service.proto";
 import "tensorboard/uploader/proto/scalar.proto";
+import "tensorboard/uploader/proto/tensor.proto";
 import "tensorboard/compat/proto/summary.proto";
 
 // Service for writing data to TensorBoard.dev.
@@ -25,8 +26,10 @@ service TensorBoardWriterService {
   // Request that unreachable data be purged. Used only for testing;
   // disabled in production.
   rpc PurgeData(PurgeDataRequest) returns (PurgeDataResponse) {}
-  // Request additional data be stored in TensorBoard.dev.
+  // Request additional scalar data be stored in TensorBoard.dev.
   rpc WriteScalar(WriteScalarRequest) returns (WriteScalarResponse) {}
+  // Request additional tensor data be stored in TensorBoard.dev.
+  rpc WriteTensor(WriteTensorRequest) returns (WriteTensorResponse) {}
   // Request that the calling user and all their data be permanently deleted.
   // Used for testing purposes.
   rpc DeleteOwnUser(DeleteOwnUserRequest) returns (DeleteOwnUserResponse) {}
@@ -148,6 +151,45 @@ message WriteScalarResponse {
   // This is empty on purpose.
 }
 
+// Carries all that is needed to add tensor data to the hosted service.
+message WriteTensorRequest {
+  // All the data to store for one Run.  This data will be stored under the
+  // corresponding run in the hosted storage. WriteTensorRequest is merged into
+  // the data store for the keyed run. The tags and included tensors will be
+  // the union of the data sent across all WriteTensorRequests. Metadata by
+  // default uses a 'first write wins' approach.
+  message Run {
+    // The name of this run.  For example "/some/path/mnist_experiments/run1/"
+    string name = 1;
+    // Data to store for this Run/Tag combination.
+    repeated Tag tags = 2;
+  }
+
+  // All the data to store for one Tag of one Run.  This data will be stored
+  // under the corresponding run/tag in the hosted storage. A tag corresponds to
+  // a single time series.
+  message Tag {
+    // The name of this tag.  For example "loss"
+    string name = 1;
+    // Data to store for this Run/Tag combination.
+    repeated TensorPoint points = 2;
+    // The metadata of this tag.
+    .tensorboard.SummaryMetadata metadata = 3;
+  }
+
+  // Which experiment to write to - corresponding to one hosted TensorBoard URL.
+  // The requester must have authorization to write to this location.
+  string experiment_id = 1;
+  // Data to append to the existing storage at the experiment_id.
+  repeated Run runs = 2;
+}
+
+// Everything the caller needs to know about how the writing went.
+// (Currently empty)
+message WriteTensorResponse {
+  // This is empty on purpose.
+}
+
 // Requests that the calling user and all their data be permanently deleted.
 message DeleteOwnUserRequest {
   // This is empty on purpose.