diff --git a/tensorboard/uploader/proto/BUILD b/tensorboard/uploader/proto/BUILD index bb702af388..f96ce1264a 100644 --- a/tensorboard/uploader/proto/BUILD +++ b/tensorboard/uploader/proto/BUILD @@ -13,6 +13,7 @@ tb_proto_library( "export_service.proto", "scalar.proto", "server_info.proto", + "tensor.proto", "write_service.proto", ], has_services = True, diff --git a/tensorboard/uploader/proto/export_service.proto b/tensorboard/uploader/proto/export_service.proto index ab98c60512..22d5660908 100644 --- a/tensorboard/uploader/proto/export_service.proto +++ b/tensorboard/uploader/proto/export_service.proto @@ -4,6 +4,7 @@ package tensorboard.service; import "google/protobuf/timestamp.proto"; import "tensorboard/compat/proto/summary.proto"; +import "tensorboard/compat/proto/tensor.proto"; // Service for exporting data from TensorBoard.dev. service TensorBoardExporterService { @@ -96,6 +97,9 @@ message Experiment { string name = 7; // User provided description of the experiment, in markdown source format. string description = 8; + // The number of bytes used for storage of tensors in this experiment, + // across all time series, including estimated overhead. + int64 total_tensor_bytes = 9; } // Field mask for `Experiment`. The `experiment_id` field is always implicitly @@ -110,6 +114,7 @@ message ExperimentMask { bool num_tags = 6; bool name = 7; bool description = 8; + bool total_tensor_bytes = 9; } // Request to stream scalars from all the runs and tags in an experiment. @@ -123,15 +128,15 @@ message StreamExperimentDataRequest { google.protobuf.Timestamp read_timestamp = 2; } -// Streams scalars from all the runs and tags in an experiment. Each stream +// Streams data from all the runs and tags in an experiment. Each stream // result only contains data for a single tag from a single run. For example if // there are five runs and each run had two tags, the RPC will return a stream // of at least ten `StreamExperimentDataResponse`s, each one having the -// scalars for one tag. The values from a single tag may be split among multiple -// responses. Users need to aggregate information from entire stream to get -// data for the entire experiment. Empty experiments will have zero stream -// results. Empty runs that doesn't have any tags need not be supported by a -// hosted service. +// either scalars or tensors for one tag. The values from a single tag may be +// split among multiple responses. Users need to aggregate information from +// entire stream to get data for the entire experiment. Empty experiments will +// have zero stream results. Empty runs that doesn't have any tags need not +// be supported by a hosted service. message StreamExperimentDataResponse { // Name of the tag whose data is contained in this response. string tag_name = 1; @@ -141,6 +146,8 @@ message StreamExperimentDataResponse { .tensorboard.SummaryMetadata tag_metadata = 3; // Data to store for the tag `tag_name. ScalarPoints points = 4; + // Tensor data to store. + TensorPoints tensors = 5; // Data for the scalars are stored in a columnar fashion to optimize it for // exporting the data into textual formats like JSON. @@ -154,4 +161,17 @@ message StreamExperimentDataResponse { // Value of the point at this step / timestamp. repeated double values = 3; } + + // Data for the Tensors are stored in a columnar fashion to optimize it for + // exporting the data into textual formats like JSON. + // The data for the ith tensor is { steps[i], wall_times[i], values[i] }. + // The data here is sorted by step values in ascending order. + message TensorPoints { + // Step index within the run. + repeated int64 steps = 1; + // Timestamp of the creation of this point. + repeated google.protobuf.Timestamp wall_times = 2; + // Value of the point at this step / timestamp. + repeated .tensorboard.TensorProto values = 3; + } } diff --git a/tensorboard/uploader/proto/tensor.proto b/tensorboard/uploader/proto/tensor.proto new file mode 100644 index 0000000000..7dee592640 --- /dev/null +++ b/tensorboard/uploader/proto/tensor.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package tensorboard.service; + +import "google/protobuf/timestamp.proto"; +import "tensorboard/compat/proto/summary.proto"; +import "tensorboard/compat/proto/tensor.proto"; + +// One point viewable on a tensor metric plot. +message TensorPoint { + // Step index within the run. + int64 step = 1; + // Timestamp of the creation of this point. + google.protobuf.Timestamp wall_time = 2; + // Value of the point at this step / timestamp. + .tensorboard.TensorProto value = 3; +} + +// Metadata for the TensorPoints stored for one (Experiment, Run, Tag). +message TensorPointMetadata { + // Maximum step recorded for the tag. + int64 max_step = 1; + // Timestamp corresponding to the max step. + google.protobuf.Timestamp max_wall_time = 2; + // Information about the plugin which created this tensor data. + // Note: The period is required part of the type here due to the + // package name resolution logic. + .tensorboard.SummaryMetadata summary_metadata = 3; +} diff --git a/tensorboard/uploader/proto/write_service.proto b/tensorboard/uploader/proto/write_service.proto index 843a4a7ad5..b5dd3b0bce 100644 --- a/tensorboard/uploader/proto/write_service.proto +++ b/tensorboard/uploader/proto/write_service.proto @@ -7,6 +7,7 @@ package tensorboard.service; // those into one file that both services depend on. import "tensorboard/uploader/proto/export_service.proto"; import "tensorboard/uploader/proto/scalar.proto"; +import "tensorboard/uploader/proto/tensor.proto"; import "tensorboard/compat/proto/summary.proto"; // Service for writing data to TensorBoard.dev. @@ -25,8 +26,10 @@ service TensorBoardWriterService { // Request that unreachable data be purged. Used only for testing; // disabled in production. rpc PurgeData(PurgeDataRequest) returns (PurgeDataResponse) {} - // Request additional data be stored in TensorBoard.dev. + // Request additional scalar data be stored in TensorBoard.dev. rpc WriteScalar(WriteScalarRequest) returns (WriteScalarResponse) {} + // Request additional tensor data be stored in TensorBoard.dev. + rpc WriteTensor(WriteTensorRequest) returns (WriteTensorResponse) {} // Request that the calling user and all their data be permanently deleted. // Used for testing purposes. rpc DeleteOwnUser(DeleteOwnUserRequest) returns (DeleteOwnUserResponse) {} @@ -148,6 +151,45 @@ message WriteScalarResponse { // This is empty on purpose. } +// Carries all that is needed to add tensor data to the hosted service. +message WriteTensorRequest { + // All the data to store for one Run. This data will be stored under the + // corresponding run in the hosted storage. WriteTensorRequest is merged into + // the data store for the keyed run. The tags and included tensors will be + // the union of the data sent across all WriteTensorRequests. Metadata by + // default uses a 'first write wins' approach. + message Run { + // The name of this run. For example "/some/path/mnist_experiments/run1/" + string name = 1; + // Data to store for this Run/Tag combination. + repeated Tag tags = 2; + } + + // All the data to store for one Tag of one Run. This data will be stored + // under the corresponding run/tag in the hosted storage. A tag corresponds to + // a single time series. + message Tag { + // The name of this tag. For example "loss" + string name = 1; + // Data to store for this Run/Tag combination. + repeated TensorPoint points = 2; + // The metadata of this tag. + .tensorboard.SummaryMetadata metadata = 3; + } + + // Which experiment to write to - corresponding to one hosted TensorBoard URL. + // The requester must have authorization to write to this location. + string experiment_id = 1; + // Data to append to the existing storage at the experiment_id. + repeated Run runs = 2; +} + +// Everything the caller needs to know about how the writing went. +// (Currently empty) +message WriteTensorResponse { + // This is empty on purpose. +} + // Requests that the calling user and all their data be permanently deleted. message DeleteOwnUserRequest { // This is empty on purpose.