Add uploader gRPC client

.travis.yml

@@ -55,6 +55,9 @@ install:
   # TensorBoard deps.
   - pip install futures==3.1.1
   - pip install grpcio==1.6.3
+  - pip install grpcio-testing==1.24.3
+  - pip install 'google-auth >= 1.6.3, < 2'
+  - pip install 'google-auth-oauthlib >= 0.4.1, < 0.5'
   - yarn install --ignore-engines
   # Uninstall older Travis numpy to avoid upgrade-in-place issues.
   - pip uninstall -y numpy

tensorboard/BUILD

@@ -28,6 +28,7 @@ py_binary(
         ":program",
         "//tensorboard:expect_tensorflow_installed",
         "//tensorboard/plugins:base_plugin",
+        "//tensorboard/uploader:uploader_main_lib",
         "//tensorboard/util:tb_logging",
     ],
 )
@@ -310,6 +311,14 @@ py_library(
     visibility = ["//visibility:public"],
 )
 
+py_library(
+    name = "expect_grpc_testing_installed",
+    # This is a dummy rule used as a grpc_testing dependency in open-source.
+    # We expect grpc_testing to already be installed on the system, e.g. via
+    # `pip install grpcio_testing`
+    visibility = ["//visibility:public"],
+)
+
 py_library(
     name = "expect_sqlite3_installed",
     # This is a dummy rule used as a sqlite3 dependency in open-source.
@@ -342,6 +351,14 @@ py_library(
     visibility = ["//visibility:public"],
 )
 
+py_library(
+    name = "expect_absl_flags_argparse_flags_installed",
+    # This is a dummy rule used as a absl-py dependency in open-source.
+    # We expect absl-py to already be installed on the system, e.g. via
+    # `pip install absl-py`
+    visibility = ["//visibility:public"],
+)
+
 py_library(
     name = "expect_absl_logging_installed",
     # This is a dummy rule used as a absl-py dependency in open-source.
@@ -358,6 +375,22 @@ py_library(
     visibility = ["//visibility:public"],
 )
 
+py_library(
+    name = "expect_google_auth_installed",
+    # This is a dummy rule used as a google_auth dependency in open-source.
+    # We expect google_auth to already be installed on the system, e.g., via
+    # `pip install google-auth`.
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "expect_google_auth_oauthlib_installed",
+    # This is a dummy rule used as a google_auth oauthlib_dependency in open-source.
+    # We expect google_auth_oauthlib to already be installed on the system, e.g., via
+    # `pip install google-auth-oauthlib`.
+    visibility = ["//visibility:public"],
+)
+
 py_library(
     name = "expect_pkg_resources_installed",
     # This is a dummy rule used as a pkg-resources dependency in open-source.

tensorboard/main.py

@@ -41,6 +41,7 @@
 from tensorboard import program
 from tensorboard.compat import tf
 from tensorboard.plugins import base_plugin
+from tensorboard.uploader import uploader_main
 from tensorboard.util import tb_logging
 
 
@@ -56,7 +57,8 @@ def run_main():
 
   tensorboard = program.TensorBoard(
       default.get_plugins() + default.get_dynamic_plugins(),
-      program.get_default_assets_zip_provider())
+      program.get_default_assets_zip_provider(),
+      subcommands=[uploader_main.UploaderSubcommand()])
   try:
     from absl import app
     # Import this to check that app.run() will accept the flags_parser argument.

tensorboard/pip_package/setup.py

@@ -27,6 +27,8 @@
     # futures is a backport of the python 3.2+ concurrent.futures module
     'futures >= 3.1.1; python_version < "3"',
     'grpcio >= 1.6.3',
+    'google-auth >= 1.6.3, < 2',
+    'google-auth-oauthlib >= 0.4.1, < 0.5',
     'markdown >= 2.6.8',
     'numpy >= 1.12.0',
     'protobuf >= 3.6.0',

tensorboard/uploader/BUILD

@@ -0,0 +1,190 @@
+py_library(
+    name = "dev_creds",
+    srcs = ["dev_creds.py"],
+)
+
+py_library(
+    name = "exporter_lib",
+    srcs = ["exporter.py"],
+    deps = [
+        ":util",
+        "//tensorboard/uploader/proto:protos_all_py_pb2",
+    ],
+)
+
+py_test(
+    name = "exporter_test",
+    srcs = ["exporter_test.py"],
+    deps = [
+        ":exporter_lib",
+        ":test_util",
+        "//tensorboard:expect_grpc_testing_installed",
+        "//tensorboard:test",
+        "//tensorboard/compat/proto:protos_all_py_pb2",
+        "//tensorboard/uploader/proto:protos_all_py_pb2",
+        "//tensorboard/uploader/proto:protos_all_py_pb2_grpc",
+        "@org_pythonhosted_mock",
+    ],
+)
+
+py_binary(
+    name = "uploader",
+    srcs = ["uploader_main.py"],
+    main = "uploader_main.py",
+    python_version = "PY2",
+    deps = [":uploader_main_lib"],
+)
+
+py_library(
+    name = "uploader_main_lib",
+    srcs = ["uploader_main.py"],
+    visibility = ["//tensorboard:internal"],
+    deps = [
+        ":auth",
+        ":dev_creds",
+        ":exporter_lib",
+        ":uploader_lib",
+        "//tensorboard:expect_absl_app_installed",
+        "//tensorboard:expect_absl_flags_argparse_flags_installed",
+        "//tensorboard:expect_absl_flags_installed",
+        "//tensorboard:expect_absl_logging_installed",
+        "//tensorboard:expect_grpc_installed",
+        "//tensorboard:program",
+        "//tensorboard/plugins:base_plugin",
+        "//tensorboard/uploader/proto:protos_all_py_pb2_grpc",
+        "@org_pythonhosted_six",
+    ],
+)
+
+py_library(
+    name = "uploader_lib",
+    srcs = ["uploader.py"],
+    deps = [
+        ":logdir_loader",
+        ":peekable_iterator",
+        ":util",
+        "//tensorboard:data_compat",
+        "//tensorboard:expect_grpc_installed",
+        "//tensorboard/backend/event_processing:directory_loader",
+        "//tensorboard/backend/event_processing:event_file_loader",
+        "//tensorboard/backend/event_processing:io_wrapper",
+        "//tensorboard/plugins/scalar:metadata",
+        "//tensorboard/uploader/proto:protos_all_py_pb2",
+        "//tensorboard/util:grpc_util",
+        "//tensorboard/util:tb_logging",
+        "//tensorboard/util:tensor_util",
+        "@org_pythonhosted_six",
+    ],
+)
+
+py_test(
+    name = "uploader_test",
+    srcs = ["uploader_test.py"],
+    deps = [
+        ":test_util",
+        ":uploader_lib",
+        ":util",
+        "//tensorboard:expect_grpc_installed",
+        "//tensorboard:expect_grpc_testing_installed",
+        "//tensorboard:expect_tensorflow_installed",
+        "//tensorboard/compat/proto:protos_all_py_pb2",
+        "//tensorboard/plugins/histogram:summary_v2",
+        "//tensorboard/plugins/scalar:summary_v2",
+        "//tensorboard/summary:summary_v1",
+        "//tensorboard/uploader/proto:protos_all_py_pb2",
+        "//tensorboard/uploader/proto:protos_all_py_pb2_grpc",
+        "//tensorboard/util:test_util",
+        "@org_pythonhosted_mock",
+    ],
+)
+
+py_library(
+    name = "auth",
+    srcs = ["auth.py"],
+    deps = [
+        ":util",
+        "//tensorboard:expect_google_auth_installed",
+        "//tensorboard:expect_google_auth_oauthlib_installed",
+        "//tensorboard:expect_grpc_installed",
+        "//tensorboard/util:tb_logging",
+    ],
+)
+
+py_test(
+    name = "auth_test",
+    srcs = ["auth_test.py"],
+    deps = [
+        ":auth",
+        "//tensorboard:expect_google_auth_installed",
+        "//tensorboard:test",
+        "@org_pythonhosted_mock",
+    ],
+)
+
+py_library(
+    name = "logdir_loader",
+    srcs = ["logdir_loader.py"],
+    deps = [
+        "//tensorboard/backend/event_processing:directory_watcher",
+        "//tensorboard/backend/event_processing:io_wrapper",
+        "//tensorboard/util:tb_logging",
+    ],
+)
+
+py_test(
+    name = "logdir_loader_test",
+    srcs = ["logdir_loader_test.py"],
+    deps = [
+        ":logdir_loader",
+        "//tensorboard:test",
+        "//tensorboard/backend/event_processing:directory_loader",
+        "//tensorboard/backend/event_processing:event_file_loader",
+        "//tensorboard/backend/event_processing:io_wrapper",
+        "//tensorboard/util:test_util",
+        "@org_pythonhosted_six",
+    ],
+)
+
+py_library(
+    name = "test_util",
+    testonly = 1,
+    srcs = ["test_util.py"],
+    deps = [
+        "//tensorboard:expect_grpc_installed",
+        "//tensorboard/compat/proto:protos_all_py_pb2",
+        "@com_google_protobuf//:protobuf_python",
+    ],
+)
+
+py_library(
+    name = "util",
+    srcs = ["util.py"],
+)
+
+py_test(
+    name = "util_test",
+    srcs = ["util_test.py"],
+    deps = [
+        ":test_util",
+        ":util",
+        "//tensorboard:test",
+        "@com_google_protobuf//:protobuf_python",
+        "@org_pythonhosted_mock",
+    ],
+)
+
+py_library(
+    name = "peekable_iterator",
+    srcs = ["peekable_iterator.py"],
+    srcs_version = "PY2AND3",
+)
+
+py_test(
+    name = "peekable_iterator_test",
+    size = "small",
+    srcs = ["peekable_iterator_test.py"],
+    deps = [
+        ":peekable_iterator",
+        "//tensorboard:test",
+    ],
+)