From 9354292d7e1cb63a2d6074d8adc2eb043512cb12 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Mon, 28 Oct 2019 11:40:16 -0700 Subject: [PATCH] uploader: change request batch size to 128 KiB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Typical requests now include ~4700 scalars instead of ~37000 scalars (and are correspondingly processed faster by the server). This is now reliably hitting the client-side rate limiting, so we’re sending one request per 5 seconds. Test Plan: The following patch is useful for testing: ```diff diff --git a/tensorboard/uploader/uploader.py b/tensorboard/uploader/uploader.py index 1a6e7951..c3191746 100644 --- a/tensorboard/uploader/uploader.py +++ b/tensorboard/uploader/uploader.py @@ -130,6 +130,10 @@ class TensorBoardUploader(object): request_bytes = request.ByteSize() - logger.info("Trying request of %d bytes", request_bytes) + total_scalars = sum( + len(tag.points) for run in request.runs for tag in run.tags) + logger.warning( + "Trying request of %d bytes (%d scalars)", + request_bytes, total_scalars) self._upload(request) upload_duration_secs = time.time() - upload_start_time - logger.info( + logger.warning( "Upload for %d runs (%d bytes) took %.3f seconds", ``` wchargin-branch: uploader-batch-128kib --- tensorboard/uploader/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorboard/uploader/uploader.py b/tensorboard/uploader/uploader.py index 9b98d9e999..1a6e795148 100644 --- a/tensorboard/uploader/uploader.py +++ b/tensorboard/uploader/uploader.py @@ -56,7 +56,7 @@ # Deadline Exceeded errors in the RPC server. # # [1]: https://github.com/grpc/grpc/blob/e70d8582b4b0eedc45e3d25a57b58a08b94a9f4a/include/grpc/impl/codegen/grpc_types.h#L447 # pylint: disable=line-too-long -_MAX_REQUEST_LENGTH_BYTES = 1024 * 1024 +_MAX_REQUEST_LENGTH_BYTES = 1024 * 128 logger = tb_logging.get_logger()