Disable tf.gfile's GCS filesystem caching

Author: nfelt

tensorboard/backend/event_processing/event_file_loader.py

@@ -57,7 +57,8 @@ def Load(self):
           # GetNext() expects a status argument on TF <= 1.7
           with tf.errors.raise_exception_on_not_ok_status() as status:
             self._reader.GetNext(status)
-      except (tf.errors.DataLossError, tf.errors.OutOfRangeError):
+      except (tf.errors.DataLossError, tf.errors.OutOfRangeError) as e:
+        tf.logging.debug('Cannot read more events: %s', e)
         # We ignore partial read exceptions, because a record may be truncated.
         # PyRecordReader holds the offset prior to the failed read, so retrying
         # will succeed.

tensorboard/main.py

@@ -25,8 +25,18 @@
 from __future__ import division
 from __future__ import print_function
 
+# pylint: disable=g-import-not-at-top
+# Disable the TF GCS filesystem cache which interacts pathologically with the
+# pattern of reads used by TensorBoard for logdirs. See for details:
+#   https://github.com/tensorflow/tensorboard/issues/1225
+# This must be set before the first import of tensorflow.
+import os
+os.environ['GCS_READ_CACHE_DISABLED'] = '1'
+
 import tensorflow as tf
 
+# pylint: enable=g-import-not-at-top
+
 from tensorboard import default
 from tensorboard import program