Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
42a7f51
ci: remove Python 2.7 from `.travis.yml` (#3094)
wchargin Jan 2, 2020
5dc9983
Convert BaseResponse to wrappers.BaseResponse (#3230)
stephanwlee Feb 10, 2020
d78b796
pylint: remove `g-import-not-at-top` suppressions (#3022)
wchargin Dec 12, 2019
b782066
black: reformat directory tensorboard/uploader
tensorboard-gardener Dec 14, 2019
2ad7c65
black: reformat directory tensorboard/backend/event_processing
tensorboard-gardener Dec 14, 2019
2b9df53
When available, use inspect.getfullargspec instead of inspect.getargs…
fcq Jan 7, 2020
de23608
Change EventFileLoader to use tf_record_iterator when possible (#3185)
nfelt Jan 28, 2020
1a7dade
Silence deprecation warning spam about tf_record_iterator (#3319)
nfelt Mar 3, 2020
231d782
uploader: make `util_test` py3-compatible internally (#3179)
wchargin Jan 28, 2020
b76fa08
add lint check against PY2 BUILD targets (#3220)
nfelt Feb 6, 2020
272b690
Proto changes to support experiment name & description. (#3234)
bileschi Feb 12, 2020
f330db3
Update export/write services for eventual tensor storage (#3236)
ericdnielsen Feb 12, 2020
0bbb22f
Add the WriteBlob bidirectional streaming RPC to the write service. (…
davidsoergel Feb 13, 2020
fca76cb
Add wall_time field to GetOrCreateBlobSequenceRequest (#3253)
davidsoergel Feb 14, 2020
b04c2e1
Extract Experiment/ExperimentMask messages into experiment.proto (#3255)
nfelt Feb 19, 2020
c02aad6
Refactor uploader to ease adding new data types (#3215)
davidsoergel Feb 19, 2020
47f11fa
Add trailing newline (#3267)
davidsoergel Feb 20, 2020
2887aa3
Support experiment name and description in uploader (#3277)
bileschi Feb 24, 2020
2a9319b
uploader: add `PluginControl` handshake protos (#3299)
wchargin Feb 27, 2020
a381172
uploader: factor test defaults into helpers (#3304)
wchargin Feb 28, 2020
75e5d59
uploader: only send data for allowed plugins (#3300)
wchargin Feb 28, 2020
c85ddb9
uploader: fix rate limiting for logdir polling (#3305)
wchargin Feb 28, 2020
b004416
uploader: export experiments to subdirectories (#3307)
wchargin Feb 29, 2020
b772612
uploader: remove inoperative `f`-string (#3311)
wchargin Feb 29, 2020
d82e181
uploader: require experiment metadata from server (#3310)
wchargin Mar 2, 2020
e533209
uploader: export experiment metadata (#3308)
wchargin Mar 2, 2020
04401e8
Fix spuriously failing grpc test
davidsoergel Dec 6, 2019
2fd8ad7
Add 2.1.1 relnotes to RELEASE.md
bileschi Mar 2, 2020
90ef4c5
TensorBoard 2.1.1
bileschi Mar 2, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
dist: trusty
language: python
python:
- "2.7"
- "3.6"

branches:
Expand Down
13 changes: 13 additions & 0 deletions RELEASE.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,16 @@
# Release 2.1.1

## Features

- Uploader: Added ability to upload and modify experiment name and description (#3277)

## Breaking changes

- As per
https://groups.google.com/a/tensorflow.org/forum/#!topic/developers/ifEAGK3aPls
this patch does not support Python 2. Only Python 3 is supported


# Release 2.1.0

The 2.1 minor series tracks TensorFlow 2.1.
Expand Down
2 changes: 1 addition & 1 deletion tensorboard/backend/empty_path_redirect_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def setUp(self):
app = empty_path_redirect.EmptyPathRedirectMiddleware(app)
app = self._lax_strip_foo_middleware(app)
self.app = app
self.server = werkzeug_test.Client(self.app, werkzeug.BaseResponse)
self.server = werkzeug_test.Client(self.app, werkzeug.wrappers.BaseResponse)

def _lax_strip_foo_middleware(self, app):
"""Strips a `/foo` prefix if it exists; no-op otherwise."""
Expand Down
3 changes: 3 additions & 0 deletions tensorboard/backend/event_processing/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,9 @@ py_test(
deps = [
":event_file_loader",
"//tensorboard:expect_tensorflow_installed",
"//tensorboard/compat/proto:protos_all_py_pb2",
"//tensorboard/summary/writer",
"@org_pythonhosted_six",
],
)

Expand Down
244 changes: 120 additions & 124 deletions tensorboard/backend/event_processing/data_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,102 +29,98 @@


class MultiplexerDataProvider(provider.DataProvider):
def __init__(self, multiplexer, logdir):
"""Trivial initializer.
def __init__(self, multiplexer, logdir):
"""Trivial initializer.

Args:
multiplexer: A `plugin_event_multiplexer.EventMultiplexer` (note:
not a boring old `event_multiplexer.EventMultiplexer`).
logdir: The log directory from which data is being read. Only used
cosmetically. Should be a `str`.
"""
self._multiplexer = multiplexer
self._logdir = logdir

def _test_run_tag(self, run_tag_filter, run, tag):
runs = run_tag_filter.runs
if runs is not None and run not in runs:
return False
tags = run_tag_filter.tags
if tags is not None and tag not in tags:
return False
return True

def _get_first_event_timestamp(self, run_name):
try:
return self._multiplexer.FirstEventTimestamp(run_name)
except ValueError as e:
return None

def data_location(self, experiment_id):
del experiment_id # ignored
return str(self._logdir)

def list_runs(self, experiment_id):
del experiment_id # ignored for now
return [
provider.Run(
run_id=run, # use names as IDs
run_name=run,
start_time=self._get_first_event_timestamp(run),
self._multiplexer = multiplexer
self._logdir = logdir

def _test_run_tag(self, run_tag_filter, run, tag):
runs = run_tag_filter.runs
if runs is not None and run not in runs:
return False
tags = run_tag_filter.tags
if tags is not None and tag not in tags:
return False
return True

def _get_first_event_timestamp(self, run_name):
try:
return self._multiplexer.FirstEventTimestamp(run_name)
except ValueError as e:
return None

def data_location(self, experiment_id):
del experiment_id # ignored
return str(self._logdir)

def list_runs(self, experiment_id):
del experiment_id # ignored for now
return [
provider.Run(
run_id=run, # use names as IDs
run_name=run,
start_time=self._get_first_event_timestamp(run),
)
for run in self._multiplexer.Runs()
]

def list_scalars(self, experiment_id, plugin_name, run_tag_filter=None):
run_tag_content = self._multiplexer.PluginRunToTagToContent(plugin_name)
return self._list(provider.ScalarTimeSeries, run_tag_content, run_tag_filter)

def read_scalars(
self, experiment_id, plugin_name, downsample=None, run_tag_filter=None
):
# TODO(@wchargin): Downsampling not implemented, as the multiplexer
# is already downsampled. We could downsample on top of the existing
# sampling, which would be nice for testing.
del downsample # ignored for now
index = self.list_scalars(
experiment_id, plugin_name, run_tag_filter=run_tag_filter
)
for run in self._multiplexer.Runs()
]

def list_scalars(self, experiment_id, plugin_name, run_tag_filter=None):
run_tag_content = self._multiplexer.PluginRunToTagToContent(plugin_name)
return self._list(
provider.ScalarTimeSeries, run_tag_content, run_tag_filter
)

def read_scalars(
self, experiment_id, plugin_name, downsample=None, run_tag_filter=None
):
# TODO(@wchargin): Downsampling not implemented, as the multiplexer
# is already downsampled. We could downsample on top of the existing
# sampling, which would be nice for testing.
del downsample # ignored for now
index = self.list_scalars(
experiment_id, plugin_name, run_tag_filter=run_tag_filter
)

def convert_scalar_event(event):
return provider.ScalarDatum(
step=event.step,
wall_time=event.wall_time,
value=tensor_util.make_ndarray(event.tensor_proto).item(),
)

return self._read(convert_scalar_event, index)

def list_tensors(self, experiment_id, plugin_name, run_tag_filter=None):
run_tag_content = self._multiplexer.PluginRunToTagToContent(plugin_name)
return self._list(
provider.TensorTimeSeries, run_tag_content, run_tag_filter
)

def read_tensors(
self, experiment_id, plugin_name, downsample=None, run_tag_filter=None
):
# TODO(@wchargin): Downsampling not implemented, as the multiplexer
# is already downsampled. We could downsample on top of the existing
# sampling, which would be nice for testing.
del downsample # ignored for now
index = self.list_tensors(
experiment_id, plugin_name, run_tag_filter=run_tag_filter
)

def convert_tensor_event(event):
return provider.TensorDatum(
step=event.step,
wall_time=event.wall_time,
numpy=tensor_util.make_ndarray(event.tensor_proto),
)

return self._read(convert_tensor_event, index)

def _list(self, construct_time_series, run_tag_content, run_tag_filter):
"""Helper to list scalar or tensor time series.

def convert_scalar_event(event):
return provider.ScalarDatum(
step=event.step,
wall_time=event.wall_time,
value=tensor_util.make_ndarray(event.tensor_proto).item(),
)

return self._read(convert_scalar_event, index)

def list_tensors(self, experiment_id, plugin_name, run_tag_filter=None):
run_tag_content = self._multiplexer.PluginRunToTagToContent(plugin_name)
return self._list(provider.TensorTimeSeries, run_tag_content, run_tag_filter)

def read_tensors(
self, experiment_id, plugin_name, downsample=None, run_tag_filter=None
):
# TODO(@wchargin): Downsampling not implemented, as the multiplexer
# is already downsampled. We could downsample on top of the existing
# sampling, which would be nice for testing.
del downsample # ignored for now
index = self.list_tensors(
experiment_id, plugin_name, run_tag_filter=run_tag_filter
)

def convert_tensor_event(event):
return provider.TensorDatum(
step=event.step,
wall_time=event.wall_time,
numpy=tensor_util.make_ndarray(event.tensor_proto),
)

return self._read(convert_tensor_event, index)

def _list(self, construct_time_series, run_tag_content, run_tag_filter):
"""Helper to list scalar or tensor time series.

Args:
construct_time_series: `ScalarTimeSeries` or `TensorTimeSeries`.
Expand All @@ -135,34 +131,34 @@ def _list(self, construct_time_series, run_tag_content, run_tag_filter):
A list of objects of type given by `construct_time_series`,
suitable to be returned from `list_scalars` or `list_tensors`.
"""
result = {}
if run_tag_filter is None:
run_tag_filter = provider.RunTagFilter(runs=None, tags=None)
for (run, tag_to_content) in six.iteritems(run_tag_content):
result_for_run = {}
for tag in tag_to_content:
if not self._test_run_tag(run_tag_filter, run, tag):
continue
result[run] = result_for_run
max_step = None
max_wall_time = None
for event in self._multiplexer.Tensors(run, tag):
if max_step is None or max_step < event.step:
max_step = event.step
if max_wall_time is None or max_wall_time < event.wall_time:
max_wall_time = event.wall_time
summary_metadata = self._multiplexer.SummaryMetadata(run, tag)
result_for_run[tag] = construct_time_series(
max_step=max_step,
max_wall_time=max_wall_time,
plugin_content=summary_metadata.plugin_data.content,
description=summary_metadata.summary_description,
display_name=summary_metadata.display_name,
)
return result

def _read(self, convert_event, index):
"""Helper to read scalar or tensor data from the multiplexer.
result = {}
if run_tag_filter is None:
run_tag_filter = provider.RunTagFilter(runs=None, tags=None)
for (run, tag_to_content) in six.iteritems(run_tag_content):
result_for_run = {}
for tag in tag_to_content:
if not self._test_run_tag(run_tag_filter, run, tag):
continue
result[run] = result_for_run
max_step = None
max_wall_time = None
for event in self._multiplexer.Tensors(run, tag):
if max_step is None or max_step < event.step:
max_step = event.step
if max_wall_time is None or max_wall_time < event.wall_time:
max_wall_time = event.wall_time
summary_metadata = self._multiplexer.SummaryMetadata(run, tag)
result_for_run[tag] = construct_time_series(
max_step=max_step,
max_wall_time=max_wall_time,
plugin_content=summary_metadata.plugin_data.content,
description=summary_metadata.summary_description,
display_name=summary_metadata.display_name,
)
return result

def _read(self, convert_event, index):
"""Helper to read scalar or tensor data from the multiplexer.

Args:
convert_event: Takes `plugin_event_accumulator.TensorEvent` to
Expand All @@ -173,11 +169,11 @@ def _read(self, convert_event, index):
A dict of dicts of values returned by `convert_event` calls,
suitable to be returned from `read_scalars` or `read_tensors`.
"""
result = {}
for (run, tags_for_run) in six.iteritems(index):
result_for_run = {}
result[run] = result_for_run
for (tag, metadata) in six.iteritems(tags_for_run):
events = self._multiplexer.Tensors(run, tag)
result_for_run[tag] = [convert_event(e) for e in events]
return result
result = {}
for (run, tags_for_run) in six.iteritems(index):
result_for_run = {}
result[run] = result_for_run
for (tag, metadata) in six.iteritems(tags_for_run):
events = self._multiplexer.Tensors(run, tag)
result_for_run[tag] = [convert_event(e) for e in events]
return result
Loading