diff --git a/tensorboard/BUILD b/tensorboard/BUILD index 9c3bce4a11..6812c66f72 100644 --- a/tensorboard/BUILD +++ b/tensorboard/BUILD @@ -57,6 +57,19 @@ py_library( ], ) +py_test( + name = "program_test", + size = "small", + srcs = ["program_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":program", + "//tensorboard:expect_tensorflow_installed", + "//tensorboard/backend:application", + "@org_pocoo_werkzeug", + ], +) + py_library( name = "default", srcs = ["default.py"], @@ -116,7 +129,7 @@ py_library( py_library( name = "expect_futures_installed", # This is a dummy rule used as a futures dependency in open-source. - # We expect numpy to already be installed on the system, e.g. via + # We expect futures to already be installed on the system, e.g. via # `pip install futures` visibility = ["//visibility:public"], ) diff --git a/tensorboard/backend/BUILD b/tensorboard/backend/BUILD index 271fb5d018..e76f135cd8 100644 --- a/tensorboard/backend/BUILD +++ b/tensorboard/backend/BUILD @@ -83,8 +83,6 @@ py_test( srcs_version = "PY2AND3", deps = [ ":application", - "//tensorboard", - "//tensorboard:default", "//tensorboard:expect_tensorflow_installed", "//tensorboard/backend/event_processing:event_multiplexer", "//tensorboard/plugins:base_plugin", diff --git a/tensorboard/backend/application_test.py b/tensorboard/backend/application_test.py index e497d78e1b..e1530cd3a8 100644 --- a/tensorboard/backend/application_test.py +++ b/tensorboard/backend/application_test.py @@ -40,8 +40,6 @@ from werkzeug import test as werkzeug_test from werkzeug import wrappers -from tensorboard import default -from tensorboard import program as tensorboard from tensorboard.backend import application from tensorboard.backend.event_processing import plugin_event_multiplexer as event_multiplexer # pylint: disable=line-too-long from tensorboard.plugins import base_plugin @@ -115,9 +113,7 @@ def is_active(self): return self._is_active_value -class TensorboardServerTest(tf.test.TestCase): - _only_use_meta_graph = False # Server data contains only a GraphDef - +class ApplicationTest(tf.test.TestCase): def setUp(self): plugins = [ FakePlugin( @@ -150,8 +146,7 @@ def testPluginsListing(self): self.assertEqual(parsed_object, {'foo': True, 'bar': False}) -class TensorboardServerBaseUrlTest(tf.test.TestCase): - _only_use_meta_graph = False # Server data contains only a GraphDef +class ApplicationBaseUrlTest(tf.test.TestCase): path_prefix = '/test' def setUp(self): plugins = [ @@ -191,7 +186,7 @@ def testPluginsListing(self): self.assertEqual(parsed_object, {'foo': True, 'bar': False}) -class TensorboardServerPluginNameTest(tf.test.TestCase): +class ApplicationPluginNameTest(tf.test.TestCase): def _test(self, name, should_be_okay): temp_dir = tempfile.mkdtemp(prefix=self.get_temp_dir()) @@ -233,8 +228,7 @@ def testComprehensiveName(self): self._test('Scalar-Dashboard_3000.1', True) - -class TensorboardServerPluginRouteTest(tf.test.TestCase): +class ApplicationPluginRouteTest(tf.test.TestCase): def _test(self, route, should_be_okay): temp_dir = tempfile.mkdtemp(prefix=self.get_temp_dir()) @@ -267,11 +261,6 @@ def testSlashlessRoute(self): self._test('runaway', False) -class TensorboardServerUsingMetagraphOnlyTest(TensorboardServerTest): - # Tests new ability to use only the MetaGraphDef - _only_use_meta_graph = True # Server data contains only a MetaGraphDef - - class ParseEventFilesSpecTest(tf.test.TestCase): def assertPlatformSpecificLogdirParsing(self, pathObj, logdir, expected): @@ -397,6 +386,7 @@ class TensorBoardPluginsTest(tf.test.TestCase): def setUp(self): self.context = None + dummy_assets_zip_provider = lambda: None # The application should have added routes for both plugins. self.app = application.standard_tensorboard_wsgi( FakeFlags(logdir=self.get_temp_dir()), @@ -414,7 +404,7 @@ def setUp(self): routes_mapping={'/bar_route': self._bar_handler}, construction_callback=self._construction_callback)), ], - default.get_assets_zip_provider()) + dummy_assets_zip_provider) def _foo_handler(self): pass @@ -441,50 +431,7 @@ def testNameToPluginMapping(self): self.assertEqual('bar', mapping['bar'].plugin_name) -class TensorboardSimpleServerConstructionTest(tf.test.TestCase): - """Tests that the default HTTP server is constructed without error. - - Mostly useful for IPv4/IPv6 testing. This test should run with only IPv4, only - IPv6, and both IPv4 and IPv6 enabled. - """ - - class _StubApplication(object): - tag = '' - - def testMakeServerBlankHost(self): - # Test that we can bind to all interfaces without throwing an error - server, url = tensorboard.make_simple_server( - self._StubApplication(), - host='', - port=0) # Grab any available port - self.assertTrue(server) - self.assertTrue(url) - - def testSpecifiedHost(self): - one_passed = False - try: - _, url = tensorboard.make_simple_server( - self._StubApplication(), - host='127.0.0.1', - port=0) - self.assertStartsWith(actual=url, expected_start='http://127.0.0.1:') - one_passed = True - except socket.error: - # IPv4 is not supported - pass - try: - _, url = tensorboard.make_simple_server( - self._StubApplication(), - host='::1', - port=0) - self.assertStartsWith(actual=url, expected_start='http://[::1]:') - one_passed = True - except socket.error: - # IPv6 is not supported - pass - self.assertTrue(one_passed) # We expect either IPv4 or IPv6 to be supported - -class TensorBoardApplcationConstructionTest(tf.test.TestCase): +class ApplicationConstructionTest(tf.test.TestCase): def testExceptions(self): logdir = '/fake/foo' diff --git a/tensorboard/default.py b/tensorboard/default.py index d01e82a091..d962aa0303 100644 --- a/tensorboard/default.py +++ b/tensorboard/default.py @@ -51,23 +51,34 @@ logger = logging.getLogger(__name__) -PLUGIN_LOADERS = [ +_PLUGINS = [ core_plugin.CorePluginLoader(), - base_plugin.BasicLoader(beholder_plugin.BeholderPlugin), - base_plugin.BasicLoader(scalars_plugin.ScalarsPlugin), - base_plugin.BasicLoader(custom_scalars_plugin.CustomScalarsPlugin), - base_plugin.BasicLoader(images_plugin.ImagesPlugin), - base_plugin.BasicLoader(audio_plugin.AudioPlugin), - base_plugin.BasicLoader(graphs_plugin.GraphsPlugin), - base_plugin.BasicLoader(distributions_plugin.DistributionsPlugin), - base_plugin.BasicLoader(histograms_plugin.HistogramsPlugin), - base_plugin.BasicLoader(pr_curves_plugin.PrCurvesPlugin), - base_plugin.BasicLoader(projector_plugin.ProjectorPlugin), - base_plugin.BasicLoader(text_plugin.TextPlugin), + beholder_plugin.BeholderPlugin, + scalars_plugin.ScalarsPlugin, + custom_scalars_plugin.CustomScalarsPlugin, + images_plugin.ImagesPlugin, + audio_plugin.AudioPlugin, + graphs_plugin.GraphsPlugin, + distributions_plugin.DistributionsPlugin, + histograms_plugin.HistogramsPlugin, + pr_curves_plugin.PrCurvesPlugin, + projector_plugin.ProjectorPlugin, + text_plugin.TextPlugin, profile_plugin.ProfilePluginLoader(), debugger_plugin_loader.DebuggerPluginLoader(), ] +def get_plugins(): + """Returns a list specifying TensorBoard's default first-party plugins. + + Plugins are specified in this list either via a TBLoader instance to load the + plugin, or the TBPlugin class itself which will be loaded using a BasicLoader. + + This list can be passed to the `tensorboard.program.TensorBoard` API. + + :rtype: list[Union[base_plugin.TBLoader, Type[base_plugin.TBPlugin]]] + """ + return _PLUGINS[:] def get_assets_zip_provider(): """Opens stock TensorBoard web assets collection. diff --git a/tensorboard/main.py b/tensorboard/main.py index c421c4051f..3c8278bb4f 100644 --- a/tensorboard/main.py +++ b/tensorboard/main.py @@ -34,32 +34,25 @@ os.environ['GCS_READ_CACHE_DISABLED'] = '1' # pylint: enable=g-import-not-at-top -import logging import sys from tensorboard import default from tensorboard import program -logger = logging.getLogger(__name__) - def run_main(): """Initializes flags and calls main().""" program.setup_environment() - server = program.TensorBoard(default.PLUGIN_LOADERS, - default.get_assets_zip_provider()) - server.configure(sys.argv[1:]) + tensorboard = program.TensorBoard(default.get_plugins(), + default.get_assets_zip_provider()) try: from absl import app - app.run(server.main, sys.argv[:1] + server.unparsed_argv) + app.run(tensorboard.main, flags_parser=tensorboard.configure) raise AssertionError("absl.app.run() shouldn't return") except ImportError: pass - if server.unparsed_argv: - sys.stderr.write('Unknown flags: %s\nPass --help for help.\n' % - (server.unparsed_argv,)) - sys.exit(1) - sys.exit(server.main()) + tensorboard.configure(sys.argv) + sys.exit(tensorboard.main()) if __name__ == '__main__': diff --git a/tensorboard/plugins/core/core_plugin.py b/tensorboard/plugins/core/core_plugin.py index 077fe51d45..2f87aad581 100644 --- a/tensorboard/plugins/core/core_plugin.py +++ b/tensorboard/plugins/core/core_plugin.py @@ -295,7 +295,8 @@ def define_flags(self, parser): type=int, default=6006, help='''\ -Port to serve TensorBoard on (default: %(default)s)\ +Port to serve TensorBoard on. Pass 0 to request an unused port selected +by the operating system. (default: %(default)s)\ ''') parser.add_argument( diff --git a/tensorboard/program.py b/tensorboard/program.py index a658490d78..6412ccbcf5 100644 --- a/tensorboard/program.py +++ b/tensorboard/program.py @@ -22,13 +22,15 @@ modifies the set of plugins and static assets. This module does not depend on first-party plugins or the default web -server assets. Those are defined in `tensorboard.default_plugins`. +server assets. Those are defined in `tensorboard.default`. """ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from abc import ABCMeta +from abc import abstractmethod import argparse import errno import logging @@ -43,6 +45,15 @@ from tensorboard import version from tensorboard.backend import application from tensorboard.backend.event_processing import event_file_inspector as efi +from tensorboard.plugins import base_plugin + +try: + from absl import flags as absl_flags + from absl.flags import argparse_flags +except ImportError: + # Fall back to argparse with no absl flags integration. + absl_flags = None + argparse_flags = argparse logger = logging.getLogger(__name__) @@ -64,77 +75,89 @@ def setup_environment(): class TensorBoard(object): - """Class for launching TensorBoard web server. + """Class for running TensorBoard. Fields: - plugin_loaders: Set by constructor. + plugin_loaders: Set from plugins passed to constructor. assets_zip_provider: Set by constructor. - flags: An argparse.Namespace() set by the configure() method, that - is initially None. - unparsed_argv: A list of strings set by the configure() method. + server_class: Set by constructor. + flags: An argparse.Namespace set by the configure() method. """ def __init__(self, - plugin_loaders=None, + plugins=None, assets_zip_provider=None, - wsgi_middleware=None): + server_class=None): """Creates new instance. - The configure() method should be called after creating a new - instance of this classe. - Args: - plugin_loaders: A list of TBLoader plugin loader instances. If not - specified, defaults to first-party plugins. - assets_zip_provider: Delegates to TBContext or uses default if - None. - wsgi_middleware: Optional function for installing middleware - around the standard TensorBoard WSGI handler. - - :type plugin_loaders: list[base_plugin.TBLoader] + plugins: A list of TensorBoard plugins to load, as TBLoader instances or + TBPlugin classes. If not specified, defaults to first-party plugins. + assets_zip_provider: Delegates to TBContext or uses default if None. + server_class: An optional subclass of TensorBoardServer to use for serving + the TensorBoard WSGI app. + + :type plugins: list[Union[base_plugin.TBLoader, Type[base_plugin.TBPlugin]]] :type assets_zip_provider: () -> file + :type server_class: class """ - if plugin_loaders is None: + if plugins is None: from tensorboard import default - plugin_loaders = default.PLUGIN_LOADERS + plugins = default.get_plugins() if assets_zip_provider is None: from tensorboard import default assets_zip_provider = default.get_assets_zip_provider() - self.plugin_loaders = plugin_loaders + if server_class is None: + server_class = WerkzeugServer + def make_loader(plugin): + if isinstance(plugin, base_plugin.TBLoader): + return plugin + if issubclass(plugin, base_plugin.TBPlugin): + return base_plugin.BasicLoader(plugin) + raise ValueError("Not a TBLoader or TBPlugin subclass: %s" % plugin) + self.plugin_loaders = [make_loader(p) for p in plugins] self.assets_zip_provider = assets_zip_provider - self._wsgi_middleware = wsgi_middleware + self.server_class = server_class self.flags = None - self.unparsed_argv = [] - def configure(self, argv=(), **kwargs): - """Creates TensorBoard CLI flag configuration object. + def configure(self, argv=('',), **kwargs): + """Configures TensorBoard behavior via flags. - The default behavior of this method is to construct an object with - its attributes set to the default values of all flags, specified by - all plugins. + This method will populate the "flags" property with an argparse.Namespace + representing flag values parsed from the provided argv list, overriden by + explicit flags from remaining keyword arguments. Args: - argv: This can be set (to what is usually) sys.argv[1:] to parse - CLI args. + argv: Can be set to CLI args equivalent to sys.argv; the first arg is + taken to be the name of the path being executed. kwargs: Additional arguments will override what was parsed from argv. They must be passed as Python data structures, e.g. `foo=1` rather than `foo="1"`. Returns: - The result is stored to the flags and unparsed_argv fields. This - method always returns None. + Either argv[:1] if argv was non-empty, or [''] otherwise, as a mechanism + for absl.app.run() compatiblity. Raises: ValueError: If flag values are invalid. """ - parser = argparse.ArgumentParser( + parser = argparse_flags.ArgumentParser( prog='tensorboard', description=('TensorBoard is a suite of web applications for ' - 'inspectinng and understanding your TensorFlow runs ' - 'and graphs. https://github.com/tensorflow/tensorboard')) + 'inspecting and understanding your TensorFlow runs ' + 'and graphs. https://github.com/tensorflow/tensorboard ')) for loader in self.plugin_loaders: loader.define_flags(parser) - flags, unparsed_argv = parser.parse_known_args(argv) + arg0 = argv[0] if argv else '' + flags = parser.parse_args(argv[1:]) # Strip binary name from argv. + if absl_flags and arg0: + # Only expose main module Abseil flags as TensorBoard native flags. + # This is the same logic Abseil's ArgumentParser uses for determining + # which Abseil flags to include in the short helpstring. + for flag in set(absl_flags.FLAGS.get_key_flags_for_module(arg0)): + if hasattr(flags, flag.name): + raise ValueError('Conflicting Abseil flag: %s' % name) + setattr(flags, flag.name, flag.value) for k, v in kwargs.items(): if hasattr(flags, k): raise ValueError('Unknown TensorBoard flag: %s' % k) @@ -142,9 +165,9 @@ def configure(self, argv=(), **kwargs): for loader in self.plugin_loaders: loader.fix_flags(flags) self.flags = flags - self.unparsed_argv = unparsed_argv + return [arg0] - def main(self, unparsed_argv=None): + def main(self, ignored_argv=('',)): """Blocking main function for TensorBoard. This method is called by `tensorboard.main.run_main`, which is the @@ -152,7 +175,7 @@ def main(self, unparsed_argv=None): configure() method must be called first. Args: - unparsed_argv: Ignored (required for Abseil compatibility). + ignored_argv: Do not pass. Required for Abseil compatibility. Returns: Process exit code, i.e. 0 if successful or non-zero on failure. In @@ -169,14 +192,17 @@ def main(self, unparsed_argv=None): self.flags.tag) return 0 try: - server, url = self._get_server() - except socket.error: + server = self._make_server() + sys.stderr.write('TensorBoard %s at %s (Press CTRL+C to quit)\n' % + (version.VERSION, server.get_url())) + sys.stderr.flush() + server.serve_forever() + return 0 + except TensorBoardServerException as e: + logger.error(e.msg) + sys.stderr.write('ERROR: %s\n' % e.msg) + sys.stderr.flush() return -1 - sys.stderr.write('TensorBoard %s at %s (Press CTRL+C to quit)\n' % - (version.VERSION, url)) - sys.stderr.flush() - server.serve_forever() - return 0 def launch(self): """Python API for launching TensorBoard. @@ -188,90 +214,104 @@ def launch(self): Returns: The URL of the TensorBoard web server. - Raises: - socket.error: If a server could not be constructed with the host - and port specified. Also logs an error message. - :rtype: str """ # Make it easy to run TensorBoard inside other programs, e.g. Colab. - server, url = self._get_server() + server = self._make_server() thread = threading.Thread(target=server.serve_forever, name='TensorBoard') thread.daemon = True thread.start() - return url + return server.get_url() - def _get_server(self): + def _make_server(self): + """Constructs the TensorBoard WSGI app and instantiates the server.""" app = application.standard_tensorboard_wsgi(self.flags, self.plugin_loaders, self.assets_zip_provider) - if self._wsgi_middleware is not None: - app = self._wsgi_middleware(app) - return make_simple_server(app, - self.flags.host, - self.flags.port, - self.flags.path_prefix) - - -def make_simple_server(tb_app, host='', port=0, path_prefix=''): - """Create an HTTP server for TensorBoard. - - Args: - tb_app: The TensorBoard WSGI application to create a server for. - host: Indicates the interfaces to bind to ('::' or '0.0.0.0' for all - interfaces, '::1' or '127.0.0.1' for localhost). A blank value ('') - indicates protocol-agnostic all interfaces. - port: The port to bind to (0 indicates an unused port selected by the - operating system). - path_prefix: Optional relative prefix to the path, e.g. "/service/tf". - - Returns: - A tuple of (server, url): - server: An HTTP server object configured to host TensorBoard. - url: A best guess at a URL where TensorBoard will be accessible once the - server has been started. - - Raises: - socket.error: If a server could not be constructed with the host and port - specified. Also logs an error message. + return self.server_class(app, self.flags) + + +class TensorBoardServer(object): + """Class for customizing TensorBoard WSGI app serving.""" + __metaclass__ = ABCMeta + + @abstractmethod + def __init__(self, wsgi_app, flags): + """Create a flag-configured HTTP server for TensorBoard's WSGI app. + + Args: + wsgi_app: The TensorBoard WSGI application to create a server for. + flags: argparse.Namespace instance of TensorBoard flags. + """ + raise NotImplementedError() + + @abstractmethod + def serve_forever(self): + """Blocking call to start serving the TensorBoard server.""" + raise NotImplementedError() + + @abstractmethod + def get_url(self): + """Returns a URL at which this server should be reachable.""" + raise NotImplementedError() + + +class TensorBoardServerException(Exception): + """Exception raised by TensorBoardServer for user-friendly errors. + + Subclasses of TensorBoardServer can raise this exception in order to + generate a clean error message for the user rather than a stacktrace. """ - try: - if host: - # The user gave us an explicit host - server = serving.make_server(host, port, tb_app, threaded=True) - if ':' in host and not host.startswith('['): - # Display IPv6 addresses as [::1]:80 rather than ::1:80 - final_host = '[{}]'.format(host) + def __init__(self, msg): + self.msg = msg + + +class WerkzeugServer(TensorBoardServer): + """Implementation of TensorBoardServer using the Werkzeug dev server.""" + + def __init__(self, wsgi_app, flags): + host = flags.host + port = flags.port + try: + if host: + # The user gave us an explicit host + server = serving.make_server(host, port, wsgi_app, threaded=True) + if ':' in host and not host.startswith('['): + # Display IPv6 addresses as [::1]:80 rather than ::1:80 + final_host = '[{}]'.format(host) + else: + final_host = host + else: + # We've promised to bind to all interfaces on this host. However, we're + # not sure whether that means IPv4 or IPv6 interfaces. + try: + # First try passing in a blank host (meaning all interfaces). This, + # unfortunately, defaults to IPv4 even if no IPv4 interface is available + # (yielding a socket.error). + server = serving.make_server(host, port, wsgi_app, threaded=True) + except socket.error: + # If a blank host didn't work, we explicitly request IPv6 interfaces. + server = serving.make_server('::', port, wsgi_app, threaded=True) + final_host = socket.gethostname() + server.daemon_threads = True + except socket.error: + if port == 0: + msg = 'TensorBoard unable to find any open port' else: - final_host = host - else: - # We've promised to bind to all interfaces on this host. However, we're - # not sure whether that means IPv4 or IPv6 interfaces. - try: - # First try passing in a blank host (meaning all interfaces). This, - # unfortunately, defaults to IPv4 even if no IPv4 interface is available - # (yielding a socket.error). - server = serving.make_server(host, port, tb_app, threaded=True) - except socket.error: - # If a blank host didn't work, we explicitly request IPv6 interfaces. - server = serving.make_server('::', port, tb_app, threaded=True) - final_host = socket.gethostname() - server.daemon_threads = True - except socket.error: - if port == 0: - msg = 'TensorBoard unable to find any open port' - else: - msg = ( - 'TensorBoard attempted to bind to port %d, but it was already in use' - % port) - logger.error(msg) - print(msg) - raise - server.handle_error = _handle_error - final_port = server.socket.getsockname()[1] - tensorboard_url = 'http://%s:%d%s' % (final_host, final_port, - path_prefix) - return server, tensorboard_url + msg = ( + 'TensorBoard attempted to bind to port %d, but it was already in use' + % port) + raise TensorBoardServerException(msg) + server.handle_error = _handle_error + final_port = server.socket.getsockname()[1] + self._server = server + self._url = 'http://%s:%d%s' % (final_host, final_port, flags.path_prefix) + + def serve_forever(self): + self._server.serve_forever() + + def get_url(self): + return self._url # Kludge to override a SocketServer.py method so we can get rid of noisy diff --git a/tensorboard/program_test.py b/tensorboard/program_test.py new file mode 100644 index 0000000000..cfb7c34c09 --- /dev/null +++ b/tensorboard/program_test.py @@ -0,0 +1,76 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Unit tests for program package.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse + +import six +import tensorflow as tf + +from tensorboard import program + + +class WerkzeugServerTest(tf.test.TestCase): + """Tests the default Werkzeug implementation of TensorBoardServer. + + Mostly useful for IPv4/IPv6 testing. This test should run with only IPv4, only + IPv6, and both IPv4 and IPv6 enabled. + """ + + class _StubApplication(object): + pass + + def make_flags(self, **kwargs): + flags = argparse.Namespace() + for k, v in six.iteritems(kwargs): + setattr(flags, k, v) + return flags + + def testMakeServerBlankHost(self): + # Test that we can bind to all interfaces without throwing an error + server = program.WerkzeugServer( + self._StubApplication(), + self.make_flags(host='', port=0, path_prefix='')) + self.assertStartsWith(server.get_url(), 'http://') + + def testSpecifiedHost(self): + one_passed = False + try: + server = program.WerkzeugServer( + self._StubApplication(), + self.make_flags(host='127.0.0.1', port=0, path_prefix='')) + self.assertStartsWith(server.get_url(), 'http://127.0.0.1:') + one_passed = True + except program.TensorBoardServerException: + # IPv4 is not supported + pass + try: + server = program.WerkzeugServer( + self._StubApplication(), + self.make_flags(host='::1', port=0, path_prefix='')) + self.assertStartsWith(server.get_url(), 'http://[::1]:') + one_passed = True + except program.TensorBoardServerException: + # IPv6 is not supported + pass + self.assertTrue(one_passed) # We expect either IPv4 or IPv6 to be supported + + +if __name__ == '__main__': + tf.test.main()