From 3fc95138bba04af116f9b55db8e31bace069a39c Mon Sep 17 00:00:00 2001 From: William Chargin Date: Fri, 19 Apr 2019 16:21:28 -0700 Subject: [PATCH 1/3] build: register `mock` dependency with Bazel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: We’ve heretofore used the `mock` library without telling Bazel about it. This isn’t generally a problem when `tensorflow` is installed in the same virtualenv, because `tensorflow` pulls in `mock`. But tests that don’t depend on TensorFlow have no build edge to `mock` when imported into google3’s build system, causing build failures after syncing. This commit makes the `mock` dependency explict. We use `mock==1.0.0` rather than the current version (2.0.0) because the current version depends on `pbr` at runtime in a way that does not play well with Bazel: (It appears to be trying to find some globally registered `setup.cfg` file from which to read a version identifier.) Rather than vendoring and patching `mock`, declaring a fake `:expect_mock_installed` target, or hacking `PBR_VERSION=2.0.0` into the action env, we simply depend on a saner version of the library, which is also the version used in google3. Existing `mock` users have been detected and fixed up: ``` $ bazel query " > kind( > py_.*rule, > rdeps(//..., set($(git grep -l 'import mock\($\| \)')), 1) > ) > " \ > | awk '{ print "add deps @org_pythonhosted_mock|" $0 }' \ > | buildozer -f - \ > ; ``` Test Plan: All notf tests now pass in a virtualenv that does not include `mock`: ``` $ virtualenv -q -p python2 ./ve $ . ./ve/bin/activate (ve) $ pip install 'absl-py>=0.7.0' 'numpy<2.0,>=1.14.5' boto3==1.9.86 flake8==3.5.0 futures==3.1.1 grpcio==1.6.3 yamllint==1.5.0 >/dev/null 2>&1 (ve) $ python -c 'import mock' Traceback (most recent call last): File "", line 1, in File "/usr/local/buildtools/current/sitecustomize/sitecustomize.py", line 152, in SetupPathsAndImport return real_import(name, globals, locals, fromlist, level) ImportError: No module named mock (ve) $ cd ~/git/tensorboard (ve) $ bazel test //tensorboard/... --test_tag_filters=support_notf ... INFO: Build completed successfully, 12 total actions //tensorboard:lazy_test PASSED in 0.2s //tensorboard:lib_test PASSED in 0.9s //tensorboard:manager_test PASSED in 0.3s //tensorboard:plugin_util_test PASSED in 0.5s //tensorboard:program_test PASSED in 0.8s //tensorboard/backend:application_test PASSED in 1.0s //tensorboard/backend:http_util_test PASSED in 0.8s //tensorboard/backend:json_util_test PASSED in 0.8s //tensorboard/compat/tensorflow_stub:gfile_test PASSED in 0.8s //tensorboard/summary:summary_test PASSED in 0.8s //tensorboard/util:platform_util_test PASSED in 0.3s ``` Previously, `manager_test` and `application_test` failed in such an environment. wchargin-branch: mock-dep --- .travis.yml | 1 - tensorboard/BUILD | 2 ++ tensorboard/backend/BUILD | 1 + tensorboard/plugins/hparams/BUILD | 3 +++ tensorboard/plugins/interactive_inference/BUILD | 1 + .../plugins/interactive_inference/utils/BUILD | 1 + third_party/mock.BUILD | 15 +++++++++++++++ third_party/python.bzl | 11 +++++++++++ 8 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 third_party/mock.BUILD diff --git a/.travis.yml b/.travis.yml index 070d97d84b..0a37740366 100644 --- a/.travis.yml +++ b/.travis.yml @@ -102,7 +102,6 @@ install: - pip install flake8==3.5.0 - pip install futures==3.1.1 - pip install grpcio==1.6.3 - - pip install mock==2.0.0 - pip install moto==1.3.7 - pip install yamllint==1.5.0 - | diff --git a/tensorboard/BUILD b/tensorboard/BUILD index 6af3f23c2c..19d49bae6f 100644 --- a/tensorboard/BUILD +++ b/tensorboard/BUILD @@ -83,6 +83,7 @@ py_test( ":test", ":version", "//tensorboard/util:tb_logging", + "@org_pythonhosted_mock", "@org_pythonhosted_six", ], ) @@ -101,6 +102,7 @@ py_test( deps = [ ":manager", "//tensorboard:expect_tensorflow_installed", + "@org_pythonhosted_mock", "@org_pythonhosted_six", ], data = [ diff --git a/tensorboard/backend/BUILD b/tensorboard/backend/BUILD index 318b94372f..66afa8118a 100644 --- a/tensorboard/backend/BUILD +++ b/tensorboard/backend/BUILD @@ -90,6 +90,7 @@ py_test( "//tensorboard/backend/event_processing:event_multiplexer", "//tensorboard/plugins:base_plugin", "@org_pocoo_werkzeug", + "@org_pythonhosted_mock", ], ) diff --git a/tensorboard/plugins/hparams/BUILD b/tensorboard/plugins/hparams/BUILD index eefb1183ae..67c40c667d 100644 --- a/tensorboard/plugins/hparams/BUILD +++ b/tensorboard/plugins/hparams/BUILD @@ -61,6 +61,7 @@ py_test( "//tensorboard:expect_tensorflow_installed", "//tensorboard/backend/event_processing:event_accumulator", "//tensorboard/backend/event_processing:event_multiplexer", + "@org_pythonhosted_mock", ], ) @@ -73,6 +74,7 @@ py_test( deps = [ ":hparams_plugin", "//tensorboard:expect_tensorflow_installed", + "@org_pythonhosted_mock", ], ) @@ -87,6 +89,7 @@ py_test( "//tensorboard:expect_tensorflow_installed", "//tensorboard/backend/event_processing:event_accumulator", "//tensorboard/backend/event_processing:event_multiplexer", + "@org_pythonhosted_mock", ], ) diff --git a/tensorboard/plugins/interactive_inference/BUILD b/tensorboard/plugins/interactive_inference/BUILD index ce0a616978..cbba5d47c1 100644 --- a/tensorboard/plugins/interactive_inference/BUILD +++ b/tensorboard/plugins/interactive_inference/BUILD @@ -39,6 +39,7 @@ py_test( "//tensorboard/plugins/interactive_inference/utils:platform_utils", "//tensorboard/plugins/interactive_inference/utils:test_utils", "@org_pocoo_werkzeug", + "@org_pythonhosted_mock", "@org_pythonhosted_six", ], ) diff --git a/tensorboard/plugins/interactive_inference/utils/BUILD b/tensorboard/plugins/interactive_inference/utils/BUILD index 4204d69d09..f83ae69350 100644 --- a/tensorboard/plugins/interactive_inference/utils/BUILD +++ b/tensorboard/plugins/interactive_inference/utils/BUILD @@ -53,6 +53,7 @@ py_test( ":test_utils", "//tensorboard:expect_numpy_installed", "//tensorboard:expect_tensorflow_installed", + "@org_pythonhosted_mock", "@org_tensorflow_serving_api", ], ) diff --git a/third_party/mock.BUILD b/third_party/mock.BUILD new file mode 100644 index 0000000000..841af364cd --- /dev/null +++ b/third_party/mock.BUILD @@ -0,0 +1,15 @@ +# Description: +# Rolling backport of unittest.mock for all Pythons + +licenses(["notice"]) # MIT + +exports_files(["LICENSE"]) + +py_library( + name = "org_pythonhosted_mock", + srcs = [ + "mock.py", + ], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) diff --git a/third_party/python.bzl b/third_party/python.bzl index ab5aeb8d92..802c7fd460 100644 --- a/third_party/python.bzl +++ b/third_party/python.bzl @@ -76,6 +76,17 @@ def tensorboard_python_workspace(): build_file = str(Label("//third_party:werkzeug.BUILD")), ) + http_archive( + name = "org_pythonhosted_mock", + urls = [ + "https://mirror.bazel.build/files.pythonhosted.org/packages/85/60/ec8c1af81337bab0caba188b218b6758bc94f125f49062f7c5f0647749d2/mock-1.0.0.tar.gz", + "https://files.pythonhosted.org/packages/85/60/ec8c1af81337bab0caba188b218b6758bc94f125f49062f7c5f0647749d2/mock-1.0.0.tar.gz", + ], + sha256 = "2d9fbe67001d2e8f02692075257f3c11e1b0194bd838c8ce3f49b31fc6c3f033", + strip_prefix = "mock-1.0.0", + build_file = str(Label("//third_party:mock.BUILD")), + ) + http_archive( name = "org_pythonhosted_six", urls = [ From 0058bf74dc3e7464371187f88744073db6e6dcd4 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Fri, 19 Apr 2019 17:04:33 -0700 Subject: [PATCH 2/3] review: include rationale for mock version wchargin-branch: mock-dep --- third_party/python.bzl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/third_party/python.bzl b/third_party/python.bzl index 802c7fd460..1f1b604ea7 100644 --- a/third_party/python.bzl +++ b/third_party/python.bzl @@ -76,6 +76,12 @@ def tensorboard_python_workspace(): build_file = str(Label("//third_party:werkzeug.BUILD")), ) + # We use `mock==1.0.0` because later versions depend on `pbr`, which + # doesn't work well in a hermetic context (it tries to look up some + # global configuration files; see GitHub pull request #2132). + # + # This dependency can go away entirely once we drop Python 2 support + # and can just depend on `unittest.mock`. http_archive( name = "org_pythonhosted_mock", urls = [ From a6ec9abf1b7f739016718f73c7337fa57a1e4394 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Fri, 19 Apr 2019 17:36:59 -0700 Subject: [PATCH 3/3] poke: does Travis not like draft PRs? wchargin-branch: mock-dep