Enable Bzlmod in Python integration tests

Prepare for --enable_bzlmod flip: #18958

To avoid accessing BCR in integration tests, we have a default lockfile checked-in and used for tests with no MODULE.bazel file (most cases).

Every time we update MODULE.tools, we should regenerate the default lockfile with `bazel run //src/test/tools/bzlmod:update_default_lock_file`

PiperOrigin-RevId: 572424932
Change-Id: Ie6b3c95eae0c1306eac2d507132a9736b0b33a3e

MODULE.bazel

@@ -247,6 +247,7 @@ bazel_build_deps = use_extension("//:extensions.bzl", "bazel_build_deps")
 use_repo(
  bazel_build_deps,
  "bootstrap_repo_cache",
+ "bazel_tools_repo_cache",
  "debian_cc_deps",
  "openjdk_linux_aarch64_vanilla",
  "openjdk_linux_ppc64le_vanilla",

distdir.bzl

@@ -20,9 +20,15 @@ load("//tools/build_defs/repo:http.bzl", "http_archive", "http_file", "http_jar"
 _BUILD = """
 load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
 
+filegroup(
+ name="files",
+ srcs = {srcs},
+ visibility = ["//visibility:public"],
+)
+
 pkg_tar(
  name="archives",
- srcs = {srcs},
+ srcs = [":files"],
  strip_prefix = "{strip_prefix}",
  package_dir = "{dirname}",
  visibility = ["//visibility:public"],

extensions.bzl

@@ -21,13 +21,16 @@ load("//:distdir_deps.bzl", "DIST_ARCHIVE_REPOS")
 load("//:repositories.bzl", "embedded_jdk_repositories")
 load("//src/main/res:winsdk_configure.bzl", "winsdk_configure")
 load("//src/test/shell/bazel:list_source_repository.bzl", "list_source_repository")
+load("//src/tools/bzlmod:utils.bzl", "parse_bazel_module_repos")
 load("//tools/distributions/debian:deps.bzl", "debian_deps")
 
 ### Dependencies for building Bazel
 def _bazel_build_deps(_ctx):
  embedded_jdk_repositories()
  debian_deps()
  repo_cache_tar(name = "bootstrap_repo_cache", repos = DIST_ARCHIVE_REPOS, dirname = "derived/repository_cache")
+ BAZEL_TOOLS_DEPS_REPOS = parse_bazel_module_repos(_ctx, _ctx.path(Label("//src/test/tools/bzlmod:MODULE.bazel.lock")))
+ repo_cache_tar(name = "bazel_tools_repo_cache", repos = BAZEL_TOOLS_DEPS_REPOS, lockfile = "//src/test/tools/bzlmod:MODULE.bazel.lock")
 
 bazel_build_deps = module_extension(implementation = _bazel_build_deps)
 

src/BUILD

@@ -372,6 +372,7 @@ filegroup(
  "//src/test/shell:srcs",
  "//src/test/testdata/test_tls_certificate",
  "//src/test/tools:srcs",
+ "//src/test/tools/bzlmod:srcs",
  "//src/tools/android:srcs",
  "//src/tools/android/java/com/google/devtools/build/android:srcs",
  "//src/tools/bzlmod:srcs",
@@ -586,5 +587,5 @@ filegroup(
  "@remotejdk%s_%s//:WORKSPACE" % (version, os)
  for version in ("17", "20")
  for os in ("macos", "macos_aarch64", "linux", "win")
- ],
+ ] + ["@bazel_tools_repo_cache//:files"],
 )

src/test/py/bazel/BUILD

@@ -27,6 +27,7 @@ py_library(
  data = [
  ":default_repos_stanza.txt",
  ":test-deps",
+ "//src/test/tools/bzlmod:MODULE.bazel.lock",
  ],
  visibility = [
  "//src/test/py/bazel:__pkg__",

src/test/py/bazel/bazel_windows_cpp_test.py

@@ -765,6 +765,7 @@ def testBuildWithClangClByCompilerFlag(self):
  self.assertIn('clang-cl.exe', ''.join(stderr))
 
  def testBuildWithClangClByToolchainResolution(self):
+ self.DisableBzlmod()
  self.CreateWorkspaceWithDefaultRepos('WORKSPACE', [
  'register_execution_platforms(',
  ' ":windows_clang"',

src/test/py/bazel/bazel_workspace_test.py

@@ -20,6 +20,10 @@
 
 class BazelWorkspaceTest(test_base.TestBase):
 
+ def setUp(self):
+ test_base.TestBase.setUp(self)
+ self.DisableBzlmod()
+
  def testWorkspaceDotBazelFileInMainRepo(self):
  workspace_dot_bazel = self.ScratchFile("WORKSPACE.bazel")
  self.ScratchFile("BUILD", [

src/test/py/bazel/bzlmod/bazel_lockfile_test.py

@@ -60,6 +60,9 @@ def setUp(self):
  # The existence of WORKSPACE.bzlmod prevents WORKSPACE prefixes or suffixes
  # from being used; this allows us to test built-in modules actually work
  self.ScratchFile('WORKSPACE.bzlmod')
+ # TODO(pcloudy): investigate why this is needed, MODULE.bazel.lock is not
+ # deterministic?
+ os.remove(self.Path('MODULE.bazel.lock'))
 
  def testChangeModuleInRegistryWithoutLockfile(self):
  # Add module 'sss' to the registry with dep on 'aaa'

src/test/py/bazel/launcher_test.py

@@ -37,13 +37,17 @@ def _buildJavaTargets(self, bazel_bin, binary_suffix):
  if self.IsWindows():
  self.assertTrue(os.path.isfile(main_binary))
  self.AssertRunfilesManifestContains(
- os.path.join(bazel_bin,
- 'foo/foo%s.runfiles/MANIFEST' % binary_suffix),
- '__main__/bar/bar.txt')
+ os.path.join(
+ bazel_bin, 'foo/foo%s.runfiles/MANIFEST' % binary_suffix
+ ),
+ '_main/bar/bar.txt',
+ )
  else:
  self.assertTrue(
  os.path.islink(
- os.path.join(bazel_bin, 'foo/foo.runfiles/__main__/bar/bar.txt')))
+ os.path.join(bazel_bin, 'foo/foo.runfiles/_main/bar/bar.txt')
+ )
+ )
 
  _, stdout, _ = self.RunProgram([main_binary])
  self.assertEqual(len(stdout), 4)
@@ -99,25 +103,31 @@ def _buildShBinaryTargets(self, bazel_bin, bin1_suffix):
 
  if self.IsWindows():
  self.AssertRunfilesManifestContains(
- os.path.join(bazel_bin,
- 'foo/bin1.sh%s.runfiles/MANIFEST' % bin1_suffix),
- '__main__/bar/bar.txt')
+ os.path.join(
+ bazel_bin, 'foo/bin1.sh%s.runfiles/MANIFEST' % bin1_suffix
+ ),
+ '_main/bar/bar.txt',
+ )
  self.AssertRunfilesManifestContains(
  os.path.join(bazel_bin, 'foo/bin2.cmd.runfiles/MANIFEST'),
- '__main__/bar/bar.txt')
+ '_main/bar/bar.txt',
+ )
  else:
  self.assertTrue(
  os.path.islink(
- os.path.join(bazel_bin,
- 'foo/bin1.sh.runfiles/__main__/bar/bar.txt')))
+ os.path.join(bazel_bin, 'foo/bin1.sh.runfiles/_main/bar/bar.txt')
+ )
+ )
  self.assertTrue(
  os.path.islink(
- os.path.join(bazel_bin,
- 'foo/bin2.cmd.runfiles/__main__/bar/bar.txt')))
+ os.path.join(bazel_bin, 'foo/bin2.cmd.runfiles/_main/bar/bar.txt')
+ )
+ )
  self.assertTrue(
  os.path.islink(
- os.path.join(bazel_bin,
- 'foo/bin3.bat.runfiles/__main__/bar/bar.txt')))
+ os.path.join(bazel_bin, 'foo/bin3.bat.runfiles/_main/bar/bar.txt')
+ )
+ )
 
  _, stdout, _ = self.RunProgram([bin1])
  self.assertEqual(len(stdout), 3)
@@ -155,13 +165,17 @@ def _buildPyTargets(self, bazel_bin, binary_suffix):
  # Verify contents of runfiles (manifest).
  if self.IsWindows():
  self.AssertRunfilesManifestContains(
- os.path.join(bazel_bin,
- 'foo/foo%s.runfiles/MANIFEST' % binary_suffix),
- '__main__/bar/bar.txt')
+ os.path.join(
+ bazel_bin, 'foo/foo%s.runfiles/MANIFEST' % binary_suffix
+ ),
+ '_main/bar/bar.txt',
+ )
  else:
  self.assertTrue(
  os.path.islink(
- os.path.join(bazel_bin, 'foo/foo.runfiles/__main__/bar/bar.txt')))
+ os.path.join(bazel_bin, 'foo/foo.runfiles/_main/bar/bar.txt')
+ )
+ )
 
  # Try to run the built py_binary.
  _, stdout, _ = self.RunProgram([foo_bin])

src/test/py/bazel/py_test.py

@@ -58,10 +58,12 @@ def testRunfilesSymlinks(self):
  self.createSimpleFiles()
  self.RunBazel(['build', '//a:a'])
  self.assertTrue(os.path.isdir('bazel-bin/a/a.runfiles'))
- self.assertTrue(os.readlink('bazel-bin/a/a.runfiles/__main__/a/a.py')
- .endswith('/a/a.py'))
- self.assertTrue(os.readlink('bazel-bin/a/a.runfiles/__main__/a/b.py')
- .endswith('/a/b.py'))
+ self.assertTrue(
+ os.readlink('bazel-bin/a/a.runfiles/_main/a/a.py').endswith('/a/a.py')
+ )
+ self.assertTrue(
+ os.readlink('bazel-bin/a/a.runfiles/_main/a/b.py').endswith('/a/b.py')
+ )
 
 
 class TestInitPyFiles(test_base.TestBase):
@@ -93,22 +95,25 @@ def testInitPyFilesCreated(self):
  self.assertTrue(os.path.exists('bazel-bin/src/a/a.zip'))
  with zipfile.ZipFile('bazel-bin/src/a/a.zip', 'r') as z:
  zip_contents = set(z.namelist())
- self.assertIn('runfiles/__main__/src/__init__.py', zip_contents)
- self.assertIn('runfiles/__main__/src/a/__init__.py', zip_contents)
+ self.assertIn('runfiles/_main/src/__init__.py', zip_contents)
+ self.assertIn('runfiles/_main/src/a/__init__.py', zip_contents)
  else:
  self.assertTrue(
- os.path.exists('bazel-bin/src/a/a.runfiles/__main__/src/__init__.py'))
+ os.path.exists('bazel-bin/src/a/a.runfiles/_main/src/__init__.py')
+ )
  self.assertTrue(
- os.path.exists(
-  'bazel-bin/src/a/a.runfiles/__main__/src/a/__init__.py'))
+ os.path.exists('bazel-bin/src/a/a.runfiles/_main/src/a/__init__.py')
+ )
 
  def testInitPyFilesNotCreatedWhenLegacyCreateInitIsSet(self):
  self.createSimpleFiles(create_init=False)
  self.RunBazel(['build', '//src/a:a'])
  self.assertFalse(
- os.path.exists('bazel-bin/src/a/a.runfiles/__main__/src/__init__.py'))
+ os.path.exists('bazel-bin/src/a/a.runfiles/_main/src/__init__.py')
+ )
  self.assertFalse(
- os.path.exists('bazel-bin/src/a/a.runfiles/__main__/src/a/__init__.py'))
+ os.path.exists('bazel-bin/src/a/a.runfiles/_main/src/a/__init__.py')
+ )
 
  # Regression test for https://github.com/bazelbuild/bazel/pull/10119
  def testBuildingZipFileWithTargetNameWithDot(self):

src/test/py/bazel/runfiles_sandboxed_test.py

@@ -105,7 +105,8 @@ def testRunfilesLibrariesFindRunfilesWithoutEnvvars(self):
  if not stdout_lines[i * 2 + 1].startswith("rloc="):
  self._FailWithContents("wrong line for " + lang[1], stdout_lines)
  if not stdout_lines[i * 2 + 1].endswith(
- "foo_ws/bar/bar-%s-data.txt" % lang[0]):
+ "_main/bar/bar-%s-data.txt" % lang[0]
+ ):
  self._FailWithContents("wrong line for " + lang[1], stdout_lines)
 
  # Assert the contents of bar-<language>-data.txt. This indicates that

src/test/py/bazel/runfiles_test.py

@@ -206,7 +206,7 @@ def testRunfilesLibrariesFindRunfilesWithRunfilesManifestEnvvar(self):
  # Runfiles manifests use forward slashes as path separators, even on
  # Windows.
  mock_bar_dep = mock_bar_dep.replace("\\", "/")
- manifest_key = "foo_ws/bar/bar-%s-data.txt" % lang[0]
+ manifest_key = "_main/bar/bar-%s-data.txt" % lang[0]
  mock_manifest_line = manifest_key + " " + mock_bar_dep
  with open(manifest_path, "rt") as f:
  # Only rstrip newlines. Do not rstrip() completely, because that would
@@ -258,6 +258,7 @@ def testRunfilesLibrariesFindRunfilesWithRunfilesManifestEnvvar(self):
  self.assertEqual(lines[0], "mock %s data" % lang[0])
 
  def testLegacyExternalRunfilesOption(self):
+ self.DisableBzlmod()
  self.ScratchDir("A")
  self.ScratchFile("A/WORKSPACE")
  self.ScratchFile("A/BUILD", [

src/test/py/bazel/test_base.py

@@ -133,8 +133,17 @@ def setUp(self):
  if TestBase.IsDarwin():
  # For reducing SSD usage on our physical Mac machines.
  f.write('common --experimental_repository_cache_hardlinks\n')
+ f.write('common --enable_bzlmod\n')
+ self.CopyFile(
+ self.Rlocation('io_bazel/src/test/tools/bzlmod/MODULE.bazel.lock'),
+ 'MODULE.bazel.lock',
+ )
  os.chdir(self._test_cwd)
 
+ def DisableBzlmod(self):
+ with open(self._test_bazelrc, 'at') as f:
+ f.write('common --noenable_bzlmod\n')
+
  def tearDown(self):
  self.RunBazel(['shutdown'])
  super(TestBase, self).tearDown()