Extract dev_deps_repositories() from WORKSPACE and remove //private (#1656)

* Extract dev_deps_repositories() from WORKSPACE

`dev_deps_repositories()` encapsulates the instantiation of repositories
used only for `rules_scala` development. Also removes the unused
`//private` package and its `WORKSPACE` statements. Part of #1482.

Both `WORKSPACE` and Bzlmod builds can use this macro, though how Bzlmod
will use it will depend on whether we continue building with Bazel 6.

`@bazel_tools//tools/build_defs/repo:local.bzl` isn't available under
Bazel 6. To continue building with Bazel 6 under Bzlmod, we will need to
call `dev_deps_repositories()` from `WORKSPACE.bzlmod` to continue using
`native.{,new_}local_repository()`.

If we switch to Bazel 7, we can load `local.bzl` and strip the `native.`
prefix from the `{,new_}local_repository()` calls. Then we can call
`dev_deps_repositories()` from a module extension instead of from
`WORKSPACE.bzlmod`.

Another alternative would be updating the local repositories to become
proper nested repositories. Then we can call `local_repository()` from
`WORKSPACE` and call `bazel_dep()` and `local_path_override()` from
`MODULE.bazel`. In that case, we'd remove the `{,new_}local_repository`
calls from `dev_deps_dependencies()`, and remove
`proto_cross_repo_boundary_repository()` entirely.

* Add test local_repository calls to WORKSPACE

Removes `native.{,new_}local_repository()` calls in macros in favor of
`local_repository` calls from `WORKSPACE`. Part of #1482.

`native.{,new_}local_repository()` isn't available under Bzlmod,
`@bazel_tools//tools/build_defs/repo:local.bzl` with the Starlarkified
definitions isn't available under Bazel 6, and Bazel 8 compatibility
work is imminent. Redefining the repositories in this way will be
compatible with Bazel 6, 7, and 8, both under `WORKSPACE` and Bzlmod.
(`MODULE.bazel` will use a combination of `bazel_dep()` and
`local_path_override()`.)

Author: mbland

WORKSPACE

@@ -26,15 +26,6 @@ load("@rules_python//python:repositories.bzl", "py_repositories")
 
 py_repositories()
 
-_build_tools_release = "5.1.0"
-
-http_archive(
-    name = "com_github_bazelbuild_buildtools",
-    sha256 = "e3bb0dc8b0274ea1aca75f1f8c0c835adbe589708ea89bf698069d0790701ea3",
-    strip_prefix = "buildtools-%s" % _build_tools_release,
-    url = "https://github.com/bazelbuild/buildtools/archive/%s.tar.gz" % _build_tools_release,
-)
-
 load("@io_bazel_rules_scala//:scala_config.bzl", "scala_config")
 
 scala_config(enable_compiler_dependency_tracking = True)
@@ -66,7 +57,6 @@ load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 
 protobuf_deps()
 
-load("//scala:scala_cross_version.bzl", "default_maven_server_urls")
 load("//twitter_scrooge:twitter_scrooge.bzl", "twitter_scrooge")
 
 twitter_scrooge()
@@ -93,23 +83,14 @@ scalafmt_default_config()
 
 scalafmt_repositories()
 
-MAVEN_SERVER_URLS = default_maven_server_urls()
-
 # needed for the cross repo proto test
-load("//test/proto_cross_repo_boundary:repo.bzl", "proto_cross_repo_boundary_repository")
-
-proto_cross_repo_boundary_repository()
+local_repository(
+    name = "proto_cross_repo_boundary",
+    path = "test/proto_cross_repo_boundary/repo",
+)
 
-new_local_repository(
+local_repository(
     name = "test_new_local_repo",
-    build_file_content =
-        """
-filegroup(
-    name = "data",
-    srcs = glob(["**/*.txt"]),
-    visibility = ["//visibility:public"],
-)
-""",
     path = "third_party/test/new_local_repo",
 )
 
@@ -118,28 +99,6 @@ local_repository(
     path = "third_party/test/example_external_workspace",
 )
 
-load("//scala:scala_maven_import_external.bzl", "java_import_external")
-
-# bazel's java_import_external has been altered in rules_scala to be a macro based on jvm_import_external
-# in order to allow for other jvm-language imports (e.g. scala_import)
-# the 3rd-party dependency below is using the java_import_external macro
-# in order to make sure no regression with the original java_import_external
-java_import_external(
-    name = "org_apache_commons_commons_lang_3_5_without_file",
-    generated_linkable_rule_name = "linkable_org_apache_commons_commons_lang_3_5_without_file",
-    jar_sha256 = "8ac96fc686512d777fca85e144f196cd7cfe0c0aec23127229497d1a38ff651c",
-    jar_urls = ["https://repo.maven.apache.org/maven2/org/apache/commons/commons-lang3/3.5/commons-lang3-3.5.jar"],
-    licenses = ["notice"],  # Apache 2.0
-    neverlink = True,
-    testonly_ = True,
-)
-
-## Linting
-
-load("//private:format.bzl", "format_repositories")
-
-format_repositories()
-
 http_archive(
     name = "io_bazel_rules_go",
     sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996",
@@ -159,12 +118,6 @@ go_rules_dependencies()
 
 go_register_toolchains(version = "1.19.5")
 
-load("@rules_java//java:repositories.bzl", "remote_jdk8_repos")
-
-# We need to select based on platform when we use these
-# https://github.com/bazelbuild/bazel/issues/11655
-remote_jdk8_repos()
-
 http_archive(
     name = "bazelci_rules",
     sha256 = "eca21884e6f66a88c358e580fd67a6b148d30ab57b1680f62a96c00f9bc6a07e",
@@ -179,35 +132,20 @@ rbe_preconfig(
     toolchain = "ubuntu2004-bazel-java11",
 )
 
-load("//third_party/repositories:repositories.bzl", "repositories")
-
-repositories(
-    fetch_sources = False,
-    for_artifact_ids = [
-        # test adding a scala jar:
-        "com_twitter__scalding_date",
-        # test of strict deps (scalac plugin UT + E2E)
-        "com_google_guava_guava_21_0_with_file",
-        "com_github_jnr_jffi_native",
-        "org_apache_commons_commons_lang_3_5",
-        "com_google_guava_guava_21_0",
-        # test of import external
-        # scala maven import external decodes maven artifacts to its parts
-        # (group id, artifact id, packaging, version and classifier). To make sure
-        # the decoding and then the download url composition are working the artifact example
-        # must contain all the different parts and sha256s so the downloaded content will be
-        # validated against it
-        "org_springframework_spring_core",
-        "org_springframework_spring_tx",
-        "org_typelevel_kind_projector",
-        # For testing that we don't include sources jars to the classpath
-        "org_typelevel__cats_core",
-    ],
-    maven_servers = MAVEN_SERVER_URLS,
-)
+load("//scala/private/extensions:dev_deps.bzl", "dev_deps_repositories")
 
-load("//test/toolchains:jdk.bzl", "remote_jdk21_repositories", "remote_jdk21_toolchains")
+dev_deps_repositories()
 
-remote_jdk21_repositories()
+# Copied from bazel_tools/tools/jdk/remote_java_repository.bzl.
+[
+    register_toolchains(
+        "@remotejdk21_" + platform + "_toolchain_config_repo//:all",
+    )
+    for platform in [
+        "linux",
+        "macos",
+        "win",
+    ]
+]
 
-remote_jdk21_toolchains()
+register_toolchains("//test/toolchains:java21_toolchain_definition")

private/BUILD

@@ -0,0 +1,77 @@
+"""Repositories for testing rules_scala itself"""
+
+load("//scala:scala_cross_version.bzl", "default_maven_server_urls")
+load("//scala:scala_maven_import_external.bzl", "java_import_external")
+load("//test/toolchains:jdk.bzl", "remote_jdk21_repositories")
+load("//third_party/repositories:repositories.bzl", "repositories")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@rules_java//java:repositories.bzl", "remote_jdk8_repos")
+
+_BUILD_TOOLS_RELEASE = "5.1.0"
+
+def dev_deps_repositories(
+        name = "unused_dev_deps_name",
+        maven_servers = default_maven_server_urls(),
+        fetch_sources = False):
+    """Instantiates internal only repos for development and testing
+
+    Args:
+        name: unused macro parameter to satisfy Buildifier lint rules
+        maven_servers: servers to use when resolving Maven artifacts
+        fetch_sources: retrieve Maven artifact sources when True
+    """
+    http_archive(
+        name = "com_github_bazelbuild_buildtools",
+        sha256 = "e3bb0dc8b0274ea1aca75f1f8c0c835adbe589708ea89bf698069d0790701ea3",
+        strip_prefix = "buildtools-%s" % _BUILD_TOOLS_RELEASE,
+        url = (
+            "https://github.com/bazelbuild/buildtools/archive/%s.tar.gz" %
+            _BUILD_TOOLS_RELEASE
+        ),
+    )
+
+    # bazel's java_import_external has been altered in rules_scala to be a macro
+    # based on jvm_import_external in order to allow for other jvm-language
+    # imports (e.g. scala_import) the 3rd-party dependency below is using the
+    # java_import_external macro in order to make sure no regression with the
+    # original java_import_external
+    java_import_external(
+        name = "org_apache_commons_commons_lang_3_5_without_file",
+        generated_linkable_rule_name = "linkable_org_apache_commons_commons_lang_3_5_without_file",
+        jar_sha256 = "8ac96fc686512d777fca85e144f196cd7cfe0c0aec23127229497d1a38ff651c",
+        jar_urls = ["https://repo.maven.apache.org/maven2/org/apache/commons/commons-lang3/3.5/commons-lang3-3.5.jar"],
+        licenses = ["notice"],  # Apache 2.0
+        neverlink = True,
+        testonly_ = True,
+    )
+
+    # We need to select based on platform when we use these
+    # https://github.com/bazelbuild/bazel/issues/11655
+    remote_jdk8_repos()
+
+    repositories(
+        fetch_sources = fetch_sources,
+        for_artifact_ids = [
+            # test adding a scala jar:
+            "com_twitter__scalding_date",
+            # test of strict deps (scalac plugin UT + E2E)
+            "com_google_guava_guava_21_0_with_file",
+            "com_github_jnr_jffi_native",
+            "org_apache_commons_commons_lang_3_5",
+            "com_google_guava_guava_21_0",
+            # test of import external
+            # scala maven import external decodes maven artifacts to its parts
+            # (group id, artifact id, packaging, version and classifier). To make sure
+            # the decoding and then the download url composition are working the artifact example
+            # must contain all the different parts and sha256s so the downloaded content will be
+            # validated against it
+            "org_springframework_spring_core",
+            "org_springframework_spring_tx",
+            "org_typelevel_kind_projector",
+            # For testing that we don't include sources jars to the classpath
+            "org_typelevel__cats_core",
+        ],
+        maven_servers = maven_servers,
+    )
+
+    remote_jdk21_repositories()

private/format.bzl

@@ -0,0 +1 @@
+6.5.0

scala/private/extensions/dev_deps.bzl

@@ -0,0 +1 @@
+6.5.0

test/proto_cross_repo_boundary/repo.bzl

@@ -0,0 +1,5 @@
+filegroup(
+    name = "data",
+    srcs = glob(["**/*.txt"]),
+    visibility = ["//visibility:public"],
+)