Increase Pants' python recursion limit by default, and allow it to be…

… overridden. (cherrypick of #11271) (#11274)

### Problem

Pants uses a default Python recursion limit that makes some build graph shapes fail to cycle detect: see #11201.

### Solution

Increase the default recursion limit, and allow it to be overridden via an environment variable.

Although increasing the recursion limit is not always the best approach to solving a problem, having the capability to do so is frequently useful as a workaround. We also know that the default value is too low for otherwise reasonable graph shapes.

### Result

Fixes #11201.

[ci skip-rust]
[ci skip-build-wheels]

src/python/pants/bin/loader_integration_test.py

@@ -2,7 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.bin.pants_loader import PantsLoader
-from pants.testutil.pants_integration_test import run_pants
+from pants.testutil.pants_integration_test import PantsResult, run_pants
 
 
 def test_invalid_locale() -> None:
@@ -49,3 +49,17 @@ def test_alternate_entrypoint_scrubbing() -> None:
     )
     pants_run.assert_success()
     assert "PANTS_ENTRYPOINT=None" in pants_run.stdout
+
+
+def test_recursion_limit() -> None:
+    def run(limit: str) -> PantsResult:
+        return run_pants(command=["help"], extra_env={"PANTS_RECURSION_LIMIT": limit})
+
+    # Large value succeeds.
+    run("100000").assert_success()
+    # Very small value fails in an arbitrary spot.
+    small_run = run("1")
+    small_run.assert_failure()
+    assert "RecursionError" in small_run.stderr
+    # Non integer value fails.
+    run("this isn't an int").assert_failure()

src/python/pants/bin/pants_loader.py

@@ -4,6 +4,7 @@
 import importlib
 import locale
 import os
+import sys
 import warnings
 from textwrap import dedent
 
@@ -14,6 +15,8 @@ class PantsLoader:
     ENTRYPOINT_ENV_VAR = "PANTS_ENTRYPOINT"
     DEFAULT_ENTRYPOINT = "pants.bin.pants_exe:main"
 
+    RECURSION_LIMIT_ENV_VAR = "PANTS_RECURSION_LIMIT"
+
     ENCODING_IGNORE_ENV_VAR = "PANTS_IGNORE_UNRECOGNIZED_ENCODING"
 
     class InvalidLocaleError(Exception):
@@ -67,6 +70,10 @@ def ensure_locale(cls):
                 )
             )
 
+    @classmethod
+    def set_recursion_limit(cls):
+        sys.setrecursionlimit(int(os.environ.get(cls.RECURSION_LIMIT_ENV_VAR, "10000")))
+
     @staticmethod
     def determine_entrypoint(env_var, default):
         return os.environ.pop(env_var, default)
@@ -86,6 +93,7 @@ def load_and_execute(entrypoint):
     def run(cls):
         cls.setup_warnings()
         cls.ensure_locale()
+        cls.set_recursion_limit()
         entrypoint = cls.determine_entrypoint(cls.ENTRYPOINT_ENV_VAR, cls.DEFAULT_ENTRYPOINT)
         cls.load_and_execute(entrypoint)