Use interpreter constraints to verify lockfile environment, rather than invalidation inputs

3rdparty/python/lockfiles/coverage_py.txt

@@ -3,7 +3,12 @@
 #    build-support/bin/generate_all_lockfiles.sh
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
-# invalidation digest: 16ff4a564929a4be3d3d447dff1792ed4f6c625afc4cbc71cc8d303fc7f2a4c0
+# {
+#   "requirements_invalidation_digest": "1f5ba0bbee50a82d9bebf91c0e77c9eb5f4d3a0138d727ce706fcca01de7195d",
+#   "valid_for_interpreter_constraints": [
+#     "CPython<3.10,>=3.7"
+#   ]
+# }
 # --- END PANTS LOCKFILE METADATA ---
 
 coverage==5.0.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4") \

3rdparty/python/lockfiles/flake8.txt

@@ -3,7 +3,12 @@
 #    build-support/bin/generate_all_lockfiles.sh
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
-# invalidation digest: 443653bdbc8facef9b521f860e1c11e48d670630150c948dbd5ae444d0796c19
+# {
+#   "requirements_invalidation_digest": "a6b4aeeac96728225dd3399bff5eb72a582aaa53cea7b63505fcb3cecdef5723",
+#   "valid_for_interpreter_constraints": [
+#     "CPython<3.10,>=3.7"
+#   ]
+# }
 # --- END PANTS LOCKFILE METADATA ---
 
 flake8-2020==1.6.0; python_full_version >= "3.6.1" \
@@ -15,9 +20,9 @@ flake8-pantsbuild==2.0.0; python_version >= "3.6" \
 flake8==3.9.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") \
     --hash=sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907 \
     --hash=sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b
-importlib-metadata==4.6.3; python_full_version >= "3.6.1" and python_version < "3.8" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.5.0" and python_version < "3.8" and python_version >= "3.6") \
-    --hash=sha256:51c6635429c77cf1ae634c997ff9e53ca3438b495f10a55ba28594dd69764a8b \
-    --hash=sha256:0645585859e9a6689c523927a5032f2ba5919f1f7d0e84bd4533312320de1ff9
+importlib-metadata==4.6.4; python_full_version >= "3.6.1" and python_version < "3.8" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.5.0" and python_version < "3.8" and python_version >= "3.6") \
+    --hash=sha256:ed5157fef23a4bc4594615a0dd8eba94b2bb36bf2a343fa3d8bb2fa0a62a99d5 \
+    --hash=sha256:7b30a78db2922d78a6f47fb30683156a14f3c6aa5cc23f77cc8967e9ab2d002f
 mccabe==0.6.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" \
     --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
     --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f

3rdparty/python/lockfiles/pytest.txt

@@ -3,7 +3,12 @@
 #    build-support/bin/generate_all_lockfiles.sh
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
-# invalidation digest: 1ee11fe335a8425c80f54cb5879e7feb0e81c4de710883ed698e88f368f54427
+# {
+#   "requirements_invalidation_digest": "0768084ec1d13f0a8c5514291b5cffd79d8304332aa2c895369f858e3839330b",
+#   "valid_for_interpreter_constraints": [
+#     "CPython<3.10,>=3.7"
+#   ]
+# }
 # --- END PANTS LOCKFILE METADATA ---
 
 appnope==0.1.2; sys_platform == "darwin" and python_version >= "3.7" \
@@ -79,9 +84,9 @@ decorator==5.0.9; python_version >= "3.7" \
     --hash=sha256:72ecfba4320a893c53f9706bebb2d55c270c1e51a28789361aa93e4a21319ed5
 icdiff==2.0.4; python_version >= "3.6" \
     --hash=sha256:c72572e5ce087bc7a7748af2664764d4a805897caeefb665bdc12677fefb2212
-importlib-metadata==4.6.3; python_version < "3.8" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.4.0" and python_version >= "3.6" and python_version < "3.8") and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") \
-    --hash=sha256:51c6635429c77cf1ae634c997ff9e53ca3438b495f10a55ba28594dd69764a8b \
-    --hash=sha256:0645585859e9a6689c523927a5032f2ba5919f1f7d0e84bd4533312320de1ff9
+importlib-metadata==4.6.4; python_version < "3.8" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.4.0" and python_version >= "3.6" and python_version < "3.8") and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") \
+    --hash=sha256:ed5157fef23a4bc4594615a0dd8eba94b2bb36bf2a343fa3d8bb2fa0a62a99d5 \
+    --hash=sha256:7b30a78db2922d78a6f47fb30683156a14f3c6aa5cc23f77cc8967e9ab2d002f
 iniconfig==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" \
     --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \
     --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32
@@ -126,9 +131,9 @@ ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.7" \
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" \
     --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a \
     --hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3
-pygments==2.9.0; python_version >= "3.5" \
-    --hash=sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e \
-    --hash=sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f
+pygments==2.10.0; python_version >= "3.5" \
+    --hash=sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380 \
+    --hash=sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6
 pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
     --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b \
     --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1

3rdparty/python/lockfiles/setuptools.txt

@@ -3,7 +3,12 @@
 #    build-support/bin/generate_all_lockfiles.sh
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
-# invalidation digest: daab248d3d4a91fafc4a04497ef6c6d6402c41d74fba2ee72a48d938fb81702f
+# {
+#   "requirements_invalidation_digest": "a5ec2e69360b67f3262d8ecc191c0227f09216343e97e8c9e2a34ce567b07c29",
+#   "valid_for_interpreter_constraints": [
+#     "CPython<3.10,>=3.7"
+#   ]
+# }
 # --- END PANTS LOCKFILE METADATA ---
 
 setuptools==57.4.0; python_version >= "3.6" \

3rdparty/python/lockfiles/user_reqs.txt

@@ -3,7 +3,12 @@
 #    build-support/bin/generate_all_lockfiles.sh
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
-# invalidation digest: d81e918117ed40ef2bd5e63c13a4f116211891f7b38f47d065ef0b55425f0de0
+# {
+#   "requirements_invalidation_digest": "76ad65c62c8e6ded78129ae57a934c85cdf562cbe8a8746cf8502d0b07a7e25b",
+#   "valid_for_interpreter_constraints": [
+#     "CPython<3.10,>=3.7"
+#   ]
+# }
 # --- END PANTS LOCKFILE METADATA ---
 
 ansicolors==1.1.8 \

src/python/pants/backend/awslambda/python/lambdex_lockfile.txt

@@ -3,7 +3,12 @@
 #    build-support/bin/generate_all_lockfiles.sh
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
-# invalidation digest: c184f0b6fadf6b42296e03edff13e1942c853da95c6b1547930c19fa62561c08
+# {
+#   "requirements_invalidation_digest": "0593e2381cc20c92398d2d4026456418c6f79d24f3b4b9fa77bafca662f38ba3",
+#   "valid_for_interpreter_constraints": [
+#     "CPython<3.10,>=3.6"
+#   ]
+# }
 # --- END PANTS LOCKFILE METADATA ---
 
 lambdex==0.1.5; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0" and python_version < "3.10") \

src/python/pants/backend/awslambda/python/rules.py

@@ -90,7 +90,7 @@ async def package_python_awslambda(
     lockfile_hex_digest = None
     if lambdex.lockfile != "<none>":
         lockfile_request = await Get(PythonLockfileRequest, LambdexLockfileSentinel())
-        lockfile_hex_digest = lockfile_request.hex_digest
+        lockfile_hex_digest = lockfile_request.requirements_hex_digest
 
     lambdex_request = PexRequest(
         output_filename="lambdex.pex",

src/python/pants/backend/codegen/protobuf/python/mypy_protobuf_lockfile.txt

@@ -3,7 +3,12 @@
 #    build-support/bin/generate_all_lockfiles.sh
 #
 # --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
-# invalidation digest: 51261468d53ac35dacae084fd4d365bb3de09da9f8cdec3e0a022e3d261b929b
+# {
+#   "requirements_invalidation_digest": "babed61947e74aedbe0cdbaefdaec172db6d4a9d27e12acc80be5ab623e3acdf",
+#   "valid_for_interpreter_constraints": [
+#     "CPython>=3.6"
+#   ]
+# }
 # --- END PANTS LOCKFILE METADATA ---
 
 mypy-protobuf==2.4; python_version >= "3.6" \

src/python/pants/backend/codegen/protobuf/python/rules.py

@@ -100,7 +100,7 @@ async def generate_python_from_protobuf(
     lockfile_hex_digest = None
     if python_protobuf_mypy_plugin.lockfile != "<none>":
         lockfile_request = await Get(PythonLockfileRequest, MypyProtobufLockfileSentinel())
-        lockfile_hex_digest = lockfile_request.hex_digest
+        lockfile_hex_digest = lockfile_request.requirements_hex_digest
 
     protoc_gen_mypy_script = "protoc-gen-mypy"
     protoc_gen_mypy_grpc_script = "protoc-gen-mypy_grpc"

src/python/pants/backend/experimental/python/lockfile.py

@@ -112,13 +112,9 @@ def from_tool(
         )
 
     @property
-    def hex_digest(self) -> str:
-        """Produces a hex digest of this lockfile's inputs, which should uniquely specify the
-        resolution of this lockfile request.
-
-        Inputs are definted as requirements and interpreter constraints.
-        """
-        return calculate_invalidation_digest(self.requirements, self.interpreter_constraints)
+    def requirements_hex_digest(self) -> str:
+        """Produces a hex digest of the requirements input for this lockfile."""
+        return calculate_invalidation_digest(self.requirements)
 
 
 @rule(desc="Generate lockfile", level=LogLevel.DEBUG)
@@ -170,7 +166,8 @@ async def generate_lockfile(
     lockfile_digest_contents = await Get(DigestContents, Digest, poetry_export_result.output_digest)
     lockfile_with_header = lockfile_content_with_header(
         python_setup.lockfile_custom_regeneration_command or req.regenerate_command,
-        req.hex_digest,
+        req.requirements_hex_digest,
+        req.interpreter_constraints,
         lockfile_digest_contents[0].content,
     )
     final_lockfile = await Get(Digest, CreateDigest([FileContent(req.dest, lockfile_with_header)]))

src/python/pants/backend/experimental/python/lockfile_metadata.py

@@ -6,6 +6,7 @@
 import hashlib
 import json
 from dataclasses import dataclass
+from typing import Any, Callable, Iterable, TypeVar
 
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.util.ordered_set import FrozenOrderedSet
@@ -16,52 +17,129 @@
 
 @dataclass
 class LockfileMetadata:
-    invalidation_digest: str | None
+    requirements_invalidation_digest: str | None
+    valid_for_interpreter_constraints: InterpreterConstraints | None
+
+    @staticmethod
+    def from_json_bytes(json_literal: str | bytes) -> LockfileMetadata:
+        """Reads a `string`/`bytes` that has JSON-encoded a `LockfileMetadata` object, returning
+        that deserialized object."""
+        try:
+            metadata = json.loads(json_literal)
+        except json.decoder.JSONDecodeError:
+            # If this block is invalid, this should trigger error/warning behavior
+            metadata = {}
+
+        T = TypeVar("T")
+
+        def coerce(t: Callable[[Any], T], key: str) -> T | None:
+            """Gets a value from `metadata`, coercing it to type `t` if not `None`."""
+            v = metadata.get(key, None)
+            try:
+                return t(v) if v is not None else None
+            except Exception:
+                # TODO: this should trigger error/warning behavior
+                return None
+
+        return LockfileMetadata(
+            requirements_invalidation_digest=coerce(str, "requirements_invalidation_digest"),
+            valid_for_interpreter_constraints=coerce(
+                InterpreterConstraints, "valid_for_interpreter_constraints"
+            ),
+        )
+
+    def to_json_literal(self) -> str:
+        """Produces a JSON-encoded dictionary that represents the contents of this metadata."""
+        constraints = self.valid_for_interpreter_constraints
+        metadata = {
+            "requirements_invalidation_digest": self.requirements_invalidation_digest,
+            "valid_for_interpreter_constraints": [str(i) for i in constraints]
+            if constraints
+            else None,
+        }
+        return json.dumps(
+            metadata,
+            ensure_ascii=True,
+            indent=2,
+        )
+
+    def is_valid_for(
+        self,
+        expected_invalidation_digest: str | None,
+        user_interpreter_constraints: InterpreterConstraints,
+        interpreter_universe: Iterable[str],
+    ) -> bool:
+        """Returns True if this `LockfileMetadata` represents a lockfile that can be used in the
+        current execution context.
+
+        A lockfile can be used in the current execution context if `expected_invalidation_digest ==
+        requirements_invalidation_digest`, and if `user_interpreter_constraints` matches only
+        interpreters specified by `valid_for_interpreter_constraints`.
+        """
+
+        if expected_invalidation_digest is None:
+            return True
+
+        if self.requirements_invalidation_digest != expected_invalidation_digest:
+            return False
+
+        if self.valid_for_interpreter_constraints is None:
+            # This lockfile matches all interpreter constraints (TODO: check this)
+            return True
+
+        return self.valid_for_interpreter_constraints.contains(
+            user_interpreter_constraints, interpreter_universe
+        )
 
 
 def calculate_invalidation_digest(
-    requirements: FrozenOrderedSet[str], interpreter_constraints: InterpreterConstraints
+    requirements: FrozenOrderedSet[str],
 ) -> str:
-    """Returns an invalidation digest for the given requirements and interpreter constraints."""
+    """Returns an invalidation digest for the given requirements."""
     m = hashlib.sha256()
-    pres = {
+    inputs = {
         "requirements": list(requirements),
-        "interpreter_constraints": [str(i) for i in interpreter_constraints],
     }
-    m.update(json.dumps(pres).encode("utf-8"))
+    m.update(json.dumps(inputs).encode("utf-8"))
     return m.hexdigest()
 
 
 def lockfile_content_with_header(
-    regenerate_command: str, invalidation_digest: str, content: bytes
+    regenerate_command: str,
+    invalidation_digest: str,
+    interpreter_constraints: InterpreterConstraints,
+    content: bytes,
 ) -> bytes:
-    """Returns a version of the lockfile with Pants metadata prepended."""
+    """Returns a version of the lockfile with Pants metadata and usage instructions prepended."""
     regenerate_command = (
         f"# This lockfile was autogenerated by Pants. To regenerate, run:\n#\n"
         f"#    {regenerate_command}"
     )
     return b"%b\n#\n%b\n\n%b" % (
         regenerate_command.encode("utf-8"),
-        lockfile_metadata_header(invalidation_digest),
+        lockfile_metadata_header(LockfileMetadata(invalidation_digest, interpreter_constraints)),
         content,
     )
 
 
-def lockfile_metadata_header(invalidation_digest: str) -> bytes:
+def lockfile_metadata_header(metadata: LockfileMetadata) -> bytes:
     """Produces a metadata bytes object for including at the top of a lockfile.
 
     Currently, this only consists of an invalidation digest for the file, which is used when Pants
     consumes the lockfile during builds.
     """
+
+    metadata_as_comment = "\n".join(f"# {i}" for i in metadata.to_json_literal().splitlines())
+
     return (
         b"""
 %(BEGIN_LOCKFILE_HEADER)b
-# invalidation digest: %(invalidation_digest)s
+%(metadata_as_comment)s
 %(END_LOCKFILE_HEADER)b
     """
         % {
             b"BEGIN_LOCKFILE_HEADER": BEGIN_LOCKFILE_HEADER,
-            b"invalidation_digest": invalidation_digest.encode("ascii"),
+            b"metadata_as_comment": metadata_as_comment.encode("ascii"),
             b"END_LOCKFILE_HEADER": END_LOCKFILE_HEADER,
         }
     ).strip()
@@ -71,17 +149,18 @@ def read_lockfile_metadata(contents: bytes) -> LockfileMetadata:
     """Reads through `contents`, and returns the contents of the lockfile metadata block as a
     `LockfileMetadata` object."""
 
-    metadata = {}
-
-    in_metadata_block = False
-    for line in contents.splitlines():
-        line = line.strip()
-        if line == BEGIN_LOCKFILE_HEADER:
-            in_metadata_block = True
-        elif line == END_LOCKFILE_HEADER:
-            break
-        elif in_metadata_block:
-            key, value = (i.strip().decode("ascii") for i in line[1:].split(b":"))
-            metadata[key] = value
-
-    return LockfileMetadata(invalidation_digest=metadata.get("invalidation digest"))
+    def yield_metadata_lines() -> Iterable[bytes]:
+        """Splits contents into lines and yields only the lines that are strictly inside the
+        metadata header block."""
+        in_metadata_block = False
+        for line in contents.splitlines():
+            if line == BEGIN_LOCKFILE_HEADER:
+                in_metadata_block = True
+            elif line == END_LOCKFILE_HEADER:
+                break
+            elif in_metadata_block:
+                yield line[2:]
+
+    metadata_lines = b"\n".join(yield_metadata_lines())
+
+    return LockfileMetadata.from_json_bytes(metadata_lines)