feat: look up custom issuers by inbound URL (#18892)

* feat: look up custom issuers by inbound URL

When receiving an inbound JWT, handle cases where the inbound issuer is
not one of the "standard" service URLs, such as in the case of GitLab
Self-Managed or GitHub Enterprise Server instances.

* feat: enable pending gitlab publisher reification

---------

Signed-off-by: Mike Fiedler <miketheman@gmail.com>

Author: miketheman

tests/unit/oidc/models/test_gitlab.py

@@ -38,6 +38,8 @@
         ),
         ("gitlab.com/foo/bar//a.yml@/some/ref", "a.yml"),
         ("gitlab.com/foo/bar//a/b.yml@/some/ref", "a/b.yml"),
+        # Custom domain.
+        ("gitlab.example.com/foo/bar//example.yml@/some/ref", "example.yml"),
         # Malformed `ci_config_ref_uri`s.
         ("gitlab.com/foo/bar//notnested.wrongsuffix@/some/ref", None),
         ("gitlab.com/foo/bar//@/some/ref", None),
@@ -59,6 +61,7 @@ class TestGitLabPublisher:
     @pytest.mark.parametrize("environment", [None, "some_environment"])
     def test_lookup_fails_invalid_ci_config_ref_uri(self, environment):
         signed_claims = {
+            "iss": "https://gitlab.com",
             "project_path": "foo/bar",
             "ci_config_ref_uri": ("gitlab.com/foo/bar//example/.yml@refs/heads/main"),
         }
@@ -101,6 +104,7 @@ def test_lookup_succeeds_with_mixed_case_project_path(
         )
 
         signed_claims = {
+            "iss": "https://gitlab.com",
             "project_path": project_path,
             "ci_config_ref_uri": "gitlab.com/foo/bar//.gitlab-ci.yml@refs/heads/main",
             "environment": "some_environment",
@@ -129,6 +133,7 @@ def test_lookup_succeeds_with_non_lowercase_environment(
         )
 
         signed_claims = {
+            "iss": "https://gitlab.com",
             "project_path": "foo/bar",
             "ci_config_ref_uri": ("gitlab.com/foo/bar//.gitlab-ci.yml@refs/heads/main"),
             "environment": environment,
@@ -157,6 +162,7 @@ def test_lookup_is_case_sensitive_for_environment(self, db_request, environment)
         )
 
         signed_claims = {
+            "iss": "https://gitlab.com",
             "project_path": "foo/bar",
             "ci_config_ref_uri": ("gitlab.com/foo/bar//.gitlab-ci.yml@refs/heads/main"),
             "environment": environment,
@@ -194,6 +200,7 @@ def test_lookup_escapes(
 
         for workflow_filepath in (workflow_filepath_a, workflow_filepath_b):
             signed_claims = {
+                "iss": "https://gitlab.com",
                 "project_path": "foo/bar",
                 "ci_config_ref_uri": (
                     f"gitlab.com/foo/bar//{workflow_filepath}@refs/heads/main"
@@ -212,6 +219,7 @@ def test_lookup_escapes(
 
     def test_lookup_no_matching_publisher(self, db_request):
         signed_claims = {
+            "iss": "https://gitlab.com",
             "project_path": "foo/bar",
             "ci_config_ref_uri": ("gitlab.com/foo/bar//.gitlab-ci.yml@refs/heads/main"),
         }
@@ -272,6 +280,7 @@ def test_gitlab_publisher_computed_properties(self):
             namespace="fakeowner",
             workflow_filepath="subfolder/fakeworkflow.yml",
             environment="fakeenv",
+            issuer_url="https://gitlab.com",
         )
 
         for claim_name in publisher.__required_verifiable_claims__.keys():
@@ -359,6 +368,7 @@ def test_gitlab_publisher_missing_claims(self, monkeypatch, missing):
             project="fakerepo",
             namespace="fakeowner",
             workflow_filepath="subfolder/fakeworkflow.yml",
+            issuer_url="https://gitlab.com",
         )
 
         scope = pretend.stub()
@@ -394,6 +404,7 @@ def test_gitlab_publisher_missing_optional_claims(self, monkeypatch):
             namespace="fakeowner",
             workflow_filepath="subfolder/fakeworkflow.yml",
             environment="some-environment",  # The optional claim that should be present
+            issuer_url="https://gitlab.com",
         )
 
         sentry_sdk = pretend.stub(capture_message=pretend.call_recorder(lambda s: None))
@@ -429,6 +440,7 @@ def test_gitlab_publisher_verifies(self, monkeypatch, environment, missing_claim
             namespace="fakeowner",
             workflow_filepath="subfolder/fakeworkflow.yml",
             environment="environment",
+            issuer_url="https://gitlab.com",
         )
 
         noop_check = pretend.call_recorder(lambda gt, sc, ac, **kwargs: True)
@@ -661,6 +673,7 @@ def test_gitlab_publisher_ci_config_ref_uri(
             project="bar",
             namespace="foo",
             workflow_filepath="workflows/baz.yml",
+            issuer_url="https://gitlab.com",
         )
 
         check = gitlab.GitLabPublisher.__required_verifiable_claims__[
@@ -844,6 +857,7 @@ def test_gitlab_publisher_verify_url(
             namespace=namespace,
             workflow_filepath="workflow_filename.yml",
             environment="",
+            issuer_url="https://gitlab.com",
         )
         assert publisher.verify_url(url) == expected
 
@@ -854,6 +868,7 @@ def test_gitlab_publisher_attestation_identity(self, environment):
             namespace="group/subgroup",
             workflow_filepath="workflow_filename.yml",
             environment=environment,
+            issuer_url="https://gitlab.com",
         )
 
         identity = publisher.attestation_identity
@@ -922,3 +937,13 @@ def test_reify_already_exists(self, db_request):
         # it is returned and the pending publisher is marked for deletion.
         assert existing_publisher == publisher
         assert pending_publisher in db_request.db.deleted
+
+    def test_reify_with_custom_issuer_url(self, db_request):
+        custom_issuer_url = "https://gitlab.custom-domain.com"
+        pending_publisher = PendingGitLabPublisherFactory.create(
+            issuer_url=custom_issuer_url
+        )
+        publisher = pending_publisher.reify(db_request.db)
+
+        assert publisher.issuer_url == custom_issuer_url
+        assert isinstance(publisher, gitlab.GitLabPublisher)

tests/unit/oidc/test_utils.py

@@ -13,6 +13,7 @@
     GitLabPublisherFactory,
     GooglePublisherFactory,
 )
+from tests.common.db.organizations import OrganizationOIDCIssuerFactory
 from warehouse.oidc import errors, utils
 from warehouse.oidc.models import (
     ActiveStatePublisher,
@@ -21,13 +22,16 @@
     GooglePublisher,
 )
 from warehouse.oidc.utils import OIDC_PUBLISHER_CLASSES
+from warehouse.organizations.models import OIDCIssuerType
 from warehouse.utils.security_policy import principals_for
 
 
 def test_find_publisher_by_issuer_bad_issuer_url():
+    session = pretend.stub(scalar=lambda *stmt: None)
+
     with pytest.raises(errors.InvalidPublisherError):
         utils.find_publisher_by_issuer(
-            pretend.stub(), "https://fake-issuer.url", pretend.stub()
+            session, "https://fake-issuer.url", pretend.stub()
         )
 
 
@@ -140,6 +144,7 @@ def test_find_publisher_by_issuer_gitlab(db_request, environment, expected_id):
 
     signed_claims.update(
         {
+            "iss": utils.GITLAB_OIDC_ISSUER_URL,
             "project_path": "foo/bar",
             "ci_config_ref_uri": "gitlab.com/foo/bar//workflows/ci.yml@refs/heads/main",
         }
@@ -307,3 +312,45 @@ def test_oidc_maps_consistent():
         # The class mapping for pending and non-pending publisher models
         # should be distinct.
         assert class_map[True] != class_map[False]
+
+
+def test_find_publisher_by_issuer_with_custom_issuer(db_request):
+    """
+    A custom OIDC issuer URL is properly resolved to a concrete publisher.
+    """
+    # Create organization and register a custom GitLab issuer URL
+    custom_issuer_url = "https://gitlab.custom-company.com"
+    OrganizationOIDCIssuerFactory.create(
+        issuer_type=OIDCIssuerType.GitLab,
+        issuer_url=custom_issuer_url,
+    )
+
+    # Create a GitLab publisher that would match the claims
+    publisher = GitLabPublisherFactory(
+        namespace="foo",
+        project="bar",
+        workflow_filepath="workflows/ci.yml",
+        environment="",
+        issuer_url=custom_issuer_url,
+    )
+
+    # Create signed claims that would come from the custom GitLab instance
+    signed_claims = {
+        claim_name: "fake" for claim_name in GitLabPublisher.all_known_claims()
+    }
+    signed_claims.update(
+        {
+            "iss": custom_issuer_url,
+            "project_path": "foo/bar",
+            "ci_config_ref_uri": "gitlab.custom-company.com/foo/bar//workflows/ci.yml@refs/heads/main",  # noqa: E501
+        }
+    )
+
+    # This should successfully resolve the custom issuer URL to the GitLab publisher
+    result = utils.find_publisher_by_issuer(
+        db_request.db,
+        utils.GITLAB_OIDC_ISSUER_URL,
+        signed_claims,
+    )
+
+    assert result.id == publisher.id

tests/unit/oidc/test_views.py

@@ -215,11 +215,13 @@ def find_service(self, *a, **kw):
         assert err["description"] == "malformed JWT"
 
 
-def test_mint_token_from_oidc_unknown_issuer():
+def test_mint_token_from_oidc_unknown_issuer(metrics):
     class Request:
         def __init__(self):
             self.response = pretend.stub(status=None)
             self.flags = pretend.stub(disallow_oidc=lambda *a: False)
+            self.db = pretend.stub(scalar=lambda *stmt: None)
+            self.metrics = metrics
 
         @property
         def body(self):

warehouse/oidc/models/gitlab.py

@@ -80,7 +80,7 @@ def _check_ci_config_ref_uri(
     **_kwargs,
 ) -> bool:
     # We expect a string formatted as follows:
-    #   gitlab.com/OWNER/REPO//WORKFLOW_PATH/WORKFLOW_FILE.yml@REF
+    #   <gitlab.com>/OWNER/REPO//WORKFLOW_PATH/WORKFLOW_FILE.yml@REF
     # where REF is the value of the `ref_path` claim.
 
     # Defensive: GitLab should never give us an empty ci_config_ref_uri,
@@ -234,6 +234,7 @@ def _get_publisher_for_environment(
 
     @classmethod
     def lookup_by_claims(cls, session: Session, signed_claims: SignedClaims) -> Self:
+        issuer_url = signed_claims["iss"]
         project_path = signed_claims["project_path"]
         ci_config_ref_uri = signed_claims["ci_config_ref_uri"]
         namespace, project = project_path.rsplit("/", 1)
@@ -249,6 +250,7 @@ def lookup_by_claims(cls, session: Session, signed_claims: SignedClaims) -> Self
             func.lower(cls.namespace) == func.lower(namespace),
             func.lower(cls.project) == func.lower(project),
             cls.workflow_filepath == workflow_filepath,
+            cls.issuer_url == issuer_url,
         )
         publishers = query.with_session(session).all()
         if publisher := cls._get_publisher_for_environment(publishers, environment):
@@ -266,15 +268,18 @@ def sub(self) -> str:
 
     @property
     def ci_config_ref_uri(self) -> str:
-        return f"gitlab.com/{self.project_path}//{self.workflow_filepath}"
+        # Extract domain from issuer_url (remove https:// prefix)
+        domain = self.issuer_url.removeprefix("https://")
+        return f"{domain}/{self.project_path}//{self.workflow_filepath}"
 
     @property
     def publisher_name(self) -> str:
         return "GitLab"
 
     @property
     def publisher_base_url(self) -> str:
-        return f"https://gitlab.com/{self.project_path}"
+        # Use the issuer_url which already includes the scheme (https://)
+        return f"{self.issuer_url}/{self.project_path}"
 
     @property
     def jti(self) -> str:
@@ -430,6 +435,7 @@ def reify(self, session: Session) -> GitLabPublisher:
                 GitLabPublisher.project == self.project,
                 GitLabPublisher.workflow_filepath == self.workflow_filepath,
                 GitLabPublisher.environment == self.environment,
+                GitLabPublisher.issuer_url == self.issuer_url,
             )
             .one_or_none()
         )
@@ -439,7 +445,7 @@ def reify(self, session: Session) -> GitLabPublisher:
             project=self.project,
             workflow_filepath=self.workflow_filepath,
             environment=self.environment,
-            issuer_url=GITLAB_OIDC_ISSUER_URL,
+            issuer_url=self.issuer_url,
         )
 
         session.delete(self)

warehouse/oidc/services.py

@@ -77,7 +77,7 @@ def verify_jwt_signature(
 
     def find_publisher(
         self, signed_claims: SignedClaims, *, pending: bool = False
-    ) -> OIDCPublisher | PendingOIDCPublisher | None:
+    ) -> OIDCPublisher | PendingOIDCPublisher:
         # NOTE: We do NOT verify the claims against the publisher, since this
         # service is for development purposes only.
         return find_publisher_by_issuer(

warehouse/oidc/utils.py

@@ -7,6 +7,7 @@
 from dataclasses import dataclass
 
 from pyramid.authorization import Authenticated
+from sqlalchemy import select
 
 from warehouse.admin.flags import AdminFlagValue
 from warehouse.oidc.errors import InvalidPublisherError
@@ -27,10 +28,13 @@
     PendingGooglePublisher,
     PendingOIDCPublisher,
 )
+from warehouse.organizations.models import OrganizationOIDCIssuer
 
 if typing.TYPE_CHECKING:
     from sqlalchemy.orm import Session
 
+    from warehouse.organizations.models import OIDCIssuerType
+
 
 OIDC_ISSUER_SERVICE_NAMES = {
     GITHUB_OIDC_ISSUER_URL: "github",
@@ -59,6 +63,18 @@
 }
 
 
+def lookup_custom_issuer_type(
+    session: Session, issuer_url: str
+) -> OIDCIssuerType | None:
+    """
+    Look up the issuer type for an Organization's OIDC issuer URL.
+    """
+    stmt = select(OrganizationOIDCIssuer.issuer_type).where(
+        OrganizationOIDCIssuer.issuer_url == issuer_url
+    )
+    return session.scalar(stmt)
+
+
 def find_publisher_by_issuer(
     session: Session,
     issuer_url: str,
@@ -72,7 +88,7 @@ def find_publisher_by_issuer(
     to one or more projects or a `PendingOIDCPublisher`, varying with the
     `pending` parameter.
 
-    Returns `None` if no publisher can be found.
+    Raises if no publisher can be found.
     """
 
     try:

warehouse/oidc/views.py

@@ -24,7 +24,11 @@
 from warehouse.oidc.models import GitHubPublisher, OIDCPublisher, PendingOIDCPublisher
 from warehouse.oidc.models.gitlab import GitLabPublisher
 from warehouse.oidc.services import OIDCPublisherService
-from warehouse.oidc.utils import OIDC_ISSUER_ADMIN_FLAGS, OIDC_ISSUER_SERVICE_NAMES
+from warehouse.oidc.utils import (
+    OIDC_ISSUER_ADMIN_FLAGS,
+    OIDC_ISSUER_SERVICE_NAMES,
+    lookup_custom_issuer_type,
+)
 from warehouse.packaging.interfaces import IProjectService
 from warehouse.packaging.models import ProjectFactory
 from warehouse.rate_limiting.interfaces import IRateLimiter
@@ -135,8 +139,16 @@ def mint_token_from_oidc(request: Request):
         )
 
     # Associate the given issuer claim with Warehouse's OIDCPublisherService.
+    # First, try the standard issuers
     service_name = OIDC_ISSUER_SERVICE_NAMES.get(unverified_issuer)
+    # If not in global mapping, check for organization-specific custom issuer
+    if not service_name:
+        service_name = lookup_custom_issuer_type(request.db, unverified_issuer)
     if not service_name:
+        request.metrics.increment(
+            "warehouse.oidc.mint_token_from_oidc.unknown_issuer",
+            tags={"issuer_url": unverified_issuer},
+        )
         return _invalid(
             errors=[
                 {
@@ -147,7 +159,7 @@ def mint_token_from_oidc(request: Request):
             request=request,
         )
 
-    if request.flags.disallow_oidc(OIDC_ISSUER_ADMIN_FLAGS[unverified_issuer]):
+    if request.flags.disallow_oidc(OIDC_ISSUER_ADMIN_FLAGS.get(unverified_issuer)):
         return _invalid(
             errors=[
                 {