Do not crash when entity-selection-profile attribute has invalid JSON

src/pyff/builtins.py

@@ -302,7 +302,7 @@
 def _any(lst, d):
     for x in lst:
         if x in d:
             if type(d) == dict:
                 return d[x]
             else:
                 return True
@@ -832,14 +832,17 @@
         if opts[0] == 'as' and len(opts) == 2:
             name = opts[1]
 
-    entities = resolve_entities(args, lookup_fn=req.md.store.select, dedup=dedup)
+    if dedup:
+        entities = resolve_entities(args, lookup_fn=req.md.store.select, dedup=dedup)
+    else:
+        entities = resolve_entities(args, lookup_fn=req.md.store.select_with_dups, dedup=dedup)
 
     if req.state.get('match', None):  # TODO - allow this to be passed in via normal arguments
 
         match = req.state['match']
 
         if isinstance(match, six.string_types):
             query = [match.lower()]
 
         def _strings(elt):
             lst = []
@@ -1458,7 +1461,7 @@
     def _drop_xsi_type(elt):
         try:
             del elt.attrib["{%s}type" % NS["xsi"]]
         except Exception as ex:
             pass
 
     with_tree(root(req.t), _drop_xsi_type)
@@ -1571,7 +1574,7 @@
                                     "%s expires in %s" % (cert.getSubject(), dt),
                                 )
                                 log.warning("%s expires in %s" % (eid, dt))
                         except ValueError as ex:
                             annotate_entity(
                                 entity_elt,
                                 "certificate-error",

src/pyff/samlmd.py

@@ -1047,14 +1047,34 @@ def discojson_sp_attr(e):
     if b64_trustinfos is None:
         return None
 
+    entityID = e.get('entityID', None)
     sp = {}
-    sp['entityID'] = e.get('entityID', None)
+    sp['entityID'] = entityID
     sp['profiles'] = {}
+    sp['extra_md'] = {}
 
     for b64_trustinfo in b64_trustinfos:
-        str_trustinfo = b64decode(b64_trustinfo.encode('ascii'))
-        trustinfo = json.loads(str_trustinfo.decode('utf8'))
-        sp['profiles'].update(trustinfo['profiles'])
+        try:
+            str_trustinfo = b64decode(b64_trustinfo.encode('ascii'))
+            trustinfo = json.loads(str_trustinfo.decode('utf8'))
+            for profile in trustinfo['profiles']:
+                if profile in sp['profiles']:
+                    log.warning(f"SP Entity {entityID} has a duplicate trust profile {profile}")
+                else:
+                    sp['profiles'][profile] = trustinfo['profiles'][profile]
+
+            if 'extra_md' in trustinfo:
+                for extra_id in trustinfo['extra_md']:
+                    if extra_id in sp['extra_md']:
+                        log.warning(f"SP Entity {entityID} has a duplicate extra IdP metadata {extra_id}")
+                    else:
+                        sp['extra_md'][extra_id] = trustinfo['extra_md'][extra_id]
+
+        except Exception as e:
+            log.warning(f"Invalid entity-selection-profile attribute for {entityID}: {e}")
+
+    if not sp['profiles']:
+        return None
 
     return sp
 

src/pyff/store.py

@@ -4,6 +4,7 @@
 import re
 import shutil
 import time
+from collections import defaultdict
 from datetime import datetime, timedelta
 from io import BytesIO
 from threading import ThreadError
@@ -807,6 +808,7 @@ def __init__(self, *args, **kwargs):
         self.md = dict()
         self.index = dict()
         self.entities = dict()
+        self.md_entities = defaultdict(dict)
 
         for hn in DINDEX:
             self.index.setdefault(hn, {})
@@ -885,13 +887,15 @@ def update(self, t, tid=None, etag=None, lazy=True):
             self.entities[relt.get('entityID')] = relt  # TODO: merge?
             if tid is not None:
                 self.md[tid] = [relt.get('entityID')]
+                self.md_entities[tid][relt.get('entityID')] = relt
         elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']:
             if tid is None:
                 tid = relt.get('Name')
             lst = []
             for e in iter_entities(t):
-                self.update(e)
+                self.update(e, tid)
                 lst.append(e.get('entityID'))
+                self.md_entities[tid][e.get('entityID')] = e
             self.md[tid] = lst
 
     def lookup(self, key):
@@ -946,3 +950,9 @@ def _lookup(self, key):
             return lst
 
         return []
+
+    def select_with_dups(self, member):
+        if member in self.md_entities:
+            return self.md_entities[member].values()
+
+        return []

src/pyff/test/data/metadata/test-sp-trustinfo-in-attr.xml

@@ -71,6 +71,118 @@ fMou5aW0mZ+QgJNKOrxY5vFxUq6pn3OiYbBu3m1C9ajbU/nx2evzt4+qUwTfHFb+
 ZgXpOtmxRekFzVvGZ18BSPJKwAAqqZ11X7skT/NwEAhbgplVPv9WkDmDzqNvHqQJ
 nyRgD2ZqUPU9nEOjGy0gI07dciVcYZQ+CiZeSECIWgQwjDEBDuwMCVAZA6gfdz6C
 KJuN+RUSKPEcxPxle1MiB4MU0ei5X4xUbvLWKn9Ok7TOXg2BpnMAv6eON1wVo0Aa
+D265cqy6Le/toVg=</ds:X509Certificate>
+        </ds:X509Data>
+      </ds:KeyInfo>
+      <md:EncryptionMethod Algorithm="http://www.w3.org/2009/xmlenc11#aes128-gcm"/>
+      <md:EncryptionMethod Algorithm="http://www.w3.org/2009/xmlenc11#aes192-gcm"/>
+      <md:EncryptionMethod Algorithm="http://www.w3.org/2009/xmlenc11#aes256-gcm"/>
+      <md:EncryptionMethod Algorithm="http://www.w3.org/2001/04/xmlenc#aes128-cbc"/>
+      <md:EncryptionMethod Algorithm="http://www.w3.org/2001/04/xmlenc#aes192-cbc"/>
+      <md:EncryptionMethod Algorithm="http://www.w3.org/2001/04/xmlenc#aes256-cbc"/>
+      <md:EncryptionMethod Algorithm="http://www.w3.org/2001/04/xmlenc#tripledes-cbc"/>
+      <md:EncryptionMethod Algorithm="http://www.w3.org/2009/xmlenc11#rsa-oaep"/>
+      <md:EncryptionMethod Algorithm="http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p"/>
+    </md:KeyDescriptor>
+    <md:AssertionConsumerService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" Location="https://cpauth.icos-cp.eu/saml/SAML2/POST" index="1" isDefault="true"/>
+    <md:AttributeConsumingService index="1">
+      <md:ServiceName xml:lang="en">ICOS Carbon Portal SAML service</md:ServiceName>
+      <md:ServiceName xml:lang="sv">ICOS Kolportalens SAML tjänst</md:ServiceName>
+      <md:RequestedAttribute FriendlyName="givenName" Name="urn:oid:2.5.4.42" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri" isRequired="true"/>
+      <md:RequestedAttribute FriendlyName="sn" Name="urn:oid:2.5.4.4" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri" isRequired="true"/>
+      <md:RequestedAttribute FriendlyName="mail" Name="urn:oid:0.9.2342.19200300.100.1.3" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri" isRequired="true"/>
+    </md:AttributeConsumingService>
+  </md:SPSSODescriptor>
+  <md:Organization>
+    <md:OrganizationName xml:lang="en">ICOS Carbon Portal</md:OrganizationName>
+    <md:OrganizationName xml:lang="sv">ICOS Kolportalen</md:OrganizationName>
+    <md:OrganizationDisplayName xml:lang="en">Carbon Portal</md:OrganizationDisplayName>
+    <md:OrganizationDisplayName xml:lang="sv">Kolportalen</md:OrganizationDisplayName>
+    <md:OrganizationURL xml:lang="en">https://www.icos-cp.eu/</md:OrganizationURL>
+    <md:OrganizationURL xml:lang="sv">https://www.icos-cp.eu/</md:OrganizationURL>
+  </md:Organization>
+  <md:ContactPerson contactType="technical">
+    <md:GivenName>Oleg</md:GivenName>
+    <md:SurName>Mirzov</md:SurName>
+    <md:EmailAddress>mailto:oleg.mirzov@nateko.lu.se</md:EmailAddress>
+  </md:ContactPerson>
+  <md:ContactPerson contactType="administrative">
+    <md:GivenName>Alex</md:GivenName>
+    <md:SurName>Vermeulen</md:SurName>
+    <md:EmailAddress>mailto:alex.vermeulen@nateko.lu.se</md:EmailAddress>
+  </md:ContactPerson>
+</md:EntityDescriptor>
+  <md:EntityDescriptor entityID="https://example.org/shibboleth">
+  <md:Extensions>
+    <mdrpi:RegistrationInfo registrationAuthority="http://www.swamid.se/" registrationInstant="2015-02-11T11:09:51Z">
+      <mdrpi:RegistrationPolicy xml:lang="en">http://swamid.se/policy/mdrps</mdrpi:RegistrationPolicy>
+    </mdrpi:RegistrationInfo>
+    <alg:DigestMethod Algorithm="http://www.w3.org/2001/04/xmlenc#sha512"/>
+    <alg:DigestMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#sha384"/>
+    <alg:DigestMethod Algorithm="http://www.w3.org/2001/04/xmlenc#sha256"/>
+    <alg:DigestMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#sha224"/>
+    <alg:DigestMethod Algorithm="http://www.w3.org/2000/09/xmldsig#sha1"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha512"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha384"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha256"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha224"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#rsa-sha512"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#rsa-sha384"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#rsa-sha256"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2009/xmldsig11#dsa-sha256"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha1"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2000/09/xmldsig#rsa-sha1"/>
+    <alg:SigningMethod Algorithm="http://www.w3.org/2000/09/xmldsig#dsa-sha1"/>
+    <mdattr:EntityAttributes>
+      <saml:Attribute NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri" Name="http://macedir.org/entity-category">
+        <saml:AttributeValue>http://www.geant.net/uri/dataprotection-code-of-conduct/v1</saml:AttributeValue>
+      </saml:Attribute>
+      <saml:Attribute NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri" Name="https://refeds.org/entity-selection-profile">   
+        <saml:AttributeValue>invalidValueForAttribute</saml:AttributeValue>
+      </saml:Attribute>
+    </mdattr:EntityAttributes>
+  </md:Extensions>
+  <md:SPSSODescriptor protocolSupportEnumeration="urn:oasis:names:tc:SAML:2.0:protocol">
+    <md:Extensions>
+      <init:RequestInitiator Binding="urn:oasis:names:tc:SAML:profiles:SSO:request-init" Location="https://cpauth.icos-cp.eu/saml/login"/>
+      <mdui:UIInfo>
+        <mdui:DisplayName xml:lang="en">Carbon Portal authentication service</mdui:DisplayName>
+        <mdui:DisplayName xml:lang="sv">Kolportalens autentiseringstjänst</mdui:DisplayName>
+        <mdui:Description xml:lang="en">Single Sign On for services of ICOS Carbon Portal. Maintained by the Carbon Portal team at Physical Geography department (nateko.lu.se).</mdui:Description>
+        <mdui:Description xml:lang="sv">Single Sign On tjänst för ICOS Kolportalen. Hanteras av Carbon Portal teamet på INES (nateko.lu.se).</mdui:Description>
+        <mdui:PrivacyStatementURL xml:lang="en">https://cpauth.icos-cp.eu/saml/privacyStatement</mdui:PrivacyStatementURL>
+        <mdui:InformationURL xml:lang="en">https://www.icos-cp.eu/</mdui:InformationURL>
+        <mdui:InformationURL xml:lang="sv">https://www.icos-cp.eu/</mdui:InformationURL>
+        <mdui:PrivacyStatementURL xml:lang="sv">https://cpauth.icos-cp.eu/saml/privacyStatement</mdui:PrivacyStatementURL>
+      </mdui:UIInfo>
+    </md:Extensions>
+    <md:KeyDescriptor>
+      <ds:KeyInfo>
+        <ds:KeyName>cpauth.icos-cp.eu</ds:KeyName>
+        <ds:X509Data>
+          <ds:X509SubjectName>CN=cpauth.icos-cp.eu</ds:X509SubjectName>
+          <ds:X509Certificate>MIIEJzCCAw+gAwIBAgIJANC3VWNs7fbTMA0GCSqGSIb3DQEBCwUAMIGpMQswCQYD
+VQQGEwJTRTERMA8GA1UECAwIU2vDg8KlbmUxDTALBgNVBAcMBEx1bmQxGzAZBgNV
+BAoMEklDT1MgQ2FyYm9uIFBvcnRhbDEfMB0GA1UECwwWQXV0aGVudGljYXRpb24g
+U2VydmljZTEaMBgGA1UEAwwRY3BhdXRoLmljb3MtY3AuZXUxHjAcBgkqhkiG9w0B
+CQEWD2luZm9AaWNvcy1jcC5ldTAeFw0xNTAyMDUxMjI0MzZaFw0yNTAyMDIxMjI0
+MzZaMIGpMQswCQYDVQQGEwJTRTERMA8GA1UECAwIU2vDg8KlbmUxDTALBgNVBAcM
+BEx1bmQxGzAZBgNVBAoMEklDT1MgQ2FyYm9uIFBvcnRhbDEfMB0GA1UECwwWQXV0
+aGVudGljYXRpb24gU2VydmljZTEaMBgGA1UEAwwRY3BhdXRoLmljb3MtY3AuZXUx
+HjAcBgkqhkiG9w0BCQEWD2luZm9AaWNvcy1jcC5ldTCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAM2QN1jaZJeuPAH+4sVMZKk7vg4JIbUuTMKk0+KIAg5M
+XiVsRiEUjY+LtIncrvA/kf2CIySI0WkbwZMjcDd03hNj4kLWhuyxfOCwDO6DsUbG
+MbyI6HIYWXJp5ljfEEFgtMqT3dDtD5vwq8h4Zy20ukxOoIokKczrAvn4JjkMsj6Z
+0CEAFBC29o4E8PWQbUBgvt6Z+2ao+RHMLD7nZVBx98Occ9KfnYnDDd9Oi1XFe009
+zaSbcqY2RpN8I9hcW/KQf3KnGW5xZ5dr4rhGklCkYr+h0W3xKu+hin8bk91t1Dkr
+gaKl/N7M3Oof3k+7ZBlwaV97es5InWCeNgDxCGkBRNsCAwEAAaNQME4wHQYDVR0O
+BBYEFDcD7MVudooGaNRYqXBYqQi3VzGxMB8GA1UdIwQYMBaAFDcD7MVudooGaNRY
+qXBYqQi3VzGxMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBABS02eZS
+weXGMJ2fEIy2JH0VhCbjuX/rz+8Hfh9LjzNb3QwKHuwP83yvPqRulV9FYmvOoK8T
+fMou5aW0mZ+QgJNKOrxY5vFxUq6pn3OiYbBu3m1C9ajbU/nx2evzt4+qUwTfHFb+
+ZgXpOtmxRekFzVvGZ18BSPJKwAAqqZ11X7skT/NwEAhbgplVPv9WkDmDzqNvHqQJ
+nyRgD2ZqUPU9nEOjGy0gI07dciVcYZQ+CiZeSECIWgQwjDEBDuwMCVAZA6gfdz6C
+KJuN+RUSKPEcxPxle1MiB4MU0ei5X4xUbvLWKn9Ok7TOXg2BpnMAv6eON1wVo0Aa
 D265cqy6Le/toVg=</ds:X509Certificate>
         </ds:X509Data>
       </ds:KeyInfo>