Add test-anomaly tag and enforce its semantics

Signed-off-by: Daniel Hardman <daniel.hardman@gmail.com>
Fathom-Public-Benefit-Corporation · Oct 16, 2019 · 527849e · 527849e
1 parent f2988ec
commit 527849e
Show file tree

Hide file tree

Showing 66 changed files with 166 additions and 119 deletions.
diff --git a/code/link_tags.py b/code/link_tags.py
@@ -0,0 +1,29 @@
+import sys
+import re
+
+tag_line_pat = re.compile(r'^\s*-\s*[Tt]ags\s*:\s*(.*?)$', re.M)
+
+def link_tags(fname):
+    changed = False
+    with open(fname, 'rt') as f:
+        txt = f.read()
+    m = tag_line_pat.search(txt)
+    if m:
+        update = False
+        tags = [t.strip() for t in m.group(1).split(',')]
+        for i in range(len(tags)):
+            tag = tags[i]
+            if tag[0] != '[':
+                changed = True
+                tags[i] = '[' + tag + '](/tags.md#' + tag + ')'
+        if changed:
+            tags = ', '.join(tags)
+            txt = txt[:m.start(1)] + tags + txt[m.end(1):]
+            with open(fname, 'wt') as f:
+                f.write(txt)
+            print('Updated ' + fname)
+
+
+if __name__ == '__main__':
+    for fname in sys.argv[1:]:
+        link_tags(fname)
diff --git a/code/rfcs.py b/code/rfcs.py
@@ -70,7 +70,7 @@ def walk():
         m = _status_val_pat.search(status)
         if m:
             status = m.group(1)
-        tags = [x.strip() for x in fields[7].split(',')]
+        tags = [unlink_tag(x) for x in fields[7].split(',')]
         content_idx = txt.find('\n##')
         impl_table = get_impl_table(txt)
         impl_count = len(impl_table) if impl_table else 0
@@ -79,6 +79,13 @@ def walk():
         yield x
 
 
+def unlink_tag(tag):
+    tag = tag.strip()
+    return tag[1:tag.find(']')].strip() if tag.startswith('[') else tag
+
+def link_tag(tag):
+    return '[' + tag + '](/tags.md#' + tag + ')'
+
 _impl_pat = re.compile(r'^[ \t]*#+[ \t]*Implementations?[ \t]*$', re.M)
 _impl_table_head_pat = re.compile(r'^[ \t]*([|][ \t]*)?Name(.*?)[|](.*?)\n[ \t]*([|][ \t]*)?-+(.*?)*[|](.*?)\n', re.M)
 _impl_table_row_pat = re.compile(r'((.*?)[|](.*?))\n')
@@ -126,11 +133,11 @@ def describe_impl_row(row):
     return desc
 
 
-_link_to_test_results_pat = re.compile(r'\[[ \t]*test[ \t]+results[ \t]*\]\(([^\)]*)\)', re.I)
+_link_to_test_results_pat = re.compile(r'\[[ \t]*((?:MISSING[ \t]+)?test[ \t]+results)[ \t]*\]\(([^\)]*)\)', re.I)
 
 def get_test_results_link(impl_row):
     m = _link_to_test_results_pat.search(impl_row[1])
-    return m.group(1).strip() if m else None
+    return m if m else None
 
 
 def relpath(abspath):

diff --git a/code/test_rfcs.py b/code/test_rfcs.py
@@ -30,32 +30,15 @@ def test_links():
     assert check_links.main() == 0
 
 
-grandfathered = """
-features/0160-connection-protocol/: Impl "Aries Framework - .NET" needs a link to test results in its Notes column. Format = [test results](...)
-features/0160-connection-protocol/: Impl "Streetcred.id" needs a link to test results in its Notes column. Format = [test results](...)
-features/0160-connection-protocol/: Impl "Aries Cloud Agent - Python" needs a link to test results in its Notes column. Format = [test results](...)
-features/0160-connection-protocol/: Impl "Aries Static Agent - Python" needs a link to test results in its Notes column. Format = [test results](...)
-features/0095-basic-message/: Impl "Indy Cloud Agent - Python" needs a link to test results in its Notes column. Format = [test results](...)
-features/0095-basic-message/: Impl "Aries Framework - .NET" needs a link to test results in its Notes column. Format = [test results](...)
-features/0095-basic-message/: Impl "Streetcred.id" needs a link to test results in its Notes column. Format = [test results](...)
-features/0095-basic-message/: Impl "Aries Cloud Agent - Python" needs a link to test results in its Notes column. Format = [test results](...)
-features/0095-basic-message/: Impl "Aries Static Agent - Python" needs a link to test results in its Notes column. Format = [test results](...)
-features/0037-present-proof/: Test suite must be an impl for any protocol- or decorator-related RFC beyond DEMONSTRATED status.
-features/0048-trust-ping/: Impl "Indy Cloud Agent - Python" needs a link to test results in its Notes column. Format = [test results](...)
-features/0048-trust-ping/: Impl "Aries Framework - .NET" needs a link to test results in its Notes column. Format = [test results](...)
-features/0048-trust-ping/: Impl "Streetcred.id" needs a link to test results in its Notes column. Format = [test results](...)
-features/0048-trust-ping/: Impl "Aries Cloud Agent - Python" needs a link to test results in its Notes column. Format = [test results](...)
-features/0048-trust-ping/: Impl "Aries Static Agent - Python" needs a link to test results in its Notes column. Format = [test results](...)
-features/0036-issue-credential/: Test suite must be an impl for any protocol- or decorator-related RFC beyond DEMONSTRATED status.
-"""
-
-
 def test_rfc_metadata():
     errors = []
 
     def e(rfc, msg):
         errors.append(rfc.relpath.replace('README.md','') + ': ' + msg)
 
+    def warn(rfc, msg):
+        sys.stderr.write('Warning: ' + rfc.relpath.replace('README.md','') + ': ' + msg + '\n')
+
     for rfc in rfcs.walk():
         if not bool(rfc.title): e(rfc, 'no title found')
         if rfc.category not in rfc.relpath: e(rfc, 'category does not match path')
@@ -81,20 +64,36 @@ def e(rfc, msg):
             for impl in rfcs.test_suite_impls(rfc, False):
                 e(rfc, 'should not be PROPOSED if it has a non-test-suite impl')
                 break
-        elif rfc.status in ['ACCEPTED', 'ADOPTED'] and 'feature' in rfc.tags and ('protocol' in rfc.tags or 'decorator' in rfc.tags):
+        # Should this RFC have links to test results?
+        elif rfc.status in ['ACCEPTED', 'ADOPTED'] and 'feature' in rfc.tags and (
+                'protocol' in rfc.tags or 'decorator' in rfc.tags):
             found_test_suite_in_impls = False
             for row in rfcs.test_suite_impls(rfc, True):
                 found_test_suite_in_impls = True
                 break
             if not found_test_suite_in_impls:
-                e(rfc, 'Test suite must be an impl for any protocol- or decorator-related RFC beyond DEMONSTRATED status.')
+                msg = 'Test suite must be an impl for any protocol- or decorator-related RFC beyond DEMONSTRATED status.'
+                if 'test-anomaly' in rfc.tags:
+                    warn(rfc, msg)
+                else:
+                    e(rfc, msg + ' Tag "test-anomaly" to temporarily override.')
             for row in rfcs.test_suite_impls(rfc, False):
-                if not rfcs.get_test_results_link(row):
-                    e(rfc, 'Impl "%s" needs a link to test results in its Notes column. Format = [test results](...)' %
-                      rfcs.describe_impl_row(row))
-
+                m = rfcs.get_test_results_link(row)
+                # If we lack a link entirely, this is an error, period.
+                # If we have tagged the RFC with "test-anomaly", then it becomes possible to link
+                # the ugly text "MISSING test results" to the test-anomaly tag and have the result
+                # be only a warning. This ugly text+link should only be accepted when the 'test-anomaly'
+                # tag is present.
+                desc = rfcs.describe_impl_row(row)
+                if m is None:
+                    e(rfc, 'Impl "%s" needs a link to test results in its Notes column. Format = [test results](...) or, if RFC is tagged "test-anomaly", [MISSING test results](/tags.md#test-anomaly).' % desc)
+                # Are test results explicitly declared to be missing?
+                elif ('MISSING' in m.group(1) and '/tags.md#test-anomaly' in m.group(2)):
+                    if 'test-anomaly' in rfc.tags:
+                        warn(rfc, 'Impl "%s" needs to replace missing test results with something meaningful.' % desc)
+                    else:
+                        e(rfc, 'Can\'t declare missing tests without the "test-anomaly" tag to make the RFC ugly, so impl "%s" needs a link to test results in its Notes column. Format = [test results](...).' % desc)
 
-    errors = [e for e in errors if e not in grandfathered]
     if errors:
         msg = '\n' + '\n'.join(errors)
         raise BaseException(msg)
@@ -175,7 +174,6 @@ def find_refs(dict, key, value):
             offenders = '\n'.join(find_refs(base_uri_for_normalized, key, value))
             e(offenders, '\n  impl name "%s" maps to multiple sites: %s' % (key, ', '.join(['"%s"' % v for v in value])))
 
-    errors = [e for e in errors if e not in grandfathered]
     if errors:
         msg = '\n' + '\n'.join(errors)
         raise BaseException(msg)
diff --git a/concepts/0003-protocols/README.md b/concepts/0003-protocols/README.md
@@ -6,7 +6,7 @@
 - Status Note: standards track and beginning to influence many mental models, but not yet [ADOPTED](/README.md#rfc-lifecycle). 
 - Supersedes: [Indy PR #69]( https://github.com/hyperledger/indy-hipe/pull/69)
 - Start Date: 2018-12-28
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0004-agents/README.md b/concepts/0004-agents/README.md
@@ -6,7 +6,7 @@
 - Status Note: On a standards track and beginning to influence many mental models, but not yet [ADOPTED](/README.md#rfc-lifecycle). 
 - Supersedes: [Indy HIPE 0002](https://github.com/hyperledger/indy-hipe/tree/master/text/0002-agents)
 - Start Date: 2017-11-01 (approx, backdated)
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0005-didcomm/README.md b/concepts/0005-didcomm/README.md
@@ -6,7 +6,7 @@
 - Status Note: Probably almost mature enough to propose [ACCEPTED](/README.md#rfc-lifecycle) status. 
 - Supersedes: [Indy PR #98](https://github.com/hyperledger/indy-hipe/pull/98)
 - Start Date: 2018-01-05 (approx, backdated)
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0006-ssi-notation/README.md b/concepts/0006-ssi-notation/README.md
@@ -5,7 +5,7 @@
 - Since: 2018-09-01
 - Status Note: broadly accepted but not heavily used; may need greater usage before it's ready for [ADOPTED](/README.md#rfc-lifecycle) status.
 - Start Date: 2018-05-03
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0008-message-id-and-threading/README.md b/concepts/0008-message-id-and-threading/README.md
@@ -5,7 +5,7 @@
 - Since: 2018-10-01
 - Status Note: Implemented broadly in Indy, but not yet elsewhere.
 - Start Date: 2018-08-03
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0011-decorators/README.md b/concepts/0011-decorators/README.md
@@ -5,7 +5,7 @@
 - Since: 2019-01-31
 - Status Note: broadly used in Indy, but not yet harmonized with DIF work on hubs.
 - Start Date: 2018-12-14
-- Tags: concept, decorator
+- Tags: [concept](/tags.md#concept), [decorator](/tags.md#decorator)
 
 ## Summary
 

diff --git a/concepts/0013-overlays/README.md b/concepts/0013-overlays/README.md
@@ -5,7 +5,7 @@
 - Since: 2019-05-20
 - Status Note: socialized for many months; no implementations yet
 - Start Date: 2018-10-01
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0017-attachments/README.md b/concepts/0017-attachments/README.md
@@ -5,7 +5,7 @@
 - Since: 2019-01-31
 - Status Note: Used in a number of other RFCs.
 - Start Date: 2018-12-24
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0020-message-types/README.md b/concepts/0020-message-types/README.md
@@ -6,7 +6,7 @@
 - Status Note:  
 - Supersedes: [HIPE 0021 Message Types](https://github.com/hyperledger/indy-hipe/tree/master/text/0021-message-types)
 - Start Date: 2018-07-06
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0021-didcomm-message-anatomy/README.md b/concepts/0021-didcomm-message-anatomy/README.md
@@ -4,7 +4,7 @@
 - Status: [PROPOSED](/README.md#proposed)
 - Since: 2019-05-25
 - Start Date: 2019-05-25
-- Tags: concept, decorator
+- Tags: [concept](/tags.md#concept), [decorator](/tags.md#decorator)
 
 ## Summary
 

diff --git a/concepts/0029-message-trust-contexts/README.md b/concepts/0029-message-trust-contexts/README.md
@@ -6,7 +6,7 @@
 - Status Note: Only implemented in one codebase and in the sample code here. 
 - Supersedes: [Indy HIPE PR #120](https://github.com/hyperledger/indy-hipe/pull/120)
 - Start Date: 2018-02-10 (approx, backdated)
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0046-mediators-and-relays/README.md b/concepts/0046-mediators-and-relays/README.md
@@ -5,7 +5,7 @@
 - Since: 2019-02-01
 - Status Note: Socialized and broadly understood in other conceptual RFCs about routing.
 - Start Date: 2018-12-01
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0047-json-ld-compatibility/README.md b/concepts/0047-json-ld-compatibility/README.md
@@ -6,7 +6,7 @@
 - Status Note: Has guided Indy design choices for several months. Not yet ratified by greater Aries community. 
 - Supersedes: [Indy HIPE 0035](https://github.com/hyperledger/indy-hipe/tree/master/text/0035-json-ld-compatibility)
 - Start Date: 2019-01-23
-- Tags: concept, decorator
+- Tags: [concept](/tags.md#concept), [decorator](/tags.md#decorator)
 
 ## Summary
 

diff --git a/concepts/0049-repudiation/README.md b/concepts/0049-repudiation/README.md
@@ -6,7 +6,7 @@
 - Status Note: Well understood and baked into various protocol designs and DIDComm processes--but not yet ratified by the larger Aries community. 
 - Supersedes: [Indy HIPE 0037]( https://github.com/hyperledger/indy-hipe/tree/master/text/0037-repudiation)
 - Start Date: 2018-03-01 (backdated)
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0050-wallets/README.md b/concepts/0050-wallets/README.md
@@ -6,7 +6,7 @@
 - Status Note: Fully implemented in Indy SDK, but not yet socialized well in the broader Aries community. Needs some updates to promote better key management; see related RFC about [lox](../../features/0042-lox/README.md). 
 - Supersedes: [Indy HIPE 0013]( https://github.com/hyperledger/indy-hipe/tree/master/text/0013-wallets)
 - Start Date: 2018-05-22
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0051-dkms/README.md b/concepts/0051-dkms/README.md
@@ -5,7 +5,7 @@
 - Since: 2019-03-29
 - Status Note: Somewhat familiar to many in the community, and influential in the designs of other RFCs--but not yet deeply explored or socialized.
 - Start Date: 2018-09-01 (approx, backdated)
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0074-didcomm-best-practices/README.md b/concepts/0074-didcomm-best-practices/README.md
@@ -5,7 +5,7 @@
 - Since: 2019-06-10
 - Status Note: This captures some tribal knowledge from the Indy community. However, it is by no means uniformly accepted inside that group, and it has never been shared in a larger context. Therefore, the RFC is currently fodder for discussion.
 - Start Date: 2019-01-15
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0094-cross-domain-messaging/README.md b/concepts/0094-cross-domain-messaging/README.md
@@ -6,7 +6,7 @@
 - Status Note: Socialized and broadly understood in other conceptual RFCs about routing. 
 - Supersedes: [0022-Cross-Domain-Messaging](https://github.com/hyperledger/indy-RFC/blob/master/text/0022-cross-domain-messaging/README.md)
 - Start Date: 2018-08-13
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0103-indirect-identity-control/README.md b/concepts/0103-indirect-identity-control/README.md
@@ -5,7 +5,7 @@
 - Since: 2019-06-04
 - Status Note: Currently under study for implementation in various pilots and POCs. See, for example, the [Sovrin Foundation's Guardianship Task Force](https://docs.google.com/document/d/1ymWzCwu2Ud6FMGZdU8md03KCvaxmT41-gQYIRXo09Xw/edit).
 - Start Date: 2019-05-10
-- Tags: concept, credentials
+- Tags: [concept](/tags.md#concept), [credentials](/tags.md#credentials)
 
 ## Summary
 

diff --git a/concepts/0104-delegatable-credentials/README.md b/concepts/0104-delegatable-credentials/README.md
@@ -5,7 +5,7 @@
 - Since: 2019-09-09
 - Status Note: Various aspects of this are implemented, or in the process of being implemented, in [Hyperledger Ursa](https://github.com/hyperledger/ursa-rfcs/pull/14) and Hyperledger Indy. This doc will be updated based on learnings.
 - Start Date: 2019-08-01
-- Tags: concept, credentials
+- Tags: [concept](/tags.md#concept), [credentials](/tags.md#credentials)
 
 ## Summary
 

diff --git a/concepts/0167-data-consent-lifecycle/README.md b/concepts/0167-data-consent-lifecycle/README.md
@@ -6,7 +6,7 @@
 - Status Note: This RFC is under development. The reference implementation was recently added.
 - Supersedes: [Indy HIPE PR #55: Consent Receipt](https://github.com/hyperledger/indy-hipe/pull/55)
 - Start Date: 2018-11-08
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 
 ## Table of Contents

diff --git a/concepts/0207-credential-fraud-threat-model/README.md b/concepts/0207-credential-fraud-threat-model/README.md
@@ -4,7 +4,7 @@
 - Since: 2019-08-30
 - Status Note: socialized on an informal credential fraud study group (https://groups.google.com/d/forum/credential-fraud-study) since ~May 2019.  
 - Start Date: 2019-05-23
-- Tags: concept, credentials
+- Tags: [concept](/tags.md#concept), [credentials](/tags.md#credentials)
 
 ## Summary
 

diff --git a/concepts/0217-linkable-message-paths/README.md b/concepts/0217-linkable-message-paths/README.md
@@ -4,7 +4,7 @@
 - Since: 2019-09-10
 - Status Note: Purely theoretical at this point, but may be relevant to efforts to join subprotocols to superprotocols. Mentioned in [RFC 0214 "Help Me Discover" Protocol](../../features/0214-help-me-discover/README.md).
 - Start Date: 2019-08-26
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0231-biometric-service-provider/README.md b/concepts/0231-biometric-service-provider/README.md
@@ -5,7 +5,7 @@
 - Since: 2019-09-24
 - Status Note: Proposed for now based on authors joint work
 - Start Date: 2019-06-01
-- Tags: concept
+- Tags: [concept](/tags.md#concept)
 
 ## Summary
 

diff --git a/concepts/0250-rich-schemas/README.md b/concepts/0250-rich-schemas/README.md
@@ -5,7 +5,7 @@
 - Status Note: Port of [this HIPE](https://github.com/hyperledger/indy-hipe/tree/master/text/0119-rich-schemas/README.md)
 - Supersedes: [this HIPE](https://github.com/hyperledger/indy-hipe/tree/master/text/0119-rich-schemas/README.md)
 - Start Date: 2019-03-19
-- Tags: concept, rich-schemas
+- Tags: [concept](/tags.md#concept), [rich-schemas](/tags.md#rich-schemas)
 
 ## Summary
 

diff --git a/features/0015-acks/README.md b/features/0015-acks/README.md
@@ -6,7 +6,7 @@
 - Status Note: Broadly socialized and beginning to be implemented. Several protocols assume ACK behavior. May be nearing the maturity and uptake appropriate for ACCEPTED status. Note: this RFC 
 - Supersedes: [Indy HIPE PR #77](https://github.com/hyperledger/indy-hipe/pull/77)
 - Start Date: 2018-12-26
-- Tags: feature, decorator
+- Tags: [feature](/tags.md#feature), [decorator](/tags.md#decorator)
 
 ## Summary
 

diff --git a/features/0019-encryption-envelope/README.md b/features/0019-encryption-envelope/README.md
@@ -6,7 +6,7 @@
 - Status Note:  
 - Supersedes: [INDY 0028 Wire Level Format](https://github.com/hyperledger/indy-hipe/tree/master/text/0028-wire-message-format)
 - Start Date: 2018-07-10 (approximate, backdated)
-- Tags: feature
+- Tags: [feature](/tags.md#feature)
 
 ## Summary