Skip to content

Commit

Permalink
Merge pull request #859 from SmithSamuelM/fix_del_ddos
Browse files Browse the repository at this point in the history
Extensive changes to the delegation validation and escrow logic. Extensive refactoring of the classes in subing.py
  • Loading branch information
SmithSamuelM authored Sep 9, 2024
2 parents 9d9a481 + 9f64bdf commit dd6e9f7
Show file tree
Hide file tree
Showing 33 changed files with 5,457 additions and 3,241 deletions.
16 changes: 8 additions & 8 deletions src/keri/app/cli/commands/escrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ def escrows(tymth, tock=0.0, **opts):
oots = list()
key = ekey = b'' # both start same. when not same means escrows found
while True:
for ekey, edig in hby.db.getOoeItemsNextIter(key=key):
pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item
for ekey, edig in hby.db.getOoeItemIter(key=key):
pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item

try:
oots.append(eventing.loadEvent(hby.db, pre, edig))
Expand All @@ -74,8 +74,8 @@ def escrows(tymth, tock=0.0, **opts):
pwes = list()
key = ekey = b'' # both start same. when not same means escrows found
while True: # break when done
for ekey, edig in hby.db.getPweItemsNextIter(key=key):
pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item
for ekey, edig in hby.db.getPweItemIter(key=key):
pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item

try:
pwes.append(eventing.loadEvent(hby.db, pre, edig))
Expand All @@ -92,8 +92,8 @@ def escrows(tymth, tock=0.0, **opts):
pses = list()
key = ekey = b'' # both start same. when not same means escrows found
while True: # break when done
for ekey, edig in hby.db.getPseItemsNextIter(key=key):
pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item
for ekey, edig in hby.db.getPseItemIter(key=key):
pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item

try:
pses.append(eventing.loadEvent(hby.db, pre, edig))
Expand All @@ -110,8 +110,8 @@ def escrows(tymth, tock=0.0, **opts):
ldes = list()
key = ekey = b'' # both start same. when not same means escrows found
while True: # break when done
for ekey, edig in hby.db.getLdeItemsNextIter(key=key):
pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item
for ekey, edig in hby.db.getLdeItemIter(key=key):
pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item

try:
ldes.append(eventing.loadEvent(hby.db, pre, edig))
Expand Down
4 changes: 2 additions & 2 deletions src/keri/app/cli/commands/ipex/grant.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ def grantDo(self, tymth, tock=0.0):
iserder = serdering.SerderKERI(raw=bytes(iss))
seqner = coring.Seqner(sn=iserder.sn)

serder = self.hby.db.findAnchoringSealEvent(creder.sad['i'],
seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said))
serder = self.hby.db.fetchAllSealingEventByEventSeal(creder.sad['i'],
seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said))
anc = self.hby.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said)

exn, atc = protocoling.ipexGrantExn(hab=self.hab, recp=recp, message=self.message, acdc=acdc, iss=iss, anc=anc,
Expand Down
2 changes: 1 addition & 1 deletion src/keri/app/cli/commands/migrate.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ def migrateKeys(db):
digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.",
klas=(coring.Prefixer, coring.Seqner))

for pre, fn, dig in db.getFelItemAllPreIter(key=b''):
for pre, fn, dig in db.getFelItemAllPreIter():
dgkey = dbing.dgKey(pre, dig) # get message
if not (raw := db.getEvt(key=dgkey)):
raise kering.MissingEntryError("Missing event for dig={}.".format(dig))
Expand Down
2 changes: 1 addition & 1 deletion src/keri/app/connecting.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ def setImg(self, pre, typ, stream):
stream (file): file-like stream of image data
"""
self.hby.db.delTopVal(db=self.hby.db.imgs, key=pre.encode("utf-8"))
self.hby.db.delTopVal(db=self.hby.db.imgs, top=pre.encode("utf-8"))

key = f"{pre}.content-type".encode("utf-8")
self.hby.db.setVal(db=self.hby.db.imgs, key=key, val=typ.encode("utf-8"))
Expand Down
2 changes: 1 addition & 1 deletion src/keri/app/delegating.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ def processUnanchoredEscrow(self):
dkever = self.hby.kevers[kever.delpre]

seal = dict(i=serder.pre, s=serder.snh, d=serder.said)
if dserder := self.hby.db.findAnchoringSealEvent(dkever.prefixer.qb64, seal=seal):
if dserder := self.hby.db.fetchAllSealingEventByEventSeal(dkever.prefixer.qb64, seal=seal):
seqner = coring.Seqner(sn=dserder.sn)
couple = seqner.qb64b + dserder.saidb
dgkey = dbing.dgKey(kever.prefixer.qb64b, kever.serder.saidb)
Expand Down
3 changes: 2 additions & 1 deletion src/keri/app/grouping.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,8 @@ def processDelegateEscrow(self):
self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider)

else: # Not witnesser, we need to look for the anchor and then wait for receipts
if serder := self.hby.db.findAnchoringSealEvent(kever.delpre, seal=anchor):
if serder := self.hby.db.fetchAllSealingEventByEventSeal(kever.delpre,
seal=anchor):
aseq = coring.Seqner(sn=serder.sn)
couple = aseq.qb64b + serder.saidb
dgkey = dbing.dgKey(pre, saider.qb64b)
Expand Down
7 changes: 5 additions & 2 deletions src/keri/app/habbing.py
Original file line number Diff line number Diff line change
Expand Up @@ -1538,7 +1538,7 @@ def replay(self, pre=None, fn=0):

return msgs

def replayAll(self, key=b''):
def replayAll(self):
"""
Returns replay of FEL first seen event log for all pre starting at key
Expand All @@ -1547,7 +1547,7 @@ def replayAll(self, key=b''):
"""
msgs = bytearray()
for msg in self.db.cloneAllPreIter(key=key):
for msg in self.db.cloneAllPreIter():
msgs.extend(msg)
return msgs

Expand Down Expand Up @@ -2284,6 +2284,9 @@ def make(self, *, secrecies=None, iridx=0, code=coring.MtrDex.Blake3_256, dcode=
# may want db method that updates .habs. and .prefixes together
habord = basing.HabitatRecord(hid=self.pre, name=self.name, domain=self.ns)

# must add self.pre to self.prefixes before calling processEvent so that
# Kever.locallyOwned or Kever.locallyDelegated or Kever.locallyWitnessed
# evaluates correctly when processing own inception event.
if not hidden:
self.save(habord)
self.prefixes.add(self.pre)
Expand Down
3 changes: 2 additions & 1 deletion src/keri/app/notifying.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ def getItemIter(self, keys: Union[str, Iterable] = b""):
each entry in db
"""
for key, val in self.db.getAllItemIter(db=self.sdb, key=self._tokey(keys), split=False):
for key, val in self.db.getTopItemIter(db=self.sdb,
top=self._tokey(keys)):
yield self._tokeys(key), self.klas(raw=bytes(val))

def cntAll(self):
Expand Down
2 changes: 1 addition & 1 deletion src/keri/app/querying.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def recur(self, tyme, deeds=None):
return False

kever = self.hab.kevers[self.pre]
if self.hby.db.findAnchoringSealEvent(self.pre, seal=self.anchor):
if self.hby.db.fetchAllSealingEventByEventSeal(self.pre, seal=self.anchor):
self.remove([self.witq])
return True

Expand Down
111 changes: 68 additions & 43 deletions src/keri/app/storing.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,17 @@ def __init__(self, name="mbx", headDirPath=None, reopen=True, **kwa):
perm:
reopen:
kwa:
Mailboxer uses two dbs for mailbox messages these are .tpcs and .msgs.
The message index is in .tpcs (topics).
Each .tpcs index key consists of topic.on where topic is bytes
identifier or prefix/topic for message and on is serialized
ordinal number to orders the appearance of a topic message.
Eash .tpcs val is the digest of the message.
The message itself is stored in .msgs where the key is the msg digest
and the value is the serialized messag itself.
Multiple messages can share the same topic but with a different ordinal.
"""
self.tpcs = None
self.msgs = None
Expand All @@ -47,88 +58,102 @@ def reopen(self, **kwa):
:return:
"""
super(Mailboxer, self).reopen(**kwa)

self.tpcs = self.env.open_db(key=b'tpcs.', dupsort=True)
self.tpcs = subing.OnSuber(db=self, subkey='tpcs.')
self.msgs = subing.Suber(db=self, subkey='msgs.') # key states

return self.env

def delTopic(self, key):
"""
Use snKey()
Deletes value at key.
Returns True If key exists in database Else False
def delTopic(self, key, on=0):
"""Removes topic index from .tpcs without deleting message from .msgs
Returns:
result (boo): True if full key consisting of key and serialized on
exists in database so removed
False otherwise (not removed)
"""
return self.delIoSetVals(self.tpcs, key)
return self.tpcs.remOn(keys=key, on=on)

def appendToTopic(self, topic, val):
"""
Return first seen order number int, fn, of appended entry.
Computes fn as next fn after last entry.
"""Appends val to end of db entries with same topic but with on
incremented by 1 relative to last preexisting entry at topic.
Append val to end of db entries with same topic but with fn incremented by
1 relative to last preexisting entry at pre.
Returns:
on (int): order number int, on, of appended entry.
Computes on as next on after last entry.
Parameters:
topic is bytes identifier prefix/topic for message
val is event digest
topic (bytes): topic identifier for message
val (bytes): msg digest
"""
return self.appendIoSetVal(db=self.tpcs, key=topic, val=val)
return self.tpcs.appendOn(key=topic, val=val)


def getTopicMsgs(self, topic, fn=0):
"""
Returns:
ioset (oset): the insertion ordered set of values at same apparent
effective key.
Uses hidden ordinal key suffix for insertion ordering.
The suffix is appended and stripped transparently.
msgs (Iterable[bytes]): belonging to topic indices with same topic but all
on >= fn i.e. all topic.on beginning with fn
Parameters:
topic (Option(bytes|str)): Apparent effective key
fn (int) starting index
topic (Option(bytes|str)): key prefix combined with serialized on
to form full actual key. When key is empty then retrieves
whole database.
fn (int): starting index ordinal number used with onKey(pre,on)
to form key at at which to initiate retrieval
"""
if hasattr(topic, "encode"):
topic = topic.encode("utf-8")

digs = self.getIoSetVals(db=self.tpcs, key=topic, ion=fn)
msgs = []
for dig in digs:
for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn):
if msg := self.msgs.get(keys=dig):
msgs.append(msg.encode("utf-8"))
msgs.append(msg.encode()) # want bytes not str
return msgs


def storeMsg(self, topic, msg):
"""
Add exn event to mailbox of dest identifier
Add exn event to mailbox topic and on that is 1 greater than last msg
at topic.
Returns:
result (bool): True if msg successfully stored and indexed at topic
False otherwise
Parameters:
msg (bytes):
topic (qb64b):
topic (str | bytes): topic (Option(bytes|str)): key prefix combined
with serialized on to form full actual key.
msg (bytes): serialized message
"""
if hasattr(topic, "encode"):
topic = topic.encode("utf-8")

if hasattr(msg, "encode"):
msg = msg.encode("utf-8")

digb = coring.Diger(ser=msg, code=MtrDex.Blake3_256).qb64b
self.appendToTopic(topic=topic, val=digb)
on = self.tpcs.appendOn(keys=topic, val=digb)
return self.msgs.pin(keys=digb, val=msg)


def cloneTopicIter(self, topic, fn=0):
"""
Returns iterator of first seen exn messages with attachments for the
identifier prefix pre starting at first seen order number, fn.
Returns:
triple (Iterator[(on, topic, msg): iterator of messages at topic
beginning with ordinal fn.
"""
if hasattr(topic, 'encode'):
topic = topic.encode("utf-8")
topic (Option(bytes|str)): key prefix combined with serialized on
to form full actual key. When key is empty then retrieves
whole database.
fn (int): starting index ordinal number used with onKey(pre,on)
to form key at at which to initiate retrieval
for (key, dig) in self.getIoSetItemsIter(self.tpcs, key=topic, ion=fn):
topic, ion = dbing.unsuffix(key)
ToDo looks like misuse of IoSet this should not be IoSet but simply
Ordinal Numbered db. since should not be using hidden ion has not
hidden.
"""
for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn):
if msg := self.msgs.get(keys=dig):
yield ion, topic, msg.encode("utf-8")
yield (on, topic, msg.encode("utf-8"))



class Respondant(doing.DoDoer):
Expand Down
Loading

0 comments on commit dd6e9f7

Please sign in to comment.