From 0fa5e6985ba301459d58ed2ab867491e8d3f5513 Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Mon, 29 Jan 2024 09:40:59 -0500 Subject: [PATCH 001/418] Fixed spelling typo in help text for --no-backers (#661) --- src/keri/app/cli/commands/vc/registry/incept.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/app/cli/commands/vc/registry/incept.py b/src/keri/app/cli/commands/vc/registry/incept.py index 77c774f9a..ac0a01ba8 100644 --- a/src/keri/app/cli/commands/vc/registry/incept.py +++ b/src/keri/app/cli/commands/vc/registry/incept.py @@ -22,7 +22,7 @@ default=None) parser.add_argument('--nonce', help='Unique random value to seed the credential registry', default=None, required=False) -parser.add_argument("--no-backers", "-nb", help="do not allow setting up backers different from the ahcnoring KEL " +parser.add_argument("--no-backers", "-nb", help="do not allow setting up backers different from the anchoring KEL " "witnesses", default=True, action="store") parser.add_argument('--backers', help='New set of backers different from the anchoring KEL witnesses. Can ' 'appear multiple times', metavar="", default=[], action="append", From ae774b135585f850415f109c1fd0b294e823a2c9 Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Mon, 29 Jan 2024 09:41:19 -0500 Subject: [PATCH 002/418] Updated build-witness-demo command. No gleif/keri-1.1.0 on dockerhub but latest probably fulfills the same requirement (#660) --- images/witness.demo.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/witness.demo.dockerfile b/images/witness.demo.dockerfile index f19b068ac..f1d554d40 100644 --- a/images/witness.demo.dockerfile +++ b/images/witness.demo.dockerfile @@ -1,4 +1,4 @@ -FROM gleif/keri:1.1.0 +FROM gleif/keri:latest SHELL ["/bin/bash", "-c"] EXPOSE 5632 From e9c09f9b2e6946b8d5416ae9ed384d59364ffb30 Mon Sep 17 00:00:00 2001 From: clackwork <124748154+clackwork@users.noreply.github.com> Date: Mon, 12 Feb 2024 13:58:44 -0800 Subject: [PATCH 003/418] Use image python:3.10-alpine (#671) --- images/keripy.dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/images/keripy.dockerfile b/images/keripy.dockerfile index 6a155c28e..42d9acf8a 100644 --- a/images/keripy.dockerfile +++ b/images/keripy.dockerfile @@ -1,5 +1,5 @@ # Builder layer -FROM python:3.10.13-alpine3.18 as builder +FROM python:3.10-alpine as builder # Install compilation dependencies RUN apk --no-cache add \ @@ -24,7 +24,8 @@ RUN pip install --upgrade pip && \ mkdir /keripy/src # Copy Python dependency files in -COPY requirements.txt setup.py . +COPY requirements.txt setup.py ./ + # Set up Rust environment and install Python dependencies # Must source the Cargo environment for the blake3 library to see # the Rust intallation during requirements install From 891d5b32dc09f0e2029a4b33bf19cfd62534bd57 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Tue, 13 Feb 2024 06:10:58 -0800 Subject: [PATCH 004/418] Revert "Use image python:3.10-alpine (#671)" (#677) This reverts commit e9c09f9b2e6946b8d5416ae9ed384d59364ffb30. --- images/keripy.dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/images/keripy.dockerfile b/images/keripy.dockerfile index 42d9acf8a..6a155c28e 100644 --- a/images/keripy.dockerfile +++ b/images/keripy.dockerfile @@ -1,5 +1,5 @@ # Builder layer -FROM python:3.10-alpine as builder +FROM python:3.10.13-alpine3.18 as builder # Install compilation dependencies RUN apk --no-cache add \ @@ -24,8 +24,7 @@ RUN pip install --upgrade pip && \ mkdir /keripy/src # Copy Python dependency files in -COPY requirements.txt setup.py ./ - +COPY requirements.txt setup.py . # Set up Rust environment and install Python dependencies # Must source the Cargo environment for the blake3 library to see # the Rust intallation during requirements install From 3e436190d5d67e9e6ab2fde121fdfe2356d4abdb Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Wed, 14 Feb 2024 15:54:55 -0800 Subject: [PATCH 005/418] Update to Python 3.12.1 (#679) Signed-off-by: pfeairheller --- .github/workflows/python-app-ci.yml | 12 ++++++------ .readthedocs.yaml | 2 +- README.md | 4 ++-- setup.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python-app-ci.yml b/.github/workflows/python-app-ci.yml index 2df15d859..98751e757 100644 --- a/.github/workflows/python-app-ci.yml +++ b/.github/workflows/python-app-ci.yml @@ -21,10 +21,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10.4 + - name: Set up Python 3.12.1 uses: actions/setup-python@v2 with: - python-version: 3.10.4 + python-version: 3.12.1 - name: Install dependencies run: | python -m pip install --upgrade pip @@ -47,10 +47,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10.4 + - name: Set up Python 3.12.1 uses: actions/setup-python@v2 with: - python-version: 3.10.4 + python-version: 3.12.1 - name: Install dependencies run: | python -m pip install --upgrade pip @@ -68,10 +68,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10.4 + - name: Set up Python 3.12.1 uses: actions/setup-python@v2 with: - python-version: 3.10.4 + python-version: 3.12.1 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.readthedocs.yaml b/.readthedocs.yaml index d50f9361f..43e7e2fed 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,7 +11,7 @@ build: apt_packages: - libsodium23 tools: - python: "3.10.4" + python: "3.12.1" # You can also specify other tool versions: # nodejs: "16" # rust: "1.55" diff --git a/README.md b/README.md index 9e9771724..fc2add973 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Then run `docker run -it gleif/keri /bin/bash` and you can run `kli version` fro ### Dependencies #### Binaries -python 3.10.4+ +python 3.12.1+ libsodium 1.0.18+ @@ -63,7 +63,7 @@ $ pip3 install -U cbor2 ## Development ### Setup -* Ensure Python 3.10.4 is present along with venv and dev header files; +* Ensure Python 3.12.1 is present along with venv and dev header files; * Setup virtual environment: `python3 -m venv keripy` * Activate virtual environment: `source keripy/bin/activate` * Setup dependencies: `pip install -r requirements.txt` diff --git a/setup.py b/setup.py index 7341c6bff..34d89c7a8 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ keywords=[ # eg: 'keyword1', 'keyword2', 'keyword3', ], - python_requires='>=3.10.4', + python_requires='>=3.12.1', install_requires=[ 'lmdb>=1.3.0', 'pysodium>=0.7.12', From 82a202dd2aedb3c566260fb79dcf33441cbc5092 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Thu, 15 Feb 2024 10:17:07 -0800 Subject: [PATCH 006/418] Superseding Delegated Rotation and misfit escrow (#680) * renamed tests * created test to test supersedeing logic. Just setting up for now not done. * setting up superseding delegation tests * more work on superseding test * add more to supersede test work, plus some cleanup * added detailed doc string on acceptance rules for delegated events added cleanup of cue triggers * more details in doc string * fixed clarification * added additional doc string language * fixed typo * updated superseding logic to include virtual delegating event when local delegator is processing delegated event acceptance into its local copy of delegatees KEL so as not to corrupt its copy. * updated doc string * more doc string goodness * some refactoring. remaned Kever.delegator to .delpre to follow convention and the uses renamed. * added event source record Komer .esrs subdb to Baser so we can track source of event local or remote * Passing event source to processEvent from all escrow processing routines. Signed-off-by: pfeairheller * updated doc string for Baser for .esrs subdb * rebasing misfit Signed-off-by: pfeairheller * Add local persistence to escrowing methods. Signed-off-by: pfeairheller * added code in Kever to update event source record * changed parser logic to support local * Fix processEscrow events to use correct key for esr Signed-off-by: pfeairheller * updated keys for .esrs usage * clean up fix merge fail * added misfit event escrow database .mfes as CesrIoSetSuber instance. * habery .__init__ creates its default .psr parser with local = True * added method to Kevery.escrowMFEvent for escrowing misfit events into .mfes * rebasing misfit Signed-off-by: pfeairheller * fixed unit test and logic for misfit processing * some clean up refactor in Kever. Added local parameter to process receipt methods in Kevery. * some cleanup on doc strings and comments * updated locallyWitnessed to return boolean * Fix tests that parse events with a lax Kevery to now use `local=true` in the call to parse. Signed-off-by: pfeairheller * removed redundant misfit checks refactored .locallWitnessed method other clean up * removed potentially buggy redundant inject of db.prefixes into Kever.prefixes as attribute and replaced with property that uses .db.prefixes instead * added Baser.gids as oset of Group Hab pres (hids) * changed Baser.mfes to .misfits since CesrIoSetSuber attribute name does not need to match subkey .mfes x * added test for prefix in locallyOwned to not be in db.groups. Changes db.gids to .groups * some refactor of methos names to distinquish processing of the attached equivalent of receipt versus the actual rct messages enabled the new local pass in from parser for rct messages. * added code to filter out local signatures for group messages received remotely (nonlocal not protected) * refactor Kever.locallyOwned to default to self.prefixer.qb64 if no prefix passed in * updated code comment * added cue for remoteMemberedSig to notify when recieving own signature on group multisig event from unprotected source * refactor logic to set stage for delegation escrow to do the approval of delegation by delegator and any sandboxing needed or cues to business logic for approval. * added db Baser.delegables subkey .dees for delegable escrowed events awaiting delegator approval. Still needs processDelegableEscrows but now Kever escrows them * updated comments * rebasing misfit Signed-off-by: pfeairheller * started changing Number class * more unit texts for Number * added .validate method to Number to validate if number is ordinal * added more tests. Replaced some Seqner usages * cleanup Seqner and tests Added .seqner property to Number and tests * removed validateSN function. replaced with Number.validate * Fix delegator escrow test to match new expectation that delegate have delegator's pre in kevers. Signed-off-by: pfeairheller * Comment delegation script until kli delegation commands can be fixed. Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller Co-authored-by: Samuel M Smith --- scripts/demo/test_scripts.sh | 6 +- src/keri/app/agenting.py | 4 +- src/keri/app/cli/commands/delegate/confirm.py | 2 +- src/keri/app/cli/commands/delegate/request.py | 4 +- src/keri/app/cli/commands/incept.py | 4 +- .../app/cli/commands/multisig/continue.py | 2 +- src/keri/app/cli/commands/multisig/incept.py | 2 +- src/keri/app/cli/commands/multisig/rotate.py | 2 +- src/keri/app/cli/commands/rotate.py | 4 +- src/keri/app/cli/common/displaying.py | 4 +- src/keri/app/delegating.py | 6 +- src/keri/app/directing.py | 4 +- src/keri/app/forwarding.py | 2 +- src/keri/app/grouping.py | 6 +- src/keri/app/habbing.py | 44 +- src/keri/app/indirecting.py | 8 +- src/keri/core/coring.py | 185 ++- src/keri/core/eventing.py | 1297 ++++++++++++----- src/keri/core/parsing.py | 120 +- src/keri/core/routing.py | 1 + src/keri/db/basing.py | 234 +-- src/keri/db/dbing.py | 5 +- src/keri/db/subing.py | 19 +- src/keri/kering.py | 18 + src/keri/vdr/credentialing.py | 2 +- src/keri/vdr/eventing.py | 35 +- tests/app/__init__.py | 6 +- tests/app/test_delegating.py | 14 +- tests/app/test_forwarding.py | 6 +- tests/app/test_grouping.py | 80 +- tests/app/test_habbing.py | 9 +- tests/core/test_coring.py | 618 ++++++-- tests/core/test_delegating.py | 445 +++++- tests/core/test_escrow.py | 41 +- tests/core/test_eventing.py | 20 +- tests/core/test_kevery.py | 26 +- tests/core/test_keystate.py | 43 +- tests/core/test_parsing.py | 7 +- tests/core/test_reply.py | 14 +- tests/core/test_witness.py | 70 +- tests/db/test_basing.py | 61 +- tests/vc/test_protocoling.py | 1 + tests/vdr/test_eventing.py | 6 +- tests/vdr/test_txn_state.py | 14 +- 44 files changed, 2628 insertions(+), 873 deletions(-) diff --git a/scripts/demo/test_scripts.sh b/scripts/demo/test_scripts.sh index 0463eb8e5..1b40d0a79 100755 --- a/scripts/demo/test_scripts.sh +++ b/scripts/demo/test_scripts.sh @@ -47,10 +47,10 @@ printf "\n************************************\n" isSuccess printf "\n************************************\n" -printf "Running multisig-delegate-delegator.sh" +printf "Skipping multisig-delegate-delegator.sh" printf "\n************************************\n" -"${script_dir}/basic/multisig-delegate-delegator.sh" -isSuccess +#"${script_dir}/basic/multisig-delegate-delegator.sh" +#isSuccess printf "\n************************************\n" printf "Running challenge.sh" diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 5683b8cea..68dccba80 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -718,7 +718,7 @@ def msgDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield from self.parser.parsator() # process messages continuously + yield from self.parser.parsator(local=True) # process messages continuously @property def idle(self): @@ -812,7 +812,7 @@ def msgDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield from self.parser.parsator() # process messages continuously + yield from self.parser.parsator(local=True) # process messages continuously @property def idle(self): diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index c5f90b481..19fe3eea5 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -111,7 +111,7 @@ def confirmDo(self, tymth, tock=0.0): elif ilk in (coring.Ilks.drt,): typ = "rotation" dkever = self.hby.kevers[eserder.pre] - delpre = dkever.delegator + delpre = dkever.delpre else: continue diff --git a/src/keri/app/cli/commands/delegate/request.py b/src/keri/app/cli/commands/delegate/request.py index 85d6cbd0f..8cd07f0ca 100644 --- a/src/keri/app/cli/commands/delegate/request.py +++ b/src/keri/app/cli/commands/delegate/request.py @@ -86,7 +86,7 @@ def requestDo(self, tymth, tock=0.0): (seqner, saider) = esc[0] evt = hab.makeOwnEvent(sn=seqner.sn) - delpre = hab.kever.delegator # get the delegator identifier + delpre = hab.kever.delpre # get the delegator identifier if isinstance(hab, GroupHab): phab = hab.mhab @@ -99,7 +99,7 @@ def requestDo(self, tymth, tock=0.0): srdr = serdering.SerderKERI(raw=evt) # coring.Serder(raw=evt) del evt[:srdr.size] self.postman.send(src=phab.pre, dest=delpre, topic="delegate", serder=srdr, attachment=evt) - self.postman.send(src=phab.pre, dest=hab.kever.delegator, topic="delegate", serder=exn, attachment=atc) + self.postman.send(src=phab.pre, dest=hab.kever.delpre, topic="delegate", serder=exn, attachment=atc) while True: while self.postman.cues: diff --git a/src/keri/app/cli/commands/incept.py b/src/keri/app/cli/commands/incept.py index fbc59560a..29ff39cf6 100644 --- a/src/keri/app/cli/commands/incept.py +++ b/src/keri/app/cli/commands/incept.py @@ -168,7 +168,7 @@ def inceptDo(self, tymth, tock=0.0): receiptor = agenting.Receiptor(hby=self.hby) self.extend([witDoer, receiptor]) - if hab.kever.delegator: + if hab.kever.delpre: self.swain.delegation(pre=hab.pre, sn=0, proxy=self.hby.habByName(self.proxy)) print("Waiting for delegation approval...") while not self.swain.complete(hab.kever.prefixer, coring.Seqner(sn=hab.kever.sn)): @@ -183,7 +183,7 @@ def inceptDo(self, tymth, tock=0.0): while not witDoer.cues: _ = yield self.tock - if hab.kever.delegator: + if hab.kever.delpre: yield from self.postman.sendEvent(hab=hab, fn=hab.kever.sn) print(f'Prefix {hab.pre}') diff --git a/src/keri/app/cli/commands/multisig/continue.py b/src/keri/app/cli/commands/multisig/continue.py index 2a2508f97..e38746711 100644 --- a/src/keri/app/cli/commands/multisig/continue.py +++ b/src/keri/app/cli/commands/multisig/continue.py @@ -65,7 +65,7 @@ def recover(self, tymth, tock=0.0, **opts): (seqner, saider) = esc[0] src = hab.mhab.pre if isinstance(hab, GroupHab) else hab.pre anchor = dict(i=hab.pre, s=seqner.snh, d=saider.qb64) - self.witq.query(src=src, pre=hab.kever.delegator, anchor=anchor) + self.witq.query(src=src, pre=hab.kever.delpre, anchor=anchor) print(f"Checking mailboxes for any events to process") while self.hby.db.cgms.get(keys=(hab.pre, seqner.qb64)) is None: diff --git a/src/keri/app/cli/commands/multisig/incept.py b/src/keri/app/cli/commands/multisig/incept.py index 1c589ab53..30face97f 100644 --- a/src/keri/app/cli/commands/multisig/incept.py +++ b/src/keri/app/cli/commands/multisig/incept.py @@ -174,7 +174,7 @@ def inceptDo(self, tymth, tock=0.0): yield self.tock - if ghab.kever.delegator: + if ghab.kever.delpre: yield from self.postman.sendEvent(hab=ghab, fn=ghab.kever.sn) print() diff --git a/src/keri/app/cli/commands/multisig/rotate.py b/src/keri/app/cli/commands/multisig/rotate.py index 668a78267..0cb87c5c1 100644 --- a/src/keri/app/cli/commands/multisig/rotate.py +++ b/src/keri/app/cli/commands/multisig/rotate.py @@ -233,7 +233,7 @@ def rotateDo(self, tymth, tock=0.0, **opts): yield self.tock - if ghab.kever.delegator: + if ghab.kever.delpre: yield from self.postman.sendEvent(hab=ghab, fn=ghab.kever.sn) print() diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index 925bc9b3b..17a45208c 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -182,7 +182,7 @@ def rotateDo(self, tymth, tock=0.0): cuts=list(self.cuts), adds=list(self.adds), data=self.data) - if hab.kever.delegator: + if hab.kever.delpre: self.swain.delegation(pre=hab.pre, sn=hab.kever.sn) print("Waiting for delegation approval...") while not self.swain.complete(hab.kever.prefixer, coring.Seqner(sn=hab.kever.sn)): @@ -206,7 +206,7 @@ def rotateDo(self, tymth, tock=0.0): self.remove([witDoer]) - if hab.kever.delegator: + if hab.kever.delpre: yield from self.postman.sendEvent(hab=hab, fn=hab.kever.sn) print(f'Prefix {hab.pre}') diff --git a/src/keri/app/cli/common/displaying.py b/src/keri/app/cli/common/displaying.py index 65745f2cf..55ece7287 100644 --- a/src/keri/app/cli/common/displaying.py +++ b/src/keri/app/cli/common/displaying.py @@ -34,7 +34,7 @@ def printIdentifier(hby, pre, label="Identifier"): print("Seq No:\t{}".format(kever.sner.num)) if kever.delegated: print("Delegated Identifier") - sys.stdout.write(f" Delegator: {kever.delegator} ") + sys.stdout.write(f" Delegator: {kever.delpre} ") if seal: print(f"{terming.Colors.OKGREEN}{terming.Symbols.CHECKMARK} Anchored{terming.Colors.ENDC}") else: @@ -93,7 +93,7 @@ def printExternal(hby, pre, label="Identifier"): print("Seq No:\t{}".format(kever.sner.num)) if kever.delegated: print("Delegated Identifier") - sys.stdout.write(f" Delegator: {kever.delegator} ") + sys.stdout.write(f" Delegator: {kever.delpre} ") if anchor: print(f"{terming.Colors.OKGREEN}{terming.Symbols.CHECKMARK} Anchored{terming.Colors.ENDC}") else: diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 433332128..9ee457228 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -55,7 +55,7 @@ def delegation(self, pre, sn=None, proxy=None): # load the hab of the delegated identifier to anchor hab = self.hby.habs[pre] - delpre = hab.kever.delegator # get the delegator identifier + delpre = hab.kever.delpre # get the delegator identifier if delpre not in hab.kevers: raise kering.ValidationError(f"delegator {delpre} not found, unable to process delegation") @@ -81,7 +81,7 @@ def delegation(self, pre, sn=None, proxy=None): # Send exn message for notification purposes exn, atc = delegateRequestExn(phab, delpre=delpre, evt=bytes(evt), aids=smids) - self.postman.send(hab=phab, dest=hab.kever.delegator, topic="delegate", serder=exn, attachment=atc) + self.postman.send(hab=phab, dest=hab.kever.delpre, topic="delegate", serder=exn, attachment=atc) srdr = serdering.SerderKERI(raw=evt) del evt[:srdr.size] @@ -151,7 +151,7 @@ def processUnanchoredEscrow(self): """ for (pre, said), serder in self.hby.db.dune.getItemIter(): # group partial witness escrow kever = self.hby.kevers[pre] - dkever = self.hby.kevers[kever.delegator] + dkever = self.hby.kevers[kever.delpre] seal = dict(i=serder.pre, s=serder.snh, d=serder.said) if dserder := self.hby.db.findAnchoringSealEvent(dkever.prefixer.qb64, seal=seal): diff --git a/src/keri/app/directing.py b/src/keri/app/directing.py index 70bd96313..6c72fc987 100644 --- a/src/keri/app/directing.py +++ b/src/keri/app/directing.py @@ -230,7 +230,7 @@ def msgDo(self, tymth=None, tock=0.0, **opts): yield # enter context if self.parser.ims: logger.info("Client %s received:\n%s\n...\n", self.hab.name, self.parser.ims[:1024]) - done = yield from self.parser.parsator() # process messages continuously + done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close @@ -585,7 +585,7 @@ def msgDo(self, tymth=None, tock=0.0, **opts): if self.parser.ims: logger.info("Server %s: received:\n%s\n...\n", self.hab.name, self.parser.ims[:1024]) - done = yield from self.parser.parsator() # process messages continuously + done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close diff --git a/src/keri/app/forwarding.py b/src/keri/app/forwarding.py index 748a09a6a..2356ebc19 100644 --- a/src/keri/app/forwarding.py +++ b/src/keri/app/forwarding.py @@ -143,7 +143,7 @@ def sendEvent(self, hab, fn=0): del icp[:ser.size] sender = hab.mhab.pre if isinstance(hab, GroupHab) else hab.pre - self.send(src=sender, dest=hab.kever.delegator, topic="delegate", serder=ser, attachment=icp) + self.send(src=sender, dest=hab.kever.delpre, topic="delegate", serder=ser, attachment=icp) while True: if self.cues: cue = self.cues.popleft() diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 1607ee29f..0495ce9f3 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -138,9 +138,9 @@ def processPartialSignedEscrow(self): else: anchor = dict(i=pre, s=seqner.snh, d=saider.qb64) if self.proxy: - self.witq.query(hab=self.proxy, pre=kever.delegator, anchor=anchor) + self.witq.query(hab=self.proxy, pre=kever.delpre, anchor=anchor) else: - self.witq.query(src=ghab.mhab.pre, pre=kever.delegator, anchor=anchor) + self.witq.query(src=ghab.mhab.pre, pre=kever.delpre, anchor=anchor) print("Waiting for delegation approval...") self.hby.db.gdee.add(keys=(pre,), val=(seqner, saider)) @@ -175,7 +175,7 @@ def processDelegateEscrow(self): self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) else: # Not witnesser, we need to look for the anchor and then wait for receipts - if serder := self.hby.db.findAnchoringSealEvent(kever.delegator, seal=anchor): + if serder := self.hby.db.findAnchoringSealEvent(kever.delpre, seal=anchor): aseq = coring.Seqner(sn=serder.sn) couple = aseq.qb64b + serder.saidb dgkey = dbing.dgKey(pre, saider.qb64b) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 98677fd99..66b3698b0 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -129,8 +129,10 @@ class Habery: psr (parsing.Parser): parses local messages for .kvy .rvy habs (dict): Hab instances keyed by prefix. - To look up Hab by name get prefix from db.habs .prefix field using - .habByName + To look up Hab by name use use .habByName + To look up Hab by prefix us .habByPrefix + to get hab from db need name for key + hab prefix in db.habs record .hid field inited (bool): True means fully initialized wrt databases. False means not yet fully initialized @@ -221,7 +223,8 @@ def __init__(self, *, name='test', base="", temp=False, self.exc = exchanging.Exchanger(hby=self, handlers=[]) self.kvy = eventing.Kevery(db=self.db, lax=False, local=True, rvy=self.rvy) self.kvy.registerReplyRoutes(router=self.rtr) - self.psr = parsing.Parser(framed=True, kvy=self.kvy, rvy=self.rvy, exc=self.exc) + self.psr = parsing.Parser(framed=True, kvy=self.kvy, rvy=self.rvy, + exc=self.exc, local=True) self.habs = {} # empty .habs self.namespaces = {} # empty .namespaces self._signator = None @@ -235,6 +238,7 @@ def __init__(self, *, name='test', base="", temp=False, if self.db.opened and self.ks.opened: self.setup(**self._inits) # finish setup later + def setup(self, *, seed=None, aeid=None, bran=None, pidx=None, algo=None, salt=None, tier=None, free=False, temp=None, ): """ @@ -310,6 +314,7 @@ def setup(self, *, seed=None, aeid=None, bran=None, pidx=None, algo=None, self.loadHabs() self.inited = True + def loadHabs(self): """Load Habs instance from db @@ -346,9 +351,9 @@ def loadHabs(self): rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, name=name, pre=pre, temp=self.temp) - # Rules for acceptance - # if its delegated its accepted into its own local KEL even if the - # delegator has not sealed it + # Rules for acceptance: + # It is accepted into its own local KEL even if it has not been fully + # witnessed and if delegated, its delegator has not yet sealed it if not hab.accepted and not habord.mid: raise kering.ConfigurationError(f"Problem loading Hab pre=" f"{pre} name={name} from db.") @@ -382,9 +387,9 @@ def loadHabs(self): rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, name=name, ns=ns, pre=pre, temp=self.temp) - # Rules for acceptance - # if its delegated its accepted into its own local KEL even if the - # delegator has not sealed it + # Rules for acceptance: + # It is accepted into its own local KEL even if it has not been fully + # witnessed and if delegated, its delegator has not yet sealed it if not hab.accepted and not habord.mid: raise kering.ConfigurationError(f"Problem loading Hab pre=" f"{pre} name={name} from db.") @@ -401,6 +406,7 @@ def loadHabs(self): self.reconfigure() # post hab load reconfiguration + def makeHab(self, name, ns=None, cf=None, **kwa): """Make new Hab with name, pre is generated from **kwa @@ -683,6 +689,8 @@ def deleteHab(self, name): del self.habs[hab.pre] self.db.prefixes.remove(hab.pre) + if hab.pre in self.db.groups: + self.db.groups.remove(hab.pre) return True @@ -752,7 +760,8 @@ def prefixes(self): def habByPre(self, pre): """ - Returns the Hab from and namespace including the default namespace. + Returns the Hab instance from .habs or .namespace + including the default namespace. Args: pre (str): qb64 aid of hab to find @@ -773,7 +782,8 @@ def habByPre(self, pre): def habByName(self, name, ns=None): """ Returns: - hab (Hab): instance from .habs by name if any otherwise None + hab (Hab): instance by name from .habs or .namspaces + if any otherwise None Parameters: name (str): alias of Hab @@ -1026,7 +1036,7 @@ def __init__(self, ks, db, cf, mgr, rtr, rvy, kvy, psr, *, self.psr = psr # injected self.name = name - self.ns = ns + self.ns = ns # what is this? self.pre = pre # wait to setup until after db is known to be opened self.temp = True if temp else False @@ -1239,7 +1249,7 @@ def rotate(self, *, verfers=None, digers=None, isith=None, nsith=None, toad=None if not kever.ntholder.satisfy(indices): raise kering.ValidationError("invalid rotation, new key set unable to satisfy prior next signing threshold") - if kever.delegator is not None: # delegator only shows up in delcept + if kever.delpre is not None: # delegator only shows up in delcept serder = eventing.deltate(pre=kever.prefixer.qb64, keys=keys, dig=kever.serder.said, @@ -1493,8 +1503,10 @@ def witness(self, serder): indexed receipt signatures if key state of serder.pre shows that own pre is a current witness of event in serder - Before calling this must check that serder being witnessed has been - accepted as valid event into controller's KEL + ToDo XXXX add parameter to force check that serder as been accepted + as valid. Otherwise must assume that before calling this that serder + being witnessed has been accepted as valid event into this hab + controller's KEL """ if self.kever.prefixer.transferable: # not non-transferable prefix @@ -2144,6 +2156,7 @@ def processCuesIter(self, cues): # ToDo XXXX cue for kin = "psUnescrow" # ToDo XXXX cue for kin = "stream" # ToDo XXXX cue for kin = "invalid" + # ToDo XXXX cue for kin=""remoteMemberedSig"" def witnesser(self): @@ -2651,6 +2664,7 @@ def __init__(self, smids, mhab=None, rmids=None, **kwa): super(GroupHab, self).__init__(**kwa) + def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, nsith=None, toad=None, wits=None, delpre=None, estOnly=False, DnD=False, merfers, migers=None, data=None): diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 96f1f4207..c2aef5a14 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -193,7 +193,7 @@ def msgDo(self, tymth=None, tock=0.0): if self.parser.ims: logger.info("Client %s received:\n%s\n...\n", self.kvy, self.parser.ims[:1024]) - done = yield from self.parser.parsator() # process messages continuously + done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close def escrowDo(self, tymth=None, tock=0.0): @@ -382,7 +382,7 @@ def msgDo(self, tymth=None, tock=0.0): if self.parser.ims: logger.info("Client %s received:\n%s\n...\n", self.hab.pre, self.parser.ims[:1024]) - done = yield from self.parser.parsator() # process messages continuously + done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close def cueDo(self, tymth=None, tock=0.0): @@ -673,7 +673,7 @@ def msgDo(self, tymth=None, tock=0.0): self.tock = tock _ = (yield self.tock) - done = yield from self.parser.parsator() # process messages continuously + done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close def escrowDo(self, tymth=None, tock=0.0): @@ -1072,7 +1072,7 @@ def on_post(self, req, rep): msg = bytearray(serder.raw) msg.extend(cr.attachments.encode("utf-8")) - self.psr.parseOne(ims=msg) + self.psr.parseOne(ims=msg, local=True) if pre in self.hab.kevers: kever = self.hab.kevers[pre] diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 4d58794c3..aedb864d0 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -25,6 +25,8 @@ from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from cryptography.hazmat.primitives.asymmetric import ec, utils +from ..kering import MaxON + from ..kering import (EmptyMaterialError, RawMaterialError, InvalidCodeError, InvalidCodeSizeError, InvalidVarIndexError, InvalidVarSizeError, InvalidVarRawSizeError, @@ -408,7 +410,7 @@ class MatterCodex: Ed25519_Seed: str = 'A' # Ed25519 256 bit random seed for private key Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. - X25519: str = 'C' # X25519 public encryption key, converted from Ed25519 or Ed25519N. + X25519: str = 'C' # X25519 public encryption key, may be converted from Ed25519 or Ed25519N. Ed25519: str = 'D' # Ed25519 verification key basic derivation Blake3_256: str = 'E' # Blake3 256 bit digest self-addressing derivation. Blake2b_256: str = 'F' # Blake2b 256 bit digest self-addressing derivation. @@ -420,7 +422,7 @@ class MatterCodex: X448: str = 'L' # X448 public encryption key, converted from Ed448 Short: str = 'M' # Short 2 byte b2 number Big: str = 'N' # Big 8 byte b2 number - X25519_Private: str = 'O' # X25519 private decryption key converted from Ed25519 + X25519_Private: str = 'O' # X25519 private decryption key/seed, may be converted from Ed25519 X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed ECDSA_256r1_Seed: str = "Q" # ECDSA secp256r1 256 bit random Seed for private key Tall: str = 'R' # Tall 5 byte b2 number @@ -431,7 +433,8 @@ class MatterCodex: Label2: str = 'W' # Label2 as two char (bytes) field map label lead size 0 Tag3: str = 'X' # Tag3 3 B64 encoded chars for field tag or packet type, semver, trait like 'DND' Tag7: str = 'Y' # Tag7 7 B64 encoded chars for field tag or packet kind and version KERIVVV - Salt_128: str = '0A' # 128 bit random salt or 128 bit number (see Huge) + Blind: str = 'Z' # Blinding factor 256 bits, Cryptographic strength deterministically generated from random salt + Salt_128: str = '0A' # random salt/seed/nonce/private key or number of length 128 bits (Huge) Ed25519_Sig: str = '0B' # Ed25519 signature. ECDSA_256k1_Sig: str = '0C' # ECDSA secp256k1 signature. Blake3_512: str = '0D' # Blake3 512 bit digest self-addressing derivation. @@ -585,27 +588,6 @@ def __iter__(self): DigDex = DigCodex() # Make instance -@dataclass(frozen=True) -class NumCodex: - """ - NumCodex is codex of Base64 derivation codes for compactly representing - numbers across a wide rage of sizes. - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Short: str = 'M' # Short 2 byte b2 number - Long: str = '0H' # Long 4 byte b2 number - Big: str = 'N' # Big 8 byte b2 number - Huge: str = '0A' # Huge 16 byte b2 number (same as Salt_128) - - def __iter__(self): - return iter(astuple(self)) - - -NumDex = NumCodex() # Make instance - - @dataclass(frozen=True) @@ -850,6 +832,7 @@ class Matter: 'W': Sizage(hs=1, ss=0, fs=4, ls=0), 'X': Sizage(hs=1, ss=0, fs=4, ls=0), 'Y': Sizage(hs=1, ss=0, fs=8, ls=0), + 'Z': Sizage(hs=1, ss=0, fs=44, ls=0), '0A': Sizage(hs=2, ss=0, fs=24, ls=0), '0B': Sizage(hs=2, ss=0, fs=88, ls=0), '0C': Sizage(hs=2, ss=0, fs=88, ls=0), @@ -1413,15 +1396,16 @@ def _bexfil(self, qb2): class Seqner(Matter): """ - Seqner is subclass of Matter, cryptographic material, for ordinal numbers - such as sequence numbers or first seen ordering numbers. - Seqner provides fully qualified format for ordinals (sequence numbers etc) - when provided as attached cryptographic material elements. + Seqner is subclass of Matter, cryptographic material, for fully qualified + fixed serialization sized ordinal numbers such as sequence numbers or + first seen numbers. - Useful when parsing attached receipt groupings with sn from stream or database + The serialization is forced to a fixed size (single code) so that it may be + used for lexocographically ordered namespaces such as database indices. + That code is MtrDex.Salt_128 - Uses default initialization code = CryTwoDex.Salt_128 - Raises error on init if code not CryTwoDex.Salt_128 + Default initialization code = MtrDex.Salt_128 + Raises error on init if code is not MtrDex.Salt_128 Attributes: @@ -1447,10 +1431,8 @@ class Seqner(Matter): ._infil is method to compute fully qualified Base64 from .raw and .code ._exfil is method to extract .code and .raw from fully qualified Base64 - Methods: - """ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, @@ -1466,16 +1448,32 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, Parameters: - sn is int sequence number or some form of ordinal number - snh is hex string of sequence number + sn (int | str | None): some form of ordinal number int or hex str + snh (str | None): hex string of ordinal number """ if raw is None and qb64b is None and qb64 is None and qb2 is None: - if sn is None: - if snh is None: - sn = 0 - else: - sn = int(snh, 16) + try: + if sn is None: + if snh is None or snh == '': + sn = 0 + else: + sn = int(snh, 16) + + else: # sn is not None but so may be hex str + if isinstance(sn, str): # is it a hex str + if sn == '': + sn = 0 + else: + sn = int(sn, 16) + except ValueError as ex: + raise InvalidValueError(f"Not whole number={sn} .") from ex + + if not isinstance(sn, int) or sn < 0: + raise InvalidValueError(f"Not whole number={sn}.") + + if sn > MaxON: # too big for ordinal 256 ** 16 - 1 + raise ValidationError(f"Non-ordinal {sn} exceeds {MaxON}.") raw = sn.to_bytes(Matter._rawSize(MtrDex.Salt_128), 'big') @@ -1486,6 +1484,7 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, raise ValidationError("Invalid code = {} for Seqner." "".format(self.code)) + @property def sn(self): """ @@ -1503,6 +1502,34 @@ def snh(self): return f"{self.sn:x}" # "{:x}".format(self.sn) + +@dataclass(frozen=True) +class NumCodex: + """ + NumCodex is codex of Base64 derivation codes for compactly representing + numbers across a wide rage of sizes. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Short: str = 'M' # Short 2 byte b2 number + Long: str = '0H' # Long 4 byte b2 number + Tall: str = 'R' # Tall 5 byte b2 number + Big: str = 'N' # Big 8 byte b2 number + Large: str = 'S' # Large 11 byte b2 number + Great: str = 'T' # Great 14 byte b2 number + Huge: str = '0A' # Huge 16 byte b2 number (same as Salt_128) + Vast: str = 'U' # Vast 17 byte b2 number + + def __iter__(self): + return iter(astuple(self)) + + +NumDex = NumCodex() # Make instance + + + + class Number(Matter): """ Number is subclass of Matter, cryptographic material, for ordinal counting @@ -1555,6 +1582,7 @@ class Number(Matter): Methods: """ + Codex = NumDex def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, code=NumDex.Short, num=None, numh=None, **kwa): @@ -1583,8 +1611,6 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, if numh is None or numh == '': num = 0 else: - #if len(numh) > 32: - #raise InvalidValueError(f"Hex numh={numh} str too long.") num = int(numh, 16) else: # handle case where num is hex str' @@ -1592,26 +1618,30 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, if num == '': num = 0 else: - #if len(num) > 32: - #raise InvalidValueError(f"Hex num={num} str too long.") num = int(num, 16) except ValueError as ex: - raise InvalidValueError(f"Invalid whole number={num} .") from ex + raise InvalidValueError(f"Not whole number={num} .") from ex if not isinstance(num, int) or num < 0: - raise InvalidValueError(f"Invalid whole number={num}.") + raise InvalidValueError(f"Not whole number={num}.") if num <= (256 ** 2 - 1): # make short version of code code = NumDex.Short - elif num <= (256 ** 4 - 1): # make long version of code - code = code = NumDex.Long + elif num <= (256 ** 5 - 1): # make tall version of code + code = code = NumDex.Tall elif num <= (256 ** 8 - 1): # make big version of code code = code = NumDex.Big - elif num <= (256 ** 16 - 1): # make huge version of code - code = code = NumDex.Huge + elif num <= (256 ** 11 - 1): # make large version of code + code = code = NumDex.Large + + elif num <= (256 ** 14 - 1): # make great version of code + code = code = NumDex.Great + + elif num <= (256 ** 17 - 1): # make vast version of code + code = code = NumDex.Vast else: raise InvalidValueError(f"Invalid num = {num}, too large to encode.") @@ -1627,6 +1657,44 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, raise ValidationError(f"Invalid code = {self.code} for Number.") + def validate(self, inceptive=None): + """ + Returns: + self (Number): + + Raises: + ValidationError: when .num is invalid ordinal such as + sequence number or first seen number etc. + + Parameters: + inceptive(bool): raise ValidationError whan .num invalid + None means exception when .num < 0 + True means exception when .num != 0 + False means exception when .num < 1 + + """ + num = self.num + + if num > MaxON: # too big for ordinal 256 ** 16 - 1 + raise ValidationError(f"Non-ordinal {num} exceeds {MaxON}.") + + if inceptive is not None: + if inceptive: + if num != 0: + raise ValidationError(f"Nonzero num = {num} non-inceptive" + f" ordinal.") + else: + if num < 1: + raise ValidationError(f"Non-positive num = {num} not " + f"non-inceptive ordinal.") + else: + if num < 0: + raise ValidationError(f"Negative num = {num} non-ordinal.") + + return self + + + @property def num(self): """ @@ -1662,7 +1730,6 @@ def snh(self): return self.numh - @property def positive(self): """ @@ -1672,16 +1739,26 @@ def positive(self): """ return True if self.num > 0 else False + @property def inceptive(self): """ Returns True if .num == 0 False otherwise. - Because valid number .num must be non-negative, positive False means - that .num is zero. + Valid number .num must be non-negative, """ return True if self.num == 0 else False + @property + def seqner(self): + """Seqner getter. + + Returns: + seqner (Seqner): instance made from number + """ + return Seqner(sn=self.sn) + + class Dater(Matter): """ Dater is subclass of Matter, cryptographic material, for RFC-3339 profile of @@ -4666,7 +4743,7 @@ class CounterCodex: TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. - SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegators or issuers event + SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegator/issuer/transaction event TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event SadPathSig: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 94a4e7177..4ac1f003c 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -30,7 +30,9 @@ MissingWitnessSignatureError, UnverifiedReplyError, MissingDelegationError, OutOfOrderError, LikelyDuplicitousError, UnverifiedWitnessReceiptError, - UnverifiedReceiptError, UnverifiedTransferableReceiptError, QueryNotFoundError) + UnverifiedReceiptError, UnverifiedTransferableReceiptError, + QueryNotFoundError, MisfitEventSourceError, + MissingDelegableApprovalError) from ..kering import Version, Versionage from ..kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, RPY_LABELS) @@ -53,8 +55,8 @@ class TraitCodex: """ EstOnly: str = 'EO' # Only allow establishment events DoNotDelegate: str = 'DND' # Dot not allow delegated identifiers - NoBackers: str = 'NB' # Do not allow any registrar backers - Backers: str = 'RB' # Registrar backer provided in Registrar seal + NoRegistrarBackers: str = 'NB' # This should be NRB in next versionDo not allow any registrar backers + RegistrarBackers: str = 'RB' # Registrar backer provided in Registrar seal def __iter__(self): return iter(astuple(self)) @@ -98,27 +100,28 @@ def __iter__(self): # used to indicate to get the latest keys available from KEL for 'i' SealLast = namedtuple("SealLast", 'i') -# Establishment Event for Source of Message: duple (s, d) -# s = sn of event as lowercase hex string no leading zeros, -# d = SAID digest qb64 of event -# the pre is provided in the 'i' field of the message itself which is the qb64 -# of identifier prefix of KEL from which to get est, event given by 's d' +# State Establishment Event (latest current) : quadruple (s, d, br, ba) +# s = sn of latest est event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of latest establishment event +# br = backer (witness) remove list (cuts) from latest est event +# ba = backer (witness) add list (adds) from latest est event +StateEstEvent = namedtuple("StateEstEvent", 's d br ba') + +# Transaction Event Seal for Transaction Event: duple (s, d) +# s = sn of transaction event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of transaction event +# the pre is provided in the 'i' field qb64 of identifier prefix of KEL +# key event that this seal appears. # use SealSourceCouples count code for attachment -SealEst = namedtuple("SealEst", 's d') +SealTrans = namedtuple("SealTrans", 's d') -# State (latest current) Event: triple (s, t, d) +# not used should this be depricated +# State Event (latest current) : triple (s, t, d) # s = sn of latest event as lowercase hex string no leading zeros, # t = message type of latest event (ilk) # d = SAID digest qb64 of latest event StateEvent = namedtuple("StateEvent", 's t d') -# State (latest current) Establishment Event: quadruple (s, d, br, ba) -# s = sn of latest est event as lowercase hex string no leading zeros, -# d = SAID digest qb64 of latest establishment event -# br = backer (witness) remove list (cuts) from latest est event -# ba = backer (witness) add list (adds) from latest est event -StateEstEvent = namedtuple("StateEstEvent", 's d br ba') - @dataclass(frozen=True) class ColdCodex: @@ -414,44 +417,6 @@ def deTransReceiptQuintuple(data, strip=False): return esaider, sprefixer, sseqner, ssaider, siger -def validateSN(sn, inceptive=None): - """ - Returns: - sn (int): converted from sn hex str - - Raises ValueError if invalid sn - - Parameters: - sn (str): hex char sequence number of event or seal in an event - inceptive(bool): Check sn value and raise ValueError if invalid - None means check for sn < 0 - True means check for sn != 0 - False means check for sn < 1 - - """ - if len(sn) > 32: - raise ValueError("Invalid sn = {} too large.".format(sn)) - - try: - sn = int(sn, 16) - except Exception as ex: - raise ValueError("Invalid sn = {}.".format(sn)) - - if inceptive is not None: - if inceptive: - if sn != 0: - raise ValidationError("Nonzero sn = {} for inception evt." - "".format(sn)) - else: - if sn < 1: - raise ValidationError("Zero or less sn = {} for non-inception evt." - "".format(sn)) - else: - if sn < 0: - raise ValidationError("Negative sn = {} for event.".format(sn)) - - return sn - def verifySigs(raw, sigers, verfers): """ @@ -1573,10 +1538,6 @@ class Kever: qb64 identifier prefixes of own habitat identifiers. Assign db.prefixes when None When empty operate in promiscuous mode - local (bool): Injected from kevery when provided. - True means only process msgs for own events when .prefixes is not empty - False means only process msgs for not own events when .prefixes is not empty - Default is False. version (Versionage): serder.version instance of current event state version prefixer (Prefixer): instance for current event state sner (Number): instance of sequence number @@ -1602,7 +1563,7 @@ class Kever: lastEst (LastEstLoc): namedtuple of int sn .s and qb64 digest .d of last est event delegated (bool): True means delegated identifier, False not delegated - delgator (str): qb64 of delegator's prefix + delpre(str): qb64 of delegator's prefix Properties: @@ -1626,7 +1587,7 @@ class Kever: def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, db=None, estOnly=None, delseqner=None, delsaider=None, firner=None, - dater=None, cues=None, prefixes=None, local=False, check=False): + dater=None, cues=None, local=True, check=False): """ Create incepting kever and state from inception serder Verify incepting serder against sigers raises ValidationError if not @@ -1654,15 +1615,10 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, When dater provided then use dater for first seen datetime cues (Deck | None): reference to Kevery.cues Deck when provided i.e. notices of events or requests to respond to - prefixes (list | None): own prefixes for own local habitats. - May not include the prefix of this Kever's event when inception - has not yet been accepted into KEL - Some restrictions if present - If empty then effectively in promiscuous mode - local (bool): True means only process msgs for own controller's - events if .prefixes is not empty. - False means only process msgs for not own events - if .prefixes is not empty + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote check (bool): True means do not update the database in any non-idempotent way. Useful for reinitializing the Kevers from a persisted KEL without updating non-idempotent first seen .fels @@ -1676,8 +1632,7 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, db = basing.Baser(reopen=True) # default name = "main" self.db = db self.cues = cues - self.prefixes = prefixes if prefixes is not None else db.prefixes - self.local = True if local else False + local = True if local else False if state: # preload from state self.reload(state) @@ -1693,30 +1648,25 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, ilk, serder.ked)) self.ilk = ilk - labels = DIP_LABELS if ilk == Ilks.dip else ICP_LABELS - for k in labels: - if k not in serder.ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, serder.ked)) - self.incept(serder=serder) # do major event validation and state setting self.config(serder=serder, estOnly=estOnly) # assign config traits perms # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, delegator, wigers = self.valSigsDelWigs(serder=serder, + sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, sigers=sigers, verfers=serder.verfers, tholder=self.tholder, wigers=wigers, toader=self.toader, wits=self.wits, + local=local, delseqner=delseqner, delsaider=delsaider) - self.delegator = delegator - self.delegated = True if self.delegator else False + self.delpre = delpre # may be None + self.delegated = True if self.delpre else False wits = serder.backs # serder.ked["b"] # .validateSigsDelWigs above ensures thresholds met otherwise raises exception @@ -1725,7 +1675,7 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, wits=wits, first=True if not check else False, seqner=delseqner, saider=delsaider, - firner=firner, dater=dater) + firner=firner, dater=dater, local=local) if fn is not None: # first is non-idempotent for fn check mode fn is None self.fner = Number(num=fn) self.dater = Dater(dts=dts) @@ -1768,6 +1718,22 @@ def kevers(self): return self.db.kevers + @property + def prefixes(self): + """ + Returns .db.prefixes + """ + return self.db.prefixes + + + @property + def groups(self): + """ + Returns .db.gids oset of group hab ids (prefixes) + """ + return self.db.groups + + @property def transferable(self): """ @@ -1779,32 +1745,90 @@ def transferable(self): return True if self.ndigers and self.prefixer.transferable else False - def locallyOwned(self, pre=''): - """Returns True if pre is in .prefixes False otherwise. Indicates that - provided identifier prefix is controlled by a local controller from - .prefixes + def locallyOwned(self, pre: str | None = None): + """Returns True if pre is in .prefixes and not in .groups + False otherwise. + Indicates that provided identifier prefix is controlled by a local + controller from .prefixes but is not a group with local member. i.e pre is a locally owned (controlled) AID (identifier prefix) + Returns: + (bool): True if pre is local hab but not group hab + Parameters: - pre (str): qb64 identifier prefix + pre (str|None): qb64 identifier prefix if any. Default None + None means use self.prefixer.qb64 """ - return pre in self.prefixes + pre = pre if pre is not None else self.prefixer.qb64 + return pre in self.prefixes and pre not in self.groups - def locallyWitnessed(self, serder=None): + def locallyWitnessed(self, *, wits: list[str]=None, serder: (str)=None): """Returns True if a local controller is a witness of this Kever's KEL of wits in serder of if None then current wits for this Kever. i.e. self is witnessd by locally owned (controlled) AID (identifier prefix) Parameters: + wits (list[str]): qb64 identifier prefixes of witnesses serder ( SerderKERI | None): SerderKERI instace if any """ - if serder and serder.pre != self.prefixer.qb64: # same KEL as self - return False - wits = serder.backs if serder is not None else self.wits - return (oset(self.prefixes) & oset(wits)) + if not wits: + if not serder: + wits = self.wits + else: + if serder.pre != self.prefixer.qb64: # not same KEL as self + return False + wits, _, _ = self.deriveBacks(serder=serder) + + return True if (self.prefixes & oset(wits)) else False + + + def locallyMembered(self, pre: str | None = None): + """Returns True if group hab identifier prefix pre has as a contributing + member a locally owned prefix by virture of pre in .groups + + Returns: + (bool): True if pre is group hab identifier in .groups + False otherwise + + Parameters: + pre (str|None): qb64 identifier prefix if any or None + When None default to use self.prefixer.qb64 + + """ + # assumes stale group membership is taken care of by presence of groups + # i.e where once a local member but no more. + pre = pre if pre is not None else self.prefixer.qb64 + return pre in self.groups # groups + + + def locallyContributedIndices(self, verfers: list[Verfer]): + """Returns list of indices of public keys contributed by local members + to the KEL with current signing keys represented by verfers + + Using the pubs index to find members of a signing group + + Parameters: + verfers (list[Verfer]): instance for each current signing key + + Returns: + indices list[int]: list of indices of keys contributed by local members + + """ + indices = [] + + for i, verfer in enumerate(verfers): + if (couples := self.pubs.get(keys=(verfer.qb64,))) is None: + continue + + for (prefixer, seqner) in couples: + if self.locallyOwned(prefixer.qb64): # only member not group aid + indices.append(i) + break # only need one local member to exclude signature + + return indices def reload(self, state): @@ -1834,8 +1858,8 @@ def reload(self, state): self.estOnly = True if TraitCodex.EstOnly in state.c else False self.lastEst = LastEstLoc(s=int(state.ee.s, 16), d=state.ee.d) - self.delegator = state.di if state.di else None - self.delegated = True if self.delegator else False + self.delpre = state.di if state.di else None + self.delegated = True if self.delpre else False if (raw := self.db.getEvt(key=dgKey(pre=self.prefixer.qb64, dig=state.d))) is None: @@ -1940,7 +1964,7 @@ def config(self, serder, estOnly=None, doNotDelegate=None): def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, - firner=None, dater=None, check=False): + firner=None, dater=None, local=True, check=False): """ Not an inception event. Verify event serder and indexed signatures in sigers and update state @@ -1964,6 +1988,10 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, dater (Dater | None): Dater instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote check (bool): True means do not update the database in any non-idempotent way. Useful for reinitializing the Kevers from a persisted KEL without updating non-idempotent first seen .fels @@ -1981,6 +2009,7 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, self.prefixer.qb64, ked)) + local = True if local else False sner = serder.sner # Number instance ensures whole number for sequence number ilk = serder.ilk # ked["t"] @@ -1996,35 +2025,24 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # returned sigers and wigers are verified signatures # If does not validate then escrows as needed and raises ValidationError - sigers, delegator, wigers = self.valSigsDelWigs(serder=serder, + sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, sigers=sigers, verfers=serder.verfers, tholder=tholder, wigers=wigers, toader=toader, wits=wits, + local=local, delseqner=delseqner, delsaider=delsaider) - # rotation so check rotation threshold against exposed sigers versus - # prior next digers in .ndigers - #ondices = self.exposeds(sigers) - #if not self.ntholder.satisfy(indices=ondices): - #self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers) - #if delseqner and delsaider: # save in case not attached later - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) - #raise MissingSignatureError(f"Failure satisfying nsith=" - #f"{self.ntholder.sith} on sigs=" - #f"{[siger.qb64 for siger in sigers]}" - #f" for evt={serder.ked}.") - - # .validateSigsDelWigs above ensures thresholds met otherwise raises exception + # .valSigWigsDel above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, wits=wits, first=True if not check else False, seqner=delseqner, saider=delsaider, - firner=firner, dater=dater) + firner=firner, dater=dater, local=local) # nxt and signatures verify so update state self.sner = sner # sequence number Number instance @@ -2072,13 +2090,14 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, delegator, wigers = self.valSigsDelWigs(serder=serder, + sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, sigers=sigers, verfers=self.verfers, tholder=self.tholder, wigers=wigers, toader=self.toader, - wits=self.wits) + wits=self.wits, + local=local) # .validateSigsDelWigs above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen @@ -2188,61 +2207,80 @@ def rotate(self, serder): # use ordered set math ops to verify and ensure strict ordering of wits # cuts and add to ensure that indexed signatures on indexed witness # receipts work + wits, cuts, adds = self.deriveBacks(serder) + + toader = serder.bner # Number(num=ked["bt"]) # auto converts hex num to int + if wits: + if toader.num < 1 or toader.num > len(wits): # out of bounds toad + raise ValueError(f"Invalid toad = {toader.num} for backers " + f"(wits)={wits} for event={ked}.") + else: + if toader.num != 0: # invalid toad + raise ValueError(f"Invalid toad = {toader.num} for backers " + "(wits)={wits} for event={ked}.") + + return tholder, toader, wits, cuts, adds + + + def deriveBacks(self, serder): + """Derives and return tuple of (wits, cuts, adds) for backers given + current set and any changes provided by serder. + + Returns: + wca (tuple): of + wits (list[str]): prefixes of witnesses full list (backers) + cuts (list[str]): prefixes of witnesses removed in latest est evt + adds (list[str]): prefixes of witnesses added in latest est evt + + Parameters: + serder (SerderKeri): instance of current event + """ + if serder.ilk not in (Ilks.rot, Ilks.drt): # no changes + return (self.wits, self.cuts, self.adds) + witset = oset(self.wits) - cuts = serder.cuts # ked["br"] + cuts = serder.cuts cutset = oset(cuts) if len(cutset) != len(cuts): - raise ValidationError("Invalid cuts = {}, has duplicates for evt = " - "{}.".format(cuts, ked)) + raise ValidationError("Invalid cuts = {cuts}, has duplicates for " + f"evt = {serder.ked}.") if (witset & cutset) != cutset: # some cuts not in wits - raise ValidationError("Invalid cuts = {}, not all members in wits" - " for evt = {}.".format(cuts, ked)) + raise ValidationError(f"Invalid cuts = {cuts}, not all members in " + f"wits for evt = {serder.ked}.") - adds = serder.adds # ked["ba"] + adds = serder.adds addset = oset(adds) if len(addset) != len(adds): - raise ValidationError("Invalid adds = {}, has duplicates for evt = " - "{}.".format(adds, ked)) + raise ValidationError(f"Invalid adds = {adds}, has duplicates for " + f"evt = {serder.ked}.") if cutset & addset: # non empty intersection - raise ValidationError("Intersecting cuts = {} and adds = {} for " - "evt = {}.".format(cuts, adds, ked)) + raise ValidationError(f"Intersecting cuts = {cuts} and adds = {adds}" + f"for evt = {serder.ked}.") if witset & addset: # non empty intersection - raise ValidationError("Intersecting wits = {} and adds = {} for " - "evt = {}.".format(self.wits, adds, ked)) + raise ValidationError(f"Intersecting wits = {self.wits} and adds =" + f"{adds} for evt = {serder.ked}.") wits = list((witset - cutset) | addset) if len(wits) != (len(self.wits) - len(cuts) + len(adds)): # redundant? - raise ValidationError("Invalid member combination among wits = {}, cuts ={}, " - "and adds = {} for evt = {}.".format(self.wits, - cuts, - adds, - ked)) - - toader = serder.bner # Number(num=ked["bt"]) # auto converts hex num to int - if wits: - if toader.num < 1 or toader.num > len(wits): # out of bounds toad - raise ValueError(f"Invalid toad = {toader.num} for backers " - f"(wits)={wits} for event={ked}.") - else: - if toader.num != 0: # invalid toad - raise ValueError(f"Invalid toad = {toader.num} for backers " - "(wits)={wits} for event={ked}.") + raise ValidationError(f"Invalid member combination among wits = " + f"{self.wits}, cuts ={cuts}, and adds = " + f"{adds,} for evt = {serder.ked}.") + return (wits, cuts, adds) - return tholder, toader, wits, cuts, adds - def valSigsDelWigs(self, serder, sigers, verfers, tholder, - wigers, toader, wits, + def valSigsWigsDel(self, serder, sigers, verfers, tholder, + wigers, toader, wits, local=True, delseqner=None, delsaider=None): """ - Returns triple (sigers, delegator, wigers) where: + Returns triple (sigers, wigers, delegator) where: sigers is unique validated signature verified members of inputed sigers - delegator is qb64 delegator prefix if delegated else None wigers is unique validated signature verified members of inputed wigers + delegator is qb64 delegator prefix if delegated else None Validates sigers signatures by validating indexes, verifying signatures, and validating threshold sith. @@ -2262,6 +2300,10 @@ def valSigsDelWigs(self, serder, sigers, verfers, tholder, toader (Number): instance of backer witness threshold wits (list): of qb64 non-transferable prefixes of witnesses used to derive werfers for wigers + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote delseqner (Seqner | None): instance of delegating event sequence number. If this event is not delegated then seqner is ignored delsaider (Saider | None): instance of of delegating event said. @@ -2274,6 +2316,19 @@ def valSigsDelWigs(self, serder, sigers, verfers, tholder, [verfer.qb64 for verfer in verfers], serder.ked)) + # Filters sigers to remove any signatures from locally membered groups + # when not local (remote) event source. So that attacker can't source + # compromised signature remotely to satisfy threshold. + + if not local and self.locallyMembered(): # is this Kever's pre a local group + if (indices := self.locallyContributedIndices(verfers)): + for siger in list(sigers): # copy so clean del on original elements + if siger.index in indices: + del sigers[siger.index] + self.cues.push(dict(kin="remoteMemberedSig", + serder=serder, + index=siger.index)) + # get unique verified sigers and indices lists from sigers list sigers, indices = verifySigs(raw=serder.raw, sigers=sigers, verfers=verfers) # sigers now have .verfer assigned @@ -2283,26 +2338,39 @@ def valSigsDelWigs(self, serder, sigers, verfers, tholder, raise ValidationError("No verified signatures for evt = {}." "".format(serder.ked)) - werfers = [Verfer(qb64=wit) for wit in wits] # get witnes signatures + # Misfit check events that must be locally sourced (protected) get + # escrowed in order to repair the protection when appropriate + if (not local and + (self.locallyOwned() or + self.locallyWitnessed(wits=wits))): + + self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MisfitEventSourceError(f"Nonlocal source for locally owned" + f"or locally witnessed event" + f" = {serder.ked}.") + + werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers # get unique verified wigers and windices lists from wigers list wigers, windices = verifySigs(raw=serder.raw, sigers=wigers, verfers=werfers) # each wiger now has added to it a werfer of its wit in its .verfer property - # escrow if not fully signed vs threshold + # escrow if not fully signed vs signing threshold if not tholder.satisfy(indices): # at least one but not enough - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) if delseqner and delsaider: self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" f" on sigs for {[siger.qb64 for siger in sigers]}" f" for evt = {serder.ked}.") + # escrow if not fully signed vs prior next rotation threshold if serder.ilk in (Ilks.rot, Ilks.drt): # rotation so check prior next threshold # prior next threshold in .ntholder and digers in .ndigers ondices = self.exposeds(sigers) if not self.ntholder.satisfy(indices=ondices): - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) if delseqner and delsaider: # save in case not attached later self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingSignatureError(f"Failure satisfying prior nsith=" @@ -2311,36 +2379,54 @@ def valSigsDelWigs(self, serder, sigers, verfers, tholder, f" for new est evt={serder.ked}.") - delegator = self.validateDelegation(serder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider) - - # Kevery .process event logic does not prevent this from seeing event when - # not local and event pre is own pre - if not self.locallyOwned(serder.pre): # not in self.prefixes - if ((wits and not self.prefixes) or # in promiscuous mode so assume must verify toad - (wits and self.prefixes and not self.local and # not promiscuous nonlocal - not (oset(self.prefixes) & oset(wits)))): # own prefix is not a witness - # validate that event is fully witnessed - - if wits: - if toader.num < 1 or toader.num > len(wits): # out of bounds toad - raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - else: - if toader.num != 0: # invalid toad - raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - - if len(windices) < toader.num: # not fully witnessed yet - if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, - seqner=delseqner, saider=delsaider): - # cue to query for witness receipts - self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) - raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " - f"on witness sigs=" - f"{[siger.qb64 for siger in wigers]} " - f"for event={serder.ked}.") - - - return sigers, delegator, wigers + # get delegator if any + if serder.ilk == Ilks.dip: + delpre = serder.delpre # delegator from dip event + if not delpre: # empty or None + raise ValidationError(f"Empty or missing delegator for delegated" + f" inception event = {serder.ked}.") + elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation + delpre = self.delpre + else: # not delegable event icp, rot, ixn + delpre = None + + # delpre maybe None so ensure not None to pass into .locallyOwned which + # defaults to self.prefixer.qb64 when None + if not local and self.locallyOwned(delpre if delpre is not None else ''): + self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MisfitEventSourceError(f"Nonlocal source for locally" + f" delegated by {delpre} of" + f"event = {serder.ked}.") + + # short circuit witness validation when either locallyOwned or locallyWitnessed + # otherwise must validate fully witnessed + if not (self.locallyOwned() or self.locallyWitnessed(wits=wits)): + if wits: # is witnessed + if toader.num < 1 or toader.num > len(wits): # out of bounds toad + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + else: # not witnessed + if toader.num != 0: # invalid toad + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + if len(windices) < toader.num: # not fully witnessed yet + if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, + seqner=delseqner, saider=delsaider, + local=local): + # cue to query for witness receipts + self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) + raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " + f"on witness sigs=" + f"{[siger.qb64 for siger in wigers]} " + f"for event={serder.ked}.") + + if delpre: + self.validateDelegation(serder, sigers=sigers, + wigers=wigers, wits=wits, + local=local, delpre=delpre, + delseqner=delseqner, delsaider=delsaider) + + return sigers, wigers, delpre def exposeds(self, sigers): @@ -2390,9 +2476,13 @@ def exposeds(self, sigers): return odxs - def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsaider=None): + def validateDelegation(self, serder, sigers, wigers, wits, local=True, + delpre=None, delseqner=None, delsaider=None): """ Returns delegator's qb64 identifier prefix if validation successful. + Assumes that local vs remote source checks have been applied before + this function is called. + Rules: If event is not a delegated event then not valid delegation If delegatee's own event (.mine) then valid delegation @@ -2406,10 +2496,17 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai Parameters: serder (SerderKERI): instance of delegated event serder - sigers (list): of Siger instances of indexed controller sigs of + sigers (list[Siger]): of Siger instances of indexed controller sigs of delegated event. Assumes sigers is list of unique verified sigs - wigers (list | None): of optional Siger instance of indexed witness sigs of + wigers (list[Siger]): of optional Siger instance of indexed witness sigs of delegated event. Assumes wigers is list of unique verified sigs + wits (list[str]): of qb64 non-transferable prefixes of witnesses used to + derive werfers for wigers + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + delpre (str | None): qb64 prefix of delegator if any delseqner (Seqner | None): instance of delegating event sequence number. If this event is not delegated then ignored delsaider (Saider | None): instance of of delegating event digest. @@ -2418,7 +2515,106 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai Returns: (str | None): qb64 delegator prefix or None if not delegated - Superseding Recovery + Process Logic: + A delegative event is processed differently for each of four different + parties, namely, controller of event, witness to controller of event, + delegator of event , and validator of event that is not controller, + witness or delegator. Events are processed as either local (protected) + or remote. A local event may assume that the event only came via a + protected transmission path. This might be because the event is + functions locally on a device under the supervision of the controller + or was received via some protected channel using some form of MFA. + A remote event is received in an unprotected manner. The purpose of + local and remote is to allow increased security on local events where + a threshold structure is imposed. + + A witness pool may act a threshold structure for enchanced security + when each witness only accepts local events that are protected + by a unique authentication factor thereby making the controller's + signature(s) the first factor and the set of unique witness factors + a secondary threshold factor. An attacker therefore has to compromise + not merely the controller's private key(s) but also the unique second factor + on each of a threshold satisfycing number of witnesses. + + Likewise a delegator may act as a threshold structure for enhanced security + when the delegator only accepts local events for delegation that are + protected by a unique authentication factor thereby making the + controller's signatures the first factor, a threshold satisfycing + number of unique witness factors the second layer of factors, and + the delegator's unique authentication factor as the third factor. + An attacker therefore has to compromise not merely the controller's + private key(s) but also the unique second factor + on each of a threshold satisfycing number of witnesses and the unique + third factor for the delegator. + + Controller as delegatee must accept its own delegated event prior + to full witnessing or delegator approval (anchored seal) by signing the + event in order to trigger the logic to get witness receipts and + delegator approval. This means a local (protected) event may be + accepted into controller's KEL when fully signed by controller. + + Witness must accept a controller's delegated event it witnesses prior to + full witnessing or delegator approval in order to trigger its + witnessing logic. This means a local (protected) event may be + accepted into a witness' KEL when fully signed by its controller. + + Delegator may escrow or sandbox a delegated event prior to it anchoring + a seal of the event in its KEL in order to trigger its approval logic. + Alternatively the approval logic may be triggered immediately after + it is received and authenticated on it its local (protected) channel + but before it is submitted to its local Kevery for processing. + + The delegator MUST NOT accept a delegable event unless it is locally + sourced, fully signed by its controller, and fully witnessed by its + controller's designated witness pool. + A Delegator may impose additional validation logic prior to approval. + The approval logic may be handled by an escrow that only runs if + the delegable event is sourced as local. This may require a + sandboxed kel for the delegatee in order to not corrupt its pristine + copy of the delegatee's KEL with a valid delegable event from a + malicious source. The sandboxing logic may create a virtual + delegation event with seal for the purpose of checking the delegated + event superseding logic prior to acceptance. + + A malicious attacker that compromises the pre-rotated keys of the + delegatee may issue a rotation that changes its witness pool in order + to bypass the local security logic of the witness pool. The approval + logic of the delegator may choose to not automatically approve a + delegable rotation event unliess the change to the witness pool is + below the threshold. + + The logic for superseded events is NOT a requirement for acceptance in + either a delegated event controller's KEL or its witness' KEL. The + delegator's kel creates a virtual (provisional) delegating interaction + event in order to evaluate correct superseding logic so as not to + accept an invalid supderseding delegated event into its local copy + of the delegated KEL. This virtual event is needed because superseding + logic requires an anchoring seal be present before the rules can + be fully evaluated. + + Should the actual anchor be via a superseding rotation in the + delegator's KEL not via an interaction event then the delegator must + check the logic for a virtual delegating rotation instead. + In either case the delegated event does not change so the virtual + delegating checks are sufficient to accept the delegated event + into the delegator's local copy of the delegatee's KEL. + + + Any of delegated controller, delegated witness, or delegator + of delegated event may after the fact fully validate event by + processing it as a remote event. + Then the logic applied is same as validator below. + + A validator of a delegated event that is not the event's controller, + witness, or delegator must not accept the event until is is fully + signed by the controller (threshold), fully witnessed by the witness + pool (threshold) and its seal anchored in the delegator's KEL. The + rules for event superseding in the delegated controller's kel must + also be satisfied. The logic should be the same for both local and + remote event because the validator is not one of the protected parties + to the event. + + Superseding Recovery: Supersede means that after an event has already been accepted as first seen into a KEL that a different event with the same sequence number is accepted @@ -2438,18 +2634,23 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai Superseding Rules for Recovery at given SN (sequence number) - A0. Any rotation event may supersede an interaction event at the same sn. (existing rule) - A1. A non-delegated rotation may not supersede another rotation at the same sn. (modified rule) + A0. Any rotation event may supersede an interaction event at the same sn. + where that interaction event is not before any other rotation event. + (existing rule) + A1. A non-delegated rotation may not supersede another rotation at the + same sn. (modified rule) A2. An interaction event may not supersede any event. ( existing rule). (B. and C. below provide the new rules) - B. A delegated rotation may supersede another delegated rotation at the same sn - under either of the following conditions: + B. A delegated rotation may supersede the latest seen delegated rotation + at the same sn under either of the following conditions: + B1. The superseding rotation's delegating event is later than the superseded rotation's delegating event in the delegator's KEL, i.e. the sn of the superseding event's delegation is higher than the superseded event's delegation. + B2. The sn of the superseding rotation's delegating event is the same as the sn of the superseded rotation's delegating event in the delegator's KEL and the superseding rotation's delegating event is a rotation and the @@ -2459,98 +2660,154 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai delgator's KEL C. IF Neither A nor B is satisfied, then recursively apply rules A. and B. to - the delegating events of those delegating events and so on until either A. or B. - is satisfied, or the root KEL of the delegation has been reached. - C1. If neither A. nor B. is satisfied by recursive application on the - delegator's KEL (i.e. the root KEL of the delegation has been reached without - satisfaction) then the superseding rotation is discarded. The terminal case of - the recursive application will occur at the root KEL which by defintion is - non-delegated wherefore either A. or B. must be satisfied, or else the - superseding rotation must be discarded. + the delegating events of those delegating events and so on until + either A. or B. is satisfied, or the root KEL of the delegation + which must be undelegated has been reached. + + C1. If neither A. nor B. is satisfied by recursive application on the + delegator's KEL (i.e. the root KEL of the delegation has been reached + without satisfaction) then the superseding rotation is discarded. + The terminal case of the recursive application will occur at the + root KEL which by defintion is non-delegated wherefore either + A. or B. must be satisfied, or else the superseding rotation must + be discarded. + + Note: The latest seen deleagated rotation constraint means that any earlier + delegated rotations can NOT be superseded. This greatly simplifies the + validation logic and avoids a potential infinite regress of forks in the + delegated identifier's KEL. + + In order to capture control of a delegated identifier the attacker must + issue a delegated rotation that rotates to keys under the control of the + attacker that must be approved by the delegator. A recovery rotation must + therefore superseded the compromised rotation. If the attacker is able + to issue and get approved by the delegator a second rotation + that follows but does not supersede the compromising rotation then + recovery is no longer possible because the delegatee would no longer + control the privete keys needed to verifiably sign a recovery rotation. """ - if serder.ilk not in (Ilks.dip, Ilks.drt): # not delegated - return None # delegator is None - - # verify delegator and attachment pointing to delegating event - if serder.ilk == Ilks.dip: - delegator = serder.delpre # delegator from dip event - if not delegator: - raise ValidationError(f"Empty or missing delegator for delegated" - f" inception event = {serder.ked}.") - else: # serder.ilk == Ilks.drt so rotation - delegator = self.delegator - + if not delpre: # not delegable so no delegation validation needed + return # if we are the delegatee, accept the event without requiring the # delegator validation via an anchored delegation seal or by requiring - # it to be witnessed - # ToDo XXXX add local lax check after figure out dist multisig group - # ToDo XXXX add check for witness to accept so that witness will - # add to its KEL without waiting for delegation seal to be anchored in - # delegator's KEL witness cue in Kevery will then generate reciept - if self.locallyOwned(serder.pre) or self.locallyWitnessed(serder=serder): - return delegator - - # during initial delegation we just escrow the delcept event - if delseqner is None and delsaider is None and delegator is not None: - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers) - raise MissingDelegationError("No delegation seal for delegator {} " - "with evt = {}.".format(delegator, serder.ked)) - - ssn = validateSN(sn=delseqner.snh, inceptive=False) # delseqner Number should already do this - #ssn = sner.num sner is Number seqner is Seqner need to replace Seqners with Numbers + # it to be witnessed. Query witness cue in Kevery will then request witnessing. + # Controller accepts without waiting for witnessor nor for the delegation + # seal to be anchored in delegator's KEL. + # Witness accepts without waiting for delegation seal to be anchored in + # delegator's KEL. Witness cue in Kevery will then generate receipt + if self.locallyOwned() or self.locallyWitnessed(wits=wits): + return + + + if self.kevers is None or delpre not in self.kevers: # drop event + # ToDo XXXX may want to cue a trigger to get the KEL of the delegator + raise ValidationError("Missing Kever for delegator = {} for evt = {}." + "".format(delpre, serder.ked)) + + dkever = self.kevers[delpre] + if dkever.doNotDelegate: # drop event if delegation not allowed + raise ValidationError("Delegator = {} for evt = {}," + " does not allow delegation.".format(delpre, + serder.ked)) + + + # Delegator accepts here without waiting for delegation seal to be anchored in + # delegator's KEL. But has already waited for fully receipted above. + # Once fully receipted, cue in Kevery will then trigger cue to approve + # delegation + + if delseqner is None or delsaider is None: + if self.locallyOwned(delpre): # local delegator so escrow. + # Won't get to here if not local and locallyOwned(delpre) because + # valSigsWigsDel will send nonlocal sourced delegable event to + # misfit escrow first. Mistfit escrow must first + # promote to local and reprocess event before we get to here + self.escrowDelegableEvent(serder=serder, sigers=sigers, + wigers=wigers,local=local) + raise MissingDelegableApprovalError(f"Missing approval for " + f" delegation by {delpre} of" + f"event = {serder.ked}.") + + # ToDo XXXX This logic moves to the Delegable escrow processing + # ToDo XXXX create process escrow for delegable events "dees." + #in order to get delegator approval + # any virtual delegation or sandboxing logic happens there + # create virtual anchor seal so local delegator can evaluate + # superseding logic with provisional virtual seal + #dkever = self.kevers[delpre] + #dseal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said) + #dserder = interact(pre=dkever.prefixer.qb64, + #dig=dkever.serder.said, + #sn=dkever.sner.num + 1, + #data=[dseal._asdict()]) + #delseqner = coring.Seqner(snh=dserder.snh) + #delsaider = coring.Saider(qb64=dserder.said) + # ToDo XXXX need to cue task here to approve delegation by generating + # an anchoring SealEvent of serder in delegators KEL + # may include MFA and or business logic for the delegator i.e. is local + # event that designates this controller as delegator triggers + # this cue to approave delegation + #self.cues.push(dict(kin="approveDelegation", + #delegator=kever.delpre, + #serder=serder)) + + + else: # not local delegator so escrow + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) + raise MissingDelegationError(f"No delegation seal for delegator " + "{delpre} of evt = {serder.ked}.") + + #ssn = validateSN(sn=delseqner.snh, inceptive=False) # delseqner Number should already do this + ssn = Number(num=delseqner.sn).validate(inceptive=False).sn + #ssn = sner.num sner is Number seqner is Seqner + # ToDo XXXX need to replace Seqners with Numbers # get the dig of the delegating event. Using getKeLast ensures delegating # event has not already been superceded - key = snKey(pre=delegator, sn=ssn) # database key + key = snKey(pre=delpre, sn=ssn) # database key raw = self.db.getKeLast(key) # get dig of delegating event if raw is None: # no delegating event at key pre, sn - # ToDo XXXX create cue to send query to fetch delegating event from + # ToDo XXXX process this cue of query to fetch delegating event from # delegator - self.cues.push(dict(kin="query", q=dict(pre=delegator, + self.cues.push(dict(kin="query", q=dict(pre=delpre, sn=delseqner.snh, dig=delsaider.qb64))) # escrow event here inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False - sn = validateSN(sn=serder.snh, inceptive=inceptive) - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers) + #sn = validateSN(sn=serder.snh, inceptive=inceptive) + sn = Number(num=serder.sn).validate(inceptive=inceptive).sn + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError("No delegating event from {} at {} for " - "evt = {}.".format(delegator, + "evt = {}.".format(delpre, delsaider.qb64, serder.ked)) # get the delegating event from dig ddig = bytes(raw) - key = dgKey(pre=delegator, dig=ddig) # database key + key = dgKey(pre=delpre, dig=ddig) # database key raw = self.db.getEvt(key) if raw is None: # drop event raise ValidationError("Missing delegation from {} at event dig = {} for evt = {}." - "".format(delegator, ddig, serder.ked)) + "".format(delpre, ddig, serder.ked)) dserder = serdering.SerderKERI(raw=bytes(raw)) # delegating event + + # compare digests to make sure they match here if not dserder.compare(said=delsaider.qb64): # drop event raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." - "".format(delegator, ddig, serder.ked)) - - if self.kevers is None or delegator not in self.kevers: # drop event - raise ValidationError("Missing Kever for delegator = {} for evt = {}." - "".format(delegator, serder.ked)) - - dkever = self.kevers[delegator] - if dkever.doNotDelegate: # drop event - raise ValidationError("Delegator = {} for evt = {}," - " does not allow delegation.".format(delegator, - serder.ked)) + "".format(delpre, ddig, serder.ked)) found = False # find event seal of delegated event in delegating data # XXXX ToDo need to change logic here to support native CESR seals not just dicts # for JSON, CBOR, MGPK + # may want to try harder here by walking KEL for dseal in dserder.seals: # find delegating seal anchor if tuple(dseal.keys()) == SealEvent._fields: seal = SealEvent(**dseal) @@ -2561,21 +2818,25 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai break if not found: # drop event - raise ValidationError("Missing delegation from {} in {} for evt = {}." - "".format(delegator, dserder.seals, serder.ked)) + raise ValidationError(f"Missing delegation seal in designated event" + f"from {delpre} in {dserder.seals} for " + f"evt = {serder.ked}.") - # Assumes database is reverified each bootup chain-of-custody of dic broken. + # Found anchor so can assume delegation successful unless its one of + # the superseding condidtions. + # Assumes database is reverified each bootup chain-of-custody of disc broken. # Rule for non-supeding delegated rotation of rotation. # Returning delegator indicates success and eventually results acceptance # via Kever.logEvent which also writes the delgating event source couple to # db.aess so we can find it later if ((serder.ilk == Ilks.dip) or # delegated inception (serder.sner.num == self.sner.num + 1) or # inorder event - (serder.sner.num == self.sner.num and - self.ilk == Ilks.ixn and + (serder.sner.num == self.sner.num and # superseding event + self.ilk == Ilks.ixn and # superseded is ixn serder.ilk == Ilks.drt)): # recovery rotation superseding ixn - return delegator # indicates delegation valid with return of delegator + return # delegator # indicates delegation valid with return of delegator + # get to here means drt rotation superseding another drt rotation # Kever.logEvent saves authorizer (delegator) seal source couple in # db.aess data base so can use it here to recusively look up delegating # events @@ -2584,7 +2845,7 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai serfn = serder # new potentially superseding delegated event i.e. serf new bossn = dserder # new delegating event of superseding delegated event i.e. boss new serfo = self.serder # original delegated event i.e. serf original - bosso = self.fetchDelegatingEvent(delegator, serfo) + bosso = self.fetchDelegatingEvent(delpre, serfo) while (True): # superseding delegated rotation of rotation recovery rules # Only get to here if same sn for drt existing and drt superseding @@ -2592,7 +2853,7 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai if (bossn.sn > bosso.sn or # later supersedes (bossn.Ilk == Ilks.drt and bosso.Ilk == Ilks.ixn) ): # drt supersedes ixn - return delegator # valid superseding delegation + return # delegator # valid superseding delegation if bossn.said == bosso.said: # same delegating event nseals = [SealEvent(**seal) for seal in bossn.seals @@ -2604,7 +2865,7 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai if nindex > oindex: # later seal supersedes # assumes index can't be None - return delegator # valid superseding delegation + return # delegator # valid superseding delegation else: # ToDo: XXXX may want to cue up business logic for delegator @@ -2614,9 +2875,9 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai # tie condition same sn and drt so need to climb delegation chain serfn = bossn - bossn = self.fetchDelegatingEvent(delegator, serfn) + bossn = self.fetchDelegatingEvent(delpre, serfn) serfo = bosso - bosso = self.fetchDelegatingEvent(delegator, serfo) + bosso = self.fetchDelegatingEvent(delpre, serfo) # repeat @@ -2651,7 +2912,7 @@ def fetchDelegatingEvent(self, delegator, serder): def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, - seqner=None, saider=None, firner=None, dater=None): + seqner=None, saider=None, firner=None, dater=None, local=True): """ Update associated logs for verified event. Update is idempotent. Logs will not write dup at key if already exists. @@ -2675,8 +2936,14 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, dater is optional Dater instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ + local = True if local else False fn = None # None means not a first seen log event so does not return an fn + dgkeys = (serder.pre, serder.said) dgkey = dgKey(serder.preb, serder.saidb) dtsb = helping.nowIso8601().encode("utf-8") self.db.putDts(dgkey, dtsb) # idempotent do not change dts if already @@ -2686,7 +2953,18 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if wits: self.db.wits.put(keys=dgkey, vals=[coring.Prefixer(qb64=w) for w in wits]) + self.db.putEvt(dgkey, serder.raw) # idempotent (maybe already excrowed) + # update event source + if (esr := self.db.esrs.get(keys=dgkeys)): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkeys, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkeys, val=esr) + val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) for verfer in (serder.verfers if serder.verfers is not None else []): self.db.pubs.add(keys=(verfer.qb64,), val=val) @@ -2715,7 +2993,87 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, serder.preb, serder.pretty()) return (fn, dtsb.decode("utf-8")) # (fn int, dts str) if first else (None, dts str) - def escrowPSEvent(self, serder, sigers, wigers=None): + + def escrowMFEvent(self, serder, sigers, wigers=None, + seqner=None, saider=None, local=True): + """ + Update associated logs for escrow of MisFit event + + Parameters: + serder (SerderKERI): instance of event + sigers (list): of Siger instance for event + wigers (list): of witness signatures + seqner (Seqner): instance of sn of event delegatint/issuing event if any + saider (Saider): instance of dig of event delegatint/issuing event if any + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + """ + local = True if local else False + dgkey = dgKey(serder.preb, serder.saidb) + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + self.db.putEvt(dgkey, serder.raw) + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: + couple = seqner.qb64b + saider.qb64b + self.db.putPde(dgkey, couple) # idempotent + self.db.misfits.add(snKey(serder.preb, serder.sn), serder.saidb) + # log escrowed + logger.info("Kever state: escrowed misfit event=\n%s\n", + json.dumps(serder.ked, indent=1)) + + + def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): + """ + Update associated logs for escrow of Delegable event that needs delegation + via an anchored seal in delegator's KEl + + Parameters: + serder (SerderKERI): instance of event + sigers (list): of Siger instance for event + wigers (list): of witness signatures + seqner (Seqner): instance of sn of event delegatint/issuing event if any + saider (Saider): instance of dig of event delegatint/issuing event if any + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + """ + local = True if local else False + dgkey = dgKey(serder.preb, serder.saidb) + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + self.db.putEvt(dgkey, serder.raw) + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + self.db.delegables.add(snKey(serder.preb, serder.sn), serder.saidb) + # log escrowed + logger.info("Kever state: escrowed delegable event=\n%s\n", + json.dumps(serder.ked, indent=1)) + + + def escrowPSEvent(self, serder, sigers, wigers=None, local=True): """ Update associated logs for escrow of partially signed event or fully signed delegated event but not yet verified delegation. @@ -2724,19 +3082,36 @@ def escrowPSEvent(self, serder, sigers, wigers=None): serder is SerderKERI instance of event sigers is list of Siger instances of indexed controller sigs wigers is optional list of Siger instance of indexed witness sigs + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + self.db.putEvt(dgkey, serder.raw) + # update event source + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + snkey = snKey(serder.preb, serder.sn) self.db.addPse(snkey, serder.saidb) # b'EOWwyMU3XA7RtWdelFt-6waurOTH_aW_Z9VTaU-CshGk.00000000000000000000000000000001' logger.info("Kever state: Escrowed partially signed or delegated " "event = %s\n", serder.ked) - def escrowPACouple(self, serder, seqner, saider): + + def escrowPACouple(self, serder, seqner, saider, local=True): """ Update associated logs for escrow of partially authenticated issued event. Assuming signatures are provided elsewhere. Partial authentication results @@ -2751,14 +3126,21 @@ def escrowPACouple(self, serder, seqner, saider): serder is SerderKERI instance of delegated or issued event seqner is Seqner instance of sn of seal source event of delegator/issuer saider is Saider instance of said of delegator/issuer + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ + local = True if local else False # ignored since not escrowing serder here dgkey = dgKey(serder.preb, serder.saidb) couple = seqner.qb64b + saider.qb64b self.db.putPde(dgkey, couple) # idempotent logger.info("Kever state: Escrowed source couple for partially signed " "or delegated event = %s\n", serder.ked) - def escrowPWEvent(self, serder, wigers, sigers=None, seqner=None, saider=None): + + def escrowPWEvent(self, serder, wigers, sigers=None, + seqner=None, saider=None, local=True): """ Update associated logs for escrow of partially witnessed event @@ -2768,7 +3150,13 @@ def escrowPWEvent(self, serder, wigers, sigers=None, seqner=None, saider=None): sigers is optional list of Siger instances of indexed controller sigs seqner is Seqner instance of sn of seal source event of delegator/issuer saider is Diger instance of digest of delegator/issuer + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + """ + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent if wigers: @@ -2780,6 +3168,16 @@ def escrowPWEvent(self, serder, wigers, sigers=None, seqner=None, saider=None): self.db.putPde(dgkey, couple) self.db.putEvt(dgkey, serder.raw) + # update event source + if (esr := self.db.esrs.get(keys=dgkey)): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + logger.info("Kever state: Escrowed partially witnessed " "event = %s\n", serder.ked) return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) @@ -2816,7 +3214,7 @@ def state(self): toad=self.toader.num, wits=self.wits, cnfg=cnfg, - dpre=self.delegator, + dpre=self.delpre, ) ) @@ -2894,7 +3292,7 @@ def fetchLatestContribTo(self, verfers, sn: int | None = None): for digb in self.db.getKelBackIter(pre, sn): dgkey = dgKey(pre, digb) raw = self.db.getEvt(dgkey) - serder = serdering.SerderKERI(raw=bytes(raw)) + serder = serdering.SerderKERI(raw=bytes(raw), verify=False) if serder.estive: # establishment event key = serder.verfers[0].qb64 try: @@ -2947,7 +3345,7 @@ def fetchLatestContribFrom(self, verfer, sn: int | None = None): for digb in self.db.getKelBackIter(pre, sn): dgkey = dgKey(pre, digb) raw = self.db.getEvt(dgkey) - serder = serdering.SerderKERI(raw=bytes(raw)) + serder = serdering.SerderKERI(raw=bytes(raw), verify=False) if serder.estive: # establishment event keys = [verfer.qb64 for verfer in serder.verfers] try: @@ -3026,8 +3424,8 @@ def __init__(self, *, cues=None, db=None, rvy=None, lax (bool): True means operate in promiscuous (unrestricted) mode, False means operate in nonpromiscuous (restricted) mode as determined by local and prefixes - local (bool): True means only process msgs for own events if not lax - False means only process msgs for not own events if not lax + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation cloned (bool): True means cloned message stream so use attached datetimes from clone source not own. False means use current datetime @@ -3044,7 +3442,7 @@ def __init__(self, *, cues=None, db=None, rvy=None, self.db = db self.rvy = rvy self.lax = True if lax else False # promiscuous mode - self.local = True if local else False # local vs nonlocal restrictions + self.local = True if local else False # local vs nonlocal default self.cloned = True if cloned else False # process as cloned self.direct = True if direct else False # process as direct mode self.check = True if check else False # process as check mode @@ -3094,26 +3492,33 @@ def fetchWitnessState(self, pre, sn): def processEvent(self, serder, sigers, *, wigers=None, delseqner=None, delsaider=None, - firner=None, dater=None): + firner=None, dater=None, local=None): """ Process one event serder with attached indexd signatures sigers Parameters: - serder is SerderKERI instance of event to process - sigers is list of Siger instances of attached controller indexed sigs - wigers is optional list of Siger instances of attached witness indexed sigs - delseqner is Seqner instance of delegating event sequence number. + serder (SerderKERI): instance of event to process + sigers (list[Siger]): instances of attached controller indexed sigs + wigers (list[Siger]|None): instances of attached witness indexed sigs + otherwise None + delseqner (Seqner|None): instance of delegating event sequence number. If this event is not delegated then seqner is ignored - delsaider is Saider instance of of delegating event SAID. + delsaider (Saider|None): instance of of delegating event SAID. If this event is not delegated then saider is ignored - firner is optional Seqner instance of cloned first seen ordinal + firner (Seqner|None): instance of cloned first seen ordinal If cloned mode then firner maybe provided (not None) When firner provided then compare fn of dater and database and first seen if not match then log and add cue notify problem - dater is optional Dater instance of cloned replay datetime + dater (Dater|None): instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch ked ilk pre, sn, dig to see how to process pre = serder.pre ked = serder.ked @@ -3126,7 +3531,7 @@ def processEvent(self, serder, sigers, *, wigers=None, "".format(pre, ked)) from ex sn = serder.sn - ilk = serder.ilk # ked["t"] + ilk = serder.ilk # ked["t"] said = serder.said @@ -3145,11 +3550,19 @@ def processEvent(self, serder, sigers, *, wigers=None, firner=firner if self.cloned else None, dater=dater if self.cloned else None, cues=self.cues, - prefixes=self.prefixes, - local=self.local, + local=local, check=self.check) self.kevers[pre] = kever # not exception so add to kevers + # At this point the inceptive event (icp or dip) given by serder + # together with its attachments has been accepted as valid with finality. + # Events that don't get to here have either been dropped as + # invalid by raising an error or have been escrowed as not + # yet complete enough to decide their validity, i.e. are + # missing signatures, or witness receipts or delegations. + # Any actions that should be triggered as a result of final + # acceptance should follow from here. + if self.direct or self.lax or pre not in self.prefixes: # not own event when owned # create cue for receipt controller or watcher # receipt of actual type is dependent on own type of identifier @@ -3157,17 +3570,33 @@ def processEvent(self, serder, sigers, *, wigers=None, elif not self.direct: # notice of new event self.cues.push(dict(kin="notice", serder=serder)) - if kever.locallyWitnessed(): - # ToDo XXXX need to cue task here kin = "witness" + if self.local and kever.locallyWitnessed(): # + # ToDo XXXX need to cue task here kin = "witness" and process + # cued witness and then combine with reciept above so only + # one receipt is generated not two self.cues.push(dict(kin="witness", serder=serder)) - if kever.locallyOwned(kever.delegator): # delegator may be None - # ToDo XXXX need to cue task here to approve delegation by generating - # and anchoring SealEvent of serder in delegators KEL - # may include MFA business logic for the delegator i.e. is local - self.cues.push(dict(kin="approveDelegation", - delegator=kever.delegator, - serder=serder)) + if self.local and kever.locallyOwned(): + # ToDo XXXX process this cue of query to send event to delegator + # to trigger generation of anchor in delegating event + # note for remote validators there is query cue in + # validateDelegation to query for anchoring event seal + pass + + # Moved logic to kever.validateDelegation trigger for delegation escrow + #if (self.local and + #kever.locallyOwned(kever.delpre if kever.delpre is not None else '')): # delpre may be None + ## ToDo XXXX need to cue task here to approve delegation by generating + ## an anchoring SealEvent of serder in delegators KEL + ## may include MFA and or business logic for the delegator i.e. is local + ## event that designates this controller as delegator triggers + ## this cue to approave delegation + #self.cues.push(dict(kin="approveDelegation", + #delegator=kever.delpre, + #serder=serder)) + + + else: # not inception so can't verify sigs etc, add to out-of-order escrow @@ -3198,8 +3627,9 @@ def processEvent(self, serder, sigers, *, wigers=None, if sigers or wigers: # at least one verified sig or wig so log evt # this allows late arriving witness receipts or controller # signatures to be added to the databse - # not first seen inception so ignore return - kever.logEvent(serder, sigers=sigers, wigers=wigers) # idempotent update db logs + # Not first seen version of event so ignore return + # idempotent update db logs + kever.logEvent(serder, sigers=sigers, wigers=wigers) else: # escrow likely duplicitous event self.escrowLDEvent(serder=serder, sigers=sigers) @@ -3228,7 +3658,17 @@ def processEvent(self, serder, sigers, *, wigers=None, delseqner=delseqner, delsaider=delsaider, firner=firner if self.cloned else None, dater=dater if self.cloned else None, - check=self.check) + local=local, check=self.check) + + # At this point the non-inceptive event (rot, drt, or ixn) + # given by serder together with its attachments has been + # accepted as valid with finality. + # Events that don't get to here have either been dropped as + # invalid by raising an error or have been escrowed as not + # yet complete enough to decide their validity, i.e. are + # missing signatures, or witness receipts or delegations. + # Any actions that should be triggered as a result of final + # acceptance should follow from here. if self.direct or self.lax or pre not in self.prefixes: # not own event when owned # create cue for receipt controller or watcher @@ -3237,18 +3677,30 @@ def processEvent(self, serder, sigers, *, wigers=None, elif not self.direct: # notice of new event self.cues.push(dict(kin="notice", serder=serder)) - if kever.locallyWitnessed(): - # ToDo XXXX need to cue task here kin = "witness" + if self.local and kever.locallyWitnessed(): # + # ToDo XXXX need to cue task here kin = "witness" and process + # cued witness and then combine with reciept above so only + # one receipt is generated not two self.cues.push(dict(kin="witness", serder=serder)) - if kever.locallyOwned(kever.delegator): # delegator may be None - # ToDo XXXX need to cue task here to approve delegation by generating - # and anchoring SealEvent of serder in delegators KEL - # may include MFA business logic for the delegator i.e. is local - self.cues.push(dict(kin="approveDelegation", - delegator=kever.delegator, - serder=serder)) - + if self.local and kever.locallyOwned(): + # ToDo XXXX process this cue of query to send event to delegator + # to trigger generation of anchor in delegating event + # note for remote validators there is query cue in + # validateDelegation to query for anchoring event seal + pass + + # Moved logic to kever.validateDelegation trigger for delegation escrow + #if (self.local and + #kever.locallyOwned(kever.delpre if kever.delpre is not None else '')): # delpre may be None + ## ToDo XXXX need to cue task here to approve delegation by generating + ## an anchoring SealEvent of serder in delegators KEL + ## may include MFA and or business logic for the delegator i.e. is local + ## event that designates this controller as delegator triggers + ## this cue to approave delegation + #self.cues.push(dict(kin="approveDelegation", + #delegator=kever.delpre, + #serder=serder)) else: # maybe duplicitous # check if duplicate of existing valid accepted event @@ -3270,23 +3722,31 @@ def processEvent(self, serder, sigers, *, wigers=None, verfers=werfers) if sigers or wigers: # at least one verified sig or wig so log evt - # not first seen update so ignore return + # this allows late arriving witness receipts or controller + # signatures to be added to the databse + # Not first seen version of event so ignore return + # idempotent update db logs kever.logEvent(serder, sigers=sigers, wigers=wigers) # idempotent update db logs else: # escrow likely duplicitous event self.escrowLDEvent(serder=serder, sigers=sigers) raise LikelyDuplicitousError("Likely Duplicitous event={}.".format(ked)) - def processReceiptWitness(self, serder, wigers): + + def processReceiptWitness(self, serder, wigers, local=None): """ - Process one witness receipt serder with attached witness sigers + Process one witness receipt serder with attached witness wigers + (indexed signatures) Parameters: - serder is SerderKERI instance of serialized receipt message not receipted event - sigers is list of Siger instances that with witness indexed signatures + serder (SerderKERI): instance of serialized receipt message not receipted event + wigers (list[Siger]): instances that with witness indexed signatures signature in .raw. Index is offset into witness list of latest establishment event for receipted event. Signature uses key pair derived from nontrans witness prefix in associated witness list. + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . Receipt dict labels vs # version string @@ -3295,10 +3755,12 @@ def processReceiptWitness(self, serder, wigers): ilk # rct dig # qb64 digest of receipted event """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch pre dig to process ked = serder.ked pre = serder.pre - sn = serder.sn # Only accept receipt if for last seen version of event at sn @@ -3326,13 +3788,13 @@ def processReceiptWitness(self, serder, wigers): if wiger.verfer.transferable: # skip transferable verfers continue # skip invalid witness prefix - if not self.lax and wiger.verfer.qb64 in self.prefixes: # own is receiptor + if not self.lax and wiger.verfer.qb64 in self.prefixes: # own is witness if pre in self.prefixes: # skip own receiptor of own event # sign own events not receipt them logger.info("Kevery process: skipped own receipt attachment" " on own event receipt=\n%s\n", serder.pretty()) continue # skip own receipt attachment on own event - if not self.local: # own receipt on other event when not local + if not local: # so skip own receipt on other event when non-local source logger.info("Kevery process: skipped own receipt attachment" " on nonlocal event receipt=\n%s\n", serder.pretty()) continue # skip own receipt attachment on non-local event @@ -3347,14 +3809,20 @@ def processReceiptWitness(self, serder, wigers): raise UnverifiedWitnessReceiptError("Unverified witness receipt={}." "".format(ked)) - def processReceipt(self, serder, cigars): + def processReceipt(self, serder, cigars, local=None): """ Process one receipt serder with attached cigars + may or may not be a witness receipt. If prefix matches witness then + promote to indexed witness signature and store appropriately. Otherwise + signature is nontrans nonwitness endorser (watcher etc) Parameters: - serder is SerderKERI instance of serialized receipt message not receipted message - cigars is list of Cigar instances that contain receipt couple + serder (SerderKERI): rct instance of serialized receipt message + cigars (list[Cigar]): instances that contain receipt couple signature in .raw and public key in .verfer + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . Receipt dict labels vs # version string @@ -3363,6 +3831,9 @@ def processReceipt(self, serder, cigars): ilk # rct dig # qb64 digest of receipted event """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch pre dig to process ked = serder.ked pre = serder.pre @@ -3395,7 +3866,7 @@ def processReceipt(self, serder, cigars): logger.info("Kevery process: skipped own receipt attachment" " on own event receipt=\n%s\n", serder.pretty()) continue # skip own receipt attachment on own event - if not self.local: # own receipt on other event when not local + if not local: # skip own receipt on other event when not local logger.info("Kevery process: skipped own receipt attachment" " on nonlocal event receipt=\n%s\n", serder.pretty()) continue # skip own receipt attachment on non-local event @@ -3403,12 +3874,12 @@ def processReceipt(self, serder, cigars): if cigar.verfer.verify(cigar.raw, lserder.raw): wits = [wit.qb64 for wit in self.fetchWitnessState(pre, sn)] rpre = cigar.verfer.qb64 # prefix of receiptor - if rpre in wits: # its a witness receipt + if rpre in wits: # its a witness receipt write in .wigs index = wits.index(rpre) # create witness indexed signature wiger = Siger(raw=cigar.raw, index=index, verfer=cigar.verfer) self.db.addWig(key=dgkey, val=wiger.qb64b) # write to db - else: # write receipt couple to database + else: # not witness rect write receipt couple to database .rcts couple = cigar.verfer.qb64b + cigar.qb64b self.db.addRct(key=dgkey, val=couple) @@ -3416,19 +3887,32 @@ def processReceipt(self, serder, cigars): self.escrowUReceipt(serder, cigars, said=ked["d"]) # digest in receipt raise UnverifiedReceiptError("Unverified receipt={}.".format(ked)) - def processReceiptCouples(self, serder, cigars, firner=None): + + def processAttachedReceiptCouples(self, serder, cigars, firner=None, local=None): """ - Process attachment with receipt couple + Process one attachment couple that represents an endorsement from + a nontransferable AID that may or may not be a witness, maybe a watcher. + Originally may have been a non-transferable receipt or key event attachment + if signature is for witness then promote to indexed sig and store + appropriately. + The is the attachment version of .processReceipt Parameters: - serder is SerderKERI instance of receipted serialized event message - to which receipts are attached from replay - cigars is list of Cigar instances that contain receipt couple + serder (SerderKERI): instance serialized event message to which + attachments come from replay (clone) + cigars (list[Cigar]): instances that contain receipt couple signature in .raw and public key in .verfer - firner is Seqner instance of first seen ordinal, + firner (Seqner): instance of first seen ordinal, if provided lookup event by fn = firner.sn + used when in cloned replay mode + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch pre dig to process ked = serder.ked pre = serder.pre @@ -3463,7 +3947,7 @@ def processReceiptCouples(self, serder, cigars, firner=None): logger.info("Kevery process: skipped own receipt attachment" " on own event receipt=\n%s\n", serder.pretty()) continue # skip own receipt attachment on own event - if not self.local: # own receipt on other event when not local + if not local: # own receipt on other event when not local logger.info("Kevery process: skipped own receipt attachment" " on nonlocal event receipt=\n%s\n", serder.pretty()) continue # skip own receipt attachment on non-local event @@ -3480,14 +3964,19 @@ def processReceiptCouples(self, serder, cigars, firner=None): couple = cigar.verfer.qb64b + cigar.qb64b self.db.addRct(key=dgKey(pre, ldig), val=couple) - def processReceiptTrans(self, serder, tsgs): + def processReceiptTrans(self, serder, tsgs, local=None): """ Process one transferable validator receipt (chit) serder with attached sigers + (indexed signatures) who are not controllers. Controllers may only attach signatures to + the associated event not by sending signature attached to receipt of event. Parameters: - serder is chit serder (transferable validator receipt message) - tsgs is tist of tuples from extracted transferable indexed sig groups + serder (serderKERI): rct (transferable validator receipt message) + tsgs (list[tuple]): from extracted transferable indexed sig groups each converted group is tuple of (i,s,d) triple plus list of sigs + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . Receipt dict labels vs # version string @@ -3497,6 +3986,9 @@ def processReceiptTrans(self, serder, tsgs): dig # qb64 digest of receipted event """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch pre, dig,seal to process ked = serder.ked pre = serder.pre @@ -3522,18 +4014,18 @@ def processReceiptTrans(self, serder, tsgs): for sprefixer, sseqner, saider, sigers in tsgs: # iterate over each tsg if not self.lax and sprefixer.qb64 in self.prefixes: # own is receipter if pre in self.prefixes: # skip own receipter of own event - # sign own events not receipt them + # sign own events as controller not endorse them via receipt raise ValidationError("Own pre={} receipter of own event" " {}.".format(self.prefixes, serder.pretty())) - if not self.local: # skip own receipts of nonlocal events + if not local: # skip own receipts of nonlocal events raise ValidationError("Own pre={} receipter of nonlocal event " "{}.".format(self.prefixes, serder.pretty())) # receipted event in db so attempt to get receipter est evt - # retrieve dig of last event at sn of est evt of receipter. + # retrieve dig of last event at sn of est evt of receiptor. sdig = self.db.getKeLast(key=snKey(pre=sprefixer.qb64b, sn=sseqner.sn)) if sdig is None: - # receipter's est event not yet in receipters's KEL + # receiptor's est event not yet in receiptors's KEL # so need cue to discover est evt KEL for receipter from watcher etc self.escrowTReceipts(serder, sprefixer, sseqner, saider, sigers) raise UnverifiedTransferableReceiptError("Unverified receipt: " @@ -3541,7 +4033,7 @@ def processReceiptTrans(self, serder, tsgs): "receipter for event={}." "".format(ked)) - # retrieve last event itself of receipter est evt from sdig. + # retrieve last event itself of receiptor est evt from sdig. sraw = self.db.getEvt(key=dgKey(pre=sprefixer.qb64b, dig=bytes(sdig))) # assumes db ensures that sraw must not be none because sdig was in KE sserder = serdering.SerderKERI(raw=bytes(sraw)) @@ -3557,7 +4049,7 @@ def processReceiptTrans(self, serder, tsgs): " dig = {} from pre ={}, no keys." "".format(saider.qb64, sprefixer.qb64)) - for siger in sigers: + for siger in sigers: # endorser (non-controller) signatures if siger.index >= len(sverfers): raise ValidationError("Index = {} to large for keys." "".format(siger.index)) @@ -3568,16 +4060,24 @@ def processReceiptTrans(self, serder, tsgs): self.db.addVrc(key=dgKey(pre=pre, dig=ldig), val=quadruple) # dups kept - def processReceiptQuadruples(self, serder, trqs, firner=None): + def processAttachedReceiptQuadruples(self, serder, trqs, firner=None, local=None): """ - Process one attachment quadruple that comprises a transferable receipt + Process one attachment quadruple that represents an endorsement from + a transferable AID that is not the controller. Maybe a watcher. + Originally may have been a transferable receipt or key event attachment + + This is the attachement version of .processReceiptTrans Parameters: - serder is chit serder (transferable validator receipt message) - trqs is list of tuples (quadruples) of form - (prefixer, seqner, diger, siger) - firner is Seqner instance of first seen ordinal, + serder (serderKERI): instance serialized event message to which + attachments come from replay (clone) + trqs (list[tuple]): quadruples of (prefixer, seqner, diger, siger) + firner (Seqner): instance of first seen ordinal, if provided lookup event by fn = firner.sn + used when in cloned replay mode + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . Seal labels i pre # qb64 prefix of receipter @@ -3585,6 +4085,9 @@ def processReceiptQuadruples(self, serder, trqs, firner=None): d dig # qb64 digest of est event for receipter keys """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch pre, dig,seal to process ked = serder.ked pre = serder.pre @@ -3602,7 +4105,7 @@ def processReceiptQuadruples(self, serder, trqs, firner=None): raise ValidationError("Own pre={} replay attached transferable " "receipt quadruple of own event {}." "".format(self.prefixes, serder.pretty())) - if not self.local: # skip own trans receipt quadruples of nonlocal events + if not local: # skip own trans receipt quadruples of nonlocal events raise ValidationError("Own pre={} seal in replay attached " "transferable receipt quadruples of nonlocal" " event {}.".format(self.prefixes, serder.pretty())) @@ -4125,8 +4628,8 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): for msg in self.db.clonePreIter(pre=pre, fn=0): msgs.append(msg) - if kever.delegator: - cloner = self.db.clonePreIter(pre=kever.delegator, fn=0) # create iterator at 0 + if kever.delpre: + cloner = self.db.clonePreIter(pre=kever.delpre, fn=0) # create iterator at 0 for msg in cloner: msgs.append(msg) @@ -4203,7 +4706,49 @@ def fetchEstEvent(self, pre, sn): if sn < 0: # no more events return None - def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None): + + def escrowMFEvent(self, serder, sigers, wigers=None, + seqner=None, saider=None, local=True): + """ + Update associated logs for escrow of MisFit event + + Parameters: + serder (SerderKERI): instance of event + sigers (list): of Siger instance for event + seqner (Seqner): instance of sn of event delegatint/issuing event if any + saider (Saider): instance of dig of event delegatint/issuing event if any + wigers (list): of witness signatures + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + """ + local = True if local else False + dgkey = dgKey(serder.preb, serder.saidb) + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + self.db.putEvt(dgkey, serder.raw) + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: + couple = seqner.qb64b + saider.qb64b + self.db.putPde(dgkey, couple) # idempotent + self.db.misfits.add(snKey(serder.preb, serder.sn), serder.saidb) + # log escrowed + logger.info("Kevery process: escrowed misfit event=\n%s\n", + json.dumps(serder.ked, indent=1)) + + + def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None, local=True): """ Update associated logs for escrow of Out-of-Order event @@ -4213,17 +4758,31 @@ def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None): seqner (Seqner): instance of sn of event delegatint/issuing event if any saider (Saider): instance of dig of event delegatint/issuing event if any wigers (list): of witness signatures + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) self.db.putEvt(dgkey, serder.raw) - self.db.addOoe(snKey(serder.preb, serder.sn), serder.saidb) if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: couple = seqner.qb64b + saider.qb64b self.db.putPde(dgkey, couple) # idempotent + self.db.addOoe(snKey(serder.preb, serder.sn), serder.saidb) # log escrowed logger.info("Kevery process: escrowed out of order event=\n%s\n", json.dumps(serder.ked, indent=1)) @@ -4252,15 +4811,29 @@ def escrowQueryNotFoundEvent(self, prefixer, serder, sigers, cigars=None): logger.info("Kevery process: escrowed query not found event=\n%s\n", json.dumps(serder.ked, indent=1)) - def escrowLDEvent(self, serder, sigers): + def escrowLDEvent(self, serder, sigers, local=True): """ Update associated logs for escrow of Likely Duplicitous event Parameters: serder is SerderKERI instance of event sigers is list of Siger instance for event + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) self.db.putEvt(dgkey, serder.raw) @@ -4523,8 +5096,14 @@ def processEscrowOutOfOrders(self): for ekey, edig in self.db.getOoeItemsNextIter(key=key): try: pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) + # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(edig))) + dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" @@ -4573,7 +5152,7 @@ def processEscrowOutOfOrders(self): wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers) + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, local=esr.local) # If process does NOT validate event with sigs, becasue it is # still out of order then process will attempt to re-escrow @@ -4658,6 +5237,11 @@ def processEscrowPartialSigs(self): try: pre, sn = splitKeySN(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) + # check date if expired then remove escrow. dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes @@ -4707,7 +5291,7 @@ def processEscrowPartialSigs(self): delseqner, delsaider = deSourceCouple(couple) elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delegator + delpre = self.kevers[eserder.pre].delpre else: delpre = eserder.ked["di"] @@ -4722,7 +5306,7 @@ def processEscrowPartialSigs(self): # process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] self.processEvent(serder=eserder, sigers=sigers, - delseqner=delseqner, delsaider=delsaider) + delseqner=delseqner, delsaider=delsaider, local=esr.local) # If process does NOT validate sigs or delegation seal (when delegated), # but there is still one valid signature then process will @@ -4817,8 +5401,14 @@ def processEscrowPartialWigs(self): for ekey, edig in self.db.getPweItemsNextIter(key=key): try: pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) + # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(edig))) + dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" @@ -4885,7 +5475,7 @@ def processEscrowPartialWigs(self): delseqner, delsaider = deSourceCouple(couple) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider) + delseqner=delseqner, delsaider=delsaider, local=esr.local) # If process does NOT validate wigs then process will attempt # to re-escrow and then raise MissingWitnessSignatureError @@ -4986,6 +5576,7 @@ def processEscrowUnverWitness(self): for ekey, ecouple in self.db.getUweItemsNextIter(key=key): try: pre, sn = splitKeySN(ekey) # get pre and sn from escrow db key + # get escrowed receipt's rdiger of receipted event and # wiger indexed signature of receipted event rdiger, wiger = deWitnessCouple(ecouple) @@ -5305,7 +5896,13 @@ def processQueryNotFound(self): # process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - # get wigs + # ToDo XXXX get wigs and attach + # getWigs + + # ToDo XXXX get trans endorsements + # getVrcs + + # get nontrans endorsements cigars = [] cigs = self.db.getRcts(dgKey(pre, bytes(edig))) # list of wigs for cig in cigs: @@ -5672,8 +6269,14 @@ def processEscrowDuplicitous(self): for ekey, edig in self.db.getLdeItemsNextIter(key=key): try: pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) + # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(edig))) + dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" @@ -5716,7 +6319,7 @@ def processEscrowDuplicitous(self): "dig = {}.".format(bytes(edig))) sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - self.processEvent(serder=eserder, sigers=sigers) + self.processEvent(serder=eserder, sigers=sigers, local=esr.local) # If process does NOT validate event with sigs, becasue it is # still out of order then process will attempt to re-escrow diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index e3f18bb9f..2140086b8 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -89,10 +89,16 @@ class Parser: whenever stream includes pipelined count codes. kvy (Kevery): route KEL message types to this instance tvy (Tevery): route TEL message types to this instance + exc (Exchanger): route EXN message types to this instance + rvy (Revery): reply (RPY) message handler + vry (Verfifier): credential verifier with wallet storage + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation """ - def __init__(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + def __init__(self, ims=None, framed=True, pipeline=False, kvy=None, + tvy=None, exc=None, rvy=None, vry=None, local=False): """ Initialize instance: @@ -107,6 +113,8 @@ def __init__(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, ex exc (Exchanger): route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verfifier): credential verifier with wallet storage + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation """ self.ims = ims if ims is not None else bytearray() self.framed = True if framed else False # extract until end-of-stream @@ -116,6 +124,7 @@ def __init__(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, ex self.exc = exc self.rvy = rvy self.vry = vry + self.local = True if local else False @staticmethod def sniff(ims): @@ -336,7 +345,9 @@ def _nonTransReceiptCouples(self, ctr, ims, cold=Colds.txt, pipelined=False): yield cigar - def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + + def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, + exc=None, rvy=None, vry=None, local=None): """ Processes all messages from incoming message stream, ims, when provided. Otherwise process messages from .ims @@ -359,11 +370,18 @@ def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=No exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verfifier): credential verifier with wallet storage + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local New Logic: Attachments must all have counters so know if txt or bny format for attachments. So even when framed==True must still have counters. """ + local = local if local is not None else self.local + local = True if local else False + + parsator = self.allParsator(ims=ims, framed=framed, pipeline=pipeline, @@ -371,7 +389,8 @@ def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=No tvy=tvy, exc=exc, rvy=rvy, - vry=vry) + vry=vry, + local=local) while True: try: @@ -379,7 +398,9 @@ def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=No except StopIteration: break - def parseOne(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + + def parseOne(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, + exc=None, rvy=None, vry=None, local=None): """ Processes one messages from incoming message stream, ims, when provided. Otherwise process message from .ims @@ -401,11 +422,17 @@ def parseOne(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, ex tvy (Tevery): route TEL message types to this instance exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local New Logic: Attachments must all have counters so know if txt or bny format for attachments. So even when framed==True must still have counters. """ + local = local if local is not None else self.local + local = True if local else False + parsator = self.onceParsator(ims=ims, framed=framed, pipeline=pipeline, @@ -413,14 +440,17 @@ def parseOne(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, ex tvy=tvy, exc=exc, rvy=rvy, - vry=vry) + vry=vry, + local=local) while True: try: next(parsator) except StopIteration: break - def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + + def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, + tvy=None, exc=None, rvy=None, vry=None, local=None): """ Returns generator to parse all messages from incoming message stream, ims until ims is exhausted (empty) then returns. @@ -443,6 +473,9 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verfifier): credential verifier with wallet storage + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local New Logic: Attachments must all have counters so know if txt or bny format for @@ -461,6 +494,8 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc = exc if exc is not None else self.exc rvy = rvy if rvy is not None else self.rvy vry = vry if vry is not None else self.vry + local = local if local is not None else self.local + local = True if local else False while ims: # only process until ims empty try: @@ -471,7 +506,8 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, tvy=tvy, exc=exc, rvy=rvy, - vry=vry) + vry=vry, + local=local) except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream @@ -498,7 +534,9 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, return True - def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + + def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, + tvy=None, exc=None, rvy=None, vry=None, local=None): """ Returns generator to parse one message from incoming message stream, ims. If ims not provided parse messages from .ims @@ -519,6 +557,9 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verfifier): credential verifier with wallet storage + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local New Logic: Attachments must all have counters so know if txt or bny format for @@ -537,6 +578,8 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc = exc if exc is not None else self.exc rvy = rvy if rvy is not None else self.rvy vry = vry if vry is not None else self.vry + local = local if local is not None else self.local + local = True if local else False done = False while not done: @@ -548,7 +591,8 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, tvy=tvy, exc=exc, rvy=rvy, - vry=vry) + vry=vry, + local=local) except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream @@ -576,13 +620,15 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, return done - def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + + def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, + exc=None, rvy=None, vry=None, local=None): """ Returns generator to continually parse messages from incoming message - stream, ims. Empty yields when ims is emply. + stream, ims. Empty yields when ims is emply. Does not return. Useful for always running servers. One yield from per each message if any. - Continually yields while ims is empty. + Continually yields while ims is empty, i.e. does not return. If ims not provided then parse messages from .ims Parameters: @@ -601,6 +647,10 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verifier): credential processor + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local + New Logic: Attachments must all have counters so know if txt or bny format for @@ -619,6 +669,8 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc exc = exc if exc is not None else self.exc rvy = rvy if rvy is not None else self.rvy vry = vry if vry is not None else self.vry + local = local if local is not None else self.local + local = True if local else False while True: # continuous stream processing never stop try: @@ -629,7 +681,8 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc tvy=tvy, exc=exc, rvy=rvy, - vry=vry) + vry=vry, + local=local) except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream @@ -658,7 +711,7 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc def msgParsator(self, ims=None, framed=True, pipeline=False, - kvy=None, tvy=None, exc=None, rvy=None, vry=None): + kvy=None, tvy=None, exc=None, rvy=None, vry=None, local=None): """ Returns generator that upon each iteration extracts and parses msg with attached crypto material (signature etc) from incoming message @@ -685,6 +738,9 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verifier) ACDC credential processor + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local Logic: Currently only support couters on attachments not on combined or @@ -703,6 +759,9 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, """ + local = local if local is not None else self.local + local = True if local else False + serdery = serdering.Serdery(version=kering.Version) if ims is None: @@ -726,16 +785,6 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, else: # extracted and stripped successfully break # break out of while loop - - #while True: # extract and deserialize message from ims - #try: - #sadder = Sadder(raw=ims) - #except kering.ShortageError as ex: # need more bytes - #yield - #else: # extracted successfully - #del ims[:sadder.size] # strip off event from front of ims - #break - sigers = [] # list of Siger instances of attached indexed controller signatures wigers = [] # list of Siger instance of attached indexed witness signatures cigars = [] # List of cigars to hold nontrans rct couplets @@ -1010,11 +1059,6 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, if isinstance(serder, serdering.SerderKERI): ilk = serder.ilk # dispatch abased on ilk - #if sadder.proto == Protos.keri: - #serder = Serder(sad=sadder) - - #ilk = serder.ked["t"] # dispatch abased on ilk - if ilk in [Ilks.icp, Ilks.rot, Ilks.ixn, Ilks.dip, Ilks.drt]: # event msg firner, dater = frcs[-1] if frcs else (None, None) # use last one if more than one # when present assumes this is source seal of delegating event in delegator's KEL @@ -1029,12 +1073,15 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, delseqner=delseqner, delsaider=delsaider, firner=firner, - dater=dater) + dater=dater, + local=local) if cigars: - kvy.processReceiptCouples(serder, cigars, firner=firner) + kvy.processAttachedReceiptCouples(serder, cigars, + firner=firner, local=local) if trqs: - kvy.processReceiptQuadruples(serder, trqs, firner=firner) + kvy.processAttachedReceiptQuadruples(serder, trqs, + firner=firner, local=local) except AttributeError as ex: raise kering.ValidationError("No kevery to process so dropped msg" @@ -1047,13 +1094,16 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, try: if cigars: - kvy.processReceipt(serder=serder, cigars=cigars) + kvy.processReceipt(serder=serder, cigars=cigars, + local=local) if wigers: - kvy.processReceiptWitness(serder=serder, wigers=wigers) + kvy.processReceiptWitness(serder=serder, wigers=wigers, + local=local) if tsgs: - kvy.processReceiptTrans(serder=serder, tsgs=tsgs) + kvy.processReceiptTrans(serder=serder, tsgs=tsgs, + local=local) except AttributeError: raise kering.ValidationError("No kevery to process so dropped msg" diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index c67092b65..6dac3a791 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -198,6 +198,7 @@ def processReply(self, serder, cigars=None, tsgs=None): self.rtr.dispatch(serder=serder, saider=saider, cigars=cigars, tsgs=tsgs) + def acceptReply(self, serder, saider, route, aid, osaider=None, cigars=None, tsgs=None): """ Applies Best Available Data Acceptance policy to reply and signatures diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index d4263fc91..5e775767c 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -46,6 +46,33 @@ logger = help.ogler.getLogger() + +# ToDo XXXX maybe +''' +class komerdict(dict): + """ + Subclass of dict that has db as attribute and employs read through cache + from db Baser.stts of kever states to reload kever from state in database + when not found in memory as dict item. + + add method that answers is a given pre a group hab pre .localGroup(pre) + + ToDo XXXX change name of dbdict to stateDict since now have differen types + and can't subclass dict with init parameters. + but can change function by manually assigning attributes but that is ugly + need wrapper decorator to do that. So can update attributes with wrapper + on class that injects instance attributes when class is instanced + one of the injected parameters is function that that maps returned Komer to + object class + parameters are subdb (must be Komer) and function that maps retrieved dataclass + record from dataabase to class instance. if no mapping function then just + return the dataclass record as value. + """ + +''' + + +# ToDo XXXX change name to statedict since not a generic dbdict class dbdict(dict): """ Subclass of dict that has db as attribute and employs read through cache @@ -100,6 +127,9 @@ def get(self, k, default=None): return self.__getitem__(k) + + + @dataclass class RawRecord: """RawRecord is base class for dataclasses that provides private utility @@ -215,6 +245,18 @@ class KeyStateRecord(RawRecord): # baser.state di: str = '' # delegator aid qb64 if any otherwise empty '' str +@dataclass +class EventSourceRecord: # tracks source of event local or remote + """ + Keyed by dig (said) of serder of event + + Usage: + + """ + local: bool = True # True of local (protected) else False for remote (unprotected) + + def __iter__(self): + return iter(asdict(self)) @dataclass @@ -271,7 +313,7 @@ class OobiQueryRecord: # information for responding to OOBI query constraint policy for endpoint discovery . Usage: - oobiqs: dict[str, OobiQueryRecord] = field(default_factory=dict) + """ cid: str = None # qb64 role: str = None # one of kering.Roles None is any or all @@ -509,6 +551,46 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event Only one value per DB key is allowed + .esrs is named sub DB instance of Komer of EventSourceRecord + dgKey + DB is keyed by identifier prefix plus digest (said) of serialized event + Value is serialized instance of EventSourceRecord dataclass. + Only one value per DB key is allowed. + Keeps track of the source of the event. When .local is Truthy the + event was sourced in a protected way such as being generated + locally or via a protected path. When .local is Falsey the event was + NOT sourced in a protected way. The value of .local determines what + validation logic to run on the event. This database is used to track + the source when processing escrows that would otherwise be decoupled + from the original source of the event. + + .misfits is named sub DB instance of CesrIoSetSuber for misfit escrows + subkey "mfes." + snKey + DB is keyed by event controller prefix plus sn of serialized event + where sn is 32 char hex string with leading zeros + Value is serialized qb64b dig (said) of event + Misfit escrows are events with remote (nonlocal) sources that are + inappropriate (i.e. would be dropped) unless they can be promoted + to local source via some extra after the fact authentication. + Escrow processing determines if and how to promote event source to + local and then reprocess + + .delegables is named sub DB instance of CesrIoSetSuber for delegable escrows + subkey "dees." + snKey + DB is keyed by event controller prefix plus sn of serialized event + where sn is 32 char hex string with leading zeros + Value is serialized qb64b dig (said) of event + Delegable event escrows are events with local delegator that need + to be approved via the anchoring of the delegated event seal in + the delegator's KEL. Event source must be local. A nonlocal (remote) + source for a delegable event of a local delegator must first pass + through the misfit escrow and get promoted to local source. + + # delegable events escrows. events with local delegator that need approval + self.delegables = subing.CesrIoSetSuber(db=self, subkey='dees.', klas=coring.Diger) + .fels is named sub DB of first seen event log table (FEL) of digests that indexes events in first 'seen' accepted order for replay and cloning of event log. Only one value per DB key is allowed. @@ -542,8 +624,9 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event More than one value per DB key is allowed - .wigs is named sub DB of indexed witness signatures of event - Witnesses always have nontransferable indetifier prefixes. + .wigs is named sub DB of indexed witness signatures of event that may + come directly or derived from a witness receipt message. + Witnesses always have nontransferable identifier prefixes. The index is the offset of the witness into the witness list of the most recent establishment event wrt the receipted event. dgKey @@ -551,7 +634,10 @@ class Baser(dbing.LMDBer): More than one value per DB key is allowed .rcts is named sub DB of event receipt couplets from nontransferable - signers. Each couple is concatenation of fully qualified items. + signers. + These are endorsements from nontrasferable signers who are not witnesses + May be watchers or other + Each couple is concatenation of fully qualified items. These are: non-transferale prefix plus non-indexed event signature by that prefix. dgKey @@ -563,6 +649,8 @@ class Baser(dbing.LMDBer): qualified items. These are: receipted event digest, non-transferable receiptor identifier prefix, plus nonindexed receipt event signature by that prefix. + Used to manage out of order events such as escrowing + receipt couple until event receipted shows up. snKey DB is keyed by receipted event controller prefix plus sn of serialized event @@ -573,6 +661,8 @@ class Baser(dbing.LMDBer): of validator. These are: transferable prefix, plus latest establishment event sequence number plus latest establishment event digest, plus indexed event signature. + These are endorsements by transferable AIDs that are not the controller + may be watchers or others. When latest establishment event is multisig then there will be multiple quadruples one per signing key, each a dup at same db key. dgKey @@ -663,7 +753,7 @@ class Baser(dbing.LMDBer): the associated event. So one can lookup event digest, get its fn here and then use fn to fetch event by fn from .fels. - .states (stts) is named subDB instance of SerderSuber that maps a prefix + .states (subkey stts.) is named subDB instance of SerderSuber that maps a prefix to the latest keystate for that prefix. Used by ._kevers.db for read through cache of key state to reload kevers in memory @@ -746,6 +836,20 @@ class Baser(dbing.LMDBer): identical message bodies across participants in group multisig body trying to reach concensus on events or credentials. + .pubs is CatCesrIoSetSuber with subkey="pubs." of concatenated tuples + (qb64 pre, qb64 snh) indexed by qb64 of public key. Maps each signing + public key from establishment event to the events's prefix and sequence number + so can look up an event by any of its signing keys. Updated by Kever.logEvent + + .digs is CatCesrIoSetSuber with subkey="digs." of of concatenated tuples + (qb64 pre, qb64 snh) indexed by qb64 of digest of next signing public key. + Maps each next signing public key digest from establishment event to + the events's prefix and sequence number so can look up an event by any + of its next public signing key digests. Updated by Kever.logEvent + + Missing ToDo XXXX other attributes as sub dbs not documented here + such as .wits etc + Properties: kevers (dbdict): read through cache of kevers of states for KELs in db @@ -769,7 +873,8 @@ def __init__(self, headDirPath=None, reopen=False, **kwa): """ - self.prefixes = oset() + self.prefixes = oset() # should change to hids for hab ids + self.groups = oset() # group hab ids self._kevers = dbdict() self._kevers.db = self # assign db for read through cache of kevers @@ -824,6 +929,17 @@ def reopen(self, **kwa): self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) self.qnfs = self.env.open_db(key=b'qnfs.', dupsort=True) + # event source local (protected) or non-local (remote not protected) + self.esrs = koming.Komer(db=self, + schema=EventSourceRecord, + subkey='esrs.') + + # misfit escrows whose processing may change the .esrs event source record + self.misfits = subing.CesrIoSetSuber(db=self, subkey='mfes.', klas=coring.Diger) + + # delegable events escrows. events with local delegator that need approval + self.delegables = subing.CesrIoSetSuber(db=self, subkey='dees.', klas=coring.Diger) + # events as ordered by first seen ordinals self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=coring.Seqner) # Kever state made of KeyStateRecord key states @@ -1001,7 +1117,7 @@ def reopen(self, **kwa): # Resolved multifactor well known OOBI auth records. Keys by controller URL self.rmfa = koming.Komer(db=self, - subkey='mfa.', + subkey='rmfa.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. @@ -1035,11 +1151,15 @@ def reopen(self, **kwa): self.cdel = subing.CesrSuber(db=self, subkey='cdel.', klas=coring.Saider) - # public keys mapped to the AID and event seq no they appeared in + # siging public keys mapped to the AID and event seq no they appeared in so + # can look up member event designating as signing keys + # updated by Kever.logEvent. self.pubs = subing.CatCesrIoSetSuber(db=self, subkey="pubs.", klas=(coring.Prefixer, coring.Seqner)) - # next key digests mapped to the AID and event seq no they appeared in + # next key digests mapped to the AID and event seq no they appeared in so + # can look up event designating as next signing key digest + # updated by Kever.logEvent. self.digs = subing.CatCesrIoSetSuber(db=self, subkey="digs.", klas=(coring.Prefixer, coring.Seqner)) @@ -1053,6 +1173,7 @@ def reopen(self, **kwa): return self.env + def reload(self): """ Reload stored prefixes and Kevers from .habs @@ -1064,13 +1185,15 @@ def reload(self): try: kever = eventing.Kever(state=ksr, db=self, - prefixes=self.prefixes, local=True) except kering.MissingEntryError as ex: # no kel event for keystate removes.append(keys) # remove from .habs continue self.kevers[kever.prefixer.qb64] = kever self.prefixes.add(kever.prefixer.qb64) + if data.mid: # group hab + self.groups.add(data.hid) + elif data.mid is None: # in .habs but no corresponding key state and not a group so remove removes.append(keys) # no key state or KEL event for .hab record @@ -1084,13 +1207,14 @@ def reload(self): try: kever = eventing.Kever(state=ksr, db=self, - prefixes=self.prefixes, local=True) except kering.MissingEntryError as ex: # no kel event for keystate removes.append(keys) # remove from .habs continue self.kevers[kever.prefixer.qb64] = kever self.prefixes.add(kever.prefixer.qb64) + if data.mid: # group hab + self.groups.add(data.hid) elif data.mid is None: # in .habs but no corresponding key state and not a group so remove removes.append(keys) # no key state or KEL event for .hab record @@ -1112,7 +1236,7 @@ def clean(self): temp=self.temp, headDirPath=self.headDirPath, perm=self.perm, - clean=True) as copy: + clean=True) as copy: # copy is Baser instance with reopenDB(db=self, reuse=True, readonly=True): # reopen as readonly if not os.path.exists(self.path): @@ -1136,7 +1260,12 @@ def clean(self): if val.hid in copy.kevers: # only copy habs that verified copy.habs.put(keys=keys, val=val) copy.prefixes.add(val.hid) + if val.mid: # a group hab + copy.groups.add(val.hid) + # ToDo XXXX + # is this obsolete? Should this be removed or should this be + # be the Signator name not the default name of the Habery? if not copy.habs.get(keys=(self.name,)): raise ValueError("Error cloning habs, missing orig name={}." "".format(self.name)) @@ -1173,6 +1302,10 @@ def clean(self): self.prefixes.clear() self.prefixes.update(copy.prefixes) + # clear and clone .gids + self.groups.clear() + self.groups.update(copy.groups) + with reopenDB(db=self, reuse=True): # make sure can reopen if not isinstance(self.env, lmdb.Environment): raise ValueError("Error cloning, unable to reopen." @@ -1260,14 +1393,16 @@ def cloneEvtMsg(self, pre, fn, dig): count=1).qb64b) atc.extend(couple) - # add trans receipts quadruples to attachments + # add trans endorsement quadruples to attachments not controller + # may have been originally key event attachments or receipted endorsements if quads := self.getVrcs(key=dgkey): atc.extend(coring.Counter(code=coring.CtrDex.TransReceiptQuadruples, count=len(quads)).qb64b) for quad in quads: atc.extend(quad) - # add nontrans receipts couples to attachments + # add nontrans endorsement couples to attachments not witnesses + # may have been originally key event attachments or receipted endorsements if coups := self.getRcts(key=dgkey): atc.extend(coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=len(coups)).qb64b) @@ -1302,10 +1437,10 @@ def cloneDelegation(self, kever): """ if kever.delegated: - dkever = self.kevers[kever.delegator] + dkever = self.kevers[kever.delpre] yield from self.cloneDelegation(dkever) - for dmsg in self.clonePreIter(pre=kever.delegator, fn=0): + for dmsg in self.clonePreIter(pre=kever.delpre, fn=0): yield dmsg @@ -1371,73 +1506,6 @@ def findAnchoringSeal(self, pre, seal, sn=0): - def findAnchoringSealEventClone(self, pre, seal): - """ - Search through a KEL for the event that contains a specific anchored - SealEvent type of provided seal but in dict form. - Returns the Serder of the first event with the anchored SealEvent seal, - None if not found - Searchs from inception forward - - Parameters: - pre is qb64 identifier of the KEL to search - seal is dict form of SealEvent to find in anchored seals list of each event - - """ - if tuple(seal.keys()) != eventing.SealEvent._fields: # wrong type of seal - return None - #raise ValueError(f"Expected SealEvent got {seal}.") - - seal = eventing.SealEvent(**seal) #convert to namedtuple - - # getEvtPreIter getEvtLastPreIter - - for evt in self.clonePreIter(pre=pre): # all events including superseded - srdr = serdering.SerderKERI(raw=evt) - for eseal in srdr.seals or []: - if tuple(eseal.keys()) == eventing.SealEvent._fields: - eseal = eventing.SealEvent(**eseal) #convert to namedtuple - if seal == eseal and self.fullyWitnessed(srdr): - return srdr - #spre = anc["i"] - #ssn = int(anc["s"], 16) - #sdig = anc["d"] - - #if spre == seal["i"] and ssn == int(seal["s"], 16) \ - #and seal["d"] == sdig and self.fullyWitnessed(srdr): - #return srdr - - return None - - - def findAnchoringSealClone(self, pre, seal): - """ - Search through a KEL for the event that contains an anchored - Seal with same Seal type as provided seal but in dict form. - Returns the Serder of the first event with the anchored Seal seal, - None if not found - Searchs from inception forward - - Parameters: - pre is qb64 identifier of the KEL to search - seal is dict form of Seal of any type to find in anchored seals list of each event - - """ - # create generic Seal namedtuple class using keys from provided seal dict - Seal = namedtuple('Seal', seal.keys()) # matching type - - # getEvtPreIter getEvtLastPreIter - - for evt in self.clonePreIter(pre=pre): # all events including superseded - srdr = serdering.SerderKERI(raw=evt) - for eseal in srdr.seals or []: - if tuple(eseal.keys()) == Seal._fields: # same type of seal - eseal = Seal(**eseal) #convert to namedtuple - if seal == eseal and self.fullyWitnessed(srdr): - return srdr - return None - - def signingMembers(self, pre: str): """ Find signing members of a multisig group aid. diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index d3831e214..f186a0b7d 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -59,12 +59,13 @@ from hio.base import filing +from ..kering import MaxON # maximum ordinal number for seqence or first seen + from ..help import helping +#MaxON = int("f"*32, 16) # largest possible ordinal number, sequence or first seen ProemSize = 32 # does not include trailing separator MaxProem = int("f"*(ProemSize), 16) -MaxON = int("f"*32, 16) # largest possible ordinal number, sequence or first seen - SuffixSize = 32 # does not include trailing separator MaxSuffix = int("f"*(SuffixSize), 16) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 754cc553a..336a81fe7 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -278,7 +278,7 @@ class CesrSuberBase(SuberBase): Sub class of Suber where data is CESR encode/decode ducktyped subclass instance such as Matter, Indexer, Counter with .qb64b property when provided as fully qualified serialization - Automatically serializes and deserializes from qb64b to/from CESR instances + Automatically serializes and deserializes from qb64b to/from CESR instance """ @@ -319,7 +319,7 @@ class CesrSuber(CesrSuberBase, Suber): Sub class of Suber where data is CESR encode/decode ducktyped subclass instance such as Matter, Indexer, Counter with .qb64b property when provided as fully qualified serialization. - Extents Suber to support val that are ducktyped CESR serializable .qb64 .qb64b + Extends Suber to support val that are ducktyped CESR serializable .qb64 .qb64b subclasses such as coring.Matter, coring.Indexer, coring.Counter. Automatically serializes and deserializes from qb64b to/from CESR instances @@ -509,8 +509,8 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str, memoryview]): val (Union[bytes, str]): serialization Returns: - result (bool): True means unique value among duplications, - False means duplicte of same value already exists. + result (bool): True means unique value added among duplications, + False means duplicate of same value already exists. """ return (self.db.addIoSetVal(db=self.sdb, @@ -742,10 +742,9 @@ def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): class CesrIoSetSuber(CesrSuberBase, IoSetSuber): """ Subclass of CesrSuber and IoSetSuber. - Class whose values stored in db are a concatenation of the .qb64b property - from one or more subclass instances (qb64b is bytes of fully qualified - serialization) that support CESR encode/decode ducktyped subclass instance - such as Matter, Indexer, Counter + Sub class of Suber where data is CESR encode/decode ducktyped subclass + instance such as Matter, Indexer, Counter with .qb64b property when provided + as fully qualified serialization Automatically serializes and deserializes from qb64b to/from CESR instances Extends IoSetSuber with mixin methods ._ser and ._des from CesrSuberBase @@ -781,8 +780,8 @@ def __init__(self, *pa, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(CesrIoSetSuber, self).__init__(*pa, **kwa) diff --git a/src/keri/kering.py b/src/keri/kering.py index f56bc4023..b8b15ea85 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -10,6 +10,8 @@ FALSEY = (False, 0, None, "?0", "no", "false", "False", "off") TRUTHY = (True, 1, "?1", "yes" "true", "True", 'on') +MaxON = int("f"*32, 16) # 256 ** 16 - 1 maximum ordinal number, sequence or first seen etc + # Serialization Kinds Serialage = namedtuple("Serialage", 'json mgpk cbor') Serials = Serialage(json='JSON', mgpk='MGPK', cbor='CBOR') @@ -504,6 +506,22 @@ class OutOfOrderTxnStateError(ValidationError): raise OutOfOrderTxnStateError("error message") """ +class MisfitEventSourceError(ValidationError): + """ + Error referenced event missing from log so can't verify this txn state event + Usage: + raise MisfitEventSourceError("error message") + """ + +class MissingDelegableApprovalError(ValidationError): + """ + Error referenced event missing from log so can't verify this txn state event + Usage: + raise MissingDelegableApprovalError("error message") + """ + +MissingDelegableApprovalError + # Stream Parsing and Extraction Errors class ExtractionError(KeriError): diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index b9b8fc223..e73985846 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -279,7 +279,7 @@ def make(self, *, nonce=None, noBackers=True, baks=None, toad=None, estOnly=Fals if vcp is None: baks = baks if baks is not None else [] - self.cnfg = [TraitDex.NoBackers] if noBackers else [] + self.cnfg = [TraitDex.NoRegistrarBackers] if noBackers else [] if estOnly: self.cnfg.append(TraitDex.EstOnly) diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 063ebaad7..8da3cc81d 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -19,8 +19,8 @@ from .. import help from ..core import serdering, coring from ..core.coring import (MtrDex, Serials, versify, Prefixer, - Ilks, Seqner, Verfer) -from ..core.eventing import SealEvent, ample, TraitDex, verifySigs, validateSN + Ilks, Seqner, Verfer, Number) +from ..core.eventing import SealEvent, ample, TraitDex, verifySigs from ..db import basing, dbing from ..db.dbing import dgKey, snKey from ..help import helping @@ -71,7 +71,7 @@ def incept( cnfg = cnfg if cnfg is not None else [] baks = baks if baks is not None else [] - if TraitDex.NoBackers in cnfg and len(baks) > 0: + if TraitDex.NoRegistrarBackers in cnfg and len(baks) > 0: raise ValueError("{} backers specified for NB vcp, 0 allowed".format(len(baks))) if len(oset(baks)) != len(baks): @@ -644,7 +644,7 @@ class Tever: Has the following public attributes and properties: Class Attributes: - .NoBackers is Boolean + .NoRegistrarBackers is Boolean True means do not allow backers (default to witnesses of controlling KEL) False means allow backers (ignore witnesses of controlling KEL) @@ -666,7 +666,7 @@ class Tever: .noBackers is boolean trait True means do not allow backers """ - NoBackers = False + NoRegistrarBackers = False def __init__(self, cues=None, rsr=None, serder=None, seqner=None, saider=None, bigers=None, db=None, reger=None, noBackers=None, estOnly=None, @@ -765,7 +765,7 @@ def reload(self, rsr): self.toad = int(ked["bt"], 16) self.baks = ked["b"] - self.noBackers = True if TraitDex.NoBackers in ked["c"] else False + self.noBackers = True if TraitDex.NoRegistrarBackers in ked["c"] else False self.estOnly = True if TraitDex.EstOnly in ked["c"] else False if (raw := self.reger.getTvt(key=dgKey(pre=self.prefixer.qb64, @@ -786,7 +786,7 @@ def state(self): #state(self, kind=Serials.json) cnfg = [] if self.noBackers: - cnfg.append(TraitDex.NoBackers) + cnfg.append(TraitDex.NoRegistrarBackers) dgkey = dbing.dgKey(self.regk, self.serder.said) couple = self.reger.getAnc(dgkey) @@ -826,8 +826,9 @@ def incept(self, serder): raise ValidationError("Invalid prefix = {} for registry inception evt = {}." .format(self.prefixer.qb64, ked)) - sn = ked["s"] - self.sn = validateSN(sn, inceptive=True) + #sn = ked["s"] + #self.sn = validateSN(sn, inceptive=True) + self.sn = Number(numh=ked["s"]).validate(inceptive=True).sn self.cuts = [] # always empty at inception since no prev event self.adds = [] # always empty at inception since no prev event @@ -864,7 +865,7 @@ def config(self, serder, noBackers=None, estOnly=None): """ # assign traits self.noBackers = (True if (noBackers if noBackers is not None - else self.NoBackers) + else self.NoRegistrarBackers) else False) # ensure default noBackers is boolean self.estOnly = (True if (estOnly if estOnly is not None @@ -872,7 +873,7 @@ def config(self, serder, noBackers=None, estOnly=None): else False) # ensure default estOnly is boolean cnfg = serder.ked["c"] # process cnfg for traits - if TraitDex.NoBackers in cnfg: + if TraitDex.NoRegistrarBackers in cnfg: self.noBackers = True if TraitDex.EstOnly in cnfg: self.estOnly = True @@ -895,12 +896,13 @@ def update(self, serder, seqner=None, saider=None, bigers=None): ked = serder.ked ilk = ked["t"] - sn = ked["s"] + #sn = ked["s"] icp = ilk in (Ilks.iss, Ilks.bis) # validate SN for - sn = validateSN(sn, inceptive=icp) + #sn = validateSN(sn, inceptive=icp) + sn = Number(numh=ked["s"]).validate(inceptive=icp).sn if ilk in (Ilks.vrt,): if self.noBackers is True: @@ -1543,13 +1545,14 @@ def processEvent(self, serder, seqner=None, saider=None, wigers=None): regk = self.registryKey(serder) pre = serder.pre ked = serder.ked - sn = ked["s"] + #sn = ked["s"] ilk = ked["t"] inceptive = ilk in (Ilks.vcp, Ilks.iss, Ilks.bis) # validate SN for - sn = validateSN(sn, inceptive=inceptive) + #sn = validateSN(sn, inceptive=inceptive) + sn = Number(numh=ked["s"]).validate(inceptive=inceptive).sn if not self.lax: if self.local: @@ -1759,7 +1762,7 @@ def processReplyRegistryTxnState(self, *, serder, saider, route, cigars=None, ts # Load backers from either tsn or Kever of issuer cnfg = rsr.c - if TraitDex.NoBackers in cnfg: + if TraitDex.NoRegistrarBackers in cnfg: kevers = self.kevers[pre] baks = kevers.wits else: diff --git a/tests/app/__init__.py b/tests/app/__init__.py index 1264abdb8..1120d113a 100644 --- a/tests/app/__init__.py +++ b/tests/app/__init__.py @@ -58,9 +58,9 @@ def openMultiSig(prefix="test", salt=b'0123456789abcdef', temp=True, **kwa): count=3).qb64b) # attach cnt evt.extend(sigs) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev3) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev2) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev3, local=True) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev2, local=True) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) assert ghab1.pre in kev1.kevers assert ghab1.pre in kev2.kevers diff --git a/tests/app/test_delegating.py b/tests/app/test_delegating.py index 7742a1d2b..2fae792a0 100644 --- a/tests/app/test_delegating.py +++ b/tests/app/test_delegating.py @@ -12,7 +12,7 @@ from keri.db import dbing -def test_boatswain(seeder): +def test_sealer(seeder): with habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby, \ habbing.openHby(name="del", salt=coring.Salter(raw=b'0123456789ghijkl').qb64) as delHby: @@ -33,7 +33,7 @@ def test_boatswain(seeder): bts=bts ) - doers = wesDoers + [witDoer, bts, doing.doify(boatswain_test_do, **opts)] + doers = wesDoers + [witDoer, bts, doing.doify(sealer_test_do, **opts)] limit = 1.0 tock = 0.03125 @@ -60,7 +60,7 @@ def test_boatswain(seeder): assert bytes(delHby.db.getAes(dgkey)) == couple -def boatswain_test_do(tymth=None, tock=0.0, **opts): +def sealer_test_do(tymth=None, tock=0.0, **opts): yield tock # enter context wesHab = opts["wesHab"] @@ -79,8 +79,8 @@ def boatswain_test_do(tymth=None, tock=0.0, **opts): witDoer.cues.popleft() msg = next(wesHab.db.clonePreIter(pre=palHab.pre)) - kvy = eventing.Kevery(db=delHby.db, local=False) - parsing.Parser().parseOne(ims=bytearray(msg), kvy=kvy) + kvy = eventing.Kevery(db=delHby.db, local=True) + parsing.Parser().parseOne(ims=bytearray(msg), kvy=kvy, local=True) while palHab.pre not in delHby.kevers: yield tock @@ -107,8 +107,8 @@ def boatswain_test_do(tymth=None, tock=0.0, **opts): couple = coring.Seqner(sn=palHab.kever.sn).qb64b + palHab.kever.serder.saidb msg = next(wesHab.db.clonePreIter(pre=palHab.pre, fn=1)) - kvy = eventing.Kevery(db=delHby.db, local=False) - parsing.Parser().parseOne(ims=bytearray(msg), kvy=kvy) + kvy = eventing.Kevery(db=delHby.db, local=True) + parsing.Parser().parseOne(ims=bytearray(msg), kvy=kvy, local=True) # Wait for the anchor. If we timeout before that happens, assertion in test will fail while delHby.db.getAes(dgkey) != couple: diff --git a/tests/app/test_forwarding.py b/tests/app/test_forwarding.py index 7f94ae439..b5932af0e 100644 --- a/tests/app/test_forwarding.py +++ b/tests/app/test_forwarding.py @@ -29,15 +29,15 @@ def test_postman(seeder): recpIcp = recpHab.makeOwnEvent(sn=0) wesKvy = eventing.Kevery(db=wesHab.db, lax=False, local=False) - parsing.Parser().parse(ims=bytearray(recpIcp), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(recpIcp), kvy=wesKvy, local=True) assert recpHab.pre in wesKvy.kevers serder = serdering.SerderKERI(raw=recpIcp) rct = wesHab.receipt(serder) kvy = eventing.Kevery(db=hab.db) - parsing.Parser().parseOne(bytearray(recpIcp), kvy=kvy) - parsing.Parser().parseOne(bytearray(rct), kvy=kvy) + parsing.Parser().parseOne(bytearray(recpIcp), kvy=kvy, local=True) + parsing.Parser().parseOne(bytearray(rct), kvy=kvy, local=True) kvy.processEscrows() assert recpHab.pre in kvy.kevers diff --git a/tests/app/test_grouping.py b/tests/app/test_grouping.py index 5df6a805b..5e00d8824 100644 --- a/tests/app/test_grouping.py +++ b/tests/app/test_grouping.py @@ -27,14 +27,14 @@ def test_counselor(): kev3 = eventing.Kevery(db=hab3.db, lax=True, local=False) icp1 = hab1.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp1), kvy=kev2) - parsing.Parser().parse(ims=bytearray(icp1), kvy=kev3) + parsing.Parser().parse(ims=bytearray(icp1), kvy=kev2, local=True) + parsing.Parser().parse(ims=bytearray(icp1), kvy=kev3, local=True) icp2 = hab2.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp2), kvy=kev1) - parsing.Parser().parse(ims=bytearray(icp2), kvy=kev3) + parsing.Parser().parse(ims=bytearray(icp2), kvy=kev1, local=True) + parsing.Parser().parse(ims=bytearray(icp2), kvy=kev3, local=True) icp3 = hab3.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp3), kvy=kev1) - parsing.Parser().parse(ims=bytearray(icp3), kvy=kev2) + parsing.Parser().parse(ims=bytearray(icp3), kvy=kev1, local=True) + parsing.Parser().parse(ims=bytearray(icp3), kvy=kev2, local=True) smids = [hab1.pre, hab2.pre, hab3.pre] rmids = None # need to fixe this @@ -69,7 +69,7 @@ def test_counselor(): b'"a":[]}-AABBBBkMCMWP1Z2MMd6dBPlogRd1k6mv1joiHIyb8mXvp0H4kY0DHIPM' b'9O6udZ1Bbyf3klr4uGnLs07qcCcnKGI6GsH') - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 to process all sigs together counselor.processEscrows() @@ -118,7 +118,7 @@ def test_counselor(): # Create group rotation from second participant - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor.processEscrows() @@ -175,7 +175,7 @@ def test_counselor(): # Create group rotation from second participant - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor.processEscrows() @@ -229,7 +229,7 @@ def test_counselor(): # Create group rotation from second participant - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor.processEscrows() @@ -250,13 +250,13 @@ def test_the_seven(): # All the Habs, this will come in handy later # Keverys so we can process each other's inception messages. - kev1 = eventing.Kevery(db=hab1.db, lax=True, local=False) - kev2 = eventing.Kevery(db=hab2.db, lax=True, local=False) - kev3 = eventing.Kevery(db=hab3.db, lax=True, local=False) - kev4 = eventing.Kevery(db=hab4.db, lax=True, local=False) - kev5 = eventing.Kevery(db=hab5.db, lax=True, local=False) - kev6 = eventing.Kevery(db=hab6.db, lax=True, local=False) - kev7 = eventing.Kevery(db=hab7.db, lax=True, local=False) + kev1 = eventing.Kevery(db=hab1.db) + kev2 = eventing.Kevery(db=hab2.db) + kev3 = eventing.Kevery(db=hab3.db) + kev4 = eventing.Kevery(db=hab4.db) + kev5 = eventing.Kevery(db=hab5.db) + kev6 = eventing.Kevery(db=hab6.db) + kev7 = eventing.Kevery(db=hab7.db) kevs = [kev1, kev2, kev3, kev4, kev5, kev6, kev7] icps = [hab1.makeOwnEvent(sn=0), @@ -271,7 +271,7 @@ def test_the_seven(): # Introduce everyone to each other by parsing each others ICP event into our keverys for (kev, icp) in [(kev, icp) for (kdx, kev) in enumerate(kevs) for (idx, icp) in enumerate(icps) if kdx != idx]: - parsing.Parser().parse(ims=bytearray(icp), kvy=kev) + parsing.Parser().parse(ims=bytearray(icp), kvy=kev, local=True) smids = [hab1.pre, hab2.pre, hab3.pre, hab4.pre, hab5.pre, hab6.pre, hab7.pre] rmids = None # need to fixe this @@ -313,7 +313,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBBAD108k4sWtYRv8jQaRbzX6kDebjdzFNVCh3N9cOAJqXV5IzmKdi60Cr0Eu' b'MaACskw0FCi73V2VX8BgFlxO8VIK') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception ghab3 = hby3.makeGroupHab(group=f"{prefix}_group3", mhab=hab3, smids=smids, rmids=rmids, **inits) @@ -322,7 +322,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBCD6V2UkAovhY07MrJUNb-ICddDoyLde9i0FWclxfs7jes01YUEihfgbGERF' b'dKDR4kSr4WF3AskrZOPvMuXipAgP') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception ghab4 = hby4.makeGroupHab(group=f"{prefix}_group4", mhab=hab4, smids=smids, rmids=rmids, **inits) @@ -331,7 +331,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBDBCZuZSFWy0tFshGny1pTR47GphDljd0SShmGRpUSpBX_BeHB1tdIObizaA' b'4GMoOcZ2sOWIe6muJPF_RaoKedYE') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception ghab5 = hby5.makeGroupHab(group=f"{prefix}_group5", mhab=hab5, smids=smids, rmids=rmids, **inits) @@ -340,7 +340,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBEBsR6_hPId3H8fFG8EfevQVji8MsLAC72MjkkRxJp3h9v1vyFS1hAGGGxno' b'F5xSHOnpBpPwjMJwOCurAa3VrNAD') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception ghab6 = hby6.makeGroupHab(group=f"{prefix}_group6", mhab=hab6, smids=smids, rmids=rmids, **inits) @@ -349,7 +349,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBFCi5hK6Ax4aBNsdoUkh7Q_CcSWJfpwkeF68aCO34J3BDN7k483lOxiyj6pl' b'8TQIQ7VJLBkoRscUMi_mls9jbpcD') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception ghab7 = hby7.makeGroupHab(group=f"{prefix}_group7", mhab=hab7, smids=smids, rmids=rmids, **inits) @@ -358,7 +358,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBGCtPvRj00vEfT5Po6eH50DWfBWwAcQgvBaJ7LlYT7kQswkl_r-K9Lsxi5tm' b'Pvsb2xFtcMJkFf-BxamGhFo9OOcD') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 to process all sigs together @@ -409,7 +409,7 @@ def test_the_seven(): msg = eventing.messagize(serder=serder, sigers=sigers) assert msg[serder.size:] == (b'-AABABAzvHN7yC3581dp9DxFXrKuXGP_62r_pzNMXL20T6RaPQASXvnBn6sKJ78z' b'KM9o499Zaz76j940nBoMT-yb9i8N') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception # Now sign the group ROT with Hab3 and parse into Kev1. This should commit the event sigers = hab3.mgr.sign(serder.raw, verfers=hab3.kever.verfers, indexed=True, indices=[2]) @@ -417,7 +417,7 @@ def test_the_seven(): assert msg[serder.size:] == (b'-AABACB6z6LrzBAgpnrCopgiGxuki3sE-KAfY8t_rFq-2dIcQxRF4iCqCYNPKM9D' b'NbZbA1WDaQ72enSsR2UWMftX2kYD') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor.processEscrows() # Get the rest of the way through counselor. @@ -474,7 +474,7 @@ def test_the_seven(): msg = eventing.messagize(serder=serder, sigers=sigers) assert msg[serder.size:] == (b'-AABABC4sYnDXCpO87BMXO21ofqHZKntPSdEXlBPlq1H8NOHD3KV-GHGWrXyrElK' b'BkQNBbNr9_yg-nSnBq7N9rAxEFcK') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception # Now sign the group ROT with Hab3 and parse into Kev1. This should commit the event sigers = hab3.mgr.sign(serder.raw, verfers=hab3.kever.verfers, indexed=True, indices=[2]) @@ -482,7 +482,7 @@ def test_the_seven(): assert msg[serder.size:] == (b'-AABACAXyUueUfXC-ccUxBZTgnyHTXOy1wUYgQrhlk8FMJGQPiaOOdAzhaW71JeF' b'0By8Se-tKKuPP1xG41DblgXIwNkE') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor.processEscrows() # Get the rest of the way through counselor. @@ -502,7 +502,7 @@ def test_the_seven(): msgs = [hab1.replay(), hab2.replay(), hab3.replay(), ghab.replay()] kevs = [kev4, kev5, kev6, kev7] for (kev, msg) in [(kev, msg) for kev in kevs for msg in msgs]: - parsing.Parser().parse(ims=bytearray(msg), kvy=kev) + parsing.Parser().parse(ims=bytearray(msg), kvy=kev, local=True) assert kev4.kevers[ghab.pre] is not None assert kev5.kevers[ghab.pre] is not None @@ -549,7 +549,7 @@ def test_the_seven(): msg = eventing.messagize(serder=serder, sigers=sigers) assert msg[serder.size:] == (b'-AAB2AABAEDSs99oM-KOhJ8q3H8lqGqPE3EvZxCHvCjZFvWHLzhqm91YlcskGqvK' b'8DwCg9dj8wRZP54ienzD52EIKvJWWh4J') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev4) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev4, local=True) # parse second signed group inception # Now sign the group ROT with Hab6 and parse into Kev4. This should commit the event sigers = hab6.mgr.sign(serder.raw, verfers=hab6.kever.verfers, indexed=True, indices=[2], ondices=[5]) @@ -557,7 +557,7 @@ def test_the_seven(): assert msg[serder.size:] == (b'-AAB2AACAFBNVTM0Gw4rSd-S5HQ_KpmBfDedi7XNvB24ijMjQaekIfKlcdguPS8p' b'ax9ht7EE3SiTj9fSO_3f4SVUfJMPmHIK') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev4) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev4, local=True) # parse second signed group inception kev4.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor4.processEscrows() # Get the rest of the way through counselor. @@ -582,14 +582,14 @@ def openMultiSig(prefix="test", salt=b'0123456789abcdef', temp=True, **kwa): kev3 = eventing.Kevery(db=hab3.db, lax=True, local=False) icp1 = hab1.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp1), kvy=kev2) - parsing.Parser().parse(ims=bytearray(icp1), kvy=kev3) + parsing.Parser().parse(ims=bytearray(icp1), kvy=kev2, local=True) + parsing.Parser().parse(ims=bytearray(icp1), kvy=kev3, local=True) icp2 = hab2.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp2), kvy=kev1) - parsing.Parser().parse(ims=bytearray(icp2), kvy=kev3) + parsing.Parser().parse(ims=bytearray(icp2), kvy=kev1, local=True) + parsing.Parser().parse(ims=bytearray(icp2), kvy=kev3, local=True) icp3 = hab3.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp3), kvy=kev1) - parsing.Parser().parse(ims=bytearray(icp3), kvy=kev2) + parsing.Parser().parse(ims=bytearray(icp3), kvy=kev1, local=True) + parsing.Parser().parse(ims=bytearray(icp3), kvy=kev2, local=True) smids = [hab1.pre, hab2.pre, hab3.pre] rmids = None @@ -620,9 +620,9 @@ def openMultiSig(prefix="test", salt=b'0123456789abcdef', temp=True, **kwa): count=3).qb64b) # attach cnt evt.extend(sigs) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev3) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev2) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev3, local=True) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev2, local=True) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) assert ghab1.pre in kev1.kevers assert ghab1.pre in kev2.kevers diff --git a/tests/app/test_habbing.py b/tests/app/test_habbing.py index 69e0d65a4..104ae29c7 100644 --- a/tests/app/test_habbing.py +++ b/tests/app/test_habbing.py @@ -13,12 +13,9 @@ from keri import kering from keri import help from keri.app import habbing, keeping, configing -from keri.core.coring import MtrDex from keri.db import basing from keri.core import coring, eventing, parsing from keri.help import helping -from keri.peer import exchanging -from keri.vdr.eventing import incept def test_habery(): @@ -892,7 +889,7 @@ def test_postman_endsfor(): kvy = eventing.Kevery(db=hab.db, lax=False, local=False) icpMsg = hab.makeOwnInception() rctMsgs = [] # list of receipts from each witness - parsing.Parser().parse(ims=bytearray(icpMsg), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(icpMsg), kvy=wesKvy, local=True) assert wesKvy.kevers[hab.pre].sn == 0 # accepted event assert len(wesKvy.cues) >= 1 # assunmes includes queued receipt cue # better to find cue in cues and confirm exactly @@ -901,11 +898,11 @@ def test_postman_endsfor(): rctMsgs.append(rctMsg) for msg in rctMsgs: # process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=kvy, local=True) assert wesHab.pre in kvy.kevers agentIcpMsg = agentHab.makeOwnInception() - parsing.Parser().parse(ims=bytearray(agentIcpMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(agentIcpMsg), kvy=kvy, local=True) assert agentHab.pre in kvy.kevers msgs = bytearray() diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 7ec33df4a..5e6b8357c 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -41,7 +41,7 @@ from keri.help import helping from keri.kering import (EmptyMaterialError, RawMaterialError, DerivationError, ShortageError, InvalidCodeSizeError, InvalidVarIndexError, - InvalidValueError, DeserializeError) + InvalidValueError, DeserializeError, ValidationError) from keri.kering import Version, Versionage, VersionError from keri.kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, RPY_LABELS) @@ -67,7 +67,7 @@ def test_protos(): """End Test""" -def test_prodex(): +def test_protocol_genus_codex(): """ Test genera in ProDex as instance of ProtocolGenusCodex @@ -347,89 +347,90 @@ def test_matter(): Test Matter class """ assert dataclasses.asdict(MtrDex) == { - 'Ed25519_Seed': 'A', - 'Ed25519N': 'B', - 'X25519': 'C', - 'Ed25519': 'D', - 'Blake3_256': 'E', - 'Blake2b_256': 'F', - 'Blake2s_256': 'G', - 'SHA3_256': 'H', - 'SHA2_256': 'I', - 'ECDSA_256k1_Seed': 'J', - 'Ed448_Seed': 'K', - 'X448': 'L', - 'Short': 'M', - 'Big': 'N', - 'X25519_Private': 'O', - 'X25519_Cipher_Seed': 'P', - 'ECDSA_256r1_Seed': 'Q', - 'Tall': 'R', - 'Large': 'S', - 'Great': 'T', - 'Vast': 'U', - 'Label1': 'V', - 'Label2': 'W', - 'Tag3': 'X', - 'Tag7': 'Y', - 'Salt_128': '0A', - 'Ed25519_Sig': '0B', - 'ECDSA_256k1_Sig': '0C', - 'Blake3_512': '0D', - 'Blake2b_512': '0E', - 'SHA3_512': '0F', - 'SHA2_512': '0G', - 'Long': '0H', - 'ECDSA_256r1_Sig': '0I', - 'Tag1': '0J', - 'Tag2': '0K', - 'Tag5': '0L', - 'Tag6': '0M', - 'ECDSA_256k1N': '1AAA', - 'ECDSA_256k1': '1AAB', - 'Ed448N': '1AAC', - 'Ed448': '1AAD', - 'Ed448_Sig': '1AAE', - 'Tag4': '1AAF', - 'DateTime': '1AAG', - 'X25519_Cipher_Salt': '1AAH', - 'ECDSA_256r1N': '1AAI', - 'ECDSA_256r1': '1AAJ', - 'Null': '1AAK', - 'Yes': '1AAL', - 'No': '1AAM', - 'TBD1': '2AAA', - 'TBD2': '3AAA', - 'StrB64_L0': '4A', - 'StrB64_L1': '5A', - 'StrB64_L2': '6A', - 'StrB64_Big_L0': '7AAA', - 'StrB64_Big_L1': '8AAA', - 'StrB64_Big_L2': '9AAA', - 'Bytes_L0': '4B', - 'Bytes_L1': '5B', - 'Bytes_L2': '6B', - 'Bytes_Big_L0': '7AAB', - 'Bytes_Big_L1': '8AAB', - 'Bytes_Big_L2': '9AAB', - 'X25519_Cipher_L0': '4C', - 'X25519_Cipher_L1': '5C', - 'X25519_Cipher_L2': '6C', - 'X25519_Cipher_Big_L0': '7AAC', - 'X25519_Cipher_Big_L1': '8AAC', - 'X25519_Cipher_Big_L2': '9AAC', - 'X25519_Cipher_QB64_L0': '4D', - 'X25519_Cipher_QB64_L1': '5D', - 'X25519_Cipher_QB64_L2': '6D', - 'X25519_Cipher_QB64_Big_L0': '7AAD', - 'X25519_Cipher_QB64_Big_L1': '8AAD', - 'X25519_Cipher_QB64_Big_L2': '9AAD', - 'X25519_Cipher_QB2_L0': '4D', - 'X25519_Cipher_QB2_L1': '5D', - 'X25519_Cipher_QB2_L2': '6D', - 'X25519_Cipher_QB2_Big_L0': '7AAD', - 'X25519_Cipher_QB2_Big_L1': '8AAD', - 'X25519_Cipher_QB2_Big_L2': '9AAD' + 'Ed25519_Seed': 'A', + 'Ed25519N': 'B', + 'X25519': 'C', + 'Ed25519': 'D', + 'Blake3_256': 'E', + 'Blake2b_256': 'F', + 'Blake2s_256': 'G', + 'SHA3_256': 'H', + 'SHA2_256': 'I', + 'ECDSA_256k1_Seed': 'J', + 'Ed448_Seed': 'K', + 'X448': 'L', + 'Short': 'M', + 'Big': 'N', + 'X25519_Private': 'O', + 'X25519_Cipher_Seed': 'P', + 'ECDSA_256r1_Seed': 'Q', + 'Tall': 'R', + 'Large': 'S', + 'Great': 'T', + 'Vast': 'U', + 'Label1': 'V', + 'Label2': 'W', + 'Tag3': 'X', + 'Tag7': 'Y', + 'Blind': 'Z', + 'Salt_128': '0A', + 'Ed25519_Sig': '0B', + 'ECDSA_256k1_Sig': '0C', + 'Blake3_512': '0D', + 'Blake2b_512': '0E', + 'SHA3_512': '0F', + 'SHA2_512': '0G', + 'Long': '0H', + 'ECDSA_256r1_Sig': '0I', + 'Tag1': '0J', + 'Tag2': '0K', + 'Tag5': '0L', + 'Tag6': '0M', + 'ECDSA_256k1N': '1AAA', + 'ECDSA_256k1': '1AAB', + 'Ed448N': '1AAC', + 'Ed448': '1AAD', + 'Ed448_Sig': '1AAE', + 'Tag4': '1AAF', + 'DateTime': '1AAG', + 'X25519_Cipher_Salt': '1AAH', + 'ECDSA_256r1N': '1AAI', + 'ECDSA_256r1': '1AAJ', + 'Null': '1AAK', + 'Yes': '1AAL', + 'No': '1AAM', + 'TBD1': '2AAA', + 'TBD2': '3AAA', + 'StrB64_L0': '4A', + 'StrB64_L1': '5A', + 'StrB64_L2': '6A', + 'StrB64_Big_L0': '7AAA', + 'StrB64_Big_L1': '8AAA', + 'StrB64_Big_L2': '9AAA', + 'Bytes_L0': '4B', + 'Bytes_L1': '5B', + 'Bytes_L2': '6B', + 'Bytes_Big_L0': '7AAB', + 'Bytes_Big_L1': '8AAB', + 'Bytes_Big_L2': '9AAB', + 'X25519_Cipher_L0': '4C', + 'X25519_Cipher_L1': '5C', + 'X25519_Cipher_L2': '6C', + 'X25519_Cipher_Big_L0': '7AAC', + 'X25519_Cipher_Big_L1': '8AAC', + 'X25519_Cipher_Big_L2': '9AAC', + 'X25519_Cipher_QB64_L0': '4D', + 'X25519_Cipher_QB64_L1': '5D', + 'X25519_Cipher_QB64_L2': '6D', + 'X25519_Cipher_QB64_Big_L0': '7AAD', + 'X25519_Cipher_QB64_Big_L1': '8AAD', + 'X25519_Cipher_QB64_Big_L2': '9AAD', + 'X25519_Cipher_QB2_L0': '4D', + 'X25519_Cipher_QB2_L1': '5D', + 'X25519_Cipher_QB2_L2': '6D', + 'X25519_Cipher_QB2_Big_L0': '7AAD', + 'X25519_Cipher_QB2_Big_L1': '8AAD', + 'X25519_Cipher_QB2_Big_L2': '9AAD' } @@ -474,6 +475,7 @@ def test_matter(): 'W': Sizage(hs=1, ss=0, fs=4, ls=0), 'X': Sizage(hs=1, ss=0, fs=4, ls=0), 'Y': Sizage(hs=1, ss=0, fs=8, ls=0), + 'Z': Sizage(hs=1, ss=0, fs=44, ls=0), '0A': Sizage(hs=2, ss=0, fs=24, ls=0), '0B': Sizage(hs=2, ss=0, fs=88, ls=0), '0C': Sizage(hs=2, ss=0, fs=88, ls=0), @@ -2441,6 +2443,7 @@ def test_counter(): 'KERIProtocolStack': '--AAA', } + # new version 2 #assert dataclasses.asdict(CtrDex) == { #'ControllerIdxSigs': '-A', #'WitnessIdxSigs': '-B', @@ -2816,7 +2819,7 @@ def test_counter(): def test_seqner(): """ - Test Seqner sequence number subclass of CryMat + Test Seqner sequence number subclass Matter """ number = Seqner() # defaults to zero assert number.raw == b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' @@ -2835,6 +2838,12 @@ def test_seqner(): with pytest.raises(RawMaterialError): number = Seqner(raw=b'') + with pytest.raises(InvalidValueError): # negative + number = Seqner(sn=-1) + + with pytest.raises(ValidationError): # too big + number = Seqner(sn=(256 ** 16)) + number = Seqner(qb64b=snqb64b) assert number.raw == snraw assert number.code == MtrDex.Salt_128 @@ -2977,9 +2986,27 @@ def test_number(): """ Test Number subclass of Matter """ + + assert dataclasses.asdict(NumDex) == { + 'Short': 'M', + 'Long': '0H', + 'Tall': 'R', + 'Big': 'N', + 'Large': 'S', + 'Great': 'T', + 'Huge': '0A', + 'Vast': 'U' + } + + assert Number.Codex == NumDex + + with pytest.raises(RawMaterialError): number = Number(raw=b'') + with pytest.raises(InvalidValueError): + number = Number(num=-1) + number = Number() # test None defaults to zero assert number.code == NumDex.Short assert number.raw == b'\x00\x00' @@ -3056,6 +3083,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(num=numh) # num can be hext str too @@ -3069,6 +3098,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(numh=numh) @@ -3082,6 +3113,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(qb64=nqb64) assert number.code == code @@ -3094,6 +3127,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(qb2=nqb2) assert number.code == code @@ -3106,6 +3141,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(raw=raw, code=code) assert number.code == code @@ -3118,15 +3155,17 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn - num = (256 ** 4 - 1) - assert num == 4294967295 + num = (256 ** 5 - 1) + assert num == 1099511627775 numh = f"{num:x}" - assert numh == 'ffffffff' - raw = b'\xff\xff\xff\xff' - code = NumDex.Long - nqb64 = '0HD_____' # '0H_____w' - nqb2 = b'\xd0p\xff\xff\xff\xff' # b'\xd0\x7f\xff\xff\xff\xf0' + assert numh == 'ffffffffff' + raw = b'\xff\xff\xff\xff\xff' + code = NumDex.Tall + nqb64 = 'RP______' # '0HD_____' # '0H_____w' + nqb2 = b'D\xff\xff\xff\xff\xff' # b'\xd0p\xff\xff\xff\xff' number = Number(num=num) assert number.code == code @@ -3139,6 +3178,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(numh=numh) assert number.code == code @@ -3151,6 +3192,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(qb64=nqb64) assert number.code == code @@ -3163,6 +3206,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(qb2=nqb2) assert number.code == code @@ -3173,6 +3218,8 @@ def test_number(): assert number.num == num assert number.numh == numh assert number.positive + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(raw=raw, code=code) assert number.code == code @@ -3185,6 +3232,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn num = (256 ** 8 - 1) assert num == 18446744073709551615 @@ -3193,7 +3242,7 @@ def test_number(): raw = b'\xff\xff\xff\xff\xff\xff\xff\xff' code = NumDex.Big nqb64 = 'NP__________' # 'N__________8' - nqb2 = b'4\xff\xff\xff\xff\xff\xff\xff\xff' # b'7\xff\xff\xff\xff\xff\xff\xff\xfc' + nqb2 = b'4\xff\xff\xff\xff\xff\xff\xff\xff' number = Number(num=num) assert number.code == code @@ -3206,6 +3255,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(numh=numh) assert number.code == code @@ -3218,6 +3269,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(qb64=nqb64) assert number.code == code @@ -3230,6 +3283,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(qb2=nqb2) assert number.code == code @@ -3242,6 +3297,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn number = Number(raw=raw, code=code) assert number.code == code @@ -3254,16 +3311,178 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn - num = (256 ** 16 - 1) - assert num == 340282366920938463463374607431768211455 + num = (256 ** 11 - 1) + assert num == 309485009821345068724781055 numh = f"{num:x}" - assert numh == 'ffffffffffffffffffffffffffffffff' - raw = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' - code = NumDex.Huge - nqb64 = '0AD_____________________' # '0A_____________________w' - nqb2 = b'\xd0\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' - #b'\xd0\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf0' + assert numh == 'ffffffffffffffffffffff' + raw = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + code = NumDex.Large + nqb64 = 'SP______________' # 'NP__________' # 'N__________8' + nqb2 = b'H\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' # b'4\xff\xff\xff\xff\xff\xff\xff\xff' + + number = Number(num=num) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn + + number = Number(numh=numh) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn + + number = Number(qb64=nqb64) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn + + number = Number(qb2=nqb2) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn + + number = Number(raw=raw, code=code) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn + + num = (256 ** 14 - 1) + assert num == 5192296858534827628530496329220095 + numh = f"{num:x}" + assert numh == 'ffffffffffffffffffffffffffff' + raw = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + code = NumDex.Great + nqb64 = 'TP__________________' # '0AD_____________________' + nqb2 = b'L\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + # b'\xd0\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + + number = Number(num=num) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn + + number = Number(numh=numh) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn + + number = Number(qb64=nqb64) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn + + number = Number(qb2=nqb2) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn + + + number = Number(raw=raw, code=code) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + assert isinstance(number.seqner, Seqner) + assert number.seqner.sn == number.sn + + num = (256 ** 17 - 1) + assert num == 87112285931760246646623899502532662132735 + numh = f"{num:x}" + assert numh == 'ffffffffffffffffffffffffffffffffff' + raw = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + code = NumDex.Vast + nqb64 = 'UP______________________' #'TP__________________' + nqb2 = b'P\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + # b'L\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' number = Number(num=num) assert number.code == code @@ -3276,6 +3495,9 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + with pytest.raises(ValidationError): # too big to be ordinal + number.seqner + number = Number(numh=numh) assert number.code == code @@ -3288,6 +3510,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + with pytest.raises(ValidationError): # too big to be ordinal + number.seqner number = Number(qb64=nqb64) assert number.code == code @@ -3300,6 +3524,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + with pytest.raises(ValidationError): # too big to be ordinal + number.seqner number = Number(qb2=nqb2) assert number.code == code @@ -3312,6 +3538,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + with pytest.raises(ValidationError): # too big to be ordinal + number.seqner number = Number(raw=raw, code=code) @@ -3325,6 +3553,9 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + with pytest.raises(ValidationError): # too big to be ordinal + number.seqner + # tests with wrong size raw for code short num = (256 ** 2 - 1) @@ -3388,7 +3619,44 @@ def test_number(): bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - # raw to small for code raises error + # raw too small for code raises error + raw2bad = b'\xff' + assert raw != raw2bad + assert len(raw2bad) < len(raw) + + with pytest.raises(RawMaterialError): + number = Number(raw=raw2bad, code=code) + + + # tests with wrong size raw for code large + num = (256 ** 5 - 1) + assert num == 1099511627775 + numh = f"{num:x}" + assert numh == 'ffffffffff' + raw = b'\xff\xff\xff\xff\xff' + code = NumDex.Tall + nqb64 = 'RP______' # '0HD_____' # '0H_____w' + nqb2 = b'D\xff\xff\xff\xff\xff' # b'\xd0p\xff\xff\xff\xff' + + + # raw to large for code, then truncates + raw2bad = b'\xff\xff\xff\xff\xff\xff' + assert raw != raw2bad + assert len(raw2bad) > len(raw) + + number = Number(raw=raw2bad, code=code) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + + # raw too small for code raises error raw2bad = b'\xff' assert raw != raw2bad assert len(raw2bad) < len(raw) @@ -3468,6 +3736,85 @@ def test_number(): with pytest.raises(RawMaterialError): number = Number(raw=raw2bad, code=code) + + # tests with wrong size raw for code huge + num = (256 ** 17 - 1) + assert num == 87112285931760246646623899502532662132735 + numh = f"{num:x}" + assert numh == 'ffffffffffffffffffffffffffffffffff' + raw = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + code = NumDex.Vast + nqb64 = 'UP______________________' #'TP__________________' + nqb2 = b'P\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + + + # raw to large for code, then truncates + raw2bad = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xffxff' + assert raw != raw2bad + assert len(raw2bad) > len(raw) + + number = Number(raw=raw2bad, code=code) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + + # raw to small for code raises error + raw2bad = b'\xff' + assert raw != raw2bad + assert len(raw2bad) < len(raw) + + with pytest.raises(RawMaterialError): + number = Number(raw=raw2bad, code=code) + + + # test with negative num + num = -1 + numh = f"{num:x}" + assert numh == '-1' + code = NumDex.Short + + with pytest.raises(InvalidValueError): + number = Number(num=num) + + with pytest.raises(InvalidValueError): + number = Number(numh=numh) + + + + # test using num to initialize Number + num = 0 + numh = f"{num:x}" + assert numh == '0' + code = NumDex.Short + raw = b'\x00\x00' + nqb64 = 'MAAA' + nqb2 = b'0\x00\x00' + assert hex(int.from_bytes(nqb2, 'big')) == '0x300000' + + number = Number(num=num) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert not number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + # test validate() + assert number.validate() == number # default inceptive = None + assert number.validate(inceptive=True) == number # inceptive = True + with pytest.raises(ValidationError): + number.validate(inceptive=False) # inceptive = False + num = 1 numh = f"{num:x}" assert numh == '1' @@ -3488,7 +3835,67 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + # test validate + assert number.validate() == number # default inceptive = None + with pytest.raises(ValidationError): + number.validate(inceptive=True) # inceptive = True + assert number.validate(inceptive=False) == number # inceptive = False + + + num = 65536 + numh = f"{num:x}" + assert numh == '10000' # hex + code = NumDex.Tall + raw = b'\x00\x00\x01\x00\x00' + nqb64 = 'RAAAAQAA' + nqb2 = b'D\x00\x00\x01\x00\x00' + assert hex(int.from_bytes(nqb2, 'big')) == '0x440000010000' + + number = Number(num=num) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + # test validate + assert number.validate() == number # default inceptive = None + with pytest.raises(ValidationError): + number.validate(inceptive=True) # inceptive = True + assert number.validate(inceptive=False) == number # inceptive = False + + # too big for ordinal + num = num = (256 ** 16) + numh = f"{num:x}" + assert numh == '100000000000000000000000000000000' # hex + code = NumDex.Vast + raw =b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + nqb64 = 'UAEAAAAAAAAAAAAAAAAAAAAA' + nqb2 = b'P\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + assert hex(int.from_bytes(nqb2, 'big')) == '0x500100000000000000000000000000000000' + number = Number(num=num) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + # test validate + with pytest.raises(ValidationError): # too big + number.validate() # default inceptive = None + with pytest.raises(ValidationError): # too big + number.validate(inceptive=True) # inceptive = True + with pytest.raises(ValidationError): # too big + number.validate(inceptive=False) # inceptive = False """ Done Test """ @@ -6144,7 +6551,8 @@ def test_tholder(): #test_counter() #test_prodex() #test_indexer() - #test_number() + test_number() + test_seqner() #test_siger() #test_signer() #test_nexter() @@ -6153,5 +6561,5 @@ def test_tholder(): #test_labels() #test_prefixer() #test_genera() - test_prodex() + #test_protocol_genus_codex() diff --git a/tests/core/test_delegating.py b/tests/core/test_delegating.py index deff595ad..b6685331d 100644 --- a/tests/core/test_delegating.py +++ b/tests/core/test_delegating.py @@ -6,7 +6,7 @@ import os from keri import help -from keri.app import keeping +from keri.app import keeping, habbing from keri.core import coring, eventing, parsing from keri.db import dbing, basing @@ -276,6 +276,449 @@ def test_delegation(): """End Test""" +def test_delegation_supersede(): + """ + Test superseding delegation rules + + Three level delegation + top is at top or root level with witness wop. top is not delegated + mid is at mid level with witness wid. mid is delegatred from top + bot is at bottom level with wintess wot. bot is delegated from mid + + + def test_load_event(mockHelpingNowUTC): + with habbing.openHby(name="tor", base="test") as torHby, \ + habbing.openHby(name="wil", base="test") as wilHby, \ + habbing.openHby(name="wan", base="test") as wanHby, \ + habbing.openHby(name="tee", base="test") as teeHby: + + + + # Create Wan the witness + wanHab = wanHby.makeHab(name="wan", transferable=False) + assert wanHab.pre == "BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP" + msg = wanHab.makeOwnEvent(sn=0) + parsing.Parser().parse(ims=msg, kvy=torKvy) + + # Create Wil the witness, we'll use him later + wilHab = wilHby.makeHab(name="wil", transferable=False) + + # Create Tor the delegaTOR and pass to witness Wan + torHab = torHby.makeHab(name="tor", icount=1, isith='1', ncount=1, nsith='1', wits=[wanHab.pre], toad=1) + assert torHab.pre == "EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP" + torIcp = torHab.makeOwnEvent(sn=0) + + wanKvy = Kevery(db=wanHby.db, lax=False, local=False) # remote events + torKvy = Kevery(db=torHby.db, lax=False, local=False) # remote events + + + """ + topSalt = coring.Salter(raw=b'0123456789abcdef').qb64 + wopSalt = coring.Salter(raw=b'0123456789abcdef').qb64 + midSalt = coring.Salter(raw=b'abcdef0123456789').qb64 + widSalt = coring.Salter(raw=b'abcdef0123456789').qb64 + botSalt = coring.Salter(raw=b'zyxwvutsrponmlkj').qb64 + wotSalt = coring.Salter(raw=b'zyxwvutsrponmlkj').qb64 + + with (habbing.openHby(name="top", base="test", salt=topSalt) as topHby, + habbing.openHby(name="wop", base="test", salt=wopSalt) as wopHby, + habbing.openHby(name="mid", base="test", salt=midSalt) as midHby, + habbing.openHby(name="wid", base="test", salt=widSalt) as widHby, + habbing.openHby(name="bot", base="test", salt=botSalt) as botHby, + habbing.openHby(name="wot", base="test", salt=wotSalt) as wotHby): + + # Create witness wop and controller top + wopHab = wopHby.makeHab(name="wop", transferable=False) # witness nontrans + # makehab also enters inception event into its own kel. + # Otherwise failed make raises exception ConfigurationError + assert wopHab.pre == 'BIDO3FhB5smF6WsTJkRRAao_wEttcbsBnDCmfQ4_f1b_' + wopRemKvy = eventing.Kevery(db=wopHby.db, lax=False, local=False) # for remote events + + topHab = topHby.makeHab(name="top", icount=1, isith='1', # single sig + ncount=1, nsith='1', # single next + wits=[wopHab.pre], toad=1) # one witness + # makehab also enters inception event into its own kel + # Otherwise failed make raises exception ConfigurationError + assert topHab.pre == 'EJcCaHg3AtW_gRzpaz6Pw03Yv49is2IJDRwYE7ey91KE' + topRemKvy = eventing.Kevery(db=topHby.db, lax=False, local=False) # for remote events + + # be witness to controller's inception + # first make inception + stream = topHab.makeOwnInception() + assert stream == (b'{"v":"KERI10JSON000159_","t":"icp","d":"EJcCaHg3AtW_gRzpaz6Pw03Y' + b'v49is2IJDRwYE7ey91KE","i":"EJcCaHg3AtW_gRzpaz6Pw03Yv49is2IJDRwYE' + b'7ey91KE","s":"0","kt":"1","k":["DPJVPYS9efLUHDOqxwG6pxISZSRACgNf' + b'uZm7qK7DzQKD"],"nt":"1","n":["EN8AnwKGnCOAyP2FRXuQMyMjbRsDjRpDd_' + b'_ZK2KuL0ID"],"bt":"1","b":["BIDO3FhB5smF6WsTJkRRAao_wEttcbsBnDCm' + b'fQ4_f1b_"],"c":[],"a":[]}-AABAADPMCL6P4DYi3qvgR4v1UcrCYMjnmRx-xJ' + b'meOfA8b8gdHZxZvVgpgLrAxFYwSEAtdhGT8LOPdGpTqxWSocCmXkH') + + # add test fail process as remote since since witness of controller + + # first process as local since witness + wopHab.psr.parse(ims=stream) # now have controller's inception in db + assert topHab.pre in wopHab.kevers # success + + serder = wopHab.kevers[topHab.pre].serder + # generate witness receipt and process + receipt = wopHab.witness(serder=serder) # now has fully witnessd controller icp + count = wopHab.db.cntWigs(dbing.dgKey(topHab.pre, serder.said)) + assert count >= 1 + + assert receipt == (b'{"v":"KERI10JSON000091_","t":"rct","d":"EJcCaHg3AtW_gRzpaz6Pw03Y' + b'v49is2IJDRwYE7ey91KE","i":"EJcCaHg3AtW_gRzpaz6Pw03Yv49is2IJDRwYE' + b'7ey91KE","s":"0"}-VAX-BABAACf4sllk4USRirj3xNlnFgDcbWHsAi6kOigNHr' + b'Ddbde06NhDELtWTeJOcz7T_rru_rpd6Uov4IN_0rthtMbxgcI') + + # add test fail process as remote since own witness + + # process receipt as local since own witness receipt. + topHab.psr.parse(ims=receipt) # now top has fully witnessed icp. + count = topHab.db.cntWigs(dbing.dgKey(topHab.pre, serder.said)) + assert count >= 1 + + # Create witness wid and delegated controller mid + widHab = widHby.makeHab(name="wid", transferable=False) # witness nontrans + # makehab also enters inception event into its own kel. + # Otherwise failed make raises exception ConfigurationError + assert widHab.pre == 'BCI95exU-RepxQ0HmGcp7USLMCPxrXKzMc1DXqfRnikP' + widRemKvy = eventing.Kevery(db=widHby.db, lax=False, local=False) # for remote events + + midHab = midHby.makeHab(name="mid", icount=1, isith='1', # single sig + ncount=1, nsith='1', # single next + wits=[widHab.pre], toad=1, # one witness + delpre=topHab.pre) # delegated + # makehab also enters inception event into its own kel + # Otherwise failed make raises exception ConfigurationError + assert midHab.pre == 'EEaTQhI7QGM-usOJtpKM9L0yQjGiBYJC3tq905aC8am4' + assert midHab.delpre == topHab.pre + midRemKvy = eventing.Kevery(db=midHby.db, lax=False, local=False) # for remote events + + # be witness to controller's inception. + # first make inception + stream = midHab.makeOwnInception() + + # add test fail process as remote since since witness of controller + + #first process as local since witness + widHab.psr.parse(ims=stream) # now have controller's inception in db + assert midHab.pre in widHab.kevers # success + + serder = widHab.kevers[midHab.pre].serder + # generate witness receipt and process + receipt = widHab.witness(serder=serder) # now has fully witnessed controller icp + count = widHab.db.cntWigs(dbing.dgKey(midHab.pre, serder.said)) + assert count >= 1 + + # add test fail process as remote since own witness + + # top process wop receipt as local since own witness receipt. + midHab.psr.parse(ims=receipt) # now top has fully witnessed icp. + count = midHab.db.cntWigs(dbing.dgKey(midHab.pre, serder.said)) + assert count >= 1 + + # Create witness wot and controller bot + wotHab = widHby.makeHab(name="wot", transferable=False) # witness nontrans + # makehab also enters inception event into its own kel. + # Otherwise failed make raises exception ConfigurationError + assert wotHab.pre == 'BDChA_O6twrlHcXKf7xu1xYee__nZxDbBa0W_XznpLQH' + wotRemKvy = eventing.Kevery(db=wotHby.db, lax=False, local=False) # for remote events + + botHab = botHby.makeHab(name="bot", icount=1, isith='1', # single sig + ncount=1, nsith='1', # single next + wits=[wotHab.pre], toad=1, # one witness + delpre=midHab.pre) # delegated + # makehab also enters inception event into its own kel + # Otherwise failed make raises exception ConfigurationError + assert botHab.pre == 'EPtHqQJwlEj2sM0e2WslvwSsAsxAflmn7JIabs-LBqJC' + assert botHab.delpre == midHab.pre + botRemKvy = eventing.Kevery(db=botHby.db, lax=False, local=False) # for remote events + + # be witness to controller's inception. + # first make inception + stream = botHab.makeOwnInception() + + # add test fail process as remote since since witness of controller + + #first process as local since witness + wotHab.psr.parse(ims=stream) # now have controller inception in db + assert botHab.pre in wotHab.kevers # success + + serder = wotHab.kevers[botHab.pre].serder + # generate witness receipt and process + receipt = wotHab.witness(serder=serder) # now has fully witnessed controller icp + count = wotHab.db.cntWigs(dbing.dgKey(botHab.pre, serder.said)) + assert count >= 1 + + # add test fail process as remote since own witness + + # top process wop receipt as local since own witness receipt. + botHab.psr.parse(ims=receipt) # now top has fully witnessed icp. + count = botHab.db.cntWigs(dbing.dgKey(botHab.pre, serder.said)) + assert count >= 1 + + + + """End Test""" + + + + + + with (basing.openDB(name="bob") as bobDB, + keeping.openKS(name="bob") as bobKS, + basing.openDB(name="del") as delDB, + keeping.openKS(name="del") as delKS): + + # Init key pair managers + bobMgr = keeping.Manager(ks=bobKS, salt=topSalt) + delMgr = keeping.Manager(ks=delKS, salt=midSalt) + + # Init Keverys + bobKvy = eventing.Kevery(db=bobDB) + delKvy = eventing.Kevery(db=delDB) + + # Setup Bob by creating inception event + verfers, digers = bobMgr.incept(stem='bob', temp=True) # algo default salty and rooted + bobSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + bob = bobSrdr.ked["i"] + assert bob == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + + bobMgr.move(old=verfers[0].qb64, new=bob) # move key pair label to prefix + + sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) + + msg = bytearray(bobSrdr.raw) + counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + assert msg == (b'{"v":"KERI10JSON00012b_","t":"icp","d":"EA_SbBUZYwqLVlAAn14d6QUB' + b'QCSReJlZ755JqTgmRhXH","i":"EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755Jq' + b'TgmRhXH","s":"0","kt":"1","k":["DKiNnDmdOkcBjcAqL2FFhMZnSlPfNyGr' + b'JlCjJmX5b1nU"],"nt":"1","n":["EMP7Lg6BtehOYZt2RwOqXLNfMUiUllejAp' + b'8G_5EiANXR"],"bt":"0","b":[],"c":[],"a":[]}-AABAAArkDBeflIAo4kBs' + b'Knc754XHJvdLnf04iq-noTFEJkbv2MeIGZtx6lIfJPmRSEmFMUkFW4otRrMeBGQ0' + b'-nlhHEE') + + # apply msg to bob's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=bobKvy) + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + bobK = bobKvy.kevers[bob] + assert bobK.prefixer.qb64 == bob + assert bobK.serder.said == bobSrdr.said + assert bobK.serder.said == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + + # apply msg to del's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=delKvy) + # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + assert bob in delKvy.kevers + + # Setup Del's inception event assuming that Bob's next event will be an ixn delegating event + verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted + + delSrdr = eventing.delcept(keys=[verfer.qb64 for verfer in verfers], + delpre=bobK.prefixer.qb64, + ndigs=[diger.qb64 for diger in digers]) + + delPre = delSrdr.ked["i"] + assert delPre == 'EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnySJj' + + delMgr.move(old=verfers[0].qb64, new=delPre) # move key pair label to prefix + assert delSrdr.said == 'EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnySJj' + + # Now create delegating event + seal = eventing.SealEvent(i=delPre, + s=delSrdr.ked["s"], + d=delSrdr.said) + bobSrdr = eventing.interact(pre=bobK.prefixer.qb64, + dig=bobK.serder.said, + sn=bobK.sn + 1, + data=[seal._asdict()]) + + assert bobSrdr.said == 'EJtQndkvwnMpVGE5oVVbLWSCm-jLviGw1AOOkzBvNwsS' + + sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) + msg = bytearray(bobSrdr.raw) + counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, + count=len(sigers)) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + assert msg == (b'{"v":"KERI10JSON00013a_","t":"ixn","d":"EJtQndkvwnMpVGE5oVVbLWSC' + b'm-jLviGw1AOOkzBvNwsS","i":"EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755Jq' + b'TgmRhXH","s":"1","p":"EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRh' + b'XH","a":[{"i":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnySJj","s"' + b':"0","d":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnySJj"}]}-AABAA' + b'DFmoctrQkBbm47vuk7ejMbQ1y5vKD0Nfo8cqzbETZAlEPdbgVRSFta1-Bpv0y1Ri' + b'DrCxa_0IOp906gYqDPXIwG') + + # apply msg to bob's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=bobKvy) + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + + # apply msg to del's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=delKvy) + # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + assert delKvy.kevers[bob].serder.said == bobSrdr.said + + # now create msg with Del's delegated inception event + sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) + + msg = bytearray(delSrdr.raw) + counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, + count=len(sigers)) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, + count=1) + msg.extend(counter.qb64b) + seqner = coring.Seqner(sn=bobK.sn) + msg.extend(seqner.qb64b) + msg.extend(bobSrdr.saidb) + + assert msg == (b'{"v":"KERI10JSON00015f_","t":"dip","d":"EHng2fV42DdKb5TLMIs6bbjF' + b'kPNmIdQ5mSFn6BTnySJj","i":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6' + b'BTnySJj","s":"0","kt":"1","k":["DLitcfMnabnLt-PNCaXdVwX45wsG93Wd' + b'8eW9QiZrlKYQ"],"nt":"1","n":["EDjXvWdaNJx7pAIr72Va6JhHxc7Pf4ScYJ' + b'G496ky8lK8"],"bt":"0","b":[],"c":[],"a":[],"di":"EA_SbBUZYwqLVlA' + b'An14d6QUBQCSReJlZ755JqTgmRhXH"}-AABAABv6Q3s-1Tif-ksrx7ul9OKyOL_Z' + b'PHHp6lB9He4n6kswjm9VvHXzWB3O7RS2OQNWhx8bd3ycg9bWRPRrcKADoYC-GAB0' + b'AAAAAAAAAAAAAAAAAAAAAABEJtQndkvwnMpVGE5oVVbLWSCm-jLviGw1AOOkzBvN' + b'wsS') + + + # apply Del's delegated inception event message to Del's own Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=delKvy) + # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + assert delPre in delKvy.kevers + delK = delKvy.kevers[delPre] + assert delK.delegated + assert delK.serder.said == delSrdr.said + couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + + # apply Del's delegated inception event message to bob's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=bobKvy) + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + assert delPre in bobKvy.kevers # successfully validated + bobDelK = bobKvy.kevers[delPre] + assert bobDelK.delegated + assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + + # Setup Del rotation event assuming that Bob's next event will be an ixn delegating event + verfers, digers = delMgr.rotate(pre=delPre, temp=True) + + delSrdr = eventing.deltate(pre=bobDelK.prefixer.qb64, + keys=[verfer.qb64 for verfer in verfers], + dig=bobDelK.serder.said, + sn=bobDelK.sn + 1, + ndigs=[diger.qb64 for diger in digers]) + + assert delSrdr.said == 'EM5fj7YtOQYH3iLyWJr6HZVVxrY5t46LRL2vkNpdnPi0' + + # Now create delegating interaction event + seal = eventing.SealEvent(i=bobDelK.prefixer.qb64, + s=delSrdr.ked["s"], + d=delSrdr.said) + bobSrdr = eventing.interact(pre=bobK.prefixer.qb64, + dig=bobK.serder.said, + sn=bobK.sn + 1, + data=[seal._asdict()]) + + sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) + + msg = bytearray(bobSrdr.raw) + counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, + count=len(sigers)) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + assert msg == (b'{"v":"KERI10JSON00013a_","t":"ixn","d":"EJaPTWDiWvay8voiJkbxkvoa' + b'buUf_1a22yk9tVdRiMVs","i":"EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755Jq' + b'TgmRhXH","s":"2","p":"EJtQndkvwnMpVGE5oVVbLWSCm-jLviGw1AOOkzBvNw' + b'sS","a":[{"i":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnySJj","s"' + b':"1","d":"EM5fj7YtOQYH3iLyWJr6HZVVxrY5t46LRL2vkNpdnPi0"}]}-AABAA' + b'C8htl4epY7F5QBjro00VdfisxZMZWRXfe6xX_nVfS5gOsv8HOkzUKYMsvAVG4TJg' + b'7n1u44IyfsiKrB2R_UeUIK') + + # apply msg to bob's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=bobKvy) + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + + # apply msg to del's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=delKvy) + # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + assert delKvy.kevers[bob].serder.said == bobSrdr.said + + # now create msg from Del's delegated rotation event + sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) + + msg = bytearray(delSrdr.raw) + counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, + count=len(sigers)) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, + count=1) + msg.extend(counter.qb64b) + seqner = coring.Seqner(sn=bobK.sn) + msg.extend(seqner.qb64b) + msg.extend(bobSrdr.saidb) + + assert msg ==(b'{"v":"KERI10JSON000160_","t":"drt","d":"EM5fj7YtOQYH3iLyWJr6HZVV' + b'xrY5t46LRL2vkNpdnPi0","i":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6' + b'BTnySJj","s":"1","p":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnyS' + b'Jj","kt":"1","k":["DE3-kGVqHrdeeKPcL83jLjYS0Ea_CWgFHogusIwf-P9P"' + b'],"nt":"1","n":["EMj2mWvNvn6w9BbGUADX1AU3vn7idcUffZIaCvAsibru"],' + b'"bt":"0","br":[],"ba":[],"a":[]}-AABAAB_x-9_FTWr-OW_xXBN5pUkFNqL' + b'pAqTTQC02sPysnP0WmBFHb8NWvog9F-o279AfpPcLMxktypg1Fz7EQFYCuwC-GAB' + b'0AAAAAAAAAAAAAAAAAAAAAACEJaPTWDiWvay8voiJkbxkvoabuUf_1a22yk9tVdR' + b'iMVs') + + # apply msg to del's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=delKvy) + # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + assert bobDelK.delegated + assert delK.serder.said == delSrdr.said + couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + + # apply Del's delegated inception event message to bob's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=bobKvy) + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + assert bobDelK.delegated + assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + + # test replay + msgs = bytearray() + for msg in delKvy.db.clonePreIter(pre=delPre, fn=0): + msgs.extend(msg) + assert len(msgs) == 1167 + assert couple in msgs + + assert not os.path.exists(delKS.path) + assert not os.path.exists(delDB.path) + assert not os.path.exists(bobKS.path) + assert not os.path.exists(bobDB.path) + + """End Test""" + if __name__ == "__main__": test_delegation() + test_delegation_supersede() diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index cac27e311..07ac3d6c0 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -388,14 +388,18 @@ def test_missing_delegator_escrow(): bobIcpMsg = msg # save for later # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy) + psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) # bobKvy.process(ims=bytearray(msg)) # process local copy of msg bobK = bobKvy.kevers[bobPre] assert bobK.prefixer.qb64 == bobPre assert bobK.serder.said == bobSrdr.said + # apply msg to del's Kevery so he knows about the AID + psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) + assert bobK.prefixer.qb64 in delKvy.kevers + # Setup Del's inception event assuming that Bob's next event will be an ixn delegating event - verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted + verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted delSrdr = eventing.delcept(keys=[verfer.qb64 for verfer in verfers], delpre=bobPre, @@ -425,7 +429,7 @@ def test_missing_delegator_escrow(): bobIxnMsg = msg # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy) + psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) # bobKvy.process(ims=bytearray(msg)) # process local copy of msg assert bobK.serder.said == bobSrdr.said # key state updated so event was validated @@ -446,7 +450,7 @@ def test_missing_delegator_escrow(): msg.extend(bobSrdr.saidb) # apply Del's delegated inception event message to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy) + psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) # bobKvy.process(ims=bytearray(msg)) # process local copy of msg assert delPre in bobKvy.kevers # successfully validated bobDelK = bobKvy.kevers[delPre] # delK in bobs kevery @@ -457,10 +461,9 @@ def test_missing_delegator_escrow(): # apply Del's inception msg to Del's Kevery # Dels event will fail but will add to its escrow - psr.parse(ims=bytearray(msg), kvy=delKvy) + psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) # delKvy.process(ims=bytearray(msg)) # process remote copy of msg assert delPre not in delKvy.kevers - assert bobPre not in delKvy.kevers escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) assert len(escrows) == 1 assert escrows[0] == delSrdr.saidb # escrow entry for event @@ -471,7 +474,6 @@ def test_missing_delegator_escrow(): # assuming not stale but nothing else has changed delKvy.processEscrowPartialSigs() assert delPre not in delKvy.kevers - assert bobPre not in delKvy.kevers escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) assert len(escrows) == 1 assert escrows[0] == delSrdr.saidb # escrow entry for event @@ -479,7 +481,7 @@ def test_missing_delegator_escrow(): assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event # apply Bob's inception to Dels' Kvy - psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy) + psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) # delKvy.process(ims=bytearray(bobIcpMsg)) # process remote copy of msg assert bobPre in delKvy.kevers # message accepted delKvy.processEscrowPartialSigs() # process escrow @@ -491,7 +493,7 @@ def test_missing_delegator_escrow(): assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event # apply Bob's delegating interaction to Dels' Kvy - psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy) + psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy, local=True) # delKvy.process(ims=bytearray(bobIxnMsg)) # process remote copy of msg delKvy.processEscrowPartialSigs() # process escrows assert delPre in delKvy.kevers # event removed from escrow @@ -534,12 +536,12 @@ def test_missing_delegator_escrow(): msg.extend(siger.qb64b) # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy) + psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) # bobKvy.process(ims=bytearray(msg)) # process local copy of msg assert bobK.serder.said == bobSrdr.said # key state updated so event was validated # apply msg to del's Kevery - psr.parse(ims=bytearray(msg), kvy=delKvy) + psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) # delKvy.process(ims=bytearray(msg)) # process remote copy of msg assert delKvy.kevers[bobPre].serder.said == bobSrdr.said @@ -560,7 +562,7 @@ def test_missing_delegator_escrow(): msg.extend(bobSrdr.saidb) # apply Del's delegated Rotation event message to del's Kevery - psr.parse(ims=bytearray(msg), kvy=delKvy) + psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) # delKvy.process(ims=bytearray(msg)) # process remote copy of msg assert delK.delegated assert delK.serder.said == delSrdr.said @@ -568,7 +570,7 @@ def test_missing_delegator_escrow(): assert couple == seqner.qb64b + bobSrdr.saidb # apply Del's delegated Rotation event message to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy) + psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) # bobKvy.process(ims=bytearray(msg)) # process local copy of msg assert bobDelK.delegated assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated @@ -585,6 +587,19 @@ def test_missing_delegator_escrow(): """End Test""" +def test_misfit_escrow(): + """ + Test misfit escrow + + """ + salt = coring.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + + # stub for now + + """End Test""" + + + def test_out_of_order_escrow(): """ Test out of order escrow diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index c6532e63f..348239a56 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -4808,6 +4808,7 @@ def test_load_event(mockHelpingNowUTC): assert wanHab.pre == "BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP" msg = wanHab.makeOwnEvent(sn=0) parsing.Parser().parse(ims=msg, kvy=torKvy) + assert wanHab.pre in torKvy.kevers # Create Wil the witness, we'll use him later wilHab = wilHby.makeHab(name="wil", transferable=False) @@ -4816,14 +4817,16 @@ def test_load_event(mockHelpingNowUTC): torHab = torHby.makeHab(name="tor", icount=1, isith='1', ncount=1, nsith='1', wits=[wanHab.pre], toad=1) assert torHab.pre == "EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP" torIcp = torHab.makeOwnEvent(sn=0) + assert torHab.pre in torHab.kvy.kevers # Try to load event before Wan has seen it with pytest.raises(ValueError): _ = eventing.loadEvent(wanHab.db, torHab.pre, torHab.pre) - parsing.Parser().parse(ims=bytearray(torIcp), kvy=wanKvy) + # tor events are locallyWitnessed by wan so must process as local + parsing.Parser().parse(ims=bytearray(torIcp), kvy=wanHab.kvy, local=True) # process as local - wanHab.processCues(wanKvy.cues) # process cue returns rct msg + wanHab.processCues(wanHab.kvy.cues) # process cue returns rct msg evt = eventing.loadEvent(wanHab.db, torHab.pre, torHab.pre) assert evt == {'ked': {'a': [], 'b': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP'], @@ -4857,8 +4860,8 @@ def test_load_event(mockHelpingNowUTC): # Anchor Tee's inception event in Tor's KEL ixn = torHab.interact(data=[dict(i=teeHab.pre, s='0', d=teeHab.kever.serder.said)]) - parsing.Parser().parse(ims=bytearray(ixn), kvy=wanKvy) - wanHab.processCues(wanKvy.cues) # process cue returns rct msg + parsing.Parser().parse(ims=bytearray(ixn), kvy=wanHab.kvy, local=True) # give to wan must be local + wanHab.processCues(wanHab.kvy.cues) # process cue returns rct msg evt = eventing.loadEvent(wanHab.db, torHab.pre, torHab.kever.serder.said) assert evt == {'ked': {'a': [{'d': 'EDnrWpxagMvr5BBCwCOh3q5M9lvurboZ66vxR-GnIgQo', @@ -4894,9 +4897,12 @@ def test_load_event(mockHelpingNowUTC): nrct = wilHab.receipt(serder=teeHab.kever.serder) # Now Wan should be ready for Tee's inception - parsing.Parser().parse(ims=bytearray(teeIcp), kvy=wanKvy) - parsing.Parser().parse(ims=bytearray(rct), kvy=wanKvy) - parsing.Parser().parse(ims=bytearray(nrct), kvy=wanKvy) + parsing.Parser().parse(ims=bytearray(teeIcp), kvy=wanKvy, local=True) # local + parsing.Parser().parse(ims=bytearray(rct), kvy=wanHab.kvy, local=True) # local + parsing.Parser().parse(ims=bytearray(nrct), kvy=wanHab.kvy, local=True) # local + # ToDo XXXX fix it so cues are durable in db so can process cues from + # both and remote sources + wanHab.processCues(wanHab.kvy.cues) # process cue returns rct msg wanHab.processCues(wanKvy.cues) # process cue returns rct msg # Endorse Tee's inception event with Wan's Hab just so we have non-trans receipts diff --git a/tests/core/test_kevery.py b/tests/core/test_kevery.py index 56dd97bff..87b71c1cc 100644 --- a/tests/core/test_kevery.py +++ b/tests/core/test_kevery.py @@ -364,30 +364,30 @@ def test_stale_event_receipts(): # Pass incept to witnesses, receipted event to bam bobIcp = bobHab.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(bobIcp), kvy=bamKvy) + parsing.Parser().parse(ims=bytearray(bobIcp), kvy=bamKvy, local=True) assert bobHab.pre not in bamKvy.kevers for witHab in awits: kvy = eventing.Kevery(db=witHab.db, lax=False, local=False) - parsing.Parser().parse(ims=bytearray(bobIcp), kvy=kvy) + parsing.Parser().parse(ims=bytearray(bobIcp), kvy=kvy, local=True) assert bobHab.pre in witHab.kevers iserder = serdering.SerderKERI(raw=bytearray(bobIcp)) msg = witHab.receipt(serder=iserder) - parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy, local=True) bamKvy.processEscrows() assert bobHab.pre in bamKvy.kevers # Rotate, pass to witnesses, send receipts from Wes and Wan to Bam rot0 = bobHab.rotate(toad=2) - parsing.Parser().parse(ims=bytearray(rot0), kvy=bamKvy) + parsing.Parser().parse(ims=bytearray(rot0), kvy=bamKvy, local=True) for witHab in [wesHab, wanHab]: kvy = eventing.Kevery(db=witHab.db, lax=False, local=False) - parsing.Parser().parse(ims=bytearray(rot0), kvy=kvy) + parsing.Parser().parse(ims=bytearray(rot0), kvy=kvy, local=True) iserder = serdering.SerderKERI(raw=bytearray(rot0)) msg = witHab.receipt(serder=iserder) - parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy, local=True) bamKvy.processEscrows() assert bamKvy.kevers[bobHab.pre].sn == 1 @@ -400,25 +400,25 @@ def test_stale_event_receipts(): # Rotate out Wil, pass to witnesses, receipted event to bam. rot1 = bobHab.rotate(cuts=[wilHab.pre], toad=2) - parsing.Parser().parse(ims=bytearray(rot1), kvy=bamKvy) + parsing.Parser().parse(ims=bytearray(rot1), kvy=bamKvy, local=True) for witHab in [wesHab, wanHab]: - kvy = eventing.Kevery(db=witHab.db, lax=False, local=False) - parsing.Parser().parse(ims=bytearray(rot1), kvy=kvy) + kvy = eventing.Kevery(db=witHab.db) + parsing.Parser().parse(ims=bytearray(rot1), kvy=kvy, local=True) iserder = serdering.SerderKERI(raw=bytearray(rot1)) msg = witHab.receipt(serder=iserder) - parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy, local=True) bamKvy.processEscrows() assert bamKvy.kevers[bobHab.pre].sn == 2 assert bamKvy.kevers[bobHab.pre].wits == [wesHab.pre, wanHab.pre] # Pass receipts from Wil for event 1 to Bam - kvy = eventing.Kevery(db=wilHab.db, lax=False, local=False) - parsing.Parser().parse(ims=bytearray(rot0), kvy=kvy) + kvy = eventing.Kevery(db=wilHab.db) + parsing.Parser().parse(ims=bytearray(rot0), kvy=kvy, local=True) iserder = serdering.SerderKERI(raw=bytearray(rot0)) msg = wilHab.receipt(serder=iserder) - parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy, local=True) # Validate that bam has 3 receipts in DB for event 1 wigs = bamHby.db.getWigs(dgkey) diff --git a/tests/core/test_keystate.py b/tests/core/test_keystate.py index b3cfb7e2d..ded410cf0 100644 --- a/tests/core/test_keystate.py +++ b/tests/core/test_keystate.py @@ -75,9 +75,9 @@ def test_keystate(mockHelpingNowUTC): assert bobHab.pre == 'EDotK23orLtF8GAU61_fNXRyFBTg49X50W0OUlP14YAK' # Create Bob's icp, pass to Wes. - wesKvy = eventing.Kevery(db=wesHby.db, lax=False, local=False) + wesKvy = eventing.Kevery(db=wesHby.db) bobIcp = bobHab.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(bobIcp), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(bobIcp), kvy=wesKvy, local=True) assert bobHab.pre in wesHab.kevers iserder = serdering.SerderKERI(raw=bytearray(bobIcp)) wesHab.receipt(serder=iserder) @@ -100,9 +100,9 @@ def test_keystate(mockHelpingNowUTC): bamRtr = routing.Router() bamRvy = routing.Revery(db=bamHby.db, rtr=bamRtr) - bamKvy = eventing.Kevery(db=bamHby.db, lax=False, local=False, rvy=bamRvy) + bamKvy = eventing.Kevery(db=bamHby.db, rvy=bamRvy) bamKvy.registerReplyRoutes(router=bamRtr) - parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy, rvy=bamRvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy, rvy=bamRvy, local=True) assert len(bamKvy.cues) == 1 cue = bamKvy.cues.popleft() @@ -113,7 +113,7 @@ def test_keystate(mockHelpingNowUTC): for msg in wesHby.db.clonePreIter(pre=bobHab.pre, fn=0): msgs.extend(msg) - parsing.Parser().parse(ims=msgs, kvy=bamKvy) + parsing.Parser().parse(ims=msgs, kvy=bamKvy, local=True) bamKvy.processEscrows() keys = (bobHab.pre, wesHab.pre) @@ -138,9 +138,9 @@ def test_keystate(mockHelpingNowUTC): # Create Bob's icp, pass to Wes. - wesKvy = eventing.Kevery(db=wesHby.db, lax=False, local=False) + wesKvy = eventing.Kevery(db=wesHby.db) bobIcp = bobHab.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(bobIcp), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(bobIcp), kvy=wesKvy, local=True) assert bobHab.pre in wesHab.kevers # Get ksr key state record from Bob and verify @@ -149,7 +149,6 @@ def test_keystate(mockHelpingNowUTC): assert ksr.s == '0' assert ksr.d == bobHab.kever.serder.said - # Get ksr key state record from Wes and verify bobKeverFromWes = wesHab.kevers[bobHab.pre] ksr = bobKeverFromWes.state() @@ -159,8 +158,6 @@ def test_keystate(mockHelpingNowUTC): msg = wesHab.reply(route="/ksn/" + wesHab.pre, data=ksr._asdict()) - #bamHab = habbing.Habitat(name="bam", ks=bamKS, db=bamDB, isith='1', icount=1, - #transferable=True, temp=True) bamHab = bamHby.makeHab(name="bam", isith='1', icount=1, transferable=True) # Set Wes has Bam's watcher @@ -170,9 +167,9 @@ def test_keystate(mockHelpingNowUTC): bamRtr = routing.Router() bamRvy = routing.Revery(db=bamHby.db, rtr=bamRtr) - bamKvy = eventing.Kevery(db=bamHby.db, lax=False, local=False, rvy=bamRvy) + bamKvy = eventing.Kevery(db=bamHby.db, rvy=bamRvy) bamKvy.registerReplyRoutes(router=bamRtr) - parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy, rvy=bamRvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy, rvy=bamRvy, local=True) assert len(bamKvy.cues) == 1 cue = bamKvy.cues.popleft() @@ -183,7 +180,7 @@ def test_keystate(mockHelpingNowUTC): for msg in wesHby.db.clonePreIter(pre=bobHab.pre, fn=0): msgs.extend(msg) - parsing.Parser().parse(ims=msgs, kvy=bamKvy) + parsing.Parser().parse(ims=msgs, kvy=bamKvy, local=True) bamKvy.processEscrows() keys = (bobHab.pre, wesHab.pre) @@ -207,9 +204,9 @@ def test_keystate(mockHelpingNowUTC): assert bobHab.pre == bobpre # Create Bob's icp, pass to Wes. - wesKvy = eventing.Kevery(db=wesHby.db, lax=False, local=False) + wesKvy = eventing.Kevery(db=wesHby.db) bobIcp = bobHab.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(bobIcp), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(bobIcp), kvy=wesKvy, local=True) assert bobHab.pre in wesHab.kevers # Get ksr key state record from Bob and verify @@ -227,8 +224,8 @@ def test_keystate(mockHelpingNowUTC): msg = wesHab.reply(route="/ksn/" + wesHab.pre, data=ksr._asdict()) - bamKvy = eventing.Kevery(db=bamHby.db, lax=False, local=False) - parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy) + bamKvy = eventing.Kevery(db=bamHby.db) + parsing.Parser().parse(ims=bytearray(msg), kvy=bamKvy, local=True) assert len(bamKvy.cues) == 0 saider = bamHby.db.knas.get(keys=keys) @@ -262,9 +259,9 @@ def test_keystate(mockHelpingNowUTC): bamRtr = routing.Router() bamRvy = routing.Revery(db=bamHby.db, rtr=bamRtr) - bamKvy = eventing.Kevery(db=bamHby.db, lax=False, local=False, rvy=bamRvy) + bamKvy = eventing.Kevery(db=bamHby.db, rvy=bamRvy) bamKvy.registerReplyRoutes(router=bamRtr) - parsing.Parser().parse(ims=bytearray(staleKsn), kvy=bamKvy, rvy=bamRvy) + parsing.Parser().parse(ims=bytearray(staleKsn), kvy=bamKvy, rvy=bamRvy, local=True) for _ in range(5): bobHab.rotate() @@ -276,13 +273,13 @@ def test_keystate(mockHelpingNowUTC): assert ksr.d == bobHab.kever.serder.said liveKsn = bobHab.reply(route="/ksn/" + bobHab.pre, data=ksr._asdict()) - parsing.Parser().parse(ims=bytearray(liveKsn), kvy=bamKvy, rvy=bamRvy) + parsing.Parser().parse(ims=bytearray(liveKsn), kvy=bamKvy, rvy=bamRvy, local=True) msgs = bytearray() # outgoing messages for msg in bobHby.db.clonePreIter(pre=bobHab.pre, fn=0): msgs.extend(msg) - parsing.Parser().parse(ims=msgs, kvy=bamKvy, rvy=bamRvy) + parsing.Parser().parse(ims=msgs, kvy=bamKvy, rvy=bamRvy, local=True) assert bobHab.pre in bamKvy.kevers @@ -296,7 +293,3 @@ def test_keystate(mockHelpingNowUTC): assert latest.s == '8' """End Test""" - - -if __name__ == "__main__": - test_keystate() diff --git a/tests/core/test_parsing.py b/tests/core/test_parsing.py index 2909effc5..ad647506a 100644 --- a/tests/core/test_parsing.py +++ b/tests/core/test_parsing.py @@ -224,6 +224,11 @@ def test_parser(): kevery = Kevery(db=valDB) parser = parsing.Parser(kvy=kevery) + assert parser.kvy == kevery + assert parser.local == False + assert parser.framed == True + assert parser.pipeline == False + assert parser.ims == bytearray() parser.parse(ims=bytearray(msgs)) # make copy assert parser.ims == bytearray(b'') # emptied @@ -236,7 +241,7 @@ def test_parser(): db_digs = [bytes(val).decode("utf-8") for val in kevery.db.getKelIter(pre)] assert db_digs == event_digs - parser = parsing.Parser() # no kevery + parser = parsing.Parser() # no kevery so drops all messages parser.parse(ims=msgs) assert parser.ims == bytearray(b'') diff --git a/tests/core/test_reply.py b/tests/core/test_reply.py index 608ae48f4..153bcd25c 100644 --- a/tests/core/test_reply.py +++ b/tests/core/test_reply.py @@ -689,20 +689,20 @@ def test_reply(mockHelpingNowUTC): # add tam kel to nel and process escrows tamicp = tamHab.makeOwnInception() - nelPrs.parse(bytearray(tamicp)) + nelPrs.parse(bytearray(tamicp), local=True) assert tamHab.pre not in nelKvy.kevers - wesPrs.parse(bytearray(tamicp)) + wesPrs.parse(bytearray(tamicp), local=True) assert tamHab.pre in wesKvy.kevers - wokPrs.parse(bytearray(tamicp)) + wokPrs.parse(bytearray(tamicp), local=True) assert tamHab.pre in wokKvy.kevers - wamPrs.parse(bytearray(tamicp)) + wamPrs.parse(bytearray(tamicp), local=True) assert tamHab.pre in wamKvy.kevers wittamicp = wesHab.witness(tamHab.iserder) - nelPrs.parse(bytearray(wittamicp)) + nelPrs.parse(bytearray(wittamicp), local=True) wittamicp = wokHab.witness(tamHab.iserder) - nelPrs.parse(bytearray(wittamicp)) + nelPrs.parse(bytearray(wittamicp), local=True) wittamicp = wamHab.witness(tamHab.iserder) - nelPrs.parse(bytearray(wittamicp)) + nelPrs.parse(bytearray(wittamicp), local=True) nelKvy.processEscrows() assert tamHab.pre in nelHab.kevers diff --git a/tests/core/test_witness.py b/tests/core/test_witness.py index 0c34e46b2..e980457ca 100644 --- a/tests/core/test_witness.py +++ b/tests/core/test_witness.py @@ -95,7 +95,7 @@ def test_indexed_witness_replay(): rctMsgs = [] # list of receipts from each witness for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy, local=True) assert kvy.kevers[camHab.pre].sn == 0 # accepted event assert len(kvy.cues) >= 1 # at least queued receipt cue # better to find receipt cue in cues exactly @@ -105,7 +105,7 @@ def test_indexed_witness_replay(): rctMsgs.append(rctMsg) for msg in rctMsgs: # process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -124,16 +124,16 @@ def test_indexed_witness_replay(): assert len(camIcpWitRctMsg) == 413 for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=kvy, local=True) assert len(kvy.db.getWigs(dgkey)) == 3 # fully witnessed assert len(kvy.cues) == 0 # no cues # send Cam icp and witness rcts to Van - parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=vanKvy, local=True) # should escrow since not witnesses assert camHab.pre not in vanKvy.kevers # process receipts - parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=vanKvy, local=True) vanKvy.processEscrows() assert camHab.pre in vanKvy.kevers # now accepted vcKvr = vanKvy.kevers[camHab.pre] @@ -145,7 +145,7 @@ def test_indexed_witness_replay(): rctMsgs = [] # list of receipts from each witness for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=kvy, local=True) assert kvy.kevers[camHab.pre].sn == 1 # accepted event assert len(kvy.cues) >= 1 # at least queued receipt cue # better to find receipt cue in cues exactly @@ -155,7 +155,7 @@ def test_indexed_witness_replay(): rctMsgs.append(rctMsg) for msg in rctMsgs: # process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -174,15 +174,15 @@ def test_indexed_witness_replay(): assert len(camIxnWitRctMsg) == 413 for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camIxnWitRctMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIxnWitRctMsg), kvy=kvy, local=True) assert len(kvy.db.getWigs(dgkey)) == 3 # fully witnessed assert len(kvy.cues) == 0 # no cues # send Cam ixn's witness rcts to Van first then send Cam ixn - parsing.Parser().parse(ims=bytearray(camIxnWitRctMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIxnWitRctMsg), kvy=vanKvy, local=True) vanKvy.processEscrows() assert vcKvr.sn == 0 - parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=vanKvy, local=True) assert vcKvr.sn == 0 vanKvy.processEscrows() assert vcKvr.sn == 1 @@ -192,13 +192,13 @@ def test_indexed_witness_replay(): # Cam update itself with Wil receipts including Wils inception camReplayMsg = camHab.replay() assert len(camReplayMsg) == 2038 - parsing.Parser().parse(ims=bytearray(camReplayMsg), kvy=wilKvy) + parsing.Parser().parse(ims=bytearray(camReplayMsg), kvy=wilKvy, local=True) assert camHab.pre in wilKvy.kevers assert wilKvy.kevers[camHab.pre].sn == 1 # asscepted both events assert len(wilKvy.cues) == 2 wilRctMsg = wilHab.processCues(wilKvy.cues) # process cue returns rct msg assert len(wilKvy.cues) == 0 - parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=camKvy, local=True) assert wilHab.pre in camKvy.kevers # Cam rotation with witness rotation @@ -214,7 +214,7 @@ def test_indexed_witness_replay(): rctMsgs = [] # list of receipt msgs from each witness for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=kvy, local=True) assert kvy.kevers[camHab.pre].sn == 2 # accepted event assert len(kvy.cues) >= 1 # at least queued receipt cue # better to find receipt cue in cues exactly @@ -224,7 +224,7 @@ def test_indexed_witness_replay(): rctMsgs.append(rctMsg) for msg in rctMsgs: # process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -243,7 +243,7 @@ def test_indexed_witness_replay(): assert len(camRotWitRctMsg) == 413 for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camRotWitRctMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camRotWitRctMsg), kvy=kvy, local=True) # kvy.process(ims=bytearray(camRotWitRctMsg)) # send copy of witness rcts assert len(kvy.db.getWigs(dgkey)) == 3 # fully witnessed assert len(kvy.cues) == 0 # no cues @@ -256,10 +256,10 @@ def test_indexed_witness_replay(): # assert vcKvr.wits == camHab.kever.wits # send Cam rot's witness rcts to Van first then send Cam rot - parsing.Parser().parse(ims=bytearray(camRotWitRctMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camRotWitRctMsg), kvy=vanKvy, local=True) vanKvy.processEscrows() assert vcKvr.sn == 1 - parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=vanKvy, local=True) assert vcKvr.sn == 1 vanKvy.processEscrows() assert vcKvr.sn == 2 @@ -361,7 +361,7 @@ def test_nonindexed_witness_receipts(): camIcpMsg = camHab.makeOwnInception() rctMsgs = [] # list of receipts from each witness for i, kvy in enumerate(camWitKvys): - parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy, local=True) # accepted event with cam sigs since own witness assert kvy.kevers[camHab.pre].sn == 0 assert len(kvy.cues) >= 1 # at least queued receipt cue @@ -371,7 +371,7 @@ def test_nonindexed_witness_receipts(): rctMsgs.append(rctMsg) for msg in rctMsgs: # Cam process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -382,7 +382,7 @@ def test_nonindexed_witness_receipts(): snkey = dbing.snKey(pre=camHab.pre, sn=camHab.kever.serder.sn) # Van process rct msgs from all witnesses for Cam's icp message for i, msg in enumerate(rctMsgs): - parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) # escrows to Ure assert vanKvy.db.cntUres(snkey) == i + 1 # escrows assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow @@ -398,7 +398,7 @@ def test_nonindexed_witness_receipts(): assert camHab.pre not in vanKvy.kevers # still not accepted # Van process icp message from Cam - parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=vanKvy, local=True) # event accepted in database with sigs but not into KEL assert vanKvy.db.cntSigs(dgkey) == len(camHab.kever.verfers) assert vanKvy.db.cntPwes(snkey) == 1 # now in partial witness escrow @@ -420,7 +420,7 @@ def test_nonindexed_witness_receipts(): camIxnMsg = camHab.interact() rctMsgs = [] # list of receipts from each witness for i, kvy in enumerate(camWitKvys): - parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=kvy, local=True) # kvy.process(ims=bytearray(camIxnMsg)) # send copy of cam icp msg to witness assert kvy.kevers[camHab.pre].sn == 1 # accepted event assert len(kvy.cues) >= 1 # at least queued receipt cue @@ -431,7 +431,7 @@ def test_nonindexed_witness_receipts(): rctMsgs.append(rctMsg) for msg in rctMsgs: # Cam process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -442,7 +442,7 @@ def test_nonindexed_witness_receipts(): snkey = dbing.snKey(pre=camHab.pre, sn=camHab.kever.serder.sn) # Van process rct msgs from all witnesses for Cam's ixn message for i, msg in enumerate(rctMsgs): - parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) # escrows to Ure assert vanKvy.db.cntUres(snkey) == i + 1 # escrows assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow @@ -457,7 +457,7 @@ def test_nonindexed_witness_receipts(): assert vcKvr.wits == wits # no change # Van process ixn message from Cam - parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=vanKvy, local=True) # event accepted in database with sigs but not into KEL assert vanKvy.db.cntSigs(dgkey) == len(camHab.kever.verfers) assert vanKvy.db.cntPwes(snkey) == 1 # now in partial witness escrow @@ -477,14 +477,14 @@ def test_nonindexed_witness_receipts(): # Cam update itself with Wil receipts including Wils inception camReplayMsg = camHab.replay() assert len(camReplayMsg) == 2038 - parsing.Parser().parse(ims=bytearray(camReplayMsg), kvy=wilKvy) + parsing.Parser().parse(ims=bytearray(camReplayMsg), kvy=wilKvy, local=True) assert camHab.pre in wilKvy.kevers assert wilKvy.kevers[camHab.pre].sn == 1 # asscepted both events assert len(wilKvy.cues) == 2 wilRctMsg = wilHab.processCues(wilKvy.cues) # process cue returns rct msg assert len(wilKvy.cues) == 0 - parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=camKvy, local=True) assert wilHab.pre in camKvy.kevers # Cam rotation with witness rotation @@ -503,7 +503,7 @@ def test_nonindexed_witness_receipts(): rctMsgs = [] # list of receipt msgs from each witness for i, kvy in enumerate(camWitKvys): - parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=kvy, local=True) assert kvy.kevers[camHab.pre].sn == 2 # accepted event assert len(kvy.cues) >= 1 # at least queued receipt cue # better to find receipt cue in cues exactly @@ -513,7 +513,7 @@ def test_nonindexed_witness_receipts(): rctMsgs.append(rctMsg) for msg in rctMsgs: # process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -524,7 +524,7 @@ def test_nonindexed_witness_receipts(): snkey = dbing.snKey(pre=camHab.pre, sn=camHab.kever.serder.sn) # Van process rct msgs from all witnesses for Cam's ixn message for i, msg in enumerate(rctMsgs): - parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) # escrows to Ure assert vanKvy.db.cntUres(snkey) == i + 1 # escrows assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow @@ -532,7 +532,7 @@ def test_nonindexed_witness_receipts(): assert vcKvr.sn == 1 # not rot yet # send stale receipts from Wil to Van - parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=vanKvy, local=True) assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # no change @@ -544,7 +544,7 @@ def test_nonindexed_witness_receipts(): assert vcKvr.wits == oldwits # no change # Van process rot message from Cam - parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=vanKvy, local=True) # event accepted in database with sigs but not into KEL assert vanKvy.db.cntSigs(dgkey) == len(camHab.kever.verfers) assert vanKvy.db.cntPwes(snkey) == 1 # now in partial witness escrow @@ -590,7 +590,7 @@ def test_out_of_order_witnessed_events(): # Create Bob's icp, pass to Wes and generate receipt. wesKvy = eventing.Kevery(db=wesHby.db, lax=False, local=False) bobIcp = bobHab.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(bobIcp), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(bobIcp), kvy=wesKvy, local=True) assert bobHab.pre in wesHab.kevers iserder = serdering.SerderKERI(raw=bytearray(bobIcp)) wesHab.receipt(serder=iserder) @@ -598,7 +598,7 @@ def test_out_of_order_witnessed_events(): # Rotate and get Bob's rot, pass to Wes and generate receipt. bobHab.rotate() bobRotMsg = bobHab.makeOwnEvent(sn=1) - parsing.Parser().parse(ims=bytearray(bobRotMsg), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(bobRotMsg), kvy=wesKvy, local=True) assert wesKvy.kevers[bobHab.pre].sn == 1 bobRot = serdering.SerderKERI(raw=bobRotMsg) wesHab.receipt(serder=bobRot) @@ -609,7 +609,7 @@ def test_out_of_order_witnessed_events(): msgs.extend(msg) bamKvy = eventing.Kevery(db=bamHby.db, lax=False, local=False) - parsing.Parser().parse(ims=msgs, kvy=bamKvy) + parsing.Parser().parse(ims=msgs, kvy=bamKvy, local=True) # Ensure the rot ended up in out-of-order escrow assert bobHab.pre not in bamKvy.kevers diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index b067985dc..14a4bd8ef 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -158,6 +158,27 @@ def test_baser(): assert db.delEvt(key) == True assert db.getEvt(key) == None + # test eventsourcerecords .srcs + record = basing.EventSourceRecord() + assert db.esrs.get(key) == None + assert db.esrs.put(key, record) == True + actual = db.esrs.get(key) + assert actual == record + record.local = False + # put does not overwrite must pin + assert db.esrs.put(key, record) == False + actual = db.esrs.get(key) + assert actual.local != record.local + assert actual != record + assert not db.esrs.get(key) == record + assert db.esrs.pin(key, record) == True + actual = db.esrs.get(key) + assert actual.local == record.local + assert db.esrs.get(key) == record + + + + # test first seen event log .fels sub db preA = b'BAKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc' preB = b'EH7Oq9oxCgYa-nnNLvwhp9sFZpALILlRYyB-6n4WDi7w' @@ -1687,7 +1708,7 @@ def test_clean_baser(): name = "nat" # with basing.openDB(name="nat") as natDB, keeping.openKS(name="nat") as natKS: with habbing.openHby(name=name) as hby: # default is temp=True - natHab = hby.makeHab(name=name, isith='2', icount=3) + natHab = hby.makeHab(name=name, isith='2', icount=3) # default Hab # setup Nat's habitat using default salt multisig already incepts #natHab = habbing.Habitat(name='nat', ks=natKS, db=natDB, #isith='2', icount=3, temp=True) @@ -2067,6 +2088,40 @@ def test_keystaterecord(): assert nksr._asdict() == ksn + """End Test""" + +def test_eventsourcerecord(): + """ + Test EventSourceRecord dataclass + """ + record = basing.EventSourceRecord() # default local is True + assert isinstance(record, basing.EventSourceRecord) + assert record.local is True + assert record.local + assert "local" in record # asdict means in is against the keys (labels) + assert (asdict(record)) == {'local': True} + + record.local = False + assert record.local is False + assert not record.local + assert (asdict(record)) == {'local': False} + + record = basing.EventSourceRecord(local=False) + assert isinstance(record, basing.EventSourceRecord) + assert record.local is False + assert not record.local + assert "local" in record # asdict means in is against the keys (labels) + assert (asdict(record)) == {'local': False} + + record = basing.EventSourceRecord(local=None) + assert isinstance(record, basing.EventSourceRecord) + assert record.local is None + assert not record.local + assert "local" in record # asdict means in is against the keys (labels) + assert (asdict(record)) == {'local': None} + + + """End Test""" @@ -2074,12 +2129,14 @@ def test_dbdict(): """ Test custom dbdict subclass of dict """ - dbd = basing.dbdict(a=1, b=2, c=3) + dbd = basing.dbdict(a=1, b=2, c=3) # init in memory so never acesses db assert dbd.db == None assert 'a' in dbd assert 'b' in dbd assert 'c' in dbd assert [(k, v) for k, v in dbd.items()] == [('a', 1), ('b', 2), ('c', 3)] + assert list(dbd.keys()) == ['a', 'b', 'c'] + assert list(dbd.values()) == [1, 2, 3] assert dbd.get('a') == 1 assert dbd['a'] == 1 diff --git a/tests/vc/test_protocoling.py b/tests/vc/test_protocoling.py index befb8dcbb..a450847f3 100644 --- a/tests/vc/test_protocoling.py +++ b/tests/vc/test_protocoling.py @@ -90,6 +90,7 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN b'","a":{"d":"EF2__B6DiLQHpdJZ_C0bddxy2o6nXIHEwchO9yylr3xx","dt":"2021-06-27T2' b'1:26:21.233257+00:00","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LE' b'I":"254900OPPU84GM83MG36"}}') + atc = bytearray(msg) atc.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) atc.extend(coring.Prefixer(qb64=iss.pre).qb64b) diff --git a/tests/vdr/test_eventing.py b/tests/vdr/test_eventing.py index 3da4f055c..079acc328 100644 --- a/tests/vdr/test_eventing.py +++ b/tests/vdr/test_eventing.py @@ -37,7 +37,7 @@ def test_incept(mockCoringRandomNonce): b'fpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') # no backers allowed - serder = eventing.incept(pre, baks=[], cnfg=[keventing.TraitDex.NoBackers], code=MtrDex.Blake3_256) + serder = eventing.incept(pre, baks=[], cnfg=[keventing.TraitDex.NoRegistrarBackers], code=MtrDex.Blake3_256) assert serder.raw == (b'{"v":"KERI10JSON000113_","t":"vcp","d":"EBoBPh3N5nr1tItAUCkXNx3vShB_Be6iiQPX' b'Bsg2LvxA","i":"EBoBPh3N5nr1tItAUCkXNx3vShB_Be6iiQPXBsg2LvxA","ii":"DAtNTPnDF' b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":["NB"],"bt":"0","b":[],"n":' @@ -45,7 +45,7 @@ def test_incept(mockCoringRandomNonce): # no backers allows, one attempted with pytest.raises(ValueError): - eventing.incept(pre, cnfg=[keventing.TraitDex.NoBackers], + eventing.incept(pre, cnfg=[keventing.TraitDex.NoRegistrarBackers], baks=[bak1]) # with backer dupes @@ -368,7 +368,7 @@ def test_prefixer(): t=Ilks.vcp, bt=0, b=[], - c=[keventing.TraitDex.NoBackers], + c=[keventing.TraitDex.NoRegistrarBackers], ) prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) assert prefixer.qb64 == 'EDz0QmMxf4Dk0C9uiP-y3okN-Bej2IAXSj8UwQgb3NsL' diff --git a/tests/vdr/test_txn_state.py b/tests/vdr/test_txn_state.py index fa24dbe2f..d2e633fa7 100644 --- a/tests/vdr/test_txn_state.py +++ b/tests/vdr/test_txn_state.py @@ -181,10 +181,6 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): # Bob is the controller # Wes is his witness # Bam is verifying the key state for Bob from Wes - #raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - #salter = coring.Salter(raw=raw) - #salt = salter.qb64 - #assert salt == '0AAFqo8tU5rp-lWcApybCEh1' # Habery.makeHab uses name as stem path for salt so different pre with (habbing.openHby(name="wes", base="test") as wesHby, habbing.openHby(name="bob", base="test") as bobHby, @@ -214,7 +210,7 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): wesKvy = Kevery(db=wesHby.db, lax=False, local=False) for msg in bobHby.db.clonePreIter(pre=bobHab.pre, fn=0): - parsing.Parser().parse(ims=bytearray(msg), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=wesKvy, local=True) iserder = serdering.SerderKERI(raw=bytearray(msg)) wesHab.receipt(serder=iserder) @@ -256,7 +252,7 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): bamReger = viring.Reger(name="bam", temp=True) bamTvy = eventing.Tevery(reger=bamReger, db=bamHby.db, lax=False, local=False, rvy=bamRvy) bamTvy.registerReplyRoutes(router=bamRtr) - parsing.Parser().parse(ims=bytearray(rpy), tvy=bamTvy, rvy=bamRvy) + parsing.Parser().parse(ims=bytearray(rpy), tvy=bamTvy, rvy=bamRvy, local=True) saider = bamReger.txnsb.escrowdb.get(keys=("registry-mae", issuer.regk, wesHab.pre)) said = b'EMoqBJpoPJCkCejSUZ8IShTeGd_WQkF0zOQc2l0HQusn' @@ -267,14 +263,14 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): assert cue['q']['pre'] == bobHab.pre wesIcp = wesHab.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(wesIcp), kvy=bamKvy) + parsing.Parser().parse(ims=bytearray(wesIcp), kvy=bamKvy, local=True) assert wesHab.pre in bamHby.db.kevers msgs = bytearray() for msg in wesHby.db.clonePreIter(pre=bobHab.pre, fn=0): msgs.extend(msg) - parsing.Parser().parse(ims=msgs, kvy=bamKvy, rvy=bamRvy) + parsing.Parser().parse(ims=msgs, kvy=bamKvy, rvy=bamRvy, local=True) assert bobHab.pre in bamHby.db.kevers bamTvy.processEscrows() @@ -287,7 +283,7 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): saider = bamReger.txnsb.escrowdb.get(keys=("registry-ooo", issuer.regk, wesHab.pre)) assert saider[0].qb64b == said - parsing.Parser().parse(ims=bytearray(tmsgs), tvy=bamTvy, rvy=bamRvy) + parsing.Parser().parse(ims=bytearray(tmsgs), tvy=bamTvy, rvy=bamRvy, local=True) assert issuer.regk in bamReger.tevers From 38d7c1663a67b5e9327c8f82e898afdc43344e18 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 16 Feb 2024 16:28:39 -0700 Subject: [PATCH 007/418] some clean up --- setup.py | 2 +- src/keri/core/coring.py | 461 ++++++++++++++++++++++++++++++++++++--- src/keri/help/helping.py | 2 +- 3 files changed, 429 insertions(+), 36 deletions(-) diff --git a/setup.py b/setup.py index 34d89c7a8..c38653b49 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ 'Operating System :: Unix', 'Operating System :: POSIX', 'Operating System :: Microsoft :: Windows', - 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', # uncomment if you test on these interpreters: # 'Programming Language :: Python :: Implementation :: PyPy', diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index aedb864d0..1fae0bd72 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -5447,7 +5447,7 @@ def pretty(self, *, size=1024): -class Tholder: +class NewTholder: """ Tholder is KERI Signing Threshold Satisfaction class .satisfy method evaluates satisfaction based on ordered list of indices of @@ -5482,8 +5482,8 @@ class Tholder: .num is int signing threshold when not ._weighted Methods: - .satisfy returns bool, True means ilist of verified signature key indices satisfies - threshold, False otherwise. + .satisfy returns bool, True means list of verified signature key indices + satisfies the threshold, False otherwise. Static Methods: weight (str): converts weight str expression into either int or Fraction @@ -5510,7 +5510,32 @@ def __init__(self, *, thold=None , limen=None, sith=None, **kwa): Accepts signing threshold in various forms so that may output correct forms for serialization and/or calculation of satisfaction. + The thold representation is meant to accept thresholds from computable + expressions for satisfaction of a threshold + + The limen representation is meant to parse threshold expressions from + CESR serializations of key event message fields or attachments. + + The sith representation is meant to parse threhold expressions from + deserializations of JSON, CBOR, or MGPK key event message fields or + the command line or configuration files. + + Parameters: + + thold is signing threshold (current or next) is suitable for computing + the satisfaction of a threshold and is expressed as either: + int of threshold number (M of N) + fractional weight clauses which may be expressed as either: + an iterable of Fractions or + an iterable of iterables of Fractions. + + limen is qualified signing threshold (current or next) expressed as either: + Number.qb64 or .qb64b of integer threshold or + Bexter.qb64 or .qb64b of fractional weight clauses which may be either: + Base64 delimited clauses of fractions + Base64 delimited clauses of fractions + sith is signing threshold (current or next) expressed as either: non-negative int of threshold number (M-of-N threshold) next threshold may be zero @@ -5523,12 +5548,386 @@ def __init__(self, *, thold=None , limen=None, sith=None, **kwa): list of rational number fraction strings >= 0 and <= 1 or list of list of rational number fraction strings >= 0 and <= 1 + """ + if thold is not None: + self._processThold(thold=thold) - limen is qualified signing threshold (current or next) expressed as either: - Number.qb64 or .qb64b of integer threshold or - Bexter.qb64 or .qb64b of fractional weight clauses which may be either: - Base64 delimited clauses of fractions - Base64 delimited clauses of fractions + elif limen is not None: + self._processLimen(limen=limen, **kwa) # kwa for strip + + elif sith is not None: + if isinstance(sith, str) and not sith: # empty str + raise EmptyMaterialError("Empty threshold expression.") + + self._processSith(sith=sith) + + else: + raise EmptyMaterialError("Missing threshold expression.") + + + @property + def weighted(self): + """ weighted property getter """ + return self._weighted + + @property + def thold(self): + """ thold property getter """ + return self._thold + + @property + def size(self): + """ size property getter """ + return self._size + + @property + def limen(self): + """ limen property getter """ + return self._bexter.qb64b if self._weighted else self._number.qb64b + + @property + def sith(self): + """ sith property getter """ + # make sith expression of thold + if self.weighted: + sith = [[f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" + for f in clause] + for clause in self.thold] + if len(sith) == 1: + sith = sith[0] # simplify list of one clause to clause + else: + sith = f"{self.thold:x}" + + return sith + + @property + def json(self): + """Returns json serialization of sith expression + + Essentially JSON list of lists of strings + """ + return json.dumps(self.sith) + + + @property + def num(self): + """ sith property getter """ + return self.thold if not self._weighted else None + + + + def _processThold(self, thold: int | Iterable): + """Process thold input + + Parameters: + thold (int | Iterable): computable thold expression + """ + if isinstance(thold, int): + self._processUnweighted(thold=thold) + + else: + self._processWeighted(thold=thold) + + + def _processLimen(self, limen: str | bytes, **kwa): + """Process limen input + + Parameters: + limen (str): CESR encoded qb64 threshold (weighted or unweighted) + """ + matter = Matter(qb64b=limen, **kwa) # kwa for strip of stream + if matter.code in NumDex: + number = Number(raw=matter.raw, code=matter.code, **kwa) + self._processUnweighted(thold=number.num) + + elif matter.code in BexDex: + # Convert to fractional thold expression + bexter = Bexter(raw=matter.raw, code=matter.code, **kwa) + t = bexter.bext.replace('s', '/') + # get clauses + thold = [clause.split('c') for clause in t.split('a')] + thold = [[self.weight(w) for w in clause] for clause in thold] + self._processWeighted(thold=thold) + + else: + raise InvalidCodeError(f"Invalid code for limen = {matter.code}.") + + + def _processSith(self, sith: int | str | Iterable): + """ + Process attributes for fractionall weighted threshold sith + + Parameters: + sith is signing threshold (current or next) expressed as either: + non-negative int of threshold number (M-of-N threshold) + next threshold may be zero + non-negative hex string of threshold number (M-of-N threshold) + next threshold may be zero + fractional weight clauses which may be expressed as either: + an iterable of rational number fraction weight str or int str + each denoted w where 0 <= w <= 1 + an iterable of iterables of rational number fraction weight + or int str + each denoted w where 0 <= w <= 1>= 0 + JSON serialized str of either: + list of rational number fraction weight strings + each denoted w where 0 <= w <= 1 + list of lists of rational number fraction weight strings + each denoted w where 0 <= w <= 1 + + when any w is 0 or 1 then representation is 0 or 1 not 0/1 or 1/1 + """ + if isinstance(sith, int): + self._processUnweighted(thold=sith) + + elif isinstance(sith, str) and '[' not in sith: + self._processUnweighted(thold=int(sith, 16)) + + else: # assumes iterable of weights or iterable of iterables of weights + if isinstance(sith, str): # json of weighted sith from cli + sith = json.loads(sith) # deserialize + + if not sith: # empty iterable + raise ValueError(f"Empty weight list = {sith}.") + + # is it non str iterable if iterables? or non str iterable of strs? + # must test for emply mask because all([]) == True + mask = [nonStringIterable(c) for c in sith] + if mask and not all(mask): # not empty and not iterable of iterables + sith = [sith] # attempt to make Iterable of Iterables of strs + + for c in sith: # get each clause + mask = [isinstance(w, str) for w in c] # must be all strs + if mask and not all(mask): # not empty and not iterable of strs? + raise ValueError(f"Invalid sith = {sith} some weights in" + f"clause {c} are non string.") + + + # replace weight str expression, int str or fractional strings with + # int or fraction as appropriate. + thold = [] + for clause in sith: # convert string fractions to Fractions + # append list of weights converted from str expression + thold.append([self.weight(w) for w in clause]) + + self._processWeighted(thold=thold) + + + def _processUnweighted(self, thold=0): + """ + Process attributes for unweighted (numeric) threshold thold + + Parameters: + thold (int): non-negative threshold number M-of-N threshold + + """ + if thold < 0: + raise ValueError(f"Non-positive int threshold = {thold}.") + self._thold = thold + self._weighted = False + self._size = self._thold # used to verify that keys list size is at least size + self._satisfy = self._satisfy_numeric + self._number = Number(num=thold) + self._bexter = None + + + def _processWeighted(self, thold=[]): + """ + Process attributes for fractionall weighted threshold thold + + Parameters: + thold (iterable): iterable or iterable or iterables of + rational number fraction strings >= 0 and <= 1 + + """ + for clause in thold: # sum of fractions in clause must be >= 1 + if not (sum(clause) >= 1): + raise ValueError(f"Invalid sith clause = {thold}, all " + f"clause weight sums must be >= 1.") + + self._thold = thold + self._weighted = True + self._size = sum(len(clause) for clause in thold) + self._satisfy = self._satisfy_weighted + # make bext str of thold for .bexter for limen + bext = [[f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" + for f in clause] + for clause in thold] + bext = "a".join(["c".join(clause) for clause in bext]) + self._number = None + self._bexter = Bexter(bext=bext) + + + @staticmethod + def weight(w: str) -> Fraction: + """Returns valid weight from w else raises error (ValueError or TypeError). + w expression must evaluate to 0, 1, or strict proper rational fraction. + w expression must be 0 <= w <= 1 Else raises ValueError + w must not be float else raises TypeError + When not int w must be ratio of integers n/d else raise ValueError. + + Parameters: + w (str): threshold weight expression + """ + try: # float str or ratio str raises ValueError + if int(float(w)) != float(w): # float str + raise TypeError("Invalid weight str got float w={w}.") + w = int(w) # expression is int str + except TypeError as ex: + raise ValueError(str(ex)) from ex + + except ValueError as ex: # not float str or int str so try ration str + w = Fraction(w) + + if not 0 <= w <= 1: + raise ValueError(f"Invalid weight not 0 <= {w} <= 1.") + return w + + + def satisfy(self, indices): + """ + Returns True if indices list of verified signature key indices satisfies + threshold, False otherwise. + + Parameters: + indices is list of non-negative indices (offsets into key list) + of verified signatures. the indices may be in any order, they + are normalized herein + """ + return (self._satisfy(indices=indices)) + + + def _satisfy_numeric(self, indices): + """ + Returns True if satisfies numeric threshold False otherwise + + Parameters: + indices is list of indices (offsets into key list) of verified signatures + """ + try: + if self.thold > 0 and len(indices) >= self.thold: # at least one + return True + + except Exception as ex: + return False + + return False + + + def _satisfy_weighted(self, indices): + """ + Returns True if satifies fractional weighted threshold False otherwise + + + Parameters: + indices is list of non-negative indices (offsets into key list) + of verified signatures. the indices may be in any order, they + are normalized herein + + """ + try: + if not indices: # empty indices + return False + + # remove duplicates with set, sort low to high + indices = sorted(set(indices)) + sats = [False] * self.size # default all satifactions to False + for idx in indices: + sats[idx] = True # set verified signature index to True + + wio = 0 # weight index offset + for clause in self.thold: + cw = 0 # init clause weight + for w in clause: + if sats[wio]: # verified signature so weight applies + cw += w + wio += 1 + if cw < 1: # each clause must sum to at least 1 + return False + + return True # all clauses including final one cw >= 1 + + except Exception as ex: + return False + + return False + + +class Tholder: + """ + Tholder is KERI Signing Threshold Satisfaction class + .satisfy method evaluates satisfaction based on ordered list of indices of + verified signatures where indices correspond to offsets in key list of + associated signatures. + + Has the following public properties: + + Properties: + .weighted is Boolean True if fractional weighted threshold False if numeric + .size is int of minimum size of keys list + when weighted is size of keys list + when unweighted is size of int thold since don't have anyway + to know size of keys list in this case + + .limen is qualified b64 signing threshold suitable for CESR serialization. + either Number.qb64b or Bexter.qb64b. + The b64 portion of limen with code stripped (Bexter.bext) of + [["1/2", "1/2", "1/4", "1/4", "1/4"], ["1", "1"]] + is '1s2c1s2c1s4c1s4c1s4a1c1' basically slash is 's', comma is 'c', + and ANDed clauses are delimited by 'a'. + + .sith is original signing threshold suitable for value to be serialized + as json, cbor, mgpk in key event message as either: + non-negative hex number str or + list of str rational number fractions >= 0 and <= 1 or + list of list of str rational number fractions >= 0 and <= 1 + + .thold is parsed signing threshold suitable for calculating satisfaction. + either as int or list of Fractions + + .num is int signing threshold when not ._weighted + + Methods: + .satisfy returns bool, True means list of verified signature key indices + satisfies the threshold, False otherwise. + + Static Methods: + weight (str): converts weight str expression into either int or Fraction + else raises ValueError must satisfy 0 <= w <= 1 + Ensures strict proper rational number fraction of ints or + 0 or 1 + + Hidden: + ._weighted is Boolean, True if fractional weighted threshold False if numeric + ._size is int minimum size of of keys list + ._sith is signing threshold for .sith property + ._thold is signing threshold for .thold propery + ._bexter is Bexter instance of weighted signing threshold or None + ._number is Number instance of integer threshold or None + ._satisfy is method reference of threshold specified verification method + ._satisfy_numeric is numeric threshold verification method + ._satisfy_weighted is fractional weighted threshold verification method + + + """ + + def __init__(self, *, thold=None , limen=None, sith=None, **kwa): + """ + Accepts signing threshold in various forms so that may output correct + forms for serialization and/or calculation of satisfaction. + + The thold representation is meant to accept thresholds from computable + expressions for satisfaction of a threshold + + The limen representation is meant to parse threshold expressions from + CESR serializations of key event message fields or attachments. + + The sith representation is meant to parse threhold expressions from + deserializations of JSON, CBOR, or MGPK key event message fields or + the command line or configuration files. + + + Parameters: thold is signing threshold (current or next) is suitable for computing the satisfaction of a threshold and is expressed as either: @@ -5537,16 +5936,23 @@ def __init__(self, *, thold=None , limen=None, sith=None, **kwa): an iterable of Fractions or an iterable of iterables of Fractions. - The sith representation is meant to parse threhold expressions from - deserializations of JSON, CBOR, or MGPK key event message fields or - the command line or configuration files. - - The limen representation is meant to parse threshold expressions from - CESR serializations of key event message fields or attachments. - - The thold representation is meant to accept thresholds from computable - expressions for satisfaction of a threshold + limen is qualified signing threshold (current or next) expressed as either: + Number.qb64 or .qb64b of integer threshold or + Bexter.qb64 or .qb64b of fractional weight clauses which may be either: + Base64 delimited clauses of fractions + Base64 delimited clauses of fractions + sith is signing threshold (current or next) expressed as either: + non-negative int of threshold number (M-of-N threshold) + next threshold may be zero + non-negative hex string of threshold number (M-of-N threshold) + next threshold may be zero + fractional weight clauses which may be expressed as either: + an iterable of rational number fraction strings >= 0 and <= 1 + an iterable of iterables of rational number fraction strings >= 0 and <= 1 + JSON serialized str of either: + list of rational number fraction strings >= 0 and <= 1 or + list of list of rational number fraction strings >= 0 and <= 1 """ if thold is not None: @@ -5690,11 +6096,11 @@ def _processSith(self, sith: int | str | Iterable): if not sith: # empty iterable raise ValueError(f"Empty weight list = {sith}.") - # because all([]) == True have to also test for emply mask - # is it non str iterable of non str iterable of strs + # is it non str iterable if iterables? or non str iterable of strs? + # must test for emply mask because all([]) == True mask = [nonStringIterable(c) for c in sith] if mask and not all(mask): # not empty and not iterable of iterables - sith = [sith] # attempt to make Iterable of Iterables + sith = [sith] # attempt to make Iterable of Iterables of strs for c in sith: # get each clause mask = [isinstance(w, str) for w in c] # must be all strs @@ -5707,7 +6113,7 @@ def _processSith(self, sith: int | str | Iterable): # int or fraction as appropriate. thold = [] for clause in sith: # convert string fractions to Fractions - # append list of weights converted fromnn str expression + # append list of weights converted from str expression thold.append([self.weight(w) for w in clause]) self._processWeighted(thold=thold) @@ -5758,18 +6164,6 @@ def _processWeighted(self, thold=[]): self._bexter = Bexter(bext=bext) - @staticmethod - def _oldcheckWeight(w: Fraction) -> Fraction: - """Returns w if 0 <= w <= 1 Else raises ValueError - - Parameters: - w (Fraction): Threshold weight Fraction - """ - if not 0 <= w <= 1: - raise ValueError(f"Invalid weight not 0 <= {w} <= 1.") - return w - - @staticmethod def weight(w: str) -> Fraction: """Returns valid weight from w else raises error (ValueError or TypeError). @@ -5865,7 +6259,6 @@ def _satisfy_weighted(self, indices): return False - class Dicter: """ Dicter class is base class for objects that can be stored in a Suber diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index 8599c6751..e1715143c 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -220,7 +220,7 @@ def extractElementValues(element, values): def extractValues(ked, labels): """ Returns list of depth first recursively extracted values from elements of - key event dict ked whose flabels are in lables list + key event dict ked whose labels are in labels list Parameters: ked is key event dict From 540e781d5bdd70a61d33ef21b4d3a6a74ebe024d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 19 Feb 2024 18:32:03 -0700 Subject: [PATCH 008/418] Added support for nested weighted elements in threshold Tholder class --- src/keri/core/coring.py | 577 +++++++++----------------------------- tests/core/test_coring.py | 132 ++++++++- 2 files changed, 264 insertions(+), 445 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 1fae0bd72..1dd7c8dc9 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -6,7 +6,7 @@ import re import json from typing import Union -from collections.abc import Iterable +from collections.abc import Sequence, Mapping from dataclasses import dataclass, astuple from collections import namedtuple, deque @@ -44,7 +44,7 @@ BRV_LABELS, TSN_LABELS, CRED_TSN_LABELS) from ..help import helping -from ..help.helping import sceil, nonStringIterable +from ..help.helping import sceil, nonStringIterable, nonStringSequence Labels = Ilkage(icp=ICP_LABELS, rot=ROT_LABELS, ixn=IXN_LABELS, dip=DIP_LABELS, @@ -5447,7 +5447,7 @@ def pretty(self, *, size=1024): -class NewTholder: +class Tholder: """ Tholder is KERI Signing Threshold Satisfaction class .satisfy method evaluates satisfaction based on ordered list of indices of @@ -5542,8 +5542,8 @@ def __init__(self, *, thold=None , limen=None, sith=None, **kwa): non-negative hex string of threshold number (M-of-N threshold) next threshold may be zero fractional weight clauses which may be expressed as either: - an iterable of rational number fraction strings >= 0 and <= 1 - an iterable of iterables of rational number fraction strings >= 0 and <= 1 + an sequence of rational number fraction strings >= 0 and <= 1 + an sequence of sequences of rational number fraction strings >= 0 and <= 1 JSON serialized str of either: list of rational number fraction strings >= 0 and <= 1 or list of list of rational number fraction strings >= 0 and <= 1 @@ -5590,9 +5590,24 @@ def sith(self): """ sith property getter """ # make sith expression of thold if self.weighted: - sith = [[f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" - for f in clause] - for clause in self.thold] + sith = [] + for c in self.thold: + clause = [] + for e in c: + if isinstance(e, tuple): + f = e[0] + k = f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" + v = [f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" + for f in e[1]] + clause.append({k: v}) + else: + f = e + clause.append(f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}") + sith.append(clause) + + #sith = [[f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" + #for f in clause] + #for clause in self.thold] if len(sith) == 1: sith = sith[0] # simplify list of one clause to clause else: @@ -5616,11 +5631,11 @@ def num(self): - def _processThold(self, thold: int | Iterable): + def _processThold(self, thold: int | Sequence): """Process thold input Parameters: - thold (int | Iterable): computable thold expression + thold (int | Sequence): computable thold expression """ if isinstance(thold, int): self._processUnweighted(thold=thold) @@ -5645,421 +5660,28 @@ def _processLimen(self, limen: str | bytes, **kwa): bexter = Bexter(raw=matter.raw, code=matter.code, **kwa) t = bexter.bext.replace('s', '/') # get clauses - thold = [clause.split('c') for clause in t.split('a')] - thold = [[self.weight(w) for w in clause] for clause in thold] - self._processWeighted(thold=thold) - - else: - raise InvalidCodeError(f"Invalid code for limen = {matter.code}.") - - - def _processSith(self, sith: int | str | Iterable): - """ - Process attributes for fractionall weighted threshold sith - - Parameters: - sith is signing threshold (current or next) expressed as either: - non-negative int of threshold number (M-of-N threshold) - next threshold may be zero - non-negative hex string of threshold number (M-of-N threshold) - next threshold may be zero - fractional weight clauses which may be expressed as either: - an iterable of rational number fraction weight str or int str - each denoted w where 0 <= w <= 1 - an iterable of iterables of rational number fraction weight - or int str - each denoted w where 0 <= w <= 1>= 0 - JSON serialized str of either: - list of rational number fraction weight strings - each denoted w where 0 <= w <= 1 - list of lists of rational number fraction weight strings - each denoted w where 0 <= w <= 1 - - when any w is 0 or 1 then representation is 0 or 1 not 0/1 or 1/1 - """ - if isinstance(sith, int): - self._processUnweighted(thold=sith) - - elif isinstance(sith, str) and '[' not in sith: - self._processUnweighted(thold=int(sith, 16)) - - else: # assumes iterable of weights or iterable of iterables of weights - if isinstance(sith, str): # json of weighted sith from cli - sith = json.loads(sith) # deserialize - - if not sith: # empty iterable - raise ValueError(f"Empty weight list = {sith}.") - - # is it non str iterable if iterables? or non str iterable of strs? - # must test for emply mask because all([]) == True - mask = [nonStringIterable(c) for c in sith] - if mask and not all(mask): # not empty and not iterable of iterables - sith = [sith] # attempt to make Iterable of Iterables of strs + clauses = [clause.split('c') for clause in t.split('a')] - for c in sith: # get each clause - mask = [isinstance(w, str) for w in c] # must be all strs - if mask and not all(mask): # not empty and not iterable of strs? - raise ValueError(f"Invalid sith = {sith} some weights in" - f"clause {c} are non string.") - - - # replace weight str expression, int str or fractional strings with - # int or fraction as appropriate. thold = [] - for clause in sith: # convert string fractions to Fractions - # append list of weights converted from str expression - thold.append([self.weight(w) for w in clause]) - - self._processWeighted(thold=thold) - - - def _processUnweighted(self, thold=0): - """ - Process attributes for unweighted (numeric) threshold thold - - Parameters: - thold (int): non-negative threshold number M-of-N threshold - - """ - if thold < 0: - raise ValueError(f"Non-positive int threshold = {thold}.") - self._thold = thold - self._weighted = False - self._size = self._thold # used to verify that keys list size is at least size - self._satisfy = self._satisfy_numeric - self._number = Number(num=thold) - self._bexter = None - - - def _processWeighted(self, thold=[]): - """ - Process attributes for fractionall weighted threshold thold - - Parameters: - thold (iterable): iterable or iterable or iterables of - rational number fraction strings >= 0 and <= 1 - - """ - for clause in thold: # sum of fractions in clause must be >= 1 - if not (sum(clause) >= 1): - raise ValueError(f"Invalid sith clause = {thold}, all " - f"clause weight sums must be >= 1.") - - self._thold = thold - self._weighted = True - self._size = sum(len(clause) for clause in thold) - self._satisfy = self._satisfy_weighted - # make bext str of thold for .bexter for limen - bext = [[f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" - for f in clause] - for clause in thold] - bext = "a".join(["c".join(clause) for clause in bext]) - self._number = None - self._bexter = Bexter(bext=bext) - - - @staticmethod - def weight(w: str) -> Fraction: - """Returns valid weight from w else raises error (ValueError or TypeError). - w expression must evaluate to 0, 1, or strict proper rational fraction. - w expression must be 0 <= w <= 1 Else raises ValueError - w must not be float else raises TypeError - When not int w must be ratio of integers n/d else raise ValueError. - - Parameters: - w (str): threshold weight expression - """ - try: # float str or ratio str raises ValueError - if int(float(w)) != float(w): # float str - raise TypeError("Invalid weight str got float w={w}.") - w = int(w) # expression is int str - except TypeError as ex: - raise ValueError(str(ex)) from ex - - except ValueError as ex: # not float str or int str so try ration str - w = Fraction(w) - - if not 0 <= w <= 1: - raise ValueError(f"Invalid weight not 0 <= {w} <= 1.") - return w - - - def satisfy(self, indices): - """ - Returns True if indices list of verified signature key indices satisfies - threshold, False otherwise. - - Parameters: - indices is list of non-negative indices (offsets into key list) - of verified signatures. the indices may be in any order, they - are normalized herein - """ - return (self._satisfy(indices=indices)) - - - def _satisfy_numeric(self, indices): - """ - Returns True if satisfies numeric threshold False otherwise - - Parameters: - indices is list of indices (offsets into key list) of verified signatures - """ - try: - if self.thold > 0 and len(indices) >= self.thold: # at least one - return True - - except Exception as ex: - return False - - return False - - - def _satisfy_weighted(self, indices): - """ - Returns True if satifies fractional weighted threshold False otherwise - - - Parameters: - indices is list of non-negative indices (offsets into key list) - of verified signatures. the indices may be in any order, they - are normalized herein - - """ - try: - if not indices: # empty indices - return False - - # remove duplicates with set, sort low to high - indices = sorted(set(indices)) - sats = [False] * self.size # default all satifactions to False - for idx in indices: - sats[idx] = True # set verified signature index to True - - wio = 0 # weight index offset - for clause in self.thold: - cw = 0 # init clause weight - for w in clause: - if sats[wio]: # verified signature so weight applies - cw += w - wio += 1 - if cw < 1: # each clause must sum to at least 1 - return False - - return True # all clauses including final one cw >= 1 - - except Exception as ex: - return False - - return False - - -class Tholder: - """ - Tholder is KERI Signing Threshold Satisfaction class - .satisfy method evaluates satisfaction based on ordered list of indices of - verified signatures where indices correspond to offsets in key list of - associated signatures. - - Has the following public properties: - - Properties: - .weighted is Boolean True if fractional weighted threshold False if numeric - .size is int of minimum size of keys list - when weighted is size of keys list - when unweighted is size of int thold since don't have anyway - to know size of keys list in this case - - .limen is qualified b64 signing threshold suitable for CESR serialization. - either Number.qb64b or Bexter.qb64b. - The b64 portion of limen with code stripped (Bexter.bext) of - [["1/2", "1/2", "1/4", "1/4", "1/4"], ["1", "1"]] - is '1s2c1s2c1s4c1s4c1s4a1c1' basically slash is 's', comma is 'c', - and ANDed clauses are delimited by 'a'. - - .sith is original signing threshold suitable for value to be serialized - as json, cbor, mgpk in key event message as either: - non-negative hex number str or - list of str rational number fractions >= 0 and <= 1 or - list of list of str rational number fractions >= 0 and <= 1 - - .thold is parsed signing threshold suitable for calculating satisfaction. - either as int or list of Fractions - - .num is int signing threshold when not ._weighted - - Methods: - .satisfy returns bool, True means list of verified signature key indices - satisfies the threshold, False otherwise. - - Static Methods: - weight (str): converts weight str expression into either int or Fraction - else raises ValueError must satisfy 0 <= w <= 1 - Ensures strict proper rational number fraction of ints or - 0 or 1 - - Hidden: - ._weighted is Boolean, True if fractional weighted threshold False if numeric - ._size is int minimum size of of keys list - ._sith is signing threshold for .sith property - ._thold is signing threshold for .thold propery - ._bexter is Bexter instance of weighted signing threshold or None - ._number is Number instance of integer threshold or None - ._satisfy is method reference of threshold specified verification method - ._satisfy_numeric is numeric threshold verification method - ._satisfy_weighted is fractional weighted threshold verification method - - - """ - - def __init__(self, *, thold=None , limen=None, sith=None, **kwa): - """ - Accepts signing threshold in various forms so that may output correct - forms for serialization and/or calculation of satisfaction. - - The thold representation is meant to accept thresholds from computable - expressions for satisfaction of a threshold - - The limen representation is meant to parse threshold expressions from - CESR serializations of key event message fields or attachments. - - The sith representation is meant to parse threhold expressions from - deserializations of JSON, CBOR, or MGPK key event message fields or - the command line or configuration files. - - - Parameters: - - thold is signing threshold (current or next) is suitable for computing - the satisfaction of a threshold and is expressed as either: - int of threshold number (M of N) - fractional weight clauses which may be expressed as either: - an iterable of Fractions or - an iterable of iterables of Fractions. - - limen is qualified signing threshold (current or next) expressed as either: - Number.qb64 or .qb64b of integer threshold or - Bexter.qb64 or .qb64b of fractional weight clauses which may be either: - Base64 delimited clauses of fractions - Base64 delimited clauses of fractions - - sith is signing threshold (current or next) expressed as either: - non-negative int of threshold number (M-of-N threshold) - next threshold may be zero - non-negative hex string of threshold number (M-of-N threshold) - next threshold may be zero - fractional weight clauses which may be expressed as either: - an iterable of rational number fraction strings >= 0 and <= 1 - an iterable of iterables of rational number fraction strings >= 0 and <= 1 - JSON serialized str of either: - list of rational number fraction strings >= 0 and <= 1 or - list of list of rational number fraction strings >= 0 and <= 1 - - """ - if thold is not None: - self._processThold(thold=thold) - - elif limen is not None: - self._processLimen(limen=limen, **kwa) # kwa for strip - - elif sith is not None: - if isinstance(sith, str) and not sith: # empty str - raise EmptyMaterialError("Empty threshold expression.") - - self._processSith(sith=sith) - - else: - raise EmptyMaterialError("Missing threshold expression.") - - - @property - def weighted(self): - """ weighted property getter """ - return self._weighted - - @property - def thold(self): - """ thold property getter """ - return self._thold - - @property - def size(self): - """ size property getter """ - return self._size - - @property - def limen(self): - """ limen property getter """ - return self._bexter.qb64b if self._weighted else self._number.qb64b - - @property - def sith(self): - """ sith property getter """ - # make sith expression of thold - if self.weighted: - sith = [[f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" - for f in clause] - for clause in self.thold] - if len(sith) == 1: - sith = sith[0] # simplify list of one clause to clause - else: - sith = f"{self.thold:x}" - - return sith - - @property - def json(self): - """Returns json serialization of sith expression - - Essentially JSON list of lists of strings - """ - return json.dumps(self.sith) - - - @property - def num(self): - """ sith property getter """ - return self.thold if not self._weighted else None - - - - def _processThold(self, thold: int | Iterable): - """Process thold input + for c in clauses: + clause = [] + for e in c: + k, s, v = e.partition("k") + if s: #not empty + clause.append((self.weight(k), [self.weight(w) for w in v.split("v")])) + else: + clause.append(self.weight(k)) - Parameters: - thold (int | Iterable): computable thold expression - """ - if isinstance(thold, int): - self._processUnweighted(thold=thold) + thold.append(clause) - else: - self._processWeighted(thold=thold) - - - def _processLimen(self, limen: str | bytes, **kwa): - """Process limen input - - Parameters: - limen (str): CESR encoded qb64 threshold (weighted or unweighted) - """ - matter = Matter(qb64b=limen, **kwa) # kwa for strip of stream - if matter.code in NumDex: - number = Number(raw=matter.raw, code=matter.code, **kwa) - self._processUnweighted(thold=number.num) - - elif matter.code in BexDex: - # Convert to fractional thold expression - bexter = Bexter(raw=matter.raw, code=matter.code, **kwa) - t = bexter.bext.replace('s', '/') - # get clauses - thold = [clause.split('c') for clause in t.split('a')] - thold = [[self.weight(w) for w in clause] for clause in thold] + #thold = [[self.weight(w) for w in clause] for clause in thold] self._processWeighted(thold=thold) else: raise InvalidCodeError(f"Invalid code for limen = {matter.code}.") - def _processSith(self, sith: int | str | Iterable): + def _processSith(self, sith: int | str | Sequence): """ Process attributes for fractionall weighted threshold sith @@ -6070,9 +5692,9 @@ def _processSith(self, sith: int | str | Iterable): non-negative hex string of threshold number (M-of-N threshold) next threshold may be zero fractional weight clauses which may be expressed as either: - an iterable of rational number fraction weight str or int str + an sequence of rational number fraction weight str or int str each denoted w where 0 <= w <= 1 - an iterable of iterables of rational number fraction weight + an sequence of sequences of rational number fraction weight or int str each denoted w where 0 <= w <= 1>= 0 JSON serialized str of either: @@ -6089,32 +5711,47 @@ def _processSith(self, sith: int | str | Iterable): elif isinstance(sith, str) and '[' not in sith: self._processUnweighted(thold=int(sith, 16)) - else: # assumes iterable of weights or iterable of iterables of weights + else: # assumes sequence of weights or sequence of sequence of weights if isinstance(sith, str): # json of weighted sith from cli sith = json.loads(sith) # deserialize - if not sith: # empty iterable + if not sith: # empty or None raise ValueError(f"Empty weight list = {sith}.") - # is it non str iterable if iterables? or non str iterable of strs? + # is it non str sequence of sequences? or non str sequnce of strs? # must test for emply mask because all([]) == True - mask = [nonStringIterable(c) for c in sith] - if mask and not all(mask): # not empty and not iterable of iterables - sith = [sith] # attempt to make Iterable of Iterables of strs + mask = [nonStringSequence(c) for c in sith] # check each element + if mask and not all(mask): # not empty and not sequence of sequenes + sith = [sith] # attempt to make sequnce of sequqnces of strs for c in sith: # get each clause - mask = [isinstance(w, str) for w in c] # must be all strs - if mask and not all(mask): # not empty and not iterable of strs? + # each element of a clause must be a str or dict + mask = [(isinstance(w, str) or isinstance(w, Mapping)) for w in c] + if mask and not all(mask): # not empty and not sequence of str or dicts raise ValueError(f"Invalid sith = {sith} some weights in" f"clause {c} are non string.") - # replace weight str expression, int str or fractional strings with # int or fraction as appropriate. thold = [] - for clause in sith: # convert string fractions to Fractions - # append list of weights converted from str expression - thold.append([self.weight(w) for w in clause]) + for c in sith: # convert string fractions to Fractions + # append list of where each element is either bare weight or + # single key map with value as list of weights + # each weight is converted from its str expression + clause = [] + for e in c: # each element of clause c + if isinstance(e, Mapping): + if len(e) != 1: + raise ValueError(f"Invalid sith = {sith} nested " + f"weight map {e} in clause {c} " + f" not single key value.") + k = list(e.keys())[0] # zeroth k:v pair is used + # convert to tuple of (weight, [list of weights]) + clause.append((self.weight(k), [self.weight(w) for w in e[k]])) + else: + clause.append(self.weight(e)) + + thold.append(clause) self._processWeighted(thold=thold) @@ -6146,20 +5783,57 @@ def _processWeighted(self, thold=[]): rational number fraction strings >= 0 and <= 1 """ - for clause in thold: # sum of fractions in clause must be >= 1 - if not (sum(clause) >= 1): - raise ValueError(f"Invalid sith clause = {thold}, all " + for clause in thold: # sum of top level weights in clause must be >= 1 + # When element is dict then sum of value's weights must be >= 1 + top = [] # top level weights + for e in clause: + if isinstance(e, tuple): + top.append(e[0]) + if not (sum(e[1]) >= 1): + raise ValueError(f"Invalid sith clause = {clause}, " + f"element = {e}. All nested clause " + f"weight sums must be >= 1.") + else: + top.append(e) + if not (sum(top) >= 1): + raise ValueError(f"Invalid sith clause = {clause}, all top level" f"clause weight sums must be >= 1.") self._thold = thold self._weighted = True - self._size = sum(len(clause) for clause in thold) + #self._size = sum(len(clause) for clause in thold) + s = 0 + for clause in thold: + for e in clause: + if isinstance(e, tuple): + s += len(e[1]) + else: + s += 1 + self._size = s + self._satisfy = self._satisfy_weighted # make bext str of thold for .bexter for limen - bext = [[f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" - for f in clause] - for clause in thold] - bext = "a".join(["c".join(clause) for clause in bext]) + ta = [] # list of list of fractions and/or single element map of fractions + for c in thold: + bc = [] # list of fractions and/or single element map of fractions + for e in c: + if isinstance(e, tuple): + f = e[0] + k = f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" + v = "v".join([f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" for f in e[1]]) + kv = "k".join([k, v]) + bc.append(kv) + else: + bc.append(f"{e.numerator}s{e.denominator}" if (0 < e < 1) else f"{int(e)}") + + ta.append(bc) + + bext = "a".join(["c".join(bc) for bc in ta]) + + #bext = [[f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" + #for f in clause] + #for clause in thold] + #bext = "a".join(["c".join(clause) for clause in bext]) self._number = None self._bexter = Bexter(bext=bext) @@ -6244,14 +5918,33 @@ def _satisfy_weighted(self, indices): wio = 0 # weight index offset for clause in self.thold: cw = 0 # init clause weight - for w in clause: - if sats[wio]: # verified signature so weight applies - cw += w - wio += 1 + for e in clause: + if isinstance(e, tuple): + vw = 0 # init element value weight + for w in e[1]: # sum weights of value + if sats[wio]: + vw += w + wio += 1 + if vw >= 1: # element true + cw += e[0] # add element key weight to clause weight + else: + w = e + if sats[wio]: # verified signature so weight applies + cw += w + wio += 1 if cw < 1: # each clause must sum to at least 1 return False - return True # all clauses including final one cw >= 1 + #for clause in self.thold: + #cw = 0 # init clause weight + #for w in clause: + #if sats[wio]: # verified signature so weight applies + #cw += w + #wio += 1 + #if cw < 1: # each clause must sum to at least 1 + #return False + + return True # all clauses have cw >= 1 including final one, AND true except Exception as ex: return False diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 5e6b8357c..f5be19ef8 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -6542,6 +6542,132 @@ def test_tholder(): assert not tholder.satisfy(indices=[2, 3, 4]) assert not tholder.satisfy(indices=[]) + # test new nested weighted with Mapping dict + # 1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1 + tholder = Tholder(sith='[{"1/3":["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]') + assert tholder.weighted + assert tholder.size == 7 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), + Fraction(1, 2), + Fraction(1, 2)]), + Fraction(1, 3), + Fraction(1, 2), + (Fraction(1, 2), + [1, 1])]] + assert tholder.limen ==b'4AAIA1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1' + assert tholder.sith ==[{'1/3': ['1/2', '1/2', '1/2']}, '1/3', '1/2', {'1/2': ['1', '1']}] + assert tholder.json == '[{"1/3": ["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]' + assert tholder.num == None + assert tholder.satisfy(indices=[0, 2, 3, 6]) + assert tholder.satisfy(indices=[3, 4, 5]) + assert tholder.satisfy(indices=[1, 2, 3, 4]) + assert tholder.satisfy(indices=[4, 6]) + assert tholder.satisfy(indices=[4, 2, 0, 3]) + assert tholder.satisfy(indices=[0, 0, 1, 2, 1, 5, 6, 3]) + assert not tholder.satisfy(indices=[0, 2, 5]) + assert not tholder.satisfy(indices=[2, 3, 4]) + + tholder = Tholder(limen=b'4AAIA1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1') + assert tholder.weighted + assert tholder.size == 7 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), + Fraction(1, 2), + Fraction(1, 2)]), + Fraction(1, 3), + Fraction(1, 2), + (Fraction(1, 2), + [1, 1])]] + assert tholder.limen ==b'4AAIA1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1' + assert tholder.sith ==[{'1/3': ['1/2', '1/2', '1/2']}, '1/3', '1/2', {'1/2': ['1', '1']}] + assert tholder.json == '[{"1/3": ["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]' + assert tholder.num == None + + tholder = Tholder(thold=[[(Fraction(1, 3), + [Fraction(1, 2), + Fraction(1, 2), + Fraction(1, 2)]), + Fraction(1, 3), + Fraction(1, 2), + (Fraction(1, 2), + [1, 1])]]) + assert tholder.weighted + assert tholder.size == 7 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), + Fraction(1, 2), + Fraction(1, 2)]), + Fraction(1, 3), + Fraction(1, 2), + (Fraction(1, 2), + [1, 1])]] + assert tholder.limen ==b'4AAIA1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1' + assert tholder.sith ==[{'1/3': ['1/2', '1/2', '1/2']}, '1/3', '1/2', {'1/2': ['1', '1']}] + assert tholder.json == '[{"1/3": ["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]' + assert tholder.num == None + + # test new nested weighted with Mapping dict + + tholder = Tholder(sith='[[{"1/3":["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ["1/2", {"1/2": ["1", "1"]}]]') + assert tholder.weighted + assert tholder.size == 9 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), Fraction(1, 2), Fraction(1, 2)]), + Fraction(1, 2), + (Fraction(1, 2), [1, 1])], + [Fraction(1, 2), (Fraction(1, 2), [1, 1])]] + assert tholder.limen == b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1' + assert tholder.sith == [[{'1/3': ['1/2', '1/2', '1/2']}, '1/2', {'1/2': ['1', '1']}], + ['1/2', {'1/2': ['1', '1']}]] + + assert tholder.json == ('[[{"1/3": ["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ' + '["1/2", ''{"1/2": ["1", "1"]}]]') + assert tholder.num == None + assert tholder.satisfy(indices=[0, 2, 3, 5, 6, 7]) + assert tholder.satisfy(indices=[3, 4, 5, 6, 8]) + assert tholder.satisfy(indices=[1, 2, 3, 4, 6, 7]) + assert tholder.satisfy(indices=[4, 2, 0, 3, 8, 6]) + assert tholder.satisfy(indices=[0, 0, 1, 2, 1, 8, 3, 5, 6, 3]) + assert not tholder.satisfy(indices=[0, 2, 5]) + assert not tholder.satisfy(indices=[6, 7, 8]) + + tholder = Tholder(limen=b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1') + assert tholder.weighted + assert tholder.size == 9 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), Fraction(1, 2), Fraction(1, 2)]), + Fraction(1, 2), + (Fraction(1, 2), [1, 1])], + [Fraction(1, 2), (Fraction(1, 2), [1, 1])]] + assert tholder.limen == b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1' + assert tholder.sith == [[{'1/3': ['1/2', '1/2', '1/2']}, '1/2', {'1/2': ['1', '1']}], + ['1/2', {'1/2': ['1', '1']}]] + + assert tholder.json == ('[[{"1/3": ["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ' + '["1/2", ''{"1/2": ["1", "1"]}]]') + assert tholder.num == None + + tholder = Tholder(thold=[[(Fraction(1, 3), + [Fraction(1, 2), Fraction(1, 2), Fraction(1, 2)]), + Fraction(1, 2), + (Fraction(1, 2), [1, 1])], + [Fraction(1, 2), (Fraction(1, 2), [1, 1])]]) + assert tholder.weighted + assert tholder.size == 9 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), Fraction(1, 2), Fraction(1, 2)]), + Fraction(1, 2), + (Fraction(1, 2), [1, 1])], + [Fraction(1, 2), (Fraction(1, 2), [1, 1])]] + assert tholder.limen == b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1' + assert tholder.sith == [[{'1/3': ['1/2', '1/2', '1/2']}, '1/2', {'1/2': ['1', '1']}], + ['1/2', {'1/2': ['1', '1']}]] + + assert tholder.json == ('[[{"1/3": ["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ' + '["1/2", ''{"1/2": ["1", "1"]}]]') + assert tholder.num == None + """ Done Test """ @@ -6551,12 +6677,12 @@ def test_tholder(): #test_counter() #test_prodex() #test_indexer() - test_number() - test_seqner() + #test_number() + #test_seqner() #test_siger() #test_signer() #test_nexter() - #test_tholder() + test_tholder() #test_ilks() #test_labels() #test_prefixer() From 9c17e01e2b413ffce57fb2f59e1169d0a0fd4273 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 19 Feb 2024 18:46:58 -0700 Subject: [PATCH 009/418] added comments to include new syntax for Tholder --- src/keri/core/coring.py | 47 +++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 1dd7c8dc9..367bf1f81 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -5468,13 +5468,21 @@ class Tholder: The b64 portion of limen with code stripped (Bexter.bext) of [["1/2", "1/2", "1/4", "1/4", "1/4"], ["1", "1"]] is '1s2c1s2c1s4c1s4c1s4a1c1' basically slash is 's', comma is 'c', - and ANDed clauses are delimited by 'a'. + ANDed clauses are delimited by 'a'. + Each clause top level weight may be optionally a weighted set of weights + delimited by 'k' for the weight on the set and 'v' for the weights in + the set. + [[{'1/3': ['1/2', '1/2', '1/2']}, '1/2', {'1/2': ['1', '1']}], + ['1/2', {'1/2': ['1', '1']}]] + b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1' + .sith is original signing threshold suitable for value to be serialized as json, cbor, mgpk in key event message as either: non-negative hex number str or list of str rational number fractions >= 0 and <= 1 or list of list of str rational number fractions >= 0 and <= 1 + list of list of weighted map of weights .thold is parsed signing threshold suitable for calculating satisfaction. either as int or list of Fractions @@ -5527,8 +5535,10 @@ def __init__(self, *, thold=None , limen=None, sith=None, **kwa): the satisfaction of a threshold and is expressed as either: int of threshold number (M of N) fractional weight clauses which may be expressed as either: - an iterable of Fractions or - an iterable of iterables of Fractions. + sequence of either Fractions or tuples of Fraction and + sequence of Fractions + sequence of sequence of either Fractions or tuples of + Fraction and sequence of Fractions limen is qualified signing threshold (current or next) expressed as either: Number.qb64 or .qb64b of integer threshold or @@ -5542,11 +5552,17 @@ def __init__(self, *, thold=None , limen=None, sith=None, **kwa): non-negative hex string of threshold number (M-of-N threshold) next threshold may be zero fractional weight clauses which may be expressed as either: - an sequence of rational number fraction strings >= 0 and <= 1 - an sequence of sequences of rational number fraction strings >= 0 and <= 1 - JSON serialized str of either: - list of rational number fraction strings >= 0 and <= 1 or - list of list of rational number fraction strings >= 0 and <= 1 + sequence of rational number fraction strings >= 0 and <= 1 + sequence of either rational number fraction strings >= 0 and <= 1 or + map with key rational number string and value as sequence + of rational number fraction strings + rational number fraction string + sequence of sequences of rational number fraction strings >= 0 and <= 1 + sequence of sequnces of either rational number fraction strings or + map with key rational number fraction string with value sequence of + rationaly number fraction strings + JSON serialized str of the above: + """ if thold is not None: @@ -5674,7 +5690,6 @@ def _processLimen(self, limen: str | bytes, **kwa): thold.append(clause) - #thold = [[self.weight(w) for w in clause] for clause in thold] self._processWeighted(thold=thold) else: @@ -5829,11 +5844,6 @@ def _processWeighted(self, thold=[]): ta.append(bc) bext = "a".join(["c".join(bc) for bc in ta]) - - #bext = [[f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" - #for f in clause] - #for clause in thold] - #bext = "a".join(["c".join(clause) for clause in bext]) self._number = None self._bexter = Bexter(bext=bext) @@ -5935,15 +5945,6 @@ def _satisfy_weighted(self, indices): if cw < 1: # each clause must sum to at least 1 return False - #for clause in self.thold: - #cw = 0 # init clause weight - #for w in clause: - #if sats[wio]: # verified signature so weight applies - #cw += w - #wio += 1 - #if cw < 1: # each clause must sum to at least 1 - #return False - return True # all clauses have cw >= 1 including final one, AND true except Exception as ex: From 052d9a89910db5a88a452236f823033e768a25bb Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 20 Feb 2024 18:13:22 -0700 Subject: [PATCH 010/418] Added support for Texter Class for variable length text (bytes) with any character. --- src/keri/core/coring.py | 251 ++++++++++++++++++++++++-------------- tests/core/test_coring.py | 244 +++++++++++++++++++++++++++++++++++- 2 files changed, 398 insertions(+), 97 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 367bf1f81..08c17e53c 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -501,6 +501,8 @@ def __iter__(self): MtrDex = MatterCodex() # Make instance + + @dataclass(frozen=True) class SmallVarRawSizeCodex: """ @@ -543,52 +545,6 @@ def __iter__(self): LargeVrzDex = LargeVarRawSizeCodex() # Make instance -@dataclass(frozen=True) -class NonTransCodex: - """ - NonTransCodex is codex all non-transferable derivation codes - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. - ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. - Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. - ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. - - def __iter__(self): - return iter(astuple(self)) - - -NonTransDex = NonTransCodex() # Make instance - -# When add new to DigCodes update Saider.Digests and Serder.Digests class attr -@dataclass(frozen=True) -class DigCodex: - """ - DigCodex is codex all digest derivation codes. This is needed to ensure - delegated inception using a self-addressing derivation i.e. digest derivation - code. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Blake3_256: str = 'E' # Blake3 256 bit digest self-addressing derivation. - Blake2b_256: str = 'F' # Blake2b 256 bit digest self-addressing derivation. - Blake2s_256: str = 'G' # Blake2s 256 bit digest self-addressing derivation. - SHA3_256: str = 'H' # SHA3 256 bit digest self-addressing derivation. - SHA2_256: str = 'I' # SHA2 256 bit digest self-addressing derivation. - Blake3_512: str = '0D' # Blake3 512 bit digest self-addressing derivation. - Blake2b_512: str = '0E' # Blake2b 512 bit digest self-addressing derivation. - SHA3_512: str = '0F' # SHA3 512 bit digest self-addressing derivation. - SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. - - def __iter__(self): - return iter(astuple(self)) - - -DigDex = DigCodex() # Make instance - - - @dataclass(frozen=True) class BextCodex: @@ -632,6 +588,7 @@ def __iter__(self): TexDex = TextCodex() # Make instance + @dataclass(frozen=True) class CipherX25519VarCodex: """ @@ -742,6 +699,77 @@ def __iter__(self): +@dataclass(frozen=True) +class NonTransCodex: + """ + NonTransCodex is codex all non-transferable derivation codes + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. + ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. + Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. + ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. + + def __iter__(self): + return iter(astuple(self)) + + +NonTransDex = NonTransCodex() # Make instance + +# When add new to DigCodes update Saider.Digests and Serder.Digests class attr +@dataclass(frozen=True) +class DigCodex: + """ + DigCodex is codex all digest derivation codes. This is needed to ensure + delegated inception using a self-addressing derivation i.e. digest derivation + code. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Blake3_256: str = 'E' # Blake3 256 bit digest self-addressing derivation. + Blake2b_256: str = 'F' # Blake2b 256 bit digest self-addressing derivation. + Blake2s_256: str = 'G' # Blake2s 256 bit digest self-addressing derivation. + SHA3_256: str = 'H' # SHA3 256 bit digest self-addressing derivation. + SHA2_256: str = 'I' # SHA2 256 bit digest self-addressing derivation. + Blake3_512: str = '0D' # Blake3 512 bit digest self-addressing derivation. + Blake2b_512: str = '0E' # Blake2b 512 bit digest self-addressing derivation. + SHA3_512: str = '0F' # SHA3 512 bit digest self-addressing derivation. + SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. + + def __iter__(self): + return iter(astuple(self)) + + +DigDex = DigCodex() # Make instance + + + +@dataclass(frozen=True) +class NumCodex: + """ + NumCodex is codex of Base64 derivation codes for compactly representing + numbers across a wide rage of sizes. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Short: str = 'M' # Short 2 byte b2 number + Long: str = '0H' # Long 4 byte b2 number + Tall: str = 'R' # Tall 5 byte b2 number + Big: str = 'N' # Big 8 byte b2 number + Large: str = 'S' # Large 11 byte b2 number + Great: str = 'T' # Great 14 byte b2 number + Huge: str = '0A' # Huge 16 byte b2 number (same as Salt_128) + Vast: str = 'U' # Vast 17 byte b2 number + + def __iter__(self): + return iter(astuple(self)) + + +NumDex = NumCodex() # Make instance + + # namedtuple for size entries in Matter and Counter derivation code tables # hs is the hard size int number of chars in hard (stable) part of code # ss is the soft size int number of chars in soft (unstable) part of code @@ -1503,33 +1531,6 @@ def snh(self): -@dataclass(frozen=True) -class NumCodex: - """ - NumCodex is codex of Base64 derivation codes for compactly representing - numbers across a wide rage of sizes. - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Short: str = 'M' # Short 2 byte b2 number - Long: str = '0H' # Long 4 byte b2 number - Tall: str = 'R' # Tall 5 byte b2 number - Big: str = 'N' # Big 8 byte b2 number - Large: str = 'S' # Large 11 byte b2 number - Great: str = 'T' # Great 14 byte b2 number - Huge: str = '0A' # Huge 16 byte b2 number (same as Salt_128) - Vast: str = 'U' # Vast 17 byte b2 number - - def __iter__(self): - return iter(astuple(self)) - - -NumDex = NumCodex() # Make instance - - - - class Number(Matter): """ Number is subclass of Matter, cryptographic material, for ordinal counting @@ -1888,6 +1889,78 @@ def datetime(self): return helping.fromIso8601(self.dts) +class Texter(Matter): + """ + Texter is subclass of Matter, cryptographic material, for variable length + text strings as bytes not unicode. Unicode strings converted to bytes. + + + Attributes: + + Inherited Properties: (See Matter) + + Properties: + .text is bytes value with CESR code and leader removed. + .uext is str value with CESR code and leader removed unicode of .text + + Inherited Hidden Properties: (See Matter) + + Methods: + + Codes: + Bytes_L0: str = '4B' # Byte String lead size 0 + Bytes_L1: str = '5B' # Byte String lead size 1 + Bytes_L2: str = '6B' # Byte String lead size 2 + Bytes_Big_L0: str = '7AAB' # Byte String big lead size 0 + Bytes_Big_L1: str = '8AAB' # Byte String big lead size 1 + Bytes_Big_L2: str = '9AAB' # Byte String big lead size 2 + + """ + + def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, + code=MtrDex.Bytes_L0, text=None, **kwa): + """ + Inherited Parameters: (see Matter) + raw is bytes of unqualified crypto material usable for crypto operations + qb64b is bytes of fully qualified crypto material + qb64 is str or bytes of fully qualified crypto material + qb2 is bytes of fully qualified crypto material + code is str of derivation code + index is int of count of attached receipts for CryCntDex codes + + Parameters: + text is the variable sized text string as either bytes or str + + """ + if raw is None and qb64b is None and qb64 is None and qb2 is None: + if text is None: + raise EmptyMaterialError("Missing text string.") + if hasattr(text, "encode"): + text = text.encode("utf-8") + raw = text + + super(Texter, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, + code=code, **kwa) + if self.code not in TexDex: + raise ValidationError("Invalid code = {} for Texter." + "".format(self.code)) + + + @property + def text(self): + """ + Property text: raw as bytes + """ + return self.raw + + @property + def uext(self): + """ + Property bext: raw as str + """ + return self.raw.decode('utf-8') + + class Bexter(Matter): """ Bexter is subclass of Matter, cryptographic material, for variable length @@ -1936,28 +2009,22 @@ class Bexter(Matter): Attributes: Inherited Properties: (See Matter) - .pad is int number of pad chars given raw - - .code is str derivation code to indicate cypher suite - .raw is bytes crypto material only without code - .index is int count of attached crypto material by context (receipts) - .qb64 is str in Base64 fully qualified with derivation code + crypto mat - .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat - .qb2 is bytes in binary with derivation code + crypto material - .transferable is Boolean, True when transferable derivation code False otherwise Properties: - .text is the Base64 text value, .qb64 with text code and leader removed. + .bext is the Base64 text value, .qb64 with text code and leader removed. - Hidden: - ._pad is method to compute .pad property - ._code is str value for .code property - ._raw is bytes value for .raw property - ._index is int value for .index property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 + Inherited Hidden Properties: (See Matter) Methods: + ._rawify(self, bext) + + Codes: + StrB64_L0: str = '4A' # String Base64 Only Leader Size 0 + StrB64_L1: str = '5A' # String Base64 Only Leader Size 1 + StrB64_L2: str = '6A' # String Base64 Only Leader Size 2 + StrB64_Big_L0: str = '7AAA' # String Base64 Only Big Leader Size 0 + StrB64_Big_L1: str = '8AAA' # String Base64 Only Big Leader Size 1 + StrB64_Big_L2: str = '9AAA' # String Base64 Only Big Leader Size 2 """ @@ -1979,10 +2046,10 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, if bext is None: raise EmptyMaterialError("Missing bext string.") if hasattr(bext, "encode"): - bext = bext.encode("utf-8") + bext = bext.encode("utf-8") # convert to bytes if not Reb64.match(bext): raise ValueError("Invalid Base64.") - raw = self._rawify(bext) + raw = self._rawify(bext) # convert bytes to raw with padding super(Bexter, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, code=code, **kwa) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index f5be19ef8..05a54135f 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -27,7 +27,7 @@ from keri.core import eventing from keri.core.coring import (Ilkage, Ilks, Labels, Saids, Protos, Protocolage, Sadder, Tholder, Seqner, - NumDex, Number, Siger, Dater, Bexter) + NumDex, Number, Siger, Dater, Bexter, Texter) from keri.core.coring import Serialage, Serials, Tiers, Vstrings from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, Indexer, @@ -41,7 +41,8 @@ from keri.help import helping from keri.kering import (EmptyMaterialError, RawMaterialError, DerivationError, ShortageError, InvalidCodeSizeError, InvalidVarIndexError, - InvalidValueError, DeserializeError, ValidationError) + InvalidValueError, DeserializeError, ValidationError, + InvalidVarRawSizeError) from keri.kering import Version, Versionage, VersionError from keri.kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, RPY_LABELS) @@ -4015,6 +4016,223 @@ def test_dater(): """ Done Test """ +def test_texter(): + """ + Test Texter variable sized text (bytes) subclass of Matter + """ + with pytest.raises(EmptyMaterialError): + texter = Texter() + + text = "" + textb = b"" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L0 + assert texter.both == '4BAA' + assert texter.raw == b'' == textb + assert texter.qb64 == '4BAA' + assert texter.qb2 == b'\xe0\x10\x00' + assert texter.text == textb + assert texter.uext == text + + texter = Texter(text=textb) + assert texter.code == MtrDex.Bytes_L0 + assert texter.both == '4BAA' + assert texter.raw == b'' == textb + assert texter.qb64 == '4BAA' + assert texter.qb2 == b'\xe0\x10\x00' + assert texter.text == textb + assert texter.uext == text + + + text = "$" + textb = b"$" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAB' + assert texter.raw == b"$" == textb + assert texter.qb64 == '6BABAAAk' + assert texter.qb2 ==b'\xe8\x10\x01\x00\x00$' + assert texter.text == textb + assert texter.uext == text + + texter = Texter(text=textb) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAB' + assert texter.raw == b"$" == textb + assert texter.qb64 == '6BABAAAk' + assert texter.qb2 ==b'\xe8\x10\x01\x00\x00$' + assert texter.text == textb + assert texter.uext == text + + + text = "@!" + textb = b"@!" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L1 + assert texter.both == '5BAB' + assert texter.raw == b'@!' == textb + assert texter.qb64 == '5BABAEAh' + assert texter.qb2 ==b'\xe4\x10\x01\x00@!' + assert texter.text == textb + assert texter.uext == text + + texter = Texter(text=textb) + assert texter.code == MtrDex.Bytes_L1 + assert texter.both == '5BAB' + assert texter.raw == b'@!' == textb + assert texter.qb64 == '5BABAEAh' + assert texter.qb2 ==b'\xe4\x10\x01\x00@!' + assert texter.text == textb + assert texter.uext == text + + text = "^*#" + textb = b"^*#" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L0 + assert texter.both == '4BAB' + assert texter.raw == b"^*#" == textb + assert texter.qb64 == '4BABXioj' + assert texter.qb2 == b'\xe0\x10\x01^*#' + assert texter.text == textb + assert texter.uext == text + + texter = Texter(text=textb) + assert texter.code == MtrDex.Bytes_L0 + assert texter.both == '4BAB' + assert texter.raw == b"^*#" == textb + assert texter.qb64 == '4BABXioj' + assert texter.qb2 == b'\xe0\x10\x01^*#' + assert texter.text == textb + assert texter.uext == text + + text = "&~?%" + textb = b"&~?%" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAC' + assert texter.raw == b"&~?%" == textb + assert texter.qb64 == '6BACAAAmfj8l' + assert texter.qb2 == b'\xe8\x10\x02\x00\x00&~?%' + assert texter.text == textb + assert texter.uext == text + + texter = Texter(text=textb) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAC' + assert texter.raw == b"&~?%" == textb + assert texter.qb64 == '6BACAAAmfj8l' + assert texter.qb2 == b'\xe8\x10\x02\x00\x00&~?%' + assert texter.text == textb + assert texter.uext == text + + + text = "\n" # control character + textb = b"\n" + + assert len(text) == len(textb) == 1 + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAB' + assert texter.raw == b"\n" == textb + assert texter.qb64 == '6BABAAAK' + assert texter.qb2 ==b'\xe8\x10\x01\x00\x00\n' + assert texter.text == textb + assert texter.uext == text + + texter = Texter(text=textb) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAB' + assert texter.raw == b"\n" == textb + assert texter.qb64 == '6BABAAAK' + assert texter.qb2 ==b'\xe8\x10\x01\x00\x00\n' + assert texter.text == textb + assert texter.uext == text + + + text = "Did the lazy fox jumped over the big dog? But it's not its dog!\n" + textb = b"Did the lazy fox jumped over the big dog? But it's not its dog!\n" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAW' + assert texter.raw == textb + assert texter.qb64 == '6BAWAABEaWQgdGhlIGxhenkgZm94IGp1bXBlZCBvdmVyIHRoZSBiaWcgZG9nPyBCdXQgaXQncyBub3QgaXRzIGRvZyEK' + assert texter.qb2 ==(b"\xe8\x10\x16\x00\x00Did the lazy fox jumped over the big dog? But it's not " + b'its dog!\n') + assert texter.text == textb + assert texter.uext == text + + assert len(texter.qb64) * 3 / 4 == len(texter.qb2) + + texter = Texter(text=textb) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAW' + assert texter.raw == textb + assert texter.qb64 == '6BAWAABEaWQgdGhlIGxhenkgZm94IGp1bXBlZCBvdmVyIHRoZSBiaWcgZG9nPyBCdXQgaXQncyBub3QgaXRzIGRvZyEK' + assert texter.qb2 ==(b"\xe8\x10\x16\x00\x00Did the lazy fox jumped over the big dog? But it's not " + b'its dog!\n') + assert texter.text == textb + assert texter.uext == text + + text = "a" * ((64 ** 2) * 3) # big variable size + textb = text.encode("utf-8") + + assert len(text) // 3 > (64 ** 2 - 1) + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_Big_L0 + assert texter.both == '7AABABAA' + assert texter.raw == textb + assert len(texter.qb64) == 16392 + assert len(texter.qb2) == 12294 + assert len(texter.qb64) * 3 / 4 == len(texter.qb2) + assert texter.text == textb + assert texter.uext == text + + text = "b" * ((64 ** 2 ) * 3 + 1) # big variable size + textb = text.encode("utf-8") + + assert len(text) // 3 > (64 ** 2 - 1) + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_Big_L2 + assert texter.both == '9AABABAB' + assert texter.raw == textb + assert len(texter.qb64) == 16396 + assert len(texter.qb2) == 12297 + assert len(texter.qb64) * 3 / 4 == len(texter.qb2) + assert texter.text == textb + assert texter.uext == text + + text = "c" * ((64 ** 2 ) * 3 + 2) # big variable size + textb = text.encode("utf-8") + + assert len(text) // 3 > (64 ** 2 - 1) + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_Big_L1 + assert texter.both == '8AABABAB' + assert texter.raw == textb + assert len(texter.qb64) == 16396 + assert len(texter.qb2) == 12297 + assert len(texter.qb64) * 3 / 4 == len(texter.qb2) + assert texter.text == textb + assert texter.uext == text + + text = "c" * ((64 ** 4) * 3) # excessive variable size + with pytest.raises(InvalidVarRawSizeError): + texter = Texter(text=text) + + """ Done Test """ + + + def test_bexter(): """ Test Bexter variable sized Base64 text subclass of Matter @@ -6542,7 +6760,7 @@ def test_tholder(): assert not tholder.satisfy(indices=[2, 3, 4]) assert not tholder.satisfy(indices=[]) - # test new nested weighted with Mapping dict + # test new nested weighted with Mapping dict with one clause # 1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1 tholder = Tholder(sith='[{"1/3":["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]') assert tholder.weighted @@ -6607,7 +6825,16 @@ def test_tholder(): assert tholder.json == '[{"1/3": ["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]' assert tholder.num == None - # test new nested weighted with Mapping dict + with pytest.raises(ValueError): + tholder = Tholder(sith=[{"1/3":["1/3", "1/3", "1/4"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]) + + with pytest.raises(ValueError): + tholder = Tholder(sith=[{"1/3":["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["2/3", "1/4"]}]) + + with pytest.raises(ValueError): + tholder = Tholder(sith=[{"1/5":["1/2", "1/2", "1/2"]}, "1/4", "1/5", {"1/5": ["1", "1"]}]) + + # test new nested weighted with Mapping dict with two clauses tholder = Tholder(sith='[[{"1/3":["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ["1/2", {"1/2": ["1", "1"]}]]') assert tholder.weighted @@ -6668,12 +6895,19 @@ def test_tholder(): '["1/2", ''{"1/2": ["1", "1"]}]]') assert tholder.num == None + with pytest.raises(ValueError): + tholder = Tholder(sith=[[{"1/3":["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ["1/2", {"1/3": ["1", "1"]}]]) + + with pytest.raises(ValueError): + tholder = Tholder(sith=[[{"1/3":["1/3", "1/4", "1/3"]}, "1/2", {"1/2": ["1", "1"]}], ["1/2", {"1/2": ["1/2", "1/2"]}]]) + """ Done Test """ if __name__ == "__main__": #test_matter() + test_texter() #test_counter() #test_prodex() #test_indexer() @@ -6682,7 +6916,7 @@ def test_tholder(): #test_siger() #test_signer() #test_nexter() - test_tholder() + #test_tholder() #test_ilks() #test_labels() #test_prefixer() From 4b5565201273e57b75432ddef0c816160e9e303e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 21 Feb 2024 10:38:04 -0700 Subject: [PATCH 011/418] refactor of some Texter method more tests. --- src/keri/core/coring.py | 9 +- tests/core/test_coring.py | 183 ++++++++++++++++++++++++-------------- 2 files changed, 116 insertions(+), 76 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 08c17e53c..97b8a1838 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -1949,14 +1949,7 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, @property def text(self): """ - Property text: raw as bytes - """ - return self.raw - - @property - def uext(self): - """ - Property bext: raw as str + Property text: raw as str """ return self.raw.decode('utf-8') diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 05a54135f..ee3a49ab3 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -4023,26 +4023,35 @@ def test_texter(): with pytest.raises(EmptyMaterialError): texter = Texter() + with pytest.raises(ValidationError): + texter = Texter(raw=b'Wrong code for Texter', code=MtrDex.StrB64_L0) + + text = "" textb = b"" texter = Texter(text=text) assert texter.code == MtrDex.Bytes_L0 assert texter.both == '4BAA' - assert texter.raw == b'' == textb + assert texter.raw == textb assert texter.qb64 == '4BAA' - assert texter.qb2 == b'\xe0\x10\x00' - assert texter.text == textb - assert texter.uext == text + assert texter.text == text texter = Texter(text=textb) - assert texter.code == MtrDex.Bytes_L0 assert texter.both == '4BAA' assert texter.raw == b'' == textb - assert texter.qb64 == '4BAA' - assert texter.qb2 == b'\xe0\x10\x00' - assert texter.text == textb - assert texter.uext == text + + texter = Texter(raw=textb) + assert texter.both == '4BAA' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '4BAA' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '4BAA' + assert texter.raw == textb text = "$" @@ -4051,20 +4060,27 @@ def test_texter(): texter = Texter(text=text) assert texter.code == MtrDex.Bytes_L2 assert texter.both == '6BAB' - assert texter.raw == b"$" == textb + assert texter.raw == textb assert texter.qb64 == '6BABAAAk' assert texter.qb2 ==b'\xe8\x10\x01\x00\x00$' - assert texter.text == textb - assert texter.uext == text + assert texter.text == text texter = Texter(text=textb) - assert texter.code == MtrDex.Bytes_L2 assert texter.both == '6BAB' - assert texter.raw == b"$" == textb - assert texter.qb64 == '6BABAAAk' - assert texter.qb2 ==b'\xe8\x10\x01\x00\x00$' - assert texter.text == textb - assert texter.uext == text + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '6BAB' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '6BAB' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '6BAB' + assert texter.raw == textb + text = "@!" @@ -4073,20 +4089,26 @@ def test_texter(): texter = Texter(text=text) assert texter.code == MtrDex.Bytes_L1 assert texter.both == '5BAB' - assert texter.raw == b'@!' == textb + assert texter.raw == textb assert texter.qb64 == '5BABAEAh' assert texter.qb2 ==b'\xe4\x10\x01\x00@!' - assert texter.text == textb - assert texter.uext == text + assert texter.text == text texter = Texter(text=textb) - assert texter.code == MtrDex.Bytes_L1 assert texter.both == '5BAB' - assert texter.raw == b'@!' == textb - assert texter.qb64 == '5BABAEAh' - assert texter.qb2 ==b'\xe4\x10\x01\x00@!' - assert texter.text == textb - assert texter.uext == text + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '5BAB' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '5BAB' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '5BAB' + assert texter.raw == textb text = "^*#" textb = b"^*#" @@ -4094,20 +4116,26 @@ def test_texter(): texter = Texter(text=text) assert texter.code == MtrDex.Bytes_L0 assert texter.both == '4BAB' - assert texter.raw == b"^*#" == textb + assert texter.raw == textb assert texter.qb64 == '4BABXioj' assert texter.qb2 == b'\xe0\x10\x01^*#' - assert texter.text == textb - assert texter.uext == text + assert texter.text == text texter = Texter(text=textb) - assert texter.code == MtrDex.Bytes_L0 assert texter.both == '4BAB' - assert texter.raw == b"^*#" == textb - assert texter.qb64 == '4BABXioj' - assert texter.qb2 == b'\xe0\x10\x01^*#' - assert texter.text == textb - assert texter.uext == text + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '4BAB' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '4BAB' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '4BAB' + assert texter.raw == textb text = "&~?%" textb = b"&~?%" @@ -4115,20 +4143,26 @@ def test_texter(): texter = Texter(text=text) assert texter.code == MtrDex.Bytes_L2 assert texter.both == '6BAC' - assert texter.raw == b"&~?%" == textb + assert texter.raw == textb assert texter.qb64 == '6BACAAAmfj8l' assert texter.qb2 == b'\xe8\x10\x02\x00\x00&~?%' - assert texter.text == textb - assert texter.uext == text + assert texter.text == text texter = Texter(text=textb) - assert texter.code == MtrDex.Bytes_L2 assert texter.both == '6BAC' - assert texter.raw == b"&~?%" == textb - assert texter.qb64 == '6BACAAAmfj8l' - assert texter.qb2 == b'\xe8\x10\x02\x00\x00&~?%' - assert texter.text == textb - assert texter.uext == text + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '6BAC' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '6BAC' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '6BAC' + assert texter.raw == textb text = "\n" # control character @@ -4139,20 +4173,26 @@ def test_texter(): texter = Texter(text=text) assert texter.code == MtrDex.Bytes_L2 assert texter.both == '6BAB' - assert texter.raw == b"\n" == textb + assert texter.raw == textb assert texter.qb64 == '6BABAAAK' assert texter.qb2 ==b'\xe8\x10\x01\x00\x00\n' - assert texter.text == textb - assert texter.uext == text + assert texter.text == text texter = Texter(text=textb) - assert texter.code == MtrDex.Bytes_L2 assert texter.both == '6BAB' - assert texter.raw == b"\n" == textb - assert texter.qb64 == '6BABAAAK' - assert texter.qb2 ==b'\xe8\x10\x01\x00\x00\n' - assert texter.text == textb - assert texter.uext == text + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '6BAB' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '6BAB' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '6BAB' + assert texter.raw == textb text = "Did the lazy fox jumped over the big dog? But it's not its dog!\n" @@ -4165,20 +4205,27 @@ def test_texter(): assert texter.qb64 == '6BAWAABEaWQgdGhlIGxhenkgZm94IGp1bXBlZCBvdmVyIHRoZSBiaWcgZG9nPyBCdXQgaXQncyBub3QgaXRzIGRvZyEK' assert texter.qb2 ==(b"\xe8\x10\x16\x00\x00Did the lazy fox jumped over the big dog? But it's not " b'its dog!\n') - assert texter.text == textb - assert texter.uext == text + assert texter.text == text assert len(texter.qb64) * 3 / 4 == len(texter.qb2) texter = Texter(text=textb) - assert texter.code == MtrDex.Bytes_L2 assert texter.both == '6BAW' assert texter.raw == textb - assert texter.qb64 == '6BAWAABEaWQgdGhlIGxhenkgZm94IGp1bXBlZCBvdmVyIHRoZSBiaWcgZG9nPyBCdXQgaXQncyBub3QgaXRzIGRvZyEK' - assert texter.qb2 ==(b"\xe8\x10\x16\x00\x00Did the lazy fox jumped over the big dog? But it's not " - b'its dog!\n') - assert texter.text == textb - assert texter.uext == text + + texter = Texter(raw=textb) + assert texter.both == '6BAW' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '6BAW' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '6BAW' + assert texter.raw == textb + + text = "a" * ((64 ** 2) * 3) # big variable size textb = text.encode("utf-8") @@ -4192,8 +4239,7 @@ def test_texter(): assert len(texter.qb64) == 16392 assert len(texter.qb2) == 12294 assert len(texter.qb64) * 3 / 4 == len(texter.qb2) - assert texter.text == textb - assert texter.uext == text + assert texter.text == text text = "b" * ((64 ** 2 ) * 3 + 1) # big variable size textb = text.encode("utf-8") @@ -4207,8 +4253,7 @@ def test_texter(): assert len(texter.qb64) == 16396 assert len(texter.qb2) == 12297 assert len(texter.qb64) * 3 / 4 == len(texter.qb2) - assert texter.text == textb - assert texter.uext == text + assert texter.text == text text = "c" * ((64 ** 2 ) * 3 + 2) # big variable size textb = text.encode("utf-8") @@ -4222,8 +4267,7 @@ def test_texter(): assert len(texter.qb64) == 16396 assert len(texter.qb2) == 12297 assert len(texter.qb64) * 3 / 4 == len(texter.qb2) - assert texter.text == textb - assert texter.uext == text + assert texter.text == text text = "c" * ((64 ** 4) * 3) # excessive variable size with pytest.raises(InvalidVarRawSizeError): @@ -4240,6 +4284,9 @@ def test_bexter(): with pytest.raises(EmptyMaterialError): bexter = Bexter() + with pytest.raises(ValidationError): + bexter = Bexter(raw=b'Wrong_code_for_Bexter', code=MtrDex.Bytes_L0) + bext = "@!" with pytest.raises(ValueError): bexter = Bexter(bext=bext) From 616c05d4708534ec657fbf758407d632c7b64905 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 21 Feb 2024 14:00:15 -0700 Subject: [PATCH 012/418] Refactor the two incompatible .sniff function methods to Parser.sniff and Sadder.smell which is also what Serder.reap does. Eventually may want to move .smell to helping if it needs to be used in two places. --- src/keri/core/coring.py | 59 +++++++++++++++++++------------------- src/keri/core/serdering.py | 16 +++++++---- tests/core/test_coring.py | 4 +-- 3 files changed, 43 insertions(+), 36 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 97b8a1838..f5d787e57 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -238,34 +238,6 @@ def nabSextets(b, l): i <<= p # pad with empty bits return (i.to_bytes(n, 'big')) -MINSNIFFSIZE = 12 + VERFULLSIZE # min bytes in buffer to sniff else need more - -def sniff(raw): - """ - Returns serialization kind, version and size from serialized event raw - by investigating leading bytes that contain version string - - Parameters: - raw is bytes of serialized event - - """ - if len(raw) < MINSNIFFSIZE: - raise ShortageError("Need more bytes.") - - match = Rever.search(raw) # Rever's regex takes bytes - if not match or match.start() > 12: - raise VersionError("Invalid version string in raw = {}".format(raw)) - - proto, major, minor, kind, size = match.group("proto", "major", "minor", "kind", "size") - version = Versionage(major=int(major, 16), minor=int(minor, 16)) - kind = kind.decode("utf-8") - proto = proto.decode("utf-8") - if kind not in Serials: - raise DeserializeError("Invalid serialization kind = {}".format(kind)) - size = int(size, 16) - - return proto, kind, version, size - def dumps(ked, kind=Serials.json): """ @@ -5275,6 +5247,8 @@ class Sadder: loads and jumps of json use str whereas cbor and msgpack use bytes """ + MaxVSOffset = 12 + SmellSize = MaxVSOffset + VERFULLSIZE # min buffer size to inhale def __init__(self, raw=b'', ked=None, sad=None, kind=None, saidify=False, code=MtrDex.Blake3_256): @@ -5333,7 +5307,7 @@ def _inhale(self, raw): loads and jumps of json use str whereas cbor and msgpack use bytes """ - proto, kind, version, size = sniff(raw) + proto, kind, version, size = self.smell(raw) if version != Version: raise VersionError("Unsupported version = {}.{}, expected {}." "".format(version.major, version.minor, Version)) @@ -5389,6 +5363,33 @@ def compare(self, said=None): else: raise ValueError("Both said and saider may not be None.") + @staticmethod + def smell(raw): + """ + Returns serialization kind, version and size from serialized event raw + by investigating leading bytes that contain version string + + Parameters: + raw is bytes of serialized event + + """ + if len(raw) < Sadder.SmellSize: + raise ShortageError("Need more bytes.") + + match = Rever.search(raw) # Rever's regex takes bytes + if not match or match.start() > 12: + raise VersionError("Invalid version string in raw = {}".format(raw)) + + proto, major, minor, kind, size = match.group("proto", "major", "minor", "kind", "size") + version = Versionage(major=int(major, 16), minor=int(minor, 16)) + kind = kind.decode("utf-8") + proto = proto.decode("utf-8") + if kind not in Serials: + raise DeserializeError("Invalid serialization kind = {}".format(kind)) + size = int(size, 16) + + return proto, kind, version, size + @property def raw(self): diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index a6a464e0f..179d052d8 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -86,16 +86,22 @@ def reap(self, ims, *, version=None): """ version = version if version is not None else self.version - if len(ims) < Serder.InhaleSize: + if len(ims) < Serder.SmellSize: raise ShortageError(f"Need more raw bytes for Serdery to reap.") match = Rever.search(ims) # Rever regex takes bytes/bytearray not str if not match or match.start() > Serder.MaxVSOffset: raise VersionError(f"Invalid version string for Serder raw = " - f"{ims[: Serder.InhaleSize]}.") + f"{ims[: Serder.SmellSize]}.") reaped = Reapage(*match.group("proto", "major", "minor", "kind", "size")) + vrsn = Versionage(major=int(reaped.major, 16), minor=int(reaped.minor, 16)) + if version: # test here for compatible cod version with message vrsn + if (vrsn.major > version.major or + (vrsn.major == version.major and vrsn.minor > version.minor)): + pass # raise error here? + if reaped.proto == Protos.keri.encode("utf-8"): return SerderKERI(raw=ims, strip=True, version=version, reaped=reaped) elif reaped.proto == Protos.acdc.encode("utf-8"): @@ -181,7 +187,7 @@ class Serder: """ MaxVSOffset = 12 - InhaleSize = MaxVSOffset + VERFULLSIZE # min buffer size to inhale + SmellSize = MaxVSOffset + VERFULLSIZE # min buffer size to inhale Dummy = "#" # dummy spaceholder char for said. Must not be a valid Base64 char @@ -783,13 +789,13 @@ def _inhale(clas, raw, version=Version, reaped=None): if reaped: proto, major, minor, kind, size = reaped # tuple unpack else: - if len(raw) < clas.InhaleSize: + if len(raw) < clas.SmellSize: raise ShortageError(f"Need more raw bytes for Serder to inhale.") match = Rever.search(raw) # Rever regex takes bytes/bytearray not str if not match or match.start() > clas.MaxVSOffset: raise VersionError(f"Invalid version string in raw = " - f"{raw[:clas.InhaleSize]}.") + f"{raw[:clas.SmellSize]}.") proto, major, minor, kind, size = match.group("proto", "major", diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index ee3a49ab3..561c55a24 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -31,10 +31,10 @@ from keri.core.coring import Serialage, Serials, Tiers, Vstrings from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, Indexer, - CtrDex, Counter, sniff, ProDex) + CtrDex, Counter, ProDex) from keri.core.coring import (Verfer, Cigar, Signer, Salter, Saider, DigDex, Diger, Prefixer, Cipher, Encrypter, Decrypter) -from keri.core.coring import versify, deversify, Rever, VERFULLSIZE, MINSNIFFSIZE +from keri.core.coring import versify, deversify, Rever, VERFULLSIZE from keri.core.coring import generateSigners, generatePrivates from keri.core.coring import (intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, B64_CHARS, Reb64, nabSextets) From f4bb5a921fc0181dd0554c76d9fa75c58f9f34bf Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 22 Feb 2024 12:56:17 -0700 Subject: [PATCH 013/418] Refactor smell to be in kering and uses in Serder Serdery prelim to version the version string. --- src/keri/core/coring.py | 15 +++---- src/keri/core/serdering.py | 81 ++++++++++---------------------------- src/keri/kering.py | 53 +++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 68 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index f5d787e57..c2835a1d2 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -36,7 +36,7 @@ ShortageError, UnexpectedCodeError, DeserializeError, UnexpectedCountCodeError, UnexpectedOpCodeError) from ..kering import (Versionage, Version, VERRAWSIZE, VERFMT, VERFULLSIZE, - versify, deversify, Rever) + versify, deversify, Rever, smell) from ..kering import Serials, Serialage, Protos, Protocolage, Ilkage, Ilks from ..kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, RPY_LABELS) @@ -5363,6 +5363,7 @@ def compare(self, said=None): else: raise ValueError("Both said and saider may not be None.") + @staticmethod def smell(raw): """ @@ -5373,14 +5374,14 @@ def smell(raw): raw is bytes of serialized event """ - if len(raw) < Sadder.SmellSize: - raise ShortageError("Need more bytes.") + #if len(raw) < Sadder.SmellSize: + #raise ShortageError("Need more bytes.") - match = Rever.search(raw) # Rever's regex takes bytes - if not match or match.start() > 12: - raise VersionError("Invalid version string in raw = {}".format(raw)) + #match = Rever.search(raw) # Rever's regex takes bytes + #if not match or match.start() > 12: + #raise VersionError("Invalid version string in raw = {}".format(raw)) - proto, major, minor, kind, size = match.group("proto", "major", "minor", "kind", "size") + proto, major, minor, kind, size = smell(raw) version = Versionage(major=int(major, 16), minor=int(minor, 16)) kind = kind.decode("utf-8") proto = proto.decode("utf-8") diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 179d052d8..196f550a7 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -18,7 +18,8 @@ ShortageError, VersionError, ProtocolError, KindError, DeserializeError, FieldError, SerializeError) from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_1_1, - VERRAWSIZE, VERFMT, VERFULLSIZE) + VERRAWSIZE, VERFMT, VERFULLSIZE, + SMELLSIZE, Smellage, smell) from ..kering import Protos, Serials, Rever, versify, deversify, Ilks from ..core import coring from .coring import MtrDex, DigDex, PreDex, Saids, Digestage @@ -42,17 +43,6 @@ Fieldage = namedtuple("Fieldage", "saids alls") #values are dicts -""" -Reapage - proto (str): protocol type value of Protos examples 'KERI', 'ACDC' - major (str): single char hex string of major version number - minor (str): single char hex string of minor version number - kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' - -""" -Reapage = namedtuple("Reapage", "proto major minor kind size") - - class Serdery: """Serder factory class for generating serder instances by protocol type from an incoming message stream. @@ -86,28 +76,14 @@ def reap(self, ims, *, version=None): """ version = version if version is not None else self.version - if len(ims) < Serder.SmellSize: - raise ShortageError(f"Need more raw bytes for Serdery to reap.") - - match = Rever.search(ims) # Rever regex takes bytes/bytearray not str - if not match or match.start() > Serder.MaxVSOffset: - raise VersionError(f"Invalid version string for Serder raw = " - f"{ims[: Serder.SmellSize]}.") - - reaped = Reapage(*match.group("proto", "major", "minor", "kind", "size")) + smellage = smell(ims, version=version) - vrsn = Versionage(major=int(reaped.major, 16), minor=int(reaped.minor, 16)) - if version: # test here for compatible cod version with message vrsn - if (vrsn.major > version.major or - (vrsn.major == version.major and vrsn.minor > version.minor)): - pass # raise error here? - - if reaped.proto == Protos.keri.encode("utf-8"): - return SerderKERI(raw=ims, strip=True, version=version, reaped=reaped) - elif reaped.proto == Protos.acdc.encode("utf-8"): - return SerderACDC(raw=ims, strip=True, version=version, reaped=reaped) + if smellage.proto == Protos.keri.encode("utf-8"): + return SerderKERI(raw=ims, strip=True, version=version, smellage=smellage) + elif smellage.proto == Protos.acdc.encode("utf-8"): + return SerderACDC(raw=ims, strip=True, version=version, smellage=smellage) else: - raise ProtocolError(f"Unsupported protocol type = {reaped.proto}.") + raise ProtocolError(f"Unsupported protocol type = {smellage.proto}.") @@ -138,8 +114,6 @@ class Serder: generation and verification in addition to the required fields. Class Attributes: - MaxVSOffset (int): Maximum Version String Offset in bytes/chars - InhaleSize (int): Minimum raw buffer size needed to inhale Labels (dict): Protocol specific dict of field labels keyed by ilk (packet type string value). None is default key when no ilk needed. Each entry is a @@ -185,10 +159,6 @@ class Serder: loads and jumps of json use str whereas cbor and msgpack use bytes """ - - MaxVSOffset = 12 - SmellSize = MaxVSOffset + VERFULLSIZE # min buffer size to inhale - Dummy = "#" # dummy spaceholder char for said. Must not be a valid Base64 char # should be same set of codes as in coring.DigestCodex coring.DigDex so @@ -357,7 +327,7 @@ class Serder: def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, - reaped=None, verify=True, makify=False, + smellage=None, verify=True, makify=False, proto=None, vrsn=None, kind=None, ilk=None, saids=None): """Deserialize raw if provided. Update properties from deserialized raw. Verifies said(s) embedded in sad as given by labels. @@ -377,10 +347,10 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, Assumes that raw is bytearray when strip is True. version (Versionage | None): instance supported protocol version None means do not enforce a supported version - reaped (Reapage | None): instance of deconstructed version string + smellage (Smellage | None): instance of deconstructed version string elements. If none or empty ignore otherwise assume that raw already had its version string extracted (reaped) into the - elements of reaped. + elements of smellage. verify (bool): True means verify said(s) of given raw or sad. Raises ValidationError if verification fails Ignore when raw not provided or when raw and saidify is True @@ -410,7 +380,7 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, # self._inhale works because it only references class attributes sad, proto, vrsn, kind, size = self._inhale(raw=raw, version=version, - reaped=reaped) + smellage=smellage) self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray self._sad = sad self._proto = proto @@ -757,7 +727,7 @@ def makify(self, sad, *, version=None, @classmethod - def _inhale(clas, raw, version=Version, reaped=None): + def _inhale(clas, raw, version=Version, smellage=None): """Deserializes raw. Parses serilized event ser of serialization kind and assigns to instance attributes and returns tuple of associated elements. @@ -776,32 +746,21 @@ def _inhale(clas, raw, version=Version, reaped=None): Parameters: raw (bytes): serialized sad message version (Versionage): instance supported protocol version - reaped (Reapage | None): instance of deconstructed version string + smellage (Smellage | None): instance of deconstructed version string elements. If none or empty ignore otherwise assume that raw already had its version string extracted (reaped) into the - elements of reaped. + elements of smellage. Note: loads and jumps of json use str whereas cbor and msgpack use bytes Assumes only supports Version """ - if reaped: - proto, major, minor, kind, size = reaped # tuple unpack - else: - if len(raw) < clas.SmellSize: - raise ShortageError(f"Need more raw bytes for Serder to inhale.") - - match = Rever.search(raw) # Rever regex takes bytes/bytearray not str - if not match or match.start() > clas.MaxVSOffset: - raise VersionError(f"Invalid version string in raw = " - f"{raw[:clas.SmellSize]}.") + if smellage: # passed in so don't need to smell raw again + proto, major, minor, kind, size = smellage # tuple unpack + else: # not passed in so smell raw + proto, major, minor, kind, size = smell(raw, version=version) - proto, major, minor, kind, size = match.group("proto", - "major", - "minor", - "kind", - "size") proto = proto.decode("utf-8") if proto not in Protos: @@ -822,7 +781,7 @@ def _inhale(clas, raw, version=Version, reaped=None): sad = clas.loads(raw=raw, size=size, kind=kind) - if "v" not in sad: + if "v" not in sad: # Regex does not check for version string label itself raise FieldError(f"Missing version string field in {sad}.") return sad, proto, vrsn, kind, size diff --git a/src/keri/kering.py b/src/keri/kering.py index b8b15ea85..c769aa225 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -92,6 +92,57 @@ def deversify(vs, version=None): raise ValueError("Invalid version string = {}".format(vs)) + + +MAXVSOFFSET = 12 +SMELLSIZE = MAXVSOFFSET + VERFULLSIZE # min buffer size to inhale + +""" +Smellage (results of smelling a version string such as in a Serder) + proto (str): protocol type value of Protos examples 'KERI', 'ACDC' + major (str): single char hex string of major version number + minor (str): single char hex string of minor version number + kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' + size (str): hex string of size of raw serialization + +""" +Smellage = namedtuple("Smellage", "proto major minor kind size") + +def smell(raw, *, version=None): + """Extract and return Smellage from version string inside serialized raw. + + Returns: + smellage (Smellage): named Tuple of proto, major, minor, kind, size + + Parameters: + raw (bytearray) of serialized incoming message stream. Assumes start + of stream is JSON, CBOR, or MGPK field map with first field + is labeled 'v' and value is version string. + version (Versionage | None): instance supported protocol version + None means do not enforce a supported version + """ + if len(raw) < SMELLSIZE: + raise ShortageError(f"Need more raw bytes to smell full version string.") + + match = Rever.search(raw) # Rever regex takes bytes/bytearray not str + if not match or match.start() > MAXVSOFFSET: + raise VersionError(f"Invalid version string from smelled raw = " + f"{raw[: SMELLSIZE]}.") + + smellage = Smellage(*match.group("proto", "major", "minor", "kind", "size")) + + # Global version compatibility check. Serder instances also peform version check + vrsn = Versionage(major=int(smellage.major, 16), minor=int(smellage.minor, 16)) + if version: # test here for compatible code version with message vrsn + if (vrsn.major > version.major or + (vrsn.major == version.major and vrsn.minor > version.minor)): + pass # raise error here? + + return smellage + + + + """ ilk is short for packet or message type for a given protocol icp = incept, inception @@ -166,6 +217,8 @@ def deversify(vs, version=None): CRED_TSN_LABELS = ["v", "i", "s", "d", "ri", "a", "ra"] + +# Exception Subclasses class KeriError(Exception): """ Base Class for keri exceptions From dccf1ac360b7f0dff5c37114768cc367144f3b7a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 22 Feb 2024 13:31:36 -0700 Subject: [PATCH 014/418] finished refactoring smell to be in kering.py now can be versioned i.e. smell is ready to be changed so it can recognize both 1.X and 2.X version strings. --- src/keri/core/coring.py | 34 ++-------------- src/keri/core/serdering.py | 40 +++++++++---------- src/keri/kering.py | 80 +++++++++++++++++++++++++++++++++----- 3 files changed, 93 insertions(+), 61 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index c2835a1d2..78c449e3e 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -5307,12 +5307,12 @@ def _inhale(self, raw): loads and jumps of json use str whereas cbor and msgpack use bytes """ - proto, kind, version, size = self.smell(raw) + proto, version, kind, size = smell(raw) if version != Version: raise VersionError("Unsupported version = {}.{}, expected {}." "".format(version.major, version.minor, Version)) - if len(raw) < size: - raise ShortageError("Need more bytes.") + #if len(raw) < size: + #raise ShortageError("Need more bytes.") ked = loads(raw=raw, size=size, kind=kind) @@ -5364,34 +5364,6 @@ def compare(self, said=None): raise ValueError("Both said and saider may not be None.") - @staticmethod - def smell(raw): - """ - Returns serialization kind, version and size from serialized event raw - by investigating leading bytes that contain version string - - Parameters: - raw is bytes of serialized event - - """ - #if len(raw) < Sadder.SmellSize: - #raise ShortageError("Need more bytes.") - - #match = Rever.search(raw) # Rever's regex takes bytes - #if not match or match.start() > 12: - #raise VersionError("Invalid version string in raw = {}".format(raw)) - - proto, major, minor, kind, size = smell(raw) - version = Versionage(major=int(major, 16), minor=int(minor, 16)) - kind = kind.decode("utf-8") - proto = proto.decode("utf-8") - if kind not in Serials: - raise DeserializeError("Invalid serialization kind = {}".format(kind)) - size = int(size, 16) - - return proto, kind, version, size - - @property def raw(self): """ raw property getter """ diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 196f550a7..ae1264887 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -78,9 +78,9 @@ def reap(self, ims, *, version=None): smellage = smell(ims, version=version) - if smellage.proto == Protos.keri.encode("utf-8"): + if smellage.protocol == Protos.keri: return SerderKERI(raw=ims, strip=True, version=version, smellage=smellage) - elif smellage.proto == Protos.acdc.encode("utf-8"): + elif smellage.protocol == Protos.acdc: return SerderACDC(raw=ims, strip=True, version=version, smellage=smellage) else: raise ProtocolError(f"Unsupported protocol type = {smellage.proto}.") @@ -347,9 +347,9 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, Assumes that raw is bytearray when strip is True. version (Versionage | None): instance supported protocol version None means do not enforce a supported version - smellage (Smellage | None): instance of deconstructed version string - elements. If none or empty ignore otherwise assume that raw - already had its version string extracted (reaped) into the + smellage (Smellage | None): instance of deconstructed and converted + version string elements. If none or empty ignore otherwise assume + that raw already had its version string extracted (reaped) into the elements of smellage. verify (bool): True means verify said(s) of given raw or sad. Raises ValidationError if verification fails @@ -757,27 +757,27 @@ def _inhale(clas, raw, version=Version, smellage=None): """ if smellage: # passed in so don't need to smell raw again - proto, major, minor, kind, size = smellage # tuple unpack + proto, vrsn, kind, size = smellage # tuple unpack else: # not passed in so smell raw - proto, major, minor, kind, size = smell(raw, version=version) + proto, vrsn, kind, size = smell(raw, version=version) - proto = proto.decode("utf-8") - if proto not in Protos: - raise ProtocolError(f"Invalid protocol type = {proto}.") + #proto = proto.decode("utf-8") + #if proto not in Protos: + #raise ProtocolError(f"Invalid protocol type = {proto}.") - vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) - if version is not None and vrsn != version: - raise VersionError(f"Expected version = {version}, got " - f"{vrsn.major}.{vrsn.minor}.") + #vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) + #if version is not None and vrsn != version: + #raise VersionError(f"Expected version = {version}, got " + #f"{vrsn.major}.{vrsn.minor}.") - kind = kind.decode("utf-8") - if kind not in Serials: - raise KindError(f"Invalid serialization kind = {kind}.") + #kind = kind.decode("utf-8") + #if kind not in Serials: + #raise KindError(f"Invalid serialization kind = {kind}.") - size = int(size, 16) - if len(raw) < size: - raise ShortageError(f"Need more bytes.") + #size = int(size, 16) + #if len(raw) < size: + #raise ShortageError(f"Need more bytes.") sad = clas.loads(raw=raw, size=size, kind=kind) diff --git a/src/keri/kering.py b/src/keri/kering.py index c769aa225..c9fbc7fac 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -97,22 +97,85 @@ def deversify(vs, version=None): MAXVSOFFSET = 12 SMELLSIZE = MAXVSOFFSET + VERFULLSIZE # min buffer size to inhale +#""" +#Smellage (results of smelling a version string such as in a Serder) + #proto (str): protocol type value of Protos examples 'KERI', 'ACDC' + #major (str): single char hex string of major version number + #minor (str): single char hex string of minor version number + #kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' + #size (str): hex string of size of raw serialization + +#""" +#Smellage = namedtuple("Smellage", "proto major minor kind size") + """ Smellage (results of smelling a version string such as in a Serder) - proto (str): protocol type value of Protos examples 'KERI', 'ACDC' - major (str): single char hex string of major version number - minor (str): single char hex string of minor version number + protocol (str): protocol type value of Protos examples 'KERI', 'ACDC' + version (Versionage): named tuple (major, minor) ints of major minor version kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' - size (str): hex string of size of raw serialization + size (str): int size of raw serialization """ -Smellage = namedtuple("Smellage", "proto major minor kind size") +Smellage = namedtuple("Smellage", "protocol version kind size") def smell(raw, *, version=None): - """Extract and return Smellage from version string inside serialized raw. + """Extract and return instance of Smellage from version string inside + raw serialization. Returns: - smellage (Smellage): named Tuple of proto, major, minor, kind, size + smellage (Smellage): named Tuple of (protocol, version, kind, size) + + Parameters: + raw (bytearray) of serialized incoming message stream. Assumes start + of stream is JSON, CBOR, or MGPK field map with first field + is labeled 'v' and value is version string. + version (Versionage | None): instance supported protocol version + None means do not enforce a supported version + """ + if len(raw) < SMELLSIZE: + raise ShortageError(f"Need more raw bytes to smell full version string.") + + match = Rever.search(raw) # Rever regex takes bytes/bytearray not str + if not match or match.start() > MAXVSOFFSET: + raise VersionError(f"Invalid version string from smelled raw = " + f"{raw[: SMELLSIZE]}.") + + proto, major, minor, kind, size = match.group("proto", "major", "minor", "kind", "size") + + # use length of version string matched to determine if version 1.x or 2.x + # so can convert major, minor, and size correctly hex vs B64 numbers + + # Global version compatibility check. Serder instances also peform version check + major = int(major, 16) + minor = int(minor, 16) + vrsn = Versionage(major=major, minor=minor) + if version is not None: # test here for compatible code version with message vrsn + if (vrsn.major > version.major or + (vrsn.major == version.major and vrsn.minor > version.minor)): + pass # raise error here? + + + protocol = proto.decode("utf-8") + if protocol not in Protos: + raise ProtocolError(f"Invalid protocol type = {protocol}.") + + kind = kind.decode("utf-8") + if kind not in Serials: + raise KindError(f"Invalid serialization kind = {kind}.") + + size = int(size, 16) + if len(raw) < size: + raise ShortageError(f"Need more bytes.") + + return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) + + +def smelly(raw, *, version=None): + """Extract and return instance of Smellage from version string inside + raw serialization. + + Returns: + smellage (Smellage): named Tuple of (protocol, version, kind, size) Parameters: raw (bytearray) of serialized incoming message stream. Assumes start @@ -140,9 +203,6 @@ def smell(raw, *, version=None): return smellage - - - """ ilk is short for packet or message type for a given protocol icp = incept, inception From 176e7aac8ffd66465f14319f4101ba0a90b4af18 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 22 Feb 2024 14:38:57 -0700 Subject: [PATCH 015/418] refactor sniff to be in kering instead of Parser class method. This is to support new Streamer class for SPAC wrapping that needs to use sniff without circular import. --- src/keri/app/httping.py | 2 +- src/keri/core/parsing.py | 95 ++----------------------------- src/keri/kering.py | 119 ++++++++++++++++++++++++++------------- 3 files changed, 87 insertions(+), 129 deletions(-) diff --git a/src/keri/app/httping.py b/src/keri/app/httping.py index 6f815d4d8..eb1ce6c48 100644 --- a/src/keri/app/httping.py +++ b/src/keri/app/httping.py @@ -168,7 +168,7 @@ def streamCESRRequests(client, ims, dest, path=None): """ path = path if path is not None else "/" - cold = parsing.Parser.sniff(ims) # check for spurious counters at front of stream + cold = kering.sniff(ims) # check for spurious counters at front of stream if cold in (parsing.Colds.txt, parsing.Colds.bny): # not message error out to flush stream # replace with pipelining here once CESR message format supported. raise kering.ColdStartError("Expecting message counter tritet={}" diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index 2140086b8..72207101b 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -15,58 +15,11 @@ from . import serdering from .. import help from .. import kering +from ..kering import ColdDex, Colds, sniff logger = help.ogler.getLogger() -@dataclass(frozen=True) -class ColdCodex: - """ - ColdCodex is codex of cold stream start tritets of first byte - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - - First three bits: - 0o0 = 000 free - 0o1 = 001 cntcode B64 - 0o2 = 010 opcode B64 - 0o3 = 011 json - 0o4 = 100 mgpk - 0o5 = 101 cbor - 0o6 = 110 mgpk - 007 = 111 cntcode or opcode B2 - - status is one of ('evt', 'txt', 'bny' ) - 'evt' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) - 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) - 'bny' if tritet in (ColdDex.CtOpB2,) - - otherwise raise ColdStartError - - x = bytearray([0x2d, 0x5f]) - x == bytearray(b'-_') - x[0] >> 5 == 0o1 - True - """ - Free: int = 0o0 # not taken - CtB64: int = 0o1 # CountCode Base64 - OpB64: int = 0o2 # OpCode Base64 - JSON: int = 0o3 # JSON Map Event Start - MGPK1: int = 0o4 # MGPK Fixed Map Event Start - CBOR: int = 0o5 # CBOR Map Event Start - MGPK2: int = 0o6 # MGPK Big 16 or 32 Map Event Start - CtOpB2: int = 0o7 # CountCode or OpCode Base2 - - def __iter__(self): - return iter(astuple(self)) - - -ColdDex = ColdCodex() # Make instance - -Coldage = namedtuple("Coldage", 'msg txt bny') # stream cold start status -Colds = Coldage(msg='msg', txt='txt', bny='bny') - - class Parser: """ Parser is stream parser that processes an incoming message stream. @@ -126,45 +79,7 @@ def __init__(self, ims=None, framed=True, pipeline=False, kvy=None, self.vry = vry self.local = True if local else False - @staticmethod - def sniff(ims): - """ - Returns status string of cold start of stream ims bytearray by looking - at first triplet of first byte to determin if message or counter code - and if counter code whether Base64 or Base2 representation - - First three bits: - 0o0 = 000 free - 0o1 = 001 cntcode B64 - 0o2 = 010 opcode B64 - 0o3 = 011 json - 0o4 = 100 mgpk - 0o5 = 101 cbor - 0o6 = 110 mgpk - 007 = 111 cntcode or opcode B2 - - counter B64 in (0o1, 0o2) return 'txt' - counter B2 in (0o7) return 'bny' - event in (0o3, 0o4, 0o5, 0o6) return 'evt' - unexpected in (0o0) raise ColdStartError - Colds = Coldage(msg='msg', txt='txt', bny='bny') - - 'msg' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) - 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) - 'bny' if tritet in (ColdDex.CtOpB2,) - """ - if not ims: - raise kering.ShortageError("Need more bytes.") - - tritet = ims[0] >> 5 - if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2): - return Colds.msg - if tritet in (ColdDex.CtB64, ColdDex.OpB64): - return Colds.txt - if tritet in (ColdDex.CtOpB2,): - return Colds.bny - raise kering.ColdStartError("Unexpected tritet={} at stream start.".format(tritet)) @staticmethod def extract(ims, klas, cold=Colds.txt): @@ -770,7 +685,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, while not ims: yield - cold = self.sniff(ims) # check for spurious counters at front of stream + cold = sniff(ims) # check for spurious counters at front of stream if cold in (Colds.txt, Colds.bny): # not message error out to flush stream # replace with pipelining here once CESR message format supported. raise kering.ColdStartError("Expecting message counter tritet={}" @@ -811,7 +726,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, # extract attachments must start with counter so know if txt or bny. while not ims: yield - cold = self.sniff(ims) # expect counter at front of attachments + cold = sniff(ims) # expect counter at front of attachments if cold != Colds.msg: # not new message so process attachments ctr = yield from self._extractor(ims=ims, klas=Counter, cold=cold) if ctr.code == CtrDex.AttachedMaterialQuadlets: # pipeline ctr? @@ -1036,7 +951,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, # group may switch stream state txt or bny if not ims: # end of frame break - cold = self.sniff(ims) + cold = sniff(ims) if cold == Colds.msg: # new message so attachments done break # finished attachments since new message else: # process until next message @@ -1044,7 +959,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, # group may switch stream state txt or bny while not ims: yield # no frame so must wait for next message - cold = self.sniff(ims) # ctr or msg + cold = sniff(ims) # ctr or msg if cold == Colds.msg: # new message break # finished attachments since new message diff --git a/src/keri/kering.py b/src/keri/kering.py index c9fbc7fac..84900e88a 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -5,7 +5,7 @@ import sys import re from collections import namedtuple - +from dataclasses import dataclass, astuple FALSEY = (False, 0, None, "?0", "no", "false", "False", "off") TRUTHY = (True, 1, "?1", "yes" "true", "True", 'on') @@ -93,21 +93,9 @@ def deversify(vs, version=None): raise ValueError("Invalid version string = {}".format(vs)) - MAXVSOFFSET = 12 SMELLSIZE = MAXVSOFFSET + VERFULLSIZE # min buffer size to inhale -#""" -#Smellage (results of smelling a version string such as in a Serder) - #proto (str): protocol type value of Protos examples 'KERI', 'ACDC' - #major (str): single char hex string of major version number - #minor (str): single char hex string of minor version number - #kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' - #size (str): hex string of size of raw serialization - -#""" -#Smellage = namedtuple("Smellage", "proto major minor kind size") - """ Smellage (results of smelling a version string such as in a Serder) protocol (str): protocol type value of Protos examples 'KERI', 'ACDC' @@ -170,38 +158,93 @@ def smell(raw, *, version=None): return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) -def smelly(raw, *, version=None): - """Extract and return instance of Smellage from version string inside - raw serialization. +@dataclass(frozen=True) +class ColdCodex: + """ + ColdCodex is codex of cold stream start tritets of first byte + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. - Returns: - smellage (Smellage): named Tuple of (protocol, version, kind, size) + First three bits: + 0o0 = 000 free + 0o1 = 001 cntcode B64 + 0o2 = 010 opcode B64 + 0o3 = 011 json + 0o4 = 100 mgpk + 0o5 = 101 cbor + 0o6 = 110 mgpk + 007 = 111 cntcode or opcode B2 - Parameters: - raw (bytearray) of serialized incoming message stream. Assumes start - of stream is JSON, CBOR, or MGPK field map with first field - is labeled 'v' and value is version string. - version (Versionage | None): instance supported protocol version - None means do not enforce a supported version + status is one of ('evt', 'txt', 'bny' ) + 'evt' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) + 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) + 'bny' if tritet in (ColdDex.CtOpB2,) + + otherwise raise ColdStartError + + x = bytearray([0x2d, 0x5f]) + x == bytearray(b'-_') + x[0] >> 5 == 0o1 + True """ - if len(raw) < SMELLSIZE: - raise ShortageError(f"Need more raw bytes to smell full version string.") + Free: int = 0o0 # not taken + CtB64: int = 0o1 # CountCode Base64 + OpB64: int = 0o2 # OpCode Base64 + JSON: int = 0o3 # JSON Map Event Start + MGPK1: int = 0o4 # MGPK Fixed Map Event Start + CBOR: int = 0o5 # CBOR Map Event Start + MGPK2: int = 0o6 # MGPK Big 16 or 32 Map Event Start + CtOpB2: int = 0o7 # CountCode or OpCode Base2 - match = Rever.search(raw) # Rever regex takes bytes/bytearray not str - if not match or match.start() > MAXVSOFFSET: - raise VersionError(f"Invalid version string from smelled raw = " - f"{raw[: SMELLSIZE]}.") + def __iter__(self): + return iter(astuple(self)) - smellage = Smellage(*match.group("proto", "major", "minor", "kind", "size")) - # Global version compatibility check. Serder instances also peform version check - vrsn = Versionage(major=int(smellage.major, 16), minor=int(smellage.minor, 16)) - if version: # test here for compatible code version with message vrsn - if (vrsn.major > version.major or - (vrsn.major == version.major and vrsn.minor > version.minor)): - pass # raise error here? +ColdDex = ColdCodex() # Make instance + +Coldage = namedtuple("Coldage", 'msg txt bny') # stream cold start status +Colds = Coldage(msg='msg', txt='txt', bny='bny') + + +def sniff(ims): + """ + Returns status string of cold start of stream ims bytearray by looking + at first triplet of first byte to determin if message or counter code + and if counter code whether Base64 or Base2 representation + + First three bits: + 0o0 = 000 free + 0o1 = 001 cntcode B64 + 0o2 = 010 opcode B64 + 0o3 = 011 json + 0o4 = 100 mgpk + 0o5 = 101 cbor + 0o6 = 110 mgpk + 007 = 111 cntcode or opcode B2 + + counter B64 in (0o1, 0o2) return 'txt' + counter B2 in (0o7) return 'bny' + event in (0o3, 0o4, 0o5, 0o6) return 'evt' + unexpected in (0o0) raise ColdStartError + Colds = Coldage(msg='msg', txt='txt', bny='bny') + + 'msg' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) + 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) + 'bny' if tritet in (ColdDex.CtOpB2,) + """ + if not ims: + raise ShortageError("Need more bytes.") + + tritet = ims[0] >> 5 + if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2): + return Colds.msg + if tritet in (ColdDex.CtB64, ColdDex.OpB64): + return Colds.txt + if tritet in (ColdDex.CtOpB2,): + return Colds.bny + + raise ColdStartError("Unexpected tritet={} at stream start.".format(tritet)) - return smellage """ ilk is short for packet or message type for a given protocol From 3e838f89f64be0af7286f67752507f41ac03032a Mon Sep 17 00:00:00 2001 From: Lance <2byrds@gmail.com> Date: Thu, 22 Feb 2024 21:15:28 -0500 Subject: [PATCH 016/418] fix cloneCreds usage of cloneTvtAt to be consistent with grant, etc. and add some coverage (#682) Signed-off-by: 2byrds <2byrds@gmail.com> --- src/keri/vdr/viring.py | 2 +- tests/vdr/test_verifying.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 01a1ce676..96eec603a 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -402,7 +402,7 @@ def cloneCreds(self, saids, db): atc = bytearray(signing.serialize(creder, prefixer, seqner, saider)) del atc[0:creder.size] - iss = bytearray(self.cloneTvtAt(pre=prefixer.qb64, sn=seqner.sn)) + iss = bytearray(self.cloneTvtAt(creder.said)) iserder = serdering.SerderKERI(raw=iss) issatc = bytes(iss[iserder.size:]) diff --git a/tests/vdr/test_verifying.py b/tests/vdr/test_verifying.py index 9a5fbb578..70a420f57 100644 --- a/tests/vdr/test_verifying.py +++ b/tests/vdr/test_verifying.py @@ -108,6 +108,12 @@ def test_verifier(seeder): assert saider[0].qb64 == creder.said saider = regery.reger.schms.get("EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC") assert saider[0].qb64 == creder.said + + # also try it via the cloneCreds function + creds = regery.reger.cloneCreds(saids=saider, db=hab.db) + + for idx, cred in enumerate(creds): + assert dcre.sad == cred["sad"] """End Test""" From e494e6ee65ad93a7f16bc326b9f46b219c4b19ab Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 23 Feb 2024 10:08:32 -0700 Subject: [PATCH 017/418] Added placeholder Streamer class --- src/keri/core/coring.py | 76 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 78c449e3e..a87ccf413 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -1861,6 +1861,82 @@ def datetime(self): return helping.fromIso8601(self.dts) +class Streamer: + """ + Streamer is CESR sniffable stream class + + + Has the following public properties: + + Properties: + + + Methods: + + + Hidden: + + + + """ + + def __init__(self, stream): + """Initialize instance + + + Parameters: + stream (bytes | bytearray): sniffable CESR stream + + + """ + self._stream = bytes(stream) + + + @property + def stream(self): + """stream property getter + """ + return self._stream + + @property + def text(self): + """expanded stream as qb64 text + Returns: + stream (bytes): expanded text qb64 version of stream + + """ + return self._stream + + @property + def binary(self): + """compacted stream as qb2 binary + Returns: + stream (bytes): compacted binary qb2 version of stream + + """ + return self._stream + + @property + def texter(self): + """expanded stream as Texter instance + Returns: + texter (Texter): Texter primitive of stream suitable wrapping + + """ + return self._stream + + @property + def bexter(self): + """expanded stream as Bexter instance + Returns: + bexter (Bexter): Bexter primitive as stream suitable wrapping + + """ + return self._stream + + + + class Texter(Matter): """ Texter is subclass of Matter, cryptographic material, for variable length From 6a03aeefb9e7cfd948574ecd2d5235c7ae2925b1 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 23 Feb 2024 10:11:02 -0700 Subject: [PATCH 018/418] syntax cleanup 2 blank lines between methods --- src/keri/core/parsing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index 72207101b..3a72354ea 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -80,7 +80,6 @@ def __init__(self, ims=None, framed=True, pipeline=False, kvy=None, self.local = True if local else False - @staticmethod def extract(ims, klas, cold=Colds.txt): """ @@ -95,6 +94,7 @@ def extract(ims, klas, cold=Colds.txt): else: raise kering.ColdStartError("Invalid stream state cold={}.".format(cold)) + @staticmethod def _extractor(ims, klas, cold=Colds.txt, abort=False): """ @@ -122,6 +122,7 @@ def _extractor(ims, klas, cold=Colds.txt, abort=False): raise # bad pipelined frame so abort by raising error yield + def _sadPathSigGroup(self, ctr, ims, root=None, cold=Colds.txt, pipelined=False): """ @@ -174,6 +175,7 @@ def _sadPathSigGroup(self, ctr, ims, root=None, cold=Colds.txt, pipelined=False) "count code={}.Expected code={}." "".format(ctr.code, CtrDex.ControllerIdxSigs)) + def _transIdxSigGroups(self, ctr, ims, cold=Colds.txt, pipelined=False): """ Extract attaced trans indexed sig groups each made of @@ -228,6 +230,7 @@ def _transIdxSigGroups(self, ctr, ims, cold=Colds.txt, pipelined=False): yield prefixer, seqner, saider, isigers + def _nonTransReceiptCouples(self, ctr, ims, cold=Colds.txt, pipelined=False): """ Extract attached rct couplets into list of sigvers From 98cbbacee3da039dd00032b57bfd257b0938ba4b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 23 Feb 2024 11:41:04 -0700 Subject: [PATCH 019/418] Moved tests relevant to kering.py to test_kering.py Removed field labels constants from kering.py as tests for these are handled by Serder.verify. This makes the tests versioned to the field label by version instead of hard coded. --- src/keri/core/coring.py | 84 ++-------- src/keri/core/eventing.py | 10 +- src/keri/core/routing.py | 8 +- src/keri/core/serdering.py | 18 --- src/keri/kering.py | 28 ---- src/keri/vdr/eventing.py | 38 ++--- tests/core/test_coring.py | 307 +------------------------------------ tests/test_kering.py | 290 +++++++++++++++++++++++++++++++++++ 8 files changed, 336 insertions(+), 447 deletions(-) create mode 100644 tests/test_kering.py diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index a87ccf413..dffec01ff 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -38,20 +38,12 @@ from ..kering import (Versionage, Version, VERRAWSIZE, VERFMT, VERFULLSIZE, versify, deversify, Rever, smell) from ..kering import Serials, Serialage, Protos, Protocolage, Ilkage, Ilks -from ..kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, - RPY_LABELS) -from ..kering import (VCP_LABELS, VRT_LABELS, ISS_LABELS, BIS_LABELS, REV_LABELS, - BRV_LABELS, TSN_LABELS, CRED_TSN_LABELS) + from ..help import helping from ..help.helping import sceil, nonStringIterable, nonStringSequence -Labels = Ilkage(icp=ICP_LABELS, rot=ROT_LABELS, ixn=IXN_LABELS, dip=DIP_LABELS, - drt=DRT_LABELS, rct=[], qry=[], rpy=RPY_LABELS, - exn=[], pro=[], bar=[], - vcp=VCP_LABELS, vrt=VRT_LABELS, iss=ISS_LABELS, rev=REV_LABELS, - bis=BIS_LABELS, brv=BRV_LABELS) DSS_SIG_MODE = "fips-186-3" @@ -59,10 +51,6 @@ ECDSA_256k1_SEEDBYTES = 32 -Vstrings = Serialage(json=versify(kind=Serials.json, size=0), - mgpk=versify(kind=Serials.mgpk, size=0), - cbor=versify(kind=Serials.cbor, size=0)) - # SAID field labels Saidage = namedtuple("Saidage", "dollar at id_ i d") @@ -304,7 +292,8 @@ def loads(raw, size=None, kind=Serials.json): return ked - +# deprecated don't use anymore need to fix demo tests that use +# use with context instead def generateSigners(salt=None, count=8, transferable=True): """ Returns list of Signers for Ed25519 @@ -2840,51 +2829,6 @@ def _secp256k1(ser, seed, verfer, index, only=False, ondex=None, **kwa): ondex=ondex, verfer=verfer,) - # def derive_index_code(code, index, only=False, ondex=None, **kwa): - # # should add Indexer class method to get ms main index size for given code - # if only: # only main index ondex not used - # ondex = None - # if index <= 63: # (64 ** ms - 1) where ms is main index size, use small current only - # if code == MtrDex.Ed25519_Seed: - # indxSigCode = IdrDex.Ed25519_Crt_Sig - # elif code == MtrDex.ECDSA_256r1_Seed: - # indxSigCode = IdrDex.ECDSA_256r1_Crt_Sig - # elif code == MtrDex.ECDSA_256k1_Seed: - # indxSigCode = IdrDex.ECDSA_256k1_Crt_Sig - # else: - # raise ValueError("Unsupported signer code = {}.".format(code)) - # else: # use big current only - # if code == MtrDex.Ed25519_Seed: - # indxSigCode = IdrDex.Ed25519_Big_Crt_Sig - # elif code == MtrDex.ECDSA_256r1_Seed: - # indxSigCode = IdrDex.ECDSA_256r1_Big_Crt_Sig - # elif code == MtrDex.ECDSA_256k1_Seed: - # indxSigCode = IdrDex.ECDSA_256k1_Big_Crt_Sig - # else: - # raise ValueError("Unsupported signer code = {}.".format(code)) - # else: # both - # if ondex == None: - # ondex = index # enable default to be same - # if ondex == index and index <= 63: # both same and small so use small both same - # if code == MtrDex.Ed25519_Seed: - # indxSigCode = IdrDex.Ed25519_Sig - # elif code == MtrDex.ECDSA_256r1_Seed: - # indxSigCode = IdrDex.ECDSA_256r1_Sig - # elif code == MtrDex.ECDSA_256k1_Seed: - # indxSigCode = IdrDex.ECDSA_256k1_Sig - # else: - # raise ValueError("Unsupported signer code = {}.".format(code)) - # else: # otherwise big or both not same so use big both - # if code == MtrDex.Ed25519_Seed: - # indxSigCode = IdrDex.Ed25519_Big_Sig - # elif code == MtrDex.ECDSA_256r1_Seed: - # indxSigCode = IdrDex.ECDSA_256r1_Big_Sig - # elif code == MtrDex.ECDSA_256k1_Seed: - # indxSigCode = IdrDex.ECDSA_256k1_Big_Sig - # else: - # raise ValueError("Unsupported signer code = {}.".format(code)) - - # return (indxSigCode, ondex) class Salter(Matter): """ @@ -3618,11 +3562,12 @@ def derive(self, ked): if ilk not in (Ilks.icp, Ilks.dip, Ilks.vcp, Ilks.iss): raise ValueError("Nonincepting ilk={} for prefix derivation.".format(ilk)) - labels = getattr(Labels, ilk) - for k in labels: - if k not in ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, ked)) + # Serder now does this check + #labels = getattr(Labels, ilk) + #for k in labels: + #if k not in ked: + #raise ValidationError("Missing element = {} from {} event for " + #"evt = {}.".format(k, ilk, ked)) return (self._derive(ked=ked)) @@ -3639,11 +3584,12 @@ def verify(self, ked, prefixed=False): if ilk not in (Ilks.icp, Ilks.dip, Ilks.vcp, Ilks.iss): raise ValueError("Nonincepting ilk={} for prefix derivation.".format(ilk)) - labels = getattr(Labels, ilk) - for k in labels: - if k not in ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, ked)) + # Serder now does this check + #labels = getattr(Labels, ilk) + #for k in labels: + #if k not in ked: + #raise ValidationError("Missing element = {} from {} event for " + #"evt = {}.".format(k, ilk, ked)) return (self._verify(ked=ked, pre=self.qb64, prefixed=prefixed)) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 4ac1f003c..3a5a3fac2 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -34,8 +34,6 @@ QueryNotFoundError, MisfitEventSourceError, MissingDelegableApprovalError) from ..kering import Version, Versionage -from ..kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, - RPY_LABELS) from ..help import helping @@ -2071,10 +2069,10 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, raise ValidationError("Unexpected non-establishment event = {}." "".format(serder.ked)) - for k in IXN_LABELS: - if k not in ked: - raise ValidationError("Missing element = {} from {} event." - " evt = {}.".format(k, Ilks.ixn, ked)) + #for k in IXN_LABELS: + #if k not in ked: + #raise ValidationError("Missing element = {} from {} event." + #" evt = {}.".format(k, Ilks.ixn, ked)) if not sner.num == (self.sner.num + 1): # sn not in order raise ValidationError("Invalid sn = {} expecting = {} for evt " diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 6dac3a791..2b3856b41 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -183,10 +183,10 @@ def processReply(self, serder, cigars=None, tsgs=None): Escrow process logic is route dependent and is dispatched by route, i.e. route is address of buffer with route specific handler of escrow. """ - for k in eventing.RPY_LABELS: - if k not in serder.ked: - raise kering.ValidationError(f"Missing element={k} from {coring.Ilks.rpy}" - f" msg={serder.ked}.") + #for k in eventing.RPY_LABELS: + #if k not in serder.ked: + #raise kering.ValidationError(f"Missing element={k} from {coring.Ilks.rpy}" + #f" msg={serder.ked}.") # fetch from serder to process ked = serder.ked diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index ae1264887..d3bb8940d 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -761,24 +761,6 @@ def _inhale(clas, raw, version=Version, smellage=None): else: # not passed in so smell raw proto, vrsn, kind, size = smell(raw, version=version) - - #proto = proto.decode("utf-8") - #if proto not in Protos: - #raise ProtocolError(f"Invalid protocol type = {proto}.") - - #vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) - #if version is not None and vrsn != version: - #raise VersionError(f"Expected version = {version}, got " - #f"{vrsn.major}.{vrsn.minor}.") - - #kind = kind.decode("utf-8") - #if kind not in Serials: - #raise KindError(f"Invalid serialization kind = {kind}.") - - #size = int(size, 16) - #if len(raw) < size: - #raise ShortageError(f"Need more bytes.") - sad = clas.loads(raw=raw, size=size, kind=kind) if "v" not in sad: # Regex does not check for version string label itself diff --git a/src/keri/kering.py b/src/keri/kering.py index 84900e88a..10cb737cd 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -292,34 +292,6 @@ def sniff(ims): watcher='watcher', judge='judge', juror='juror', peer='peer', mailbox="mailbox", agent="agent") -ICP_LABELS = ["v", "t", "d", "i", "s", "kt", "k", "nt", "n", - "bt", "b", "c", "a"] -DIP_LABELS = ["v", "d", "i", "s", "t", "kt", "k", "nt", "n", - "bt", "b", "c", "a", "di"] -ROT_LABELS = ["v", "d", "i", "s", "t", "p", "kt", "k", "nt", "n", - "bt", "br", "ba", "a"] -DRT_LABELS = ["v", "d", "i", "s", "t", "p", "kt", "k", "nt", "n", - "bt", "br", "ba", "a"] -IXN_LABELS = ["v", "d", "i", "s", "t", "p", "a"] - -#KSN_LABELS = ["v", "d", "i", "s", "p", "d", "f", "dt", "et", "kt", "k", "nt", "n", - #"bt", "b", "c", "ee", "di"] - -RPY_LABELS = ["v", "d", "t", "d", "dt", "r", "a"] - -VCP_LABELS = ["v", "d", "i", "s", "t", "bt", "b", "c"] -VRT_LABELS = ["v", "d", "i", "s", "t", "p", "bt", "b", "ba", "br"] - -ISS_LABELS = ["v", "i", "s", "t", "ri", "dt"] -BIS_LABELS = ["v", "i", "s", "t", "ra", "dt"] - -REV_LABELS = ["v", "i", "s", "t", "p", "dt"] -BRV_LABELS = ["v", "i", "s", "t", "ra", "p", "dt"] - -TSN_LABELS = ["v", "i", "s", "d", "ii", "a", "et", "bt", "b", "c", "br", "ba"] -CRED_TSN_LABELS = ["v", "i", "s", "d", "ri", "a", "ra"] - - # Exception Subclasses class KeriError(Exception): diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 8da3cc81d..47a1b54be 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -26,8 +26,6 @@ from ..help import helping from ..kering import (MissingWitnessSignatureError, Version, MissingAnchorError, ValidationError, OutOfOrderError, LikelyDuplicitousError) -from ..kering import (VCP_LABELS, VRT_LABELS, ISS_LABELS, BIS_LABELS, REV_LABELS, - BRV_LABELS, TSN_LABELS, CRED_TSN_LABELS) from ..vdr import viring logger = help.ogler.getLogger() @@ -718,11 +716,11 @@ def __init__(self, cues=None, rsr=None, serder=None, seqner=None, saider=None, raise ValidationError("Expected ilk {} got {} for evt: {}".format(Ilks.vcp, ilk, serder)) self.ilk = ilk - labels = VCP_LABELS - for k in labels: - if k not in serder.ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, serder.ked)) + #labels = VCP_LABELS + #for k in labels: + #if k not in serder.ked: + #raise ValidationError("Missing element = {} from {} event for " + #"evt = {}.".format(k, ilk, serder.ked)) self.incept(serder=serder) self.config(serder=serder, noBackers=noBackers, estOnly=estOnly) @@ -963,8 +961,8 @@ def rotate(self, serder, sn): ilk = ked["t"] dig = ked["p"] - # XXXX should there be validation of labels here - labels = VRT_LABELS # assumes ilk == Ilks.vrt + + #labels = VRT_LABELS # assumes ilk == Ilks.vrt #for k in labels: #if k not in ked: #raise ValidationError("Missing element = {} from {} event for " @@ -1053,12 +1051,11 @@ def issue(self, serder, seqner, saider, sn, bigers=None): ilk = ked["t"] vci = vcpre - labels = ISS_LABELS if ilk == Ilks.iss else BIS_LABELS - - for k in labels: - if k not in ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, ked)) + #labels = ISS_LABELS if ilk == Ilks.iss else BIS_LABELS + #for k in labels: + #if k not in ked: + #raise ValidationError("Missing element = {} from {} event for " + #"evt = {}.".format(k, ilk, ked)) if ilk == Ilks.iss: # simple issue if self.noBackers is False: @@ -1120,12 +1117,11 @@ def revoke(self, serder, seqner, saider, sn, bigers=None): vcpre = ked["i"] ilk = ked["t"] - labels = REV_LABELS if ilk == Ilks.rev else BRV_LABELS - - for k in labels: - if k not in ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, ked)) + #labels = REV_LABELS if ilk == Ilks.rev else BRV_LABELS + #for k in labels: + #if k not in ked: + #raise ValidationError("Missing element = {} from {} event for " + #"evt = {}.".format(k, ilk, ked)) # have to compare with VC issuance serder vci = vcpre diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 561c55a24..79bda8661 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -25,10 +25,10 @@ from keri.core import coring from keri.core import eventing -from keri.core.coring import (Ilkage, Ilks, Labels, Saids, Protos, Protocolage, +from keri.core.coring import (Ilkage, Ilks, Saids, Protos, Protocolage, Sadder, Tholder, Seqner, NumDex, Number, Siger, Dater, Bexter, Texter) -from keri.core.coring import Serialage, Serials, Tiers, Vstrings +from keri.core.coring import Serialage, Serials, Tiers from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, Indexer, CtrDex, Counter, ProDex) @@ -44,29 +44,13 @@ InvalidValueError, DeserializeError, ValidationError, InvalidVarRawSizeError) from keri.kering import Version, Versionage, VersionError -from keri.kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, - RPY_LABELS) -from keri.kering import (VCP_LABELS, VRT_LABELS, ISS_LABELS, BIS_LABELS, REV_LABELS, - BRV_LABELS, TSN_LABELS, CRED_TSN_LABELS) +#from keri.kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, + #RPY_LABELS) +#from keri.kering import (VCP_LABELS, VRT_LABELS, ISS_LABELS, BIS_LABELS, REV_LABELS, + #BRV_LABELS, TSN_LABELS, CRED_TSN_LABELS) -def test_protos(): - """ - Test protocols namedtuple instance Protos - """ - - assert isinstance(Protos, Protocolage) - - assert Protos.keri == 'KERI' - assert Protos.crel == 'CREL' - assert Protos.acdc == 'ACDC' - - assert 'KERI' in Protos - assert 'CREL' in Protos - assert 'ACDC' in Protos - - """End Test""" def test_protocol_genus_codex(): """ @@ -87,101 +71,6 @@ def test_protocol_genus_codex(): """End Test""" -def test_ilks(): - """ - Test Ilkage namedtuple instance Ilks - """ - assert Ilks == Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', - rct='rct', qry='qry', rpy='rpy', - exn='exn', pro='pro', bar='bar', - vcp='vcp', vrt='vrt', - iss='iss', rev='rev', bis='bis', brv='brv', ) - - assert isinstance(Ilks, Ilkage) - - for fld in Ilks._fields: - assert fld == getattr(Ilks, fld) - - assert 'icp' in Ilks - assert Ilks.icp == 'icp' - assert 'rot' in Ilks - assert Ilks.rot == 'rot' - assert 'ixn' in Ilks - assert Ilks.ixn == 'ixn' - assert 'dip' in Ilks - assert Ilks.dip == 'dip' - assert 'drt' in Ilks - assert Ilks.drt == 'drt' - assert 'rct' in Ilks - assert Ilks.rct == 'rct' - assert 'qry' in Ilks - assert Ilks.qry == 'qry' - assert 'rpy' in Ilks - assert Ilks.rpy == 'rpy' - assert 'exn' in Ilks - assert Ilks.exn == 'exn' - - - assert 'pro' in Ilks - assert Ilks.pro == 'pro' - assert 'bar' in Ilks - assert Ilks.bar == 'bar' - - - assert 'vcp' in Ilks - assert Ilks.vcp == 'vcp' - assert 'vrt' in Ilks - assert Ilks.vrt == 'vrt' - assert 'iss' in Ilks - assert Ilks.iss == 'iss' - assert 'rev' in Ilks - assert Ilks.rev == 'rev' - assert 'bis' in Ilks - assert Ilks.bis == 'bis' - assert 'brv' in Ilks - assert Ilks.brv == 'brv' - - """End Test """ - -def test_labels(): - """ - Test Ilkage namedtuple instance Labels - """ - assert Labels == Ilkage(icp=ICP_LABELS, rot=ROT_LABELS, ixn=IXN_LABELS, - dip=DIP_LABELS, drt=DRT_LABELS, - rct=[], qry=[], rpy=RPY_LABELS, - exn=[], pro=[], bar=[], - vcp=VCP_LABELS, vrt=VRT_LABELS, iss=ISS_LABELS, - rev=REV_LABELS, bis=BIS_LABELS, brv=BRV_LABELS) - - assert isinstance(Labels, Ilkage) - - for fld in Labels._fields: - assert isinstance(getattr(Labels, fld), list) - - assert Labels.icp == ICP_LABELS - assert Labels.rot == ROT_LABELS - assert Labels.ixn == IXN_LABELS - assert Labels.dip == DIP_LABELS - assert Labels.drt == DRT_LABELS - assert Labels.rct == [] - assert Labels.qry == [] - assert Labels.rpy == RPY_LABELS - assert Labels.exn == [] - assert Labels.pro == [] - assert Labels.bar == [] - - assert Labels.vcp == VCP_LABELS - assert Labels.vrt == VRT_LABELS - assert Labels.iss == ISS_LABELS - assert Labels.rev == REV_LABELS - assert Labels.bis == BIS_LABELS - assert Labels.brv == BRV_LABELS - - """End Test """ - - - def test_b64_conversions(): """ @@ -6254,190 +6143,6 @@ def test_saider(): """Done Test""" -def test_serials(): - """ - Test Serializations namedtuple instance Serials - """ - assert Version == Versionage(major=1, minor=0) - - assert isinstance(Serials, Serialage) - - assert Serials.json == 'JSON' - assert Serials.mgpk == 'MGPK' - assert Serials.cbor == 'CBOR' - - assert 'JSON' in Serials - assert 'MGPK' in Serials - assert 'CBOR' in Serials - - assert Vstrings.json == 'KERI10JSON000000_' - assert Vstrings.mgpk == 'KERI10MGPK000000_' - assert Vstrings.cbor == 'KERI10CBOR000000_' - - icp = dict(vs=Vstrings.json, - pre='AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM', - sn='0001', - ilk='icp', - dig='DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfS', - sith=1, - keys=['AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM'], - nxt='DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM', - toad=0, - wits=[], - cnfg=[], - ) - - rot = dict(vs=Vstrings.json, - pre='AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM', - sn='0001', - ilk='rot', - dig='DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfS', - sith=1, - keys=['AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM'], - nxt='DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM', - toad=0, - cuts=[], - adds=[], - data=[], - ) - - icps = json.dumps(icp, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - assert len(icps) == 303 - assert icps == (b'{"vs":"KERI10JSON000000_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' - b'","sn":"0001","ilk":"icp","dig":"DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' - b'S","sith":1,"keys":["AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"],"nxt":"' - b'DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM","toad":0,"wits":[],"cnfg":[]}') - - match = Rever.search(icps) - assert match.group() == Vstrings.json.encode("utf-8") - - rots = json.dumps(rot, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - assert len(rots) == 313 - assert rots == (b'{"vs":"KERI10JSON000000_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' - b'","sn":"0001","ilk":"rot","dig":"DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' - b'S","sith":1,"keys":["AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"],"nxt":"' - b'DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM","toad":0,"cuts":[],"adds":[],"' - b'data":[]}') - - match = Rever.search(rots) - assert match.group() == Vstrings.json.encode("utf-8") - - icp["vs"] = Vstrings.mgpk - icps = msgpack.dumps(icp) - assert len(icps) == 264 - assert icps == (b'\x8b\xa2vs\xb1KERI10MGPK000000_\xa3pre\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' - b'SVPzhzS6b5CM\xa2sn\xa40001\xa3ilk\xa3icp\xa3dig\xd9,DVPzhzS6b5CMaU6JR2nmwy' - b'Z-i0d8JZAoTNZH3ULvYAfS\xa4sith\x01\xa4keys\x91\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZ' - b'H3ULvYAfSVPzhzS6b5CM\xa3nxt\xd9,DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5' - b'CM\xa4toad\x00\xa4wits\x90\xa4cnfg\x90') - - match = Rever.search(icps) - assert match.group() == Vstrings.mgpk.encode("utf-8") - - rot["vs"] = Vstrings.mgpk - rots = msgpack.dumps(rot) - assert len(rots) == 270 - assert rots == (b'\x8c\xa2vs\xb1KERI10MGPK000000_\xa3pre\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' - b'SVPzhzS6b5CM\xa2sn\xa40001\xa3ilk\xa3rot\xa3dig\xd9,DVPzhzS6b5CMaU6JR2nmwy' - b'Z-i0d8JZAoTNZH3ULvYAfS\xa4sith\x01\xa4keys\x91\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZ' - b'H3ULvYAfSVPzhzS6b5CM\xa3nxt\xd9,DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5' - b'CM\xa4toad\x00\xa4cuts\x90\xa4adds\x90\xa4data\x90') - - match = Rever.search(rots) - assert match.group() == Vstrings.mgpk.encode("utf-8") - - icp["vs"] = Vstrings.cbor - icps = cbor.dumps(icp) - assert len(icps) == 264 - assert icps == (b'\xabbvsqKERI10CBOR000000_cprex,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' - b'bsnd0001cilkcicpcdigx,DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSdsith\x01' - b'dkeys\x81x,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CMcnxtx,DZ-i0d8JZAoTNZ' - b'H3ULvaU6JR2nmwyYAfSVPzhzS6b5CMdtoad\x00dwits\x80dcnfg\x80') - - match = Rever.search(icps) - assert match.group() == Vstrings.cbor.encode("utf-8") - - rot["vs"] = Vstrings.cbor - rots = cbor.dumps(rot) - assert len(rots) == 270 - assert rots == (b'\xacbvsqKERI10CBOR000000_cprex,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' - b'bsnd0001cilkcrotcdigx,DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSdsith\x01' - b'dkeys\x81x,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CMcnxtx,DZ-i0d8JZAoTNZ' - b'H3ULvaU6JR2nmwyYAfSVPzhzS6b5CMdtoad\x00dcuts\x80dadds\x80ddata\x80') - - match = Rever.search(rots) - assert match.group() == Vstrings.cbor.encode("utf-8") - """Done Test""" - - -def test_versify(): - """ - Test Versify support - """ - vs = versify(kind=Serials.json, size=0) - assert vs == "KERI10JSON000000_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.json - assert version == Version - assert size == 0 - - vs = versify(kind=Serials.json, size=65) - assert vs == "KERI10JSON000041_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.json - assert version == Version - assert size == 65 - - vs = versify(proto=Protos.acdc, kind=Serials.json, size=86) - assert vs == "ACDC10JSON000056_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.acdc - assert kind == Serials.json - assert version == Version - assert size == 86 - - vs = versify(kind=Serials.mgpk, size=0) - assert vs == "KERI10MGPK000000_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.mgpk - assert version == Version - assert size == 0 - - vs = versify(kind=Serials.mgpk, size=65) - assert vs == "KERI10MGPK000041_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.mgpk - assert version == Version - assert size == 65 - - vs = versify(kind=Serials.cbor, size=0) - assert vs == "KERI10CBOR000000_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.cbor - assert version == Version - assert size == 0 - - vs = versify(kind=Serials.cbor, size=65) - assert vs == "KERI10CBOR000041_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.cbor - assert version == Version - assert size == 65 - """End Test""" - def test_tholder(): diff --git a/tests/test_kering.py b/tests/test_kering.py new file mode 100644 index 000000000..51cbb9230 --- /dev/null +++ b/tests/test_kering.py @@ -0,0 +1,290 @@ +# -*- encoding: utf-8 -*- +""" +tests.test_kering module + +""" + +import json + +import cbor2 as cbor +import msgpack + +from keri import kering +from keri.kering import Protocolage, Protos +from keri.kering import Serialage, Serials +from keri.kering import (Versionage, Version, VERFULLSIZE, + versify, deversify, Rever) +from keri.kering import Ilkage, Ilks + + + + +def test_protos(): + """ + Test protocols namedtuple instance Protos + """ + + assert isinstance(Protos, Protocolage) + + assert Protos.keri == 'KERI' + assert Protos.crel == 'CREL' + assert Protos.acdc == 'ACDC' + + assert 'KERI' in Protos + assert 'CREL' in Protos + assert 'ACDC' in Protos + + """End Test""" + + +def test_serials(): + """ + Test Serializations namedtuple instance Serials + """ + Vstrings = Serialage(json=versify(kind=Serials.json, size=0), + mgpk=versify(kind=Serials.mgpk, size=0), + cbor=versify(kind=Serials.cbor, size=0)) + + assert Version == Versionage(major=1, minor=0) + + assert isinstance(Serials, Serialage) + + assert Serials.json == 'JSON' + assert Serials.mgpk == 'MGPK' + assert Serials.cbor == 'CBOR' + + assert 'JSON' in Serials + assert 'MGPK' in Serials + assert 'CBOR' in Serials + + assert Vstrings.json == 'KERI10JSON000000_' + assert Vstrings.mgpk == 'KERI10MGPK000000_' + assert Vstrings.cbor == 'KERI10CBOR000000_' + + icp = dict(vs=Vstrings.json, + pre='AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM', + sn='0001', + ilk='icp', + dig='DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfS', + sith=1, + keys=['AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM'], + nxt='DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM', + toad=0, + wits=[], + cnfg=[], + ) + + rot = dict(vs=Vstrings.json, + pre='AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM', + sn='0001', + ilk='rot', + dig='DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfS', + sith=1, + keys=['AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM'], + nxt='DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM', + toad=0, + cuts=[], + adds=[], + data=[], + ) + + icps = json.dumps(icp, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + assert len(icps) == 303 + assert icps == (b'{"vs":"KERI10JSON000000_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' + b'","sn":"0001","ilk":"icp","dig":"DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' + b'S","sith":1,"keys":["AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"],"nxt":"' + b'DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM","toad":0,"wits":[],"cnfg":[]}') + + match = Rever.search(icps) + assert match.group() == Vstrings.json.encode("utf-8") + + rots = json.dumps(rot, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + assert len(rots) == 313 + assert rots == (b'{"vs":"KERI10JSON000000_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' + b'","sn":"0001","ilk":"rot","dig":"DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' + b'S","sith":1,"keys":["AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"],"nxt":"' + b'DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM","toad":0,"cuts":[],"adds":[],"' + b'data":[]}') + + match = Rever.search(rots) + assert match.group() == Vstrings.json.encode("utf-8") + + icp["vs"] = Vstrings.mgpk + icps = msgpack.dumps(icp) + assert len(icps) == 264 + assert icps == (b'\x8b\xa2vs\xb1KERI10MGPK000000_\xa3pre\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' + b'SVPzhzS6b5CM\xa2sn\xa40001\xa3ilk\xa3icp\xa3dig\xd9,DVPzhzS6b5CMaU6JR2nmwy' + b'Z-i0d8JZAoTNZH3ULvYAfS\xa4sith\x01\xa4keys\x91\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZ' + b'H3ULvYAfSVPzhzS6b5CM\xa3nxt\xd9,DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5' + b'CM\xa4toad\x00\xa4wits\x90\xa4cnfg\x90') + + match = Rever.search(icps) + assert match.group() == Vstrings.mgpk.encode("utf-8") + + rot["vs"] = Vstrings.mgpk + rots = msgpack.dumps(rot) + assert len(rots) == 270 + assert rots == (b'\x8c\xa2vs\xb1KERI10MGPK000000_\xa3pre\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' + b'SVPzhzS6b5CM\xa2sn\xa40001\xa3ilk\xa3rot\xa3dig\xd9,DVPzhzS6b5CMaU6JR2nmwy' + b'Z-i0d8JZAoTNZH3ULvYAfS\xa4sith\x01\xa4keys\x91\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZ' + b'H3ULvYAfSVPzhzS6b5CM\xa3nxt\xd9,DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5' + b'CM\xa4toad\x00\xa4cuts\x90\xa4adds\x90\xa4data\x90') + + match = Rever.search(rots) + assert match.group() == Vstrings.mgpk.encode("utf-8") + + icp["vs"] = Vstrings.cbor + icps = cbor.dumps(icp) + assert len(icps) == 264 + assert icps == (b'\xabbvsqKERI10CBOR000000_cprex,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' + b'bsnd0001cilkcicpcdigx,DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSdsith\x01' + b'dkeys\x81x,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CMcnxtx,DZ-i0d8JZAoTNZ' + b'H3ULvaU6JR2nmwyYAfSVPzhzS6b5CMdtoad\x00dwits\x80dcnfg\x80') + + match = Rever.search(icps) + assert match.group() == Vstrings.cbor.encode("utf-8") + + rot["vs"] = Vstrings.cbor + rots = cbor.dumps(rot) + assert len(rots) == 270 + assert rots == (b'\xacbvsqKERI10CBOR000000_cprex,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' + b'bsnd0001cilkcrotcdigx,DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSdsith\x01' + b'dkeys\x81x,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CMcnxtx,DZ-i0d8JZAoTNZ' + b'H3ULvaU6JR2nmwyYAfSVPzhzS6b5CMdtoad\x00dcuts\x80dadds\x80ddata\x80') + + match = Rever.search(rots) + assert match.group() == Vstrings.cbor.encode("utf-8") + """Done Test""" + + +def test_versify(): + """ + Test Versify support + """ + vs = versify(kind=Serials.json, size=0) + assert vs == "KERI10JSON000000_" + assert len(vs) == VERFULLSIZE + proto, version, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.json + assert version == Version + assert size == 0 + + vs = versify(kind=Serials.json, size=65) + assert vs == "KERI10JSON000041_" + assert len(vs) == VERFULLSIZE + proto, version, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.json + assert version == Version + assert size == 65 + + vs = versify(proto=Protos.acdc, kind=Serials.json, size=86) + assert vs == "ACDC10JSON000056_" + assert len(vs) == VERFULLSIZE + proto, version, kind, size = deversify(vs) + assert proto == Protos.acdc + assert kind == Serials.json + assert version == Version + assert size == 86 + + vs = versify(kind=Serials.mgpk, size=0) + assert vs == "KERI10MGPK000000_" + assert len(vs) == VERFULLSIZE + proto, version, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.mgpk + assert version == Version + assert size == 0 + + vs = versify(kind=Serials.mgpk, size=65) + assert vs == "KERI10MGPK000041_" + assert len(vs) == VERFULLSIZE + proto, version, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.mgpk + assert version == Version + assert size == 65 + + vs = versify(kind=Serials.cbor, size=0) + assert vs == "KERI10CBOR000000_" + assert len(vs) == VERFULLSIZE + proto, version, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.cbor + assert version == Version + assert size == 0 + + vs = versify(kind=Serials.cbor, size=65) + assert vs == "KERI10CBOR000041_" + assert len(vs) == VERFULLSIZE + proto, version, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.cbor + assert version == Version + assert size == 65 + """End Test""" + + +def test_ilks(): + """ + Test Ilkage namedtuple instance Ilks + """ + assert Ilks == Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', + rct='rct', qry='qry', rpy='rpy', + exn='exn', pro='pro', bar='bar', + vcp='vcp', vrt='vrt', + iss='iss', rev='rev', bis='bis', brv='brv', ) + + assert isinstance(Ilks, Ilkage) + + for fld in Ilks._fields: + assert fld == getattr(Ilks, fld) + + assert 'icp' in Ilks + assert Ilks.icp == 'icp' + assert 'rot' in Ilks + assert Ilks.rot == 'rot' + assert 'ixn' in Ilks + assert Ilks.ixn == 'ixn' + assert 'dip' in Ilks + assert Ilks.dip == 'dip' + assert 'drt' in Ilks + assert Ilks.drt == 'drt' + assert 'rct' in Ilks + assert Ilks.rct == 'rct' + assert 'qry' in Ilks + assert Ilks.qry == 'qry' + assert 'rpy' in Ilks + assert Ilks.rpy == 'rpy' + assert 'exn' in Ilks + assert Ilks.exn == 'exn' + + + assert 'pro' in Ilks + assert Ilks.pro == 'pro' + assert 'bar' in Ilks + assert Ilks.bar == 'bar' + + + assert 'vcp' in Ilks + assert Ilks.vcp == 'vcp' + assert 'vrt' in Ilks + assert Ilks.vrt == 'vrt' + assert 'iss' in Ilks + assert Ilks.iss == 'iss' + assert 'rev' in Ilks + assert Ilks.rev == 'rev' + assert 'bis' in Ilks + assert Ilks.bis == 'bis' + assert 'brv' in Ilks + assert Ilks.brv == 'brv' + + """End Test """ + + +if __name__ == "__main__": + test_protos() + test_serials() + test_versify() + test_ilks() From b7c28e6ccc2b12c5e2e3a7ceb2abc0ab99086cc5 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 24 Feb 2024 13:30:14 -0700 Subject: [PATCH 020/418] added regex guts for supporting v2.xx version string. Prepatory to making version string tooling support it. Added unit tests to confirm --- src/keri/kering.py | 22 +++++++- tests/test_kering.py | 129 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 4 deletions(-) diff --git a/src/keri/kering.py b/src/keri/kering.py index 10cb737cd..2dfa03ada 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -31,8 +31,22 @@ VERFMT = "{}{:x}{:x}{}{:0{}x}_" # version format string VERFULLSIZE = 17 # number of characters in full version string -VEREX = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' -Rever = re.compile(VEREX) # compile is faster +VEREX0 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' + +Rever = re.compile(VEREX0) # compile is faster + + +VER1FULLSPAN = 17 # number of characters in full version string +VER1TERM = b'_' +VEREX1 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' + +VER2FULLSPAN = 16 # number of characters in full version string +VER2TERM = b'.' +VEREX2 = b'(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[A-Z]{4})(?P[0-9A-Za-z_-]{4}).' + +VEREX = VEREX2 + b'|' + VEREX1 + +pattern = re.compile(VEREX) # compile is faster def versify(proto=Protos.keri, version=Version, kind=Serials.json, size=0): @@ -69,6 +83,9 @@ def deversify(vs, version=None): serialization kind serialization size """ + # length of matched string is sum of lengths of returned group elements + # span + match = Rever.match(vs.encode("utf-8")) # match takes bytes if match: proto, major, minor, kind, size = match.group("proto", @@ -76,6 +93,7 @@ def deversify(vs, version=None): "minor", "kind", "size") + proto = proto.decode("utf-8") vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) kind = kind.decode("utf-8") diff --git a/tests/test_kering.py b/tests/test_kering.py index 51cbb9230..e96592a4c 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -3,7 +3,7 @@ tests.test_kering module """ - +import re import json import cbor2 as cbor @@ -12,9 +12,11 @@ from keri import kering from keri.kering import Protocolage, Protos from keri.kering import Serialage, Serials +from keri.kering import Ilkage, Ilks from keri.kering import (Versionage, Version, VERFULLSIZE, versify, deversify, Rever) -from keri.kering import Ilkage, Ilks +from keri.kering import (VER1FULLSPAN, VER1TERM, VEREX1, + VER2FULLSPAN, VER2TERM, VEREX2, VEREX) @@ -36,6 +38,128 @@ def test_protos(): """End Test""" +def test_version_regex(): + """ + Test version string regexing + """ + + #VER1FULLSPAN = 17 # number of characters in full version string + #VER1TERM = b'_' + #VEREX1 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' + + #VER2FULLSPAN = 16 # number of characters in full version string + #VER2TERM = b'.' + #VEREX2 = b'(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[A-Z]{4})(?P[0-9A-Za-z_-]{4}).' + + #VEREX = VEREX2 + b'|' + VEREX1 + + pattern = re.compile(VEREX2) # compile is faster + + vs = b'KERICAAJSONAAAB.' + + match = pattern.match(vs) + assert match + + full = match.group() # not group args so returns full match + assert full == vs + span = len(full) + assert span == VER2FULLSPAN + assert VER2TERM == chr(full[-1]).encode("utf-8") + assert ord(VER2TERM) == full[-1] + + groups = match.group("proto2", + "major2", + "minor2", + "kind2", + "size2") + + assert groups == (b'KERI', b'C', b'AA', b'JSON', b'AAAB') + + + pattern = re.compile(VEREX) # compile is faster + + vs = b'KERICAAJSONAAAB.' + + match = pattern.match(vs) + assert match + + full = match.group() # not group args so returns full match + assert full == vs + span = len(full) + assert span == VER2FULLSPAN + assert VER2TERM == chr(full[-1]).encode("utf-8") + assert ord(VER2TERM) == full[-1] + + groups = match.group("proto2", + "major2", + "minor2", + "kind2", + "size2") + + assert groups == (b'KERI', b'C', b'AA', b'JSON', b'AAAB') + + vs = b'KERI10JSON000002_' + + match = pattern.match(vs) + assert match + + full = match.group() # not group args so returns full match + assert full == vs + span = len(full) + assert span == VER1FULLSPAN + assert VER1TERM == chr(full[-1]).encode("utf-8") + assert ord(VER1TERM) == full[-1] + + groups = match.group("proto1", + "major1", + "minor1", + "kind1", + "size1") + + assert groups == (b'KERI', b'1', b'0', b'JSON', b'000002') + + raw = b'{"vs":"KERICAAJSONAAAB.","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"}' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + span = len(full) + assert span == VER2FULLSPAN + assert VER2TERM == chr(full[-1]).encode("utf-8") + assert ord(VER2TERM) == full[-1] + + groups = match.group("proto2", + "major2", + "minor2", + "kind2", + "size2") + + assert groups == (b'KERI', b'C', b'AA', b'JSON', b'AAAB') + + + + raw = b'{"vs":"KERI10JSON000002_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"}' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + span = len(full) + assert span == VER1FULLSPAN + assert VER1TERM == chr(full[-1]).encode("utf-8") + assert ord(VER1TERM) == full[-1] + + groups = match.group("proto1", + "major1", + "minor1", + "kind1", + "size1") + + assert groups == (b'KERI', b'1', b'0', b'JSON', b'000002') + + """End Test""" + def test_serials(): """ @@ -285,6 +409,7 @@ def test_ilks(): if __name__ == "__main__": test_protos() + test_version_regex() test_serials() test_versify() test_ilks() From 522821b378307978f0d862525e6fb217fcfecc15 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 24 Feb 2024 13:48:18 -0700 Subject: [PATCH 021/418] Fixed streamer --- src/keri/core/coring.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index dffec01ff..b6aa76abb 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -1914,14 +1914,7 @@ def texter(self): """ return self._stream - @property - def bexter(self): - """expanded stream as Bexter instance - Returns: - bexter (Bexter): Bexter primitive as stream suitable wrapping - """ - return self._stream From de848bc6323fef003ab78cf71367c2592a6b33b0 Mon Sep 17 00:00:00 2001 From: Shoeb Ahmed Tanjim <39959228+s-a-tanjim@users.noreply.github.com> Date: Mon, 26 Feb 2024 06:46:46 +0600 Subject: [PATCH 022/418] Add loglevel as a command line argument for witness (#688) * Enable INFO level logging for keripy witnesses keripy INFO level logs cover most of the cases. hio.help.ogling module provides Ogler class for global logging in keripy. The class can print log to - console - syslog - file * Make log level configurable from command line argument --------- Co-authored-by: Rubel Hassan Mollik Co-authored-by: Daniel Hardman --- src/keri/app/cli/commands/witness/start.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/keri/app/cli/commands/witness/start.py b/src/keri/app/cli/commands/witness/start.py index f8ace796d..7fff2cd94 100644 --- a/src/keri/app/cli/commands/witness/start.py +++ b/src/keri/app/cli/commands/witness/start.py @@ -47,11 +47,12 @@ parser.add_argument("--keypath", action="store", required=False, default=None) parser.add_argument("--certpath", action="store", required=False, default=None) parser.add_argument("--cafilepath", action="store", required=False, default=None) +parser.add_argument("--loglevel", action="store", required=False, default="CRITICAL", help="Set log level to DEBUG | INFO | WARNING | ERROR | CRITICAL. Default is CRITICAL") def launch(args): - help.ogler.level = logging.CRITICAL - help.ogler.reopen(name=args.name, temp=True, clear=True) + help.ogler.level = logging.getLevelName(args.loglevel) + help.ogler.reopen(name=args.name, temp=True, clear=True) # need to configure for logging persistent file logger = help.ogler.getLogger() From 2efbc0f88272a9c3861d8c3d3953a3166280b97a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Feb 2024 10:01:38 -0700 Subject: [PATCH 023/418] fixed typos and formatting --- docs/naming.md | 146 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 115 insertions(+), 31 deletions(-) diff --git a/docs/naming.md b/docs/naming.md index 8002f29b3..038f6e0a7 100644 --- a/docs/naming.md +++ b/docs/naming.md @@ -117,6 +117,12 @@ When using CapCamelCase or mixedCase the acronyms should be treated as words when namespacing. First ref is module not package variable such as: core.behaving, core.clustering +### Public Module Level Class Instances Singletons + UpperCamelCase + +### Private Module Level Class Instances Singletons + _LeadingUnderscoreUpperCamelCase + ### Public Module Level Methods and Attributes: lowerCamelCase. Methods use verbs. @@ -124,15 +130,15 @@ When using CapCamelCase or mixedCase the acronyms should be treated as words Examples: getName setStyle, display, first, last, itemCount, entities, books, data ### Private Module Level Methods and Attributes (not exported by from import `*`): - leadingUnderscoreLowerCamelCase. + _leadingUnderscoreLowerCamelCase. Methods use verbs. Attributes that are sequences use plural nouns. Examples: `_dirtyBit` ### Constants Module Level: - UPPER_CASE_WITH_UNDERSCORES + UPPER_CASE_WITH_UNDERSCORES or ALLUPPERCASE Not meant to be changed should be numbers or strings. - Examples: MY_CONSTANT + Examples: MY_CONSTANT, SOMECONSTANT ### Dynamic Global Variables (not constants) Module Level: Capitalized_With_Underscores. @@ -158,7 +164,7 @@ When using CapCamelCase or mixedCase the acronyms should be treated as words Example: TotalInstances, Storage, Store, Redact ### Private Class Attributes, Class Methods, Static Methods: - LeadingUnderscoreUpperCamelCase, + _LeadingUnderscoreUpperCamelCase, Methods use verbs. Attributes that are sequences use plural nouns. Example: `_TotalInstances _Storage __Entries` @@ -177,7 +183,7 @@ When using CapCamelCase or mixedCase the acronyms should be treated as words ### Private Instance and Attributes (not exported with from import `*``): - leadingUnderscoreLowerCamelCase. + _leadingUnderscoreLowerCamelCase. Methods use verbs. Attributes that are sequences use plural nouns. Examples: `_getScore _setInternal _lastName _count _entries` @@ -271,44 +277,76 @@ that are sufficiently evocative. ## Suffix Mapping Adjective describing module. What does the module enable one to do. + Verb is the deed or act + Object is actor doer + Place to keep track of or create Objects container or factory + doery actorery doerage actorage + of an Doer is a doery or doage or dodom or dohood -er -or -eur -ster Agent one who does something brewer (Object Classes) + -ery a place for an actor to act factory brewery (Object Factories) + -ing action of running, wishing (Module Names) + -age state of acting actor to act brewerage + -dom state of doing acting kingdom + -hood state of being childhood + -ship quality of or state of rank of midship + -ize to make itemize + -izer Someone who makes one do itemizer + -ive having nature of active rotative inceptive + -acy -isy -ty quality of linty piracy clerisy + -ion -tion -sion act or state of action itemization + -y -ly like full of happening noisy monthly patron + patroner + patronist + patronery + patronage + patrondom + patronship + patronacy + patronhood + patronize + patronish + patronive + patronlet + patronly + patrony + ### Rules for English suffixes http://www.paulnoll.com/Books/Clear-English/English-suffixes-1.html @@ -316,51 +354,97 @@ http://www.prefixsuffix.com/rootchart.php Suffix Meaning Examples Used + able, ible capable of, worthy agreeable, comfortable, credible + age act of or state of salvage, bondage + acy, isy quality hypocrisy, piracy + al, eal, ial on account of related to, action of judicial, official arrival, refusal + ance, ence act or fact of doing state of violence, dependence allowance, insurance + ant quality of one who defiant, expectant, reliant occupant, accountant + er, or, eur agent, one who author, baker, winner, dictator, chauffeur, worker + ed past jumped, baked + ery a place to practice of condition of nunnery, cannery surgery bravery, drudgery + dom state, condition of wisdom, kingdom, martyrdom + ent having the quality of different, dependent, innocent + en made of, to make woolen, wooden, darken -er degree of comparison harder, newer, older 5440 -est highest of comparison cleanest, hardest, softest 1339 -ful full of graceful, restful, faithful 212 -hood state of being boyhood, knighthood, womanhood 61 -ible, ile, il capable of being digestible, responsible, docile, civil 783 -ier, ior one who carrier, warrior 1114 -ify to make magnify, beautify, falsify 125 -ic like, made of metallic, toxic, poetic 3774 -ing action of running, wishing 5440 -ion act or state of confusion, correction, protection 3362 -ism fact of being communism, socialism 1147 -ish like childish, sheepish, foolish 566 -ist a person who does artist, geologist 1375 -ity, ty state of majesty, chastity, humanity 4795 -itis inflammation of appendicitis, tonsillitis 124 -ive having nature of attractive, active 961 -ize to make pasteurize, motorize 637 -less without motionless, careless, childless 282 - -let small starlet, eaglet 185 -ly like, in a manner happening heavenly, remarkably, suddenly every absolutely, monthly 5440 -ment state or quality act of doing accomplishment, excitement placement, movement 680 -meter device for measuring thermometer, barometer 166 -ness state of blindness, kindness 3322 -ology study of geology, zoology, archaeology 374 -ous, ious full of joyous, marvelous, furious 1615 +er degree of comparison harder, newer, older + +est highest of comparison cleanest, hardest, softest + +ful full of graceful, restful, faithful + +hood state of being boyhood, knighthood, womanhood + +ible, ile, il capable of being digestible, responsible, docile, civil + +ier, ior one who carrier, warrior + +ify to make magnify, beautify, falsify + +ic like, made of metallic, toxic, poetic + +ing action of running, wishing + +ion act or state of confusion, correction, protection + +ism fact of being communism, socialism + +ish like childish, sheepish, foolish + +ist a person who does artist, geologist + +ity, ty state of majesty, chastity, humanity + +itis inflammation of appendicitis, tonsillitis + +ive having nature of attractive, active + +ize to make pasteurize, motorize + +less without motionless, careless, childless + + +let small starlet, eaglet + +ly like, in a manner happening heavenly, remarkably, suddenly every absolutely, monthly + +ment state or quality act of doing accomplishment, excitement placement, movement + +meter device for measuring thermometer, barometer + +ness state of blindness, kindness + +ology study of geology, zoology, archaeology + +ous, ious full of joyous, marvelous, furious + ship quality of or state of rank of friendship, leadership lordship + scope instrument for seeing telescope, microscope + some like tiresome, lonesome + tion, sion action, state of being condition, attention, fusion + ty quality or state of liberty, majesty + ward toward southward, forward + y like, full of, diminutive: noisy, sooty, kitty + ure noun from verv indicating act or office seizure prefecture + + From 206eda356a8626b471992052ecbc301b25bc1643 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Feb 2024 10:06:19 -0700 Subject: [PATCH 024/418] more clean up formatting --- docs/naming.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/naming.md b/docs/naming.md index 038f6e0a7..e1c7981c9 100644 --- a/docs/naming.md +++ b/docs/naming.md @@ -213,7 +213,7 @@ Verticle white space matters. Code should have paragraphs. Balanced brackets, indendation and blank lines demarcate paragraphs For example - +```c void display(void) { int start; @@ -223,13 +223,16 @@ void display(void) if(start == -1) return; } +``` is more readable than +```c void display(void){ int start; start = -1; if(start == -1) return; } +``` Line length matters. Which means variable length matters. Simple logic statments that wrap are no longer simple. From d6dbe690b3579c666d0d52ba9d7254625ba615c6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Feb 2024 12:35:06 -0700 Subject: [PATCH 025/418] refactor base64 stuff --- src/keri/core/coring.py | 116 +------------ src/keri/kering.py | 344 +++++++++++++++++++++++++++++++------- tests/core/test_coring.py | 162 +----------------- tests/test_kering.py | 165 ++++++++++++++++++ 4 files changed, 456 insertions(+), 331 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index b6aa76abb..dbb871ca5 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -26,6 +26,8 @@ from cryptography.hazmat.primitives.asymmetric import ec, utils from ..kering import MaxON +from ..kering import (intToB64, intToB64b, b64ToInt, B64_CHARS, + codeB64ToB2, codeB2ToB64, Reb64, nabSextets) from ..kering import (EmptyMaterialError, RawMaterialError, InvalidCodeError, InvalidCodeSizeError, InvalidVarIndexError, @@ -111,120 +113,6 @@ def sizeify(ked, kind=None, version=Version): return raw, proto, kind, ked, vrsn -# Base64 utilities -BASE64_PAD = b'=' - -# Mappings between Base64 Encode Index and Decode Characters -# B64ChrByIdx is dict where each key is a B64 index and each value is the B64 char -# B64IdxByChr is dict where each key is a B64 char and each value is the B64 index -# Map Base64 index to char -B64ChrByIdx = dict((index, char) for index, char in enumerate([chr(x) for x in range(65, 91)])) -B64ChrByIdx.update([(index + 26, char) for index, char in enumerate([chr(x) for x in range(97, 123)])]) -B64ChrByIdx.update([(index + 52, char) for index, char in enumerate([chr(x) for x in range(48, 58)])]) -B64ChrByIdx[62] = '-' -B64ChrByIdx[63] = '_' -# Map char to Base64 index -B64IdxByChr = {char: index for index, char in B64ChrByIdx.items()} -B64_CHARS = tuple(B64ChrByIdx.values()) # tuple of characters in Base64 - -B64REX = b'^[A-Za-z0-9\-\_]*\Z' -Reb64 = re.compile(B64REX) # compile is faster - - -def intToB64(i, l=1): - """ - Returns conversion of int i to Base64 str - l is min number of b64 digits left padded with Base64 0 == "A" char - """ - d = deque() # deque of characters base64 - - while l: - d.appendleft(B64ChrByIdx[i % 64]) - i = i // 64 - if not i: - break - # d.appendleft(B64ChrByIdx[i % 64]) - # i = i // 64 - for j in range(l - len(d)): # range(x) x <= 0 means do not iterate - d.appendleft("A") - return ("".join(d)) - - -def intToB64b(i, l=1): - """ - Returns conversion of int i to Base64 bytes - l is min number of b64 digits left padded with Base64 0 == "A" char - """ - return (intToB64(i=i, l=l).encode("utf-8")) - - -def b64ToInt(s): - """ - Returns conversion of Base64 str s or bytes to int - """ - if not s: - raise ValueError("Empty string, conversion undefined.") - if hasattr(s, 'decode'): - s = s.decode("utf-8") - i = 0 - for e, c in enumerate(reversed(s)): - i |= B64IdxByChr[c] << (e * 6) # same as i += B64IdxByChr[c] * (64 ** e) - return i - - -def codeB64ToB2(s): - """ - Returns conversion (decode) of Base64 chars to Base2 bytes. - Where the number of total bytes returned is equal to the minimun number of - octets sufficient to hold the total converted concatenated sextets from s, - with one sextet per each Base64 decoded char of s. Assumes no pad chars in s. - Sextets are left aligned with pad bits in last (rightmost) byte. - This is useful for decoding as bytes, code characters from the front of - a Base64 encoded string of characters. - """ - i = b64ToInt(s) - i <<= 2 * (len(s) % 4) # add 2 bits right zero padding for each sextet - n = sceil(len(s) * 3 / 4) # compute min number of ocetets to hold all sextets - return (i.to_bytes(n, 'big')) - - -def codeB2ToB64(b, l): - """ - Returns conversion (encode) of l Base2 sextets from front of b to Base64 chars. - One char for each of l sextets from front (left) of b. - This is useful for encoding as code characters, sextets from the front of - a Base2 bytes (byte string). Must provide l because of ambiguity between l=3 - and l=4. Both require 3 bytes in b. - """ - if hasattr(b, 'encode'): - b = b.encode("utf-8") # convert to bytes - n = sceil(l * 3 / 4) # number of bytes needed for l sextets - if n > len(b): - raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) - i = int.from_bytes(b[:n], 'big') # convert only first n bytes to int - # check if prepad bits are zero - tbs = 2 * (l % 4) # trailing bit size in bits - i >>= tbs # right shift out trailing bits to make right aligned - return (intToB64(i, l)) # return as B64 - - -def nabSextets(b, l): - """ - Return first l sextets from front (left) of b as bytes (byte string). - Length of bytes returned is minimum sufficient to hold all l sextets. - Last byte returned is right bit padded with zeros - b is bytes or str - """ - if hasattr(b, 'encode'): - b = b.encode("utf-8") # convert to bytes - n = sceil(l * 3 / 4) # number of bytes needed for l sextets - if n > len(b): - raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) - i = int.from_bytes(b[:n], 'big') - p = 2 * (l % 4) - i >>= p # strip of last bits - i <<= p # pad with empty bits - return (i.to_bytes(n, 'big')) def dumps(ked, kind=Serials.json): diff --git a/src/keri/kering.py b/src/keri/kering.py index 2dfa03ada..cbc12266d 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -4,14 +4,135 @@ """ import sys import re -from collections import namedtuple +from collections import namedtuple, deque from dataclasses import dataclass, astuple +from .help.helping import sceil + FALSEY = (False, 0, None, "?0", "no", "false", "False", "off") TRUTHY = (True, 1, "?1", "yes" "true", "True", 'on') MaxON = int("f"*32, 16) # 256 ** 16 - 1 maximum ordinal number, sequence or first seen etc + +# Base64 utilities +BASE64_PAD = b'=' + +# Mappings between Base64 Encode Index and Decode Characters +# B64ChrByIdx is dict where each key is a B64 index and each value is the B64 char +# B64IdxByChr is dict where each key is a B64 char and each value is the B64 index +# Map Base64 index to char +B64ChrByIdx = dict((index, char) for index, char in enumerate([chr(x) for x in range(65, 91)])) +B64ChrByIdx.update([(index + 26, char) for index, char in enumerate([chr(x) for x in range(97, 123)])]) +B64ChrByIdx.update([(index + 52, char) for index, char in enumerate([chr(x) for x in range(48, 58)])]) +B64ChrByIdx[62] = '-' +B64ChrByIdx[63] = '_' +# Map char to Base64 index +B64IdxByChr = {char: index for index, char in B64ChrByIdx.items()} +# tuple +B64_CHARS = tuple(B64ChrByIdx.values()) # tuple of characters in Base64 + + +B64REX = b'^[A-Za-z0-9\-\_]*\Z' +Reb64 = re.compile(B64REX) # compile is faster + + +def intToB64(i, l=1): + """ + Returns conversion of int i to Base64 str + l is min number of b64 digits left padded with Base64 0 == "A" char + """ + d = deque() # deque of characters base64 + + while l: + d.appendleft(B64ChrByIdx[i % 64]) + i = i // 64 + if not i: + break + for j in range(l - len(d)): # range(x) x <= 0 means do not iterate + d.appendleft("A") + return ("".join(d)) + + +def intToB64b(i, l=1): + """ + Returns conversion of int i to Base64 bytes + l is min number of b64 digits left padded with Base64 0 == "A" char + """ + return (intToB64(i=i, l=l).encode("utf-8")) + + +def b64ToInt(s): + """ + Returns conversion of Base64 str s or bytes to int + """ + if not s: + raise ValueError("Empty string, conversion undefined.") + if hasattr(s, 'decode'): + s = s.decode("utf-8") + i = 0 + for e, c in enumerate(reversed(s)): + i |= B64IdxByChr[c] << (e * 6) # same as i += B64IdxByChr[c] * (64 ** e) + return i + + + +def codeB64ToB2(s): + """ + Returns conversion (decode) of Base64 chars to Base2 bytes. + Where the number of total bytes returned is equal to the minimun number of + octets sufficient to hold the total converted concatenated sextets from s, + with one sextet per each Base64 decoded char of s. Assumes no pad chars in s. + Sextets are left aligned with pad bits in last (rightmost) byte. + This is useful for decoding as bytes, code characters from the front of + a Base64 encoded string of characters. + """ + i = b64ToInt(s) + i <<= 2 * (len(s) % 4) # add 2 bits right zero padding for each sextet + n = sceil(len(s) * 3 / 4) # compute min number of ocetets to hold all sextets + return (i.to_bytes(n, 'big')) + + +def codeB2ToB64(b, l): + """ + Returns conversion (encode) of l Base2 sextets from front of b to Base64 chars. + One char for each of l sextets from front (left) of b. + This is useful for encoding as code characters, sextets from the front of + a Base2 bytes (byte string). Must provide l because of ambiguity between l=3 + and l=4. Both require 3 bytes in b. + """ + if hasattr(b, 'encode'): + b = b.encode("utf-8") # convert to bytes + n = sceil(l * 3 / 4) # number of bytes needed for l sextets + if n > len(b): + raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) + i = int.from_bytes(b[:n], 'big') # convert only first n bytes to int + # check if prepad bits are zero + tbs = 2 * (l % 4) # trailing bit size in bits + i >>= tbs # right shift out trailing bits to make right aligned + return (intToB64(i, l)) # return as B64 + + +def nabSextets(b, l): + """ + Return first l sextets from front (left) of b as bytes (byte string). + Length of bytes returned is minimum sufficient to hold all l sextets. + Last byte returned is right bit padded with zeros + b is bytes or str + """ + if hasattr(b, 'encode'): + b = b.encode("utf-8") # convert to bytes + n = sceil(l * 3 / 4) # number of bytes needed for l sextets + if n > len(b): + raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) + i = int.from_bytes(b[:n], 'big') + p = 2 * (l % 4) + i >>= p # strip of last bits + i <<= p # pad with empty bits + return (i.to_bytes(n, 'big')) + + + # Serialization Kinds Serialage = namedtuple("Serialage", 'json mgpk cbor') Serials = Serialage(json='JSON', mgpk='MGPK', cbor='CBOR') @@ -33,7 +154,7 @@ VEREX0 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' -Rever = re.compile(VEREX0) # compile is faster +#Rever = re.compile(VEREX0) # compile is faster VER1FULLSPAN = 17 # number of characters in full version string @@ -46,7 +167,17 @@ VEREX = VEREX2 + b'|' + VEREX1 -pattern = re.compile(VEREX) # compile is faster +Rever = re.compile(VEREX) # compile is faster + +""" +Smellage (results of smelling a version string such as in a Serder) + protocol (str): protocol type value of Protos examples 'KERI', 'ACDC' + version (Versionage): named tuple (major, minor) ints of major minor version + kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' + size (str): int size of raw serialization + +""" +Smellage = namedtuple("Smellage", "protocol version kind size") def versify(proto=Protos.keri, version=Version, kind=Serials.json, size=0): @@ -67,13 +198,14 @@ def deversify(vs, version=None): Returns: tuple(proto, kind, version, size) Where: proto (str): value is protocol type identifier one of Protos (Protocolage) acdc='ACDC', keri='KERI' + + vrsn (tuple): version tuple of type Versionage kind (str): value is serialization kind, one of Serials json='JSON', mgpk='MGPK', cbor='CBOR' - vrsn (tuple): version tuple of type Versionage size (int): raw size in bytes Parameters: - vs (str): version string to extract from + vs (str | bytes): version string to extract from version (Versionage | None): supported version. None means do not check for supported version. @@ -83,46 +215,90 @@ def deversify(vs, version=None): serialization kind serialization size """ - # length of matched string is sum of lengths of returned group elements - # span + if hasattr(vs, "encode"): # match takes bytes + vs = vs.encode("utf-8") - match = Rever.match(vs.encode("utf-8")) # match takes bytes + match = Rever.match(vs) if match: - proto, major, minor, kind, size = match.group("proto", - "major", - "minor", - "kind", - "size") - - proto = proto.decode("utf-8") - vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) - kind = kind.decode("utf-8") - - if proto not in Protos: - raise ValueError("Invalid message identifier = {}".format(proto)) - if version is not None and vrsn != version: - raise ValueError(f"Expected version = {version}, got " - f"{vrsn.major}.{vrsn.minor}.") - if kind not in Serials: - raise ValueError("Invalid serialization kind = {}".format(kind)) - size = int(size, 16) - return proto, vrsn, kind, size + full = match.group() # full matched version string + if len(full) == VER2FULLSPAN and full[-1] == ord(VER2TERM): + proto, major, minor, kind, size = match.group("proto2", + "major2", + "minor2", + "kind2", + "size2") + protocol = proto.decode("utf-8") + if protocol not in Protos: + raise ProtocolError(f"Invalid protocol type = {protocol}.") + vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) + if vrsn.major < 2: # version2 vs but major < 2 + VersionError(f"Incompatible {vrsn=} with version string.") + if version is not None: # compatible version with vrsn + if (vrsn.major > version.major or + (vrsn.major == version.major and vrsn.minor > version.minor)): + raise VersionError(f"Incompatible {version=}, with " + f"{vrsn=}.") + + kind = kind.decode("utf-8") + if kind not in Serials: + raise KindError(f"Invalid serialization kind = {kind}.") + size = b64ToInt(size) + return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) + + + + elif len(full) == VER1FULLSPAN and full[-1] == ord(VER1TERM): + proto, major, minor, kind, size = match.group("proto1", + "major1", + "minor1", + "kind1", + "size1") + protocol = proto.decode("utf-8") + if protocol not in Protos: + raise ProtocolError(f"Invalid protocol type = {protocol}.") + vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) + if vrsn.major > 1: # version1 vs but major > 1 + VersionError(f"Incompatible {vrsn=} with version string.") + if version is not None and vrsn != version: + raise VersionError(f"Expected {version=}, got " + f"{vrsn=}.") + kind = kind.decode("utf-8") + if kind not in Serials: + raise KindError(f"Invalid serialization kind = {kind}.") + size = int(size, 16) + return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) + + + + raise ValueError(f"Invalid version string '{vs}'.") + + +#proto, major, minor, kind, size = match.group("proto", + #"major", + #"minor", + #"kind", + #"size") + +#proto = proto.decode("utf-8") +#vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) +#kind = kind.decode("utf-8") + +#if proto not in Protos: + #raise ValueError("Invalid message identifier = {}".format(proto)) +#if version is not None and vrsn != version: + #raise ValueError(f"Expected version = {version}, got " + #f"{vrsn.major}.{vrsn.minor}.") +#if kind not in Serials: + #raise ValueError("Invalid serialization kind = {}".format(kind)) +#size = int(size, 16) +#return proto, vrsn, kind, size - raise ValueError("Invalid version string = {}".format(vs)) MAXVSOFFSET = 12 SMELLSIZE = MAXVSOFFSET + VERFULLSIZE # min buffer size to inhale -""" -Smellage (results of smelling a version string such as in a Serder) - protocol (str): protocol type value of Protos examples 'KERI', 'ACDC' - version (Versionage): named tuple (major, minor) ints of major minor version - kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' - size (str): int size of raw serialization -""" -Smellage = namedtuple("Smellage", "protocol version kind size") def smell(raw, *, version=None): """Extract and return instance of Smellage from version string inside @@ -146,34 +322,88 @@ def smell(raw, *, version=None): raise VersionError(f"Invalid version string from smelled raw = " f"{raw[: SMELLSIZE]}.") - proto, major, minor, kind, size = match.group("proto", "major", "minor", "kind", "size") - # use length of version string matched to determine if version 1.x or 2.x - # so can convert major, minor, and size correctly hex vs B64 numbers - # Global version compatibility check. Serder instances also peform version check - major = int(major, 16) - minor = int(minor, 16) - vrsn = Versionage(major=major, minor=minor) - if version is not None: # test here for compatible code version with message vrsn - if (vrsn.major > version.major or - (vrsn.major == version.major and vrsn.minor > version.minor)): - pass # raise error here? + full = match.group() # full matched version string + if len(full) == VER2FULLSPAN and full[-1] == ord(VER2TERM): + proto, major, minor, kind, size = match.group("proto2", + "major2", + "minor2", + "kind2", + "size2") + protocol = proto.decode("utf-8") + if protocol not in Protos: + raise ProtocolError(f"Invalid protocol type = {protocol}.") + vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) + if vrsn.major < 2: # version2 vs but major < 2 + VersionError(f"Incompatible {vrsn=} with version string.") + if version is not None: # compatible version with vrsn + if (vrsn.major > version.major or + (vrsn.major == version.major and vrsn.minor > version.minor)): + raise VersionError(f"Incompatible {version=}, with " + f"{vrsn=}.") + kind = kind.decode("utf-8") + if kind not in Serials: + raise KindError(f"Invalid serialization kind = {kind}.") + size = b64ToInt(size) + return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) - protocol = proto.decode("utf-8") - if protocol not in Protos: - raise ProtocolError(f"Invalid protocol type = {protocol}.") - kind = kind.decode("utf-8") - if kind not in Serials: - raise KindError(f"Invalid serialization kind = {kind}.") - size = int(size, 16) - if len(raw) < size: - raise ShortageError(f"Need more bytes.") + elif len(full) == VER1FULLSPAN and full[-1] == ord(VER1TERM): + proto, major, minor, kind, size = match.group("proto1", + "major1", + "minor1", + "kind1", + "size1") + protocol = proto.decode("utf-8") + if protocol not in Protos: + raise ProtocolError(f"Invalid protocol type = {protocol}.") + vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) + if vrsn.major > 1: # version1 vs but major > 1 + VersionError(f"Incompatible {vrsn=} with version string.") + if version is not None and vrsn != version: + raise VersionError(f"Expected {version=}, got " + f"{vrsn=}.") + kind = kind.decode("utf-8") + if kind not in Serials: + raise KindError(f"Invalid serialization kind = {kind}.") + size = int(size, 16) + return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) + + + + + + #proto, major, minor, kind, size = match.group("proto", "major", "minor", "kind", "size") + + ## use length of version string matched to determine if version 1.x or 2.x + ## so can convert major, minor, and size correctly hex vs B64 numbers + + ## Global version compatibility check. Serder instances also peform version check + #major = int(major, 16) + #minor = int(minor, 16) + #vrsn = Versionage(major=major, minor=minor) + #if version is not None: # test here for compatible code version with message vrsn + #if (vrsn.major > version.major or + #(vrsn.major == version.major and vrsn.minor > version.minor)): + #pass # raise error here? + + + #protocol = proto.decode("utf-8") + #if protocol not in Protos: + #raise ProtocolError(f"Invalid protocol type = {protocol}.") + + #kind = kind.decode("utf-8") + #if kind not in Serials: + #raise KindError(f"Invalid serialization kind = {kind}.") + + #size = int(size, 16) + #if len(raw) < size: + #raise ShortageError(f"Need more bytes.") - return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) + #return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) @dataclass(frozen=True) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 79bda8661..012b4dcd2 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -52,6 +52,8 @@ + + def test_protocol_genus_codex(): """ Test genera in ProDex as instance of ProtocolGenusCodex @@ -72,166 +74,6 @@ def test_protocol_genus_codex(): -def test_b64_conversions(): - """ - Test Base64 conversion utility routines - """ - - cs = intToB64(0) - assert cs == "A" - i = b64ToInt(cs) - assert i == 0 - - cs = intToB64(0, l=0) - assert cs == "" - with pytest.raises(ValueError): - i = b64ToInt(cs) - - cs = intToB64(None, l=0) - assert cs == "" - with pytest.raises(ValueError): - i = b64ToInt(cs) - - cs = intToB64b(0) - assert cs == b"A" - i = b64ToInt(cs) - assert i == 0 - - cs = intToB64(27) - assert cs == "b" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64b(27) - assert cs == b"b" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64(27, l=2) - assert cs == "Ab" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64b(27, l=2) - assert cs == b"Ab" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64(80) - assert cs == "BQ" - i = b64ToInt(cs) - assert i == 80 - - cs = intToB64b(80) - assert cs == b"BQ" - i = b64ToInt(cs) - assert i == 80 - - cs = intToB64(4095) - assert cs == '__' - i = b64ToInt(cs) - assert i == 4095 - - cs = intToB64b(4095) - assert cs == b'__' - i = b64ToInt(cs) - assert i == 4095 - - cs = intToB64(4096) - assert cs == 'BAA' - i = b64ToInt(cs) - assert i == 4096 - - cs = intToB64b(4096) - assert cs == b'BAA' - i = b64ToInt(cs) - assert i == 4096 - - cs = intToB64(6011) - assert cs == "Bd7" - i = b64ToInt(cs) - assert i == 6011 - - cs = intToB64b(6011) - assert cs == b"Bd7" - i = b64ToInt(cs) - assert i == 6011 - - s = "-BAC" - b = codeB64ToB2(s[:]) - assert len(b) == 3 - assert b == b'\xf8\x10\x02' - t = codeB2ToB64(b, 4) - assert t == s[:] - i = int.from_bytes(b, 'big') - assert i == 0o76010002 - i >>= 2 * (len(s) % 4) - assert i == 0o76010002 - p = nabSextets(b, 4) - assert p == b'\xf8\x10\x02' - - b = codeB64ToB2(s[:3]) - assert len(b) == 3 - assert b == b'\xf8\x10\x00' - t = codeB2ToB64(b, 3) - assert t == s[:3] - i = int.from_bytes(b, 'big') - assert i == 0o76010000 - i >>= 2 * (len(s[:3]) % 4) - assert i == 0o760100 - p = nabSextets(b, 3) - assert p == b'\xf8\x10\x00' - - b = codeB64ToB2(s[:2]) - assert len(b) == 2 - assert b == b'\xf8\x10' - t = codeB2ToB64(b, 2) - assert t == s[:2] - i = int.from_bytes(b, 'big') - assert i == 0o174020 - i >>= 2 * (len(s[:2]) % 4) - assert i == 0o7601 - p = nabSextets(b, 2) - assert p == b'\xf8\x10' - - b = codeB64ToB2(s[:1]) - assert len(b) == 1 - assert b == b'\xf8' - t = codeB2ToB64(b, 1) - assert t == s[:1] - i = int.from_bytes(b, 'big') - assert i == 0o370 - i >>= 2 * (len(s[:1]) % 4) - assert i == 0o76 - p = nabSextets(b, 1) - assert p == b'\xf8' - - assert B64_CHARS == ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', - 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', - 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_') - assert '@' not in B64_CHARS - assert 'A' in B64_CHARS - - text = b"-A-Bg-1-3-cd" - match = Reb64.match(text) - assert match - assert match is not None - - text = b'' - match = Reb64.match(text) - assert match - assert match is not None - - text = b'123#$' - match = Reb64.match(text) - assert not match - assert match is None - - """End Test""" - - def test_matter(): """ Test Matter class diff --git a/tests/test_kering.py b/tests/test_kering.py index e96592a4c..7bbb2872a 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -9,7 +9,11 @@ import cbor2 as cbor import msgpack +import pytest + from keri import kering +from keri.kering import (intToB64, intToB64b, b64ToInt, B64_CHARS, + codeB64ToB2, codeB2ToB64, Reb64, nabSextets) from keri.kering import Protocolage, Protos from keri.kering import Serialage, Serials from keri.kering import Ilkage, Ilks @@ -21,6 +25,166 @@ +def test_b64_conversions(): + """ + Test Base64 conversion utility routines + """ + + cs = intToB64(0) + assert cs == "A" + i = b64ToInt(cs) + assert i == 0 + + cs = intToB64(0, l=0) + assert cs == "" + with pytest.raises(ValueError): + i = b64ToInt(cs) + + cs = intToB64(None, l=0) + assert cs == "" + with pytest.raises(ValueError): + i = b64ToInt(cs) + + cs = intToB64b(0) + assert cs == b"A" + i = b64ToInt(cs) + assert i == 0 + + cs = intToB64(27) + assert cs == "b" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64b(27) + assert cs == b"b" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64(27, l=2) + assert cs == "Ab" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64b(27, l=2) + assert cs == b"Ab" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64(80) + assert cs == "BQ" + i = b64ToInt(cs) + assert i == 80 + + cs = intToB64b(80) + assert cs == b"BQ" + i = b64ToInt(cs) + assert i == 80 + + cs = intToB64(4095) + assert cs == '__' + i = b64ToInt(cs) + assert i == 4095 + + cs = intToB64b(4095) + assert cs == b'__' + i = b64ToInt(cs) + assert i == 4095 + + cs = intToB64(4096) + assert cs == 'BAA' + i = b64ToInt(cs) + assert i == 4096 + + cs = intToB64b(4096) + assert cs == b'BAA' + i = b64ToInt(cs) + assert i == 4096 + + cs = intToB64(6011) + assert cs == "Bd7" + i = b64ToInt(cs) + assert i == 6011 + + cs = intToB64b(6011) + assert cs == b"Bd7" + i = b64ToInt(cs) + assert i == 6011 + + s = "-BAC" + b = codeB64ToB2(s[:]) + assert len(b) == 3 + assert b == b'\xf8\x10\x02' + t = codeB2ToB64(b, 4) + assert t == s[:] + i = int.from_bytes(b, 'big') + assert i == 0o76010002 + i >>= 2 * (len(s) % 4) + assert i == 0o76010002 + p = nabSextets(b, 4) + assert p == b'\xf8\x10\x02' + + b = codeB64ToB2(s[:3]) + assert len(b) == 3 + assert b == b'\xf8\x10\x00' + t = codeB2ToB64(b, 3) + assert t == s[:3] + i = int.from_bytes(b, 'big') + assert i == 0o76010000 + i >>= 2 * (len(s[:3]) % 4) + assert i == 0o760100 + p = nabSextets(b, 3) + assert p == b'\xf8\x10\x00' + + b = codeB64ToB2(s[:2]) + assert len(b) == 2 + assert b == b'\xf8\x10' + t = codeB2ToB64(b, 2) + assert t == s[:2] + i = int.from_bytes(b, 'big') + assert i == 0o174020 + i >>= 2 * (len(s[:2]) % 4) + assert i == 0o7601 + p = nabSextets(b, 2) + assert p == b'\xf8\x10' + + b = codeB64ToB2(s[:1]) + assert len(b) == 1 + assert b == b'\xf8' + t = codeB2ToB64(b, 1) + assert t == s[:1] + i = int.from_bytes(b, 'big') + assert i == 0o370 + i >>= 2 * (len(s[:1]) % 4) + assert i == 0o76 + p = nabSextets(b, 1) + assert p == b'\xf8' + + assert B64_CHARS == ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', + 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_') + assert '@' not in B64_CHARS + assert 'A' in B64_CHARS + + text = b"-A-Bg-1-3-cd" + match = Reb64.match(text) + assert match + assert match is not None + + text = b'' + match = Reb64.match(text) + assert match + assert match is not None + + text = b'123#$' + match = Reb64.match(text) + assert not match + assert match is None + + """End Test""" + + def test_protos(): """ Test protocols namedtuple instance Protos @@ -408,6 +572,7 @@ def test_ilks(): if __name__ == "__main__": + test_b64_conversions() test_protos() test_version_regex() test_serials() From 8aaf9765fe382eb9d06a86a97905bc79223741d3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Feb 2024 13:33:29 -0700 Subject: [PATCH 026/418] more refactoring. base64 utility conversion functions now in help.helping smell ans deversify now support both version 1.x and 2.x version string formats --- src/keri/core/coring.py | 4 +- src/keri/end/ending.py | 2 +- src/keri/help/helping.py | 231 +++++++++++++++++++++++++++---------- src/keri/kering.py | 121 +------------------ tests/core/test_coring.py | 10 +- tests/help/test_helping.py | 169 +++++++++++++++++++++++++++ tests/test_kering.py | 162 +------------------------- 7 files changed, 349 insertions(+), 350 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index dbb871ca5..0a543690e 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -26,8 +26,6 @@ from cryptography.hazmat.primitives.asymmetric import ec, utils from ..kering import MaxON -from ..kering import (intToB64, intToB64b, b64ToInt, B64_CHARS, - codeB64ToB2, codeB2ToB64, Reb64, nabSextets) from ..kering import (EmptyMaterialError, RawMaterialError, InvalidCodeError, InvalidCodeSizeError, InvalidVarIndexError, @@ -44,6 +42,8 @@ from ..help import helping from ..help.helping import sceil, nonStringIterable, nonStringSequence +from ..help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, + codeB64ToB2, codeB2ToB64, Reb64, nabSextets) diff --git a/src/keri/end/ending.py b/src/keri/end/ending.py index 9e65ab7f0..d775930fb 100644 --- a/src/keri/end/ending.py +++ b/src/keri/end/ending.py @@ -220,7 +220,7 @@ def designature(value): if "indexed" not in items: raise ValueError("Missing indexed field in Signature header signage.") - indexed = items["indexed"] not in kering.FALSEY # make bool + indexed = items["indexed"] not in helping.FALSEY # make bool del items["indexed"] if "signer" in items: diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index e1715143c..038d97203 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -7,12 +7,18 @@ import dataclasses import datetime import re +from collections import deque from collections.abc import Iterable, Sequence, Mapping import pysodium -B64REX = b'[A-Za-z-_]*' -B64Rev = re.compile(B64REX) #compile is faster +# usage: +# x not in FALSEY +# x in FALSEY +# x not in TRUTHY +# x in TRUTHY +FALSEY = (False, 0, None, "?0", "no", "false", "False", "off") +TRUTHY = (True, 1, "?1", "yes" "true", "True", 'on') # Utilities @@ -98,62 +104,6 @@ def klasify(sers: Iterable, klases: Iterable, args: Iterable = None): for ser, klas, arg in zip(sers, klases, args)) -def isBase64(sb): - try: - if hasattr(sb, "encode"): - sb = sb.encode("utf-8") - - match = B64Rev.fullmatch(sb) - if match: - return True - return False - except Exception as ex: - return False - - -def keyToKey64u(key): - """ - Returns 64u - Convert and return bytes key to unicode base64 url-file safe version - """ - return base64.urlsafe_b64encode(key).decode("utf-8") - - -def key64uToKey(key64u): - """ - Returns bytes - Convert and return unicode base64 url-file safe key64u to bytes key - """ - return base64.urlsafe_b64decode(key64u.encode("utf-8")) - - -def verifyEd25519(sig, msg, vk): - """ - Returns True if signature sig of message msg is verified with - verification key vk Otherwise False - All of sig, msg, vk are bytes - """ - try: - result = pysodium.crypto_sign_verify_detached(sig, msg, vk) - except Exception as ex: - return False - return (True if result else False) - - -def verify64uEd25519(signature, message, verkey): - """ - Returns True if signature is valid for message with respect to verification - key verkey - - signature and verkey are encoded as unicode base64 url-file strings - and message is unicode string as would be the case for a json object - - """ - sig = key64uToKey(signature) - vk = key64uToKey(verkey) - msg = message.encode("utf-8") - return (verifyEd25519(sig, msg, vk)) - def nonStringIterable(obj): """ @@ -293,3 +243,168 @@ def fromIso8601(dts): if hasattr(dts, "decode"): dts = dts.decode("utf-8") return (datetime.datetime.fromisoformat(dts)) + + + +# Base64 utilities +BASE64_PAD = b'=' + +# Mappings between Base64 Encode Index and Decode Characters +# B64ChrByIdx is dict where each key is a B64 index and each value is the B64 char +# B64IdxByChr is dict where each key is a B64 char and each value is the B64 index +# Map Base64 index to char +B64ChrByIdx = dict((index, char) for index, char in enumerate([chr(x) for x in range(65, 91)])) +B64ChrByIdx.update([(index + 26, char) for index, char in enumerate([chr(x) for x in range(97, 123)])]) +B64ChrByIdx.update([(index + 52, char) for index, char in enumerate([chr(x) for x in range(48, 58)])]) +B64ChrByIdx[62] = '-' +B64ChrByIdx[63] = '_' +# Map char to Base64 index +B64IdxByChr = {char: index for index, char in B64ChrByIdx.items()} +# tuple +B64_CHARS = tuple(B64ChrByIdx.values()) # tuple of characters in Base64 + + +B64REX = b'^[0-9A-Za-z_-]*\Z' # [A-Za-z0-9\-\_] +Reb64 = re.compile(B64REX) # compile is faster +# to use if Reb64.match(bext): + + +def intToB64(i, l=1): + """ + Returns conversion of int i to Base64 str + l is min number of b64 digits left padded with Base64 0 == "A" char + """ + d = deque() # deque of characters base64 + + while l: + d.appendleft(B64ChrByIdx[i % 64]) + i = i // 64 + if not i: + break + for j in range(l - len(d)): # range(x) x <= 0 means do not iterate + d.appendleft("A") + return ("".join(d)) + + +def intToB64b(i, l=1): + """ + Returns conversion of int i to Base64 bytes + l is min number of b64 digits left padded with Base64 0 == "A" char + """ + return (intToB64(i=i, l=l).encode("utf-8")) + + +def b64ToInt(s): + """ + Returns conversion of Base64 str s or bytes to int + """ + if not s: + raise ValueError("Empty string, conversion undefined.") + if hasattr(s, 'decode'): + s = s.decode("utf-8") + i = 0 + for e, c in enumerate(reversed(s)): + i |= B64IdxByChr[c] << (e * 6) # same as i += B64IdxByChr[c] * (64 ** e) + return i + + + +def codeB64ToB2(s): + """ + Returns conversion (decode) of Base64 chars to Base2 bytes. + Where the number of total bytes returned is equal to the minimun number of + octets sufficient to hold the total converted concatenated sextets from s, + with one sextet per each Base64 decoded char of s. Assumes no pad chars in s. + Sextets are left aligned with pad bits in last (rightmost) byte. + This is useful for decoding as bytes, code characters from the front of + a Base64 encoded string of characters. + """ + i = b64ToInt(s) + i <<= 2 * (len(s) % 4) # add 2 bits right zero padding for each sextet + n = sceil(len(s) * 3 / 4) # compute min number of ocetets to hold all sextets + return (i.to_bytes(n, 'big')) + + +def codeB2ToB64(b, l): + """ + Returns conversion (encode) of l Base2 sextets from front of b to Base64 chars. + One char for each of l sextets from front (left) of b. + This is useful for encoding as code characters, sextets from the front of + a Base2 bytes (byte string). Must provide l because of ambiguity between l=3 + and l=4. Both require 3 bytes in b. + """ + if hasattr(b, 'encode'): + b = b.encode("utf-8") # convert to bytes + n = sceil(l * 3 / 4) # number of bytes needed for l sextets + if n > len(b): + raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) + i = int.from_bytes(b[:n], 'big') # convert only first n bytes to int + # check if prepad bits are zero + tbs = 2 * (l % 4) # trailing bit size in bits + i >>= tbs # right shift out trailing bits to make right aligned + return (intToB64(i, l)) # return as B64 + + +def nabSextets(b, l): + """ + Return first l sextets from front (left) of b as bytes (byte string). + Length of bytes returned is minimum sufficient to hold all l sextets. + Last byte returned is right bit padded with zeros + b is bytes or str + """ + if hasattr(b, 'encode'): + b = b.encode("utf-8") # convert to bytes + n = sceil(l * 3 / 4) # number of bytes needed for l sextets + if n > len(b): + raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) + i = int.from_bytes(b[:n], 'big') + p = 2 * (l % 4) + i >>= p # strip of last bits + i <<= p # pad with empty bits + return (i.to_bytes(n, 'big')) + + +def keyToKey64u(key): + """ + Returns 64u + Convert and return bytes key to unicode base64 url-file safe version + """ + return base64.urlsafe_b64encode(key).decode("utf-8") + + +def key64uToKey(key64u): + """ + Returns bytes + Convert and return unicode base64 url-file safe key64u to bytes key + """ + return base64.urlsafe_b64decode(key64u.encode("utf-8")) + + +def verifyEd25519(sig, msg, vk): + """ + Returns True if signature sig of message msg is verified with + verification key vk Otherwise False + All of sig, msg, vk are bytes + """ + try: + result = pysodium.crypto_sign_verify_detached(sig, msg, vk) + except Exception as ex: + return False + return (True if result else False) + + +def verify64uEd25519(signature, message, verkey): + """ + Returns True if signature is valid for message with respect to verification + key verkey + + signature and verkey are encoded as unicode base64 url-file strings + and message is unicode string as would be the case for a json object + + """ + sig = key64uToKey(signature) + vk = key64uToKey(verkey) + msg = message.encode("utf-8") + return (verifyEd25519(sig, msg, vk)) + + diff --git a/src/keri/kering.py b/src/keri/kering.py index cbc12266d..064d1793e 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -8,131 +8,12 @@ from dataclasses import dataclass, astuple from .help.helping import sceil +from .help.helping import intToB64, intToB64b, b64ToInt -FALSEY = (False, 0, None, "?0", "no", "false", "False", "off") -TRUTHY = (True, 1, "?1", "yes" "true", "True", 'on') MaxON = int("f"*32, 16) # 256 ** 16 - 1 maximum ordinal number, sequence or first seen etc -# Base64 utilities -BASE64_PAD = b'=' - -# Mappings between Base64 Encode Index and Decode Characters -# B64ChrByIdx is dict where each key is a B64 index and each value is the B64 char -# B64IdxByChr is dict where each key is a B64 char and each value is the B64 index -# Map Base64 index to char -B64ChrByIdx = dict((index, char) for index, char in enumerate([chr(x) for x in range(65, 91)])) -B64ChrByIdx.update([(index + 26, char) for index, char in enumerate([chr(x) for x in range(97, 123)])]) -B64ChrByIdx.update([(index + 52, char) for index, char in enumerate([chr(x) for x in range(48, 58)])]) -B64ChrByIdx[62] = '-' -B64ChrByIdx[63] = '_' -# Map char to Base64 index -B64IdxByChr = {char: index for index, char in B64ChrByIdx.items()} -# tuple -B64_CHARS = tuple(B64ChrByIdx.values()) # tuple of characters in Base64 - - -B64REX = b'^[A-Za-z0-9\-\_]*\Z' -Reb64 = re.compile(B64REX) # compile is faster - - -def intToB64(i, l=1): - """ - Returns conversion of int i to Base64 str - l is min number of b64 digits left padded with Base64 0 == "A" char - """ - d = deque() # deque of characters base64 - - while l: - d.appendleft(B64ChrByIdx[i % 64]) - i = i // 64 - if not i: - break - for j in range(l - len(d)): # range(x) x <= 0 means do not iterate - d.appendleft("A") - return ("".join(d)) - - -def intToB64b(i, l=1): - """ - Returns conversion of int i to Base64 bytes - l is min number of b64 digits left padded with Base64 0 == "A" char - """ - return (intToB64(i=i, l=l).encode("utf-8")) - - -def b64ToInt(s): - """ - Returns conversion of Base64 str s or bytes to int - """ - if not s: - raise ValueError("Empty string, conversion undefined.") - if hasattr(s, 'decode'): - s = s.decode("utf-8") - i = 0 - for e, c in enumerate(reversed(s)): - i |= B64IdxByChr[c] << (e * 6) # same as i += B64IdxByChr[c] * (64 ** e) - return i - - - -def codeB64ToB2(s): - """ - Returns conversion (decode) of Base64 chars to Base2 bytes. - Where the number of total bytes returned is equal to the minimun number of - octets sufficient to hold the total converted concatenated sextets from s, - with one sextet per each Base64 decoded char of s. Assumes no pad chars in s. - Sextets are left aligned with pad bits in last (rightmost) byte. - This is useful for decoding as bytes, code characters from the front of - a Base64 encoded string of characters. - """ - i = b64ToInt(s) - i <<= 2 * (len(s) % 4) # add 2 bits right zero padding for each sextet - n = sceil(len(s) * 3 / 4) # compute min number of ocetets to hold all sextets - return (i.to_bytes(n, 'big')) - - -def codeB2ToB64(b, l): - """ - Returns conversion (encode) of l Base2 sextets from front of b to Base64 chars. - One char for each of l sextets from front (left) of b. - This is useful for encoding as code characters, sextets from the front of - a Base2 bytes (byte string). Must provide l because of ambiguity between l=3 - and l=4. Both require 3 bytes in b. - """ - if hasattr(b, 'encode'): - b = b.encode("utf-8") # convert to bytes - n = sceil(l * 3 / 4) # number of bytes needed for l sextets - if n > len(b): - raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) - i = int.from_bytes(b[:n], 'big') # convert only first n bytes to int - # check if prepad bits are zero - tbs = 2 * (l % 4) # trailing bit size in bits - i >>= tbs # right shift out trailing bits to make right aligned - return (intToB64(i, l)) # return as B64 - - -def nabSextets(b, l): - """ - Return first l sextets from front (left) of b as bytes (byte string). - Length of bytes returned is minimum sufficient to hold all l sextets. - Last byte returned is right bit padded with zeros - b is bytes or str - """ - if hasattr(b, 'encode'): - b = b.encode("utf-8") # convert to bytes - n = sceil(l * 3 / 4) # number of bytes needed for l sextets - if n > len(b): - raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) - i = int.from_bytes(b[:n], 'big') - p = 2 * (l % 4) - i >>= p # strip of last bits - i <<= p # pad with empty bits - return (i.to_bytes(n, 'big')) - - - # Serialization Kinds Serialage = namedtuple("Serialage", 'json mgpk cbor') Serials = Serialage(json='JSON', mgpk='MGPK', cbor='CBOR') diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 012b4dcd2..816810e1f 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -36,7 +36,7 @@ Diger, Prefixer, Cipher, Encrypter, Decrypter) from keri.core.coring import versify, deversify, Rever, VERFULLSIZE from keri.core.coring import generateSigners, generatePrivates -from keri.core.coring import (intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, +from keri.help.helping import (intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, B64_CHARS, Reb64, nabSextets) from keri.help import helping from keri.kering import (EmptyMaterialError, RawMaterialError, DerivationError, @@ -44,14 +44,6 @@ InvalidValueError, DeserializeError, ValidationError, InvalidVarRawSizeError) from keri.kering import Version, Versionage, VersionError -#from keri.kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, - #RPY_LABELS) -#from keri.kering import (VCP_LABELS, VRT_LABELS, ISS_LABELS, BIS_LABELS, REV_LABELS, - #BRV_LABELS, TSN_LABELS, CRED_TSN_LABELS) - - - - def test_protocol_genus_codex(): diff --git a/tests/help/test_helping.py b/tests/help/test_helping.py index 23779a74d..4d5d7baae 100644 --- a/tests/help/test_helping.py +++ b/tests/help/test_helping.py @@ -15,6 +15,8 @@ from keri.help.helping import isign, sceil from keri.help.helping import extractValues from keri.help.helping import dictify, datify, klasify +from keri.help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, + codeB64ToB2, codeB2ToB64, Reb64, nabSextets) def test_utilities(): """ @@ -273,5 +275,172 @@ def test_iso8601(): """ End Test """ + +def test_b64_conversions(): + """ + Test Base64 conversion utility routines + """ + + cs = intToB64(0) + assert cs == "A" + i = b64ToInt(cs) + assert i == 0 + + cs = intToB64(0, l=0) + assert cs == "" + with pytest.raises(ValueError): + i = b64ToInt(cs) + + cs = intToB64(None, l=0) + assert cs == "" + with pytest.raises(ValueError): + i = b64ToInt(cs) + + cs = intToB64b(0) + assert cs == b"A" + i = b64ToInt(cs) + assert i == 0 + + cs = intToB64(27) + assert cs == "b" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64b(27) + assert cs == b"b" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64(27, l=2) + assert cs == "Ab" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64b(27, l=2) + assert cs == b"Ab" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64(80) + assert cs == "BQ" + i = b64ToInt(cs) + assert i == 80 + + cs = intToB64b(80) + assert cs == b"BQ" + i = b64ToInt(cs) + assert i == 80 + + cs = intToB64(4095) + assert cs == '__' + i = b64ToInt(cs) + assert i == 4095 + + cs = intToB64b(4095) + assert cs == b'__' + i = b64ToInt(cs) + assert i == 4095 + + cs = intToB64(4096) + assert cs == 'BAA' + i = b64ToInt(cs) + assert i == 4096 + + cs = intToB64b(4096) + assert cs == b'BAA' + i = b64ToInt(cs) + assert i == 4096 + + cs = intToB64(6011) + assert cs == "Bd7" + i = b64ToInt(cs) + assert i == 6011 + + cs = intToB64b(6011) + assert cs == b"Bd7" + i = b64ToInt(cs) + assert i == 6011 + + s = "-BAC" + b = codeB64ToB2(s[:]) + assert len(b) == 3 + assert b == b'\xf8\x10\x02' + t = codeB2ToB64(b, 4) + assert t == s[:] + i = int.from_bytes(b, 'big') + assert i == 0o76010002 + i >>= 2 * (len(s) % 4) + assert i == 0o76010002 + p = nabSextets(b, 4) + assert p == b'\xf8\x10\x02' + + b = codeB64ToB2(s[:3]) + assert len(b) == 3 + assert b == b'\xf8\x10\x00' + t = codeB2ToB64(b, 3) + assert t == s[:3] + i = int.from_bytes(b, 'big') + assert i == 0o76010000 + i >>= 2 * (len(s[:3]) % 4) + assert i == 0o760100 + p = nabSextets(b, 3) + assert p == b'\xf8\x10\x00' + + b = codeB64ToB2(s[:2]) + assert len(b) == 2 + assert b == b'\xf8\x10' + t = codeB2ToB64(b, 2) + assert t == s[:2] + i = int.from_bytes(b, 'big') + assert i == 0o174020 + i >>= 2 * (len(s[:2]) % 4) + assert i == 0o7601 + p = nabSextets(b, 2) + assert p == b'\xf8\x10' + + b = codeB64ToB2(s[:1]) + assert len(b) == 1 + assert b == b'\xf8' + t = codeB2ToB64(b, 1) + assert t == s[:1] + i = int.from_bytes(b, 'big') + assert i == 0o370 + i >>= 2 * (len(s[:1]) % 4) + assert i == 0o76 + p = nabSextets(b, 1) + assert p == b'\xf8' + + assert B64_CHARS == ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', + 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_') + assert '@' not in B64_CHARS + assert 'A' in B64_CHARS + + text = b"-A-Bg-1-3-cd" + match = Reb64.match(text) + assert match + assert match is not None + + text = b'' + match = Reb64.match(text) + assert match + assert match is not None + + text = b'123#$' + match = Reb64.match(text) + assert not match + assert match is None + + """End Test""" + + if __name__ == "__main__": + test_utilities() + test_datify() + test_dictify() test_klasify() + test_extractvalues() + test_iso8601() + test_b64_conversions() diff --git a/tests/test_kering.py b/tests/test_kering.py index 7bbb2872a..ffb3aa622 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -12,8 +12,6 @@ import pytest from keri import kering -from keri.kering import (intToB64, intToB64b, b64ToInt, B64_CHARS, - codeB64ToB2, codeB2ToB64, Reb64, nabSextets) from keri.kering import Protocolage, Protos from keri.kering import Serialage, Serials from keri.kering import Ilkage, Ilks @@ -21,168 +19,12 @@ versify, deversify, Rever) from keri.kering import (VER1FULLSPAN, VER1TERM, VEREX1, VER2FULLSPAN, VER2TERM, VEREX2, VEREX) +from keri.help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, + codeB64ToB2, codeB2ToB64, Reb64, nabSextets) -def test_b64_conversions(): - """ - Test Base64 conversion utility routines - """ - - cs = intToB64(0) - assert cs == "A" - i = b64ToInt(cs) - assert i == 0 - - cs = intToB64(0, l=0) - assert cs == "" - with pytest.raises(ValueError): - i = b64ToInt(cs) - - cs = intToB64(None, l=0) - assert cs == "" - with pytest.raises(ValueError): - i = b64ToInt(cs) - - cs = intToB64b(0) - assert cs == b"A" - i = b64ToInt(cs) - assert i == 0 - - cs = intToB64(27) - assert cs == "b" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64b(27) - assert cs == b"b" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64(27, l=2) - assert cs == "Ab" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64b(27, l=2) - assert cs == b"Ab" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64(80) - assert cs == "BQ" - i = b64ToInt(cs) - assert i == 80 - - cs = intToB64b(80) - assert cs == b"BQ" - i = b64ToInt(cs) - assert i == 80 - - cs = intToB64(4095) - assert cs == '__' - i = b64ToInt(cs) - assert i == 4095 - - cs = intToB64b(4095) - assert cs == b'__' - i = b64ToInt(cs) - assert i == 4095 - - cs = intToB64(4096) - assert cs == 'BAA' - i = b64ToInt(cs) - assert i == 4096 - - cs = intToB64b(4096) - assert cs == b'BAA' - i = b64ToInt(cs) - assert i == 4096 - - cs = intToB64(6011) - assert cs == "Bd7" - i = b64ToInt(cs) - assert i == 6011 - - cs = intToB64b(6011) - assert cs == b"Bd7" - i = b64ToInt(cs) - assert i == 6011 - - s = "-BAC" - b = codeB64ToB2(s[:]) - assert len(b) == 3 - assert b == b'\xf8\x10\x02' - t = codeB2ToB64(b, 4) - assert t == s[:] - i = int.from_bytes(b, 'big') - assert i == 0o76010002 - i >>= 2 * (len(s) % 4) - assert i == 0o76010002 - p = nabSextets(b, 4) - assert p == b'\xf8\x10\x02' - - b = codeB64ToB2(s[:3]) - assert len(b) == 3 - assert b == b'\xf8\x10\x00' - t = codeB2ToB64(b, 3) - assert t == s[:3] - i = int.from_bytes(b, 'big') - assert i == 0o76010000 - i >>= 2 * (len(s[:3]) % 4) - assert i == 0o760100 - p = nabSextets(b, 3) - assert p == b'\xf8\x10\x00' - - b = codeB64ToB2(s[:2]) - assert len(b) == 2 - assert b == b'\xf8\x10' - t = codeB2ToB64(b, 2) - assert t == s[:2] - i = int.from_bytes(b, 'big') - assert i == 0o174020 - i >>= 2 * (len(s[:2]) % 4) - assert i == 0o7601 - p = nabSextets(b, 2) - assert p == b'\xf8\x10' - - b = codeB64ToB2(s[:1]) - assert len(b) == 1 - assert b == b'\xf8' - t = codeB2ToB64(b, 1) - assert t == s[:1] - i = int.from_bytes(b, 'big') - assert i == 0o370 - i >>= 2 * (len(s[:1]) % 4) - assert i == 0o76 - p = nabSextets(b, 1) - assert p == b'\xf8' - - assert B64_CHARS == ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', - 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', - 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_') - assert '@' not in B64_CHARS - assert 'A' in B64_CHARS - - text = b"-A-Bg-1-3-cd" - match = Reb64.match(text) - assert match - assert match is not None - - text = b'' - match = Reb64.match(text) - assert match - assert match is not None - - text = b'123#$' - match = Reb64.match(text) - assert not match - assert match is None - - """End Test""" def test_protos(): From e973f4234ea35de83c634e2bd06aa4a7d6ed728a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Feb 2024 14:48:03 -0700 Subject: [PATCH 027/418] refactor versify and deversify to support new version string format need tests --- src/keri/core/coring.py | 6 +- src/keri/core/serdering.py | 8 +- src/keri/kering.py | 262 +++++++++++++------------------------ src/keri/vc/proving.py | 2 +- tests/core/test_coring.py | 2 +- tests/test_kering.py | 18 +-- tests/vc/test_proving.py | 6 +- 7 files changed, 114 insertions(+), 190 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 0a543690e..cd7ca2af2 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -35,7 +35,7 @@ EmptyListError, ShortageError, UnexpectedCodeError, DeserializeError, UnexpectedCountCodeError, UnexpectedOpCodeError) -from ..kering import (Versionage, Version, VERRAWSIZE, VERFMT, VERFULLSIZE, +from ..kering import (Versionage, Version, VERRAWSIZE, VERFMT, MAXVERFULLSPAN, versify, deversify, Rever, smell) from ..kering import Serials, Serialage, Protos, Protocolage, Ilkage, Ilks @@ -103,7 +103,7 @@ def sizeify(ked, kind=None, version=Version): fore, back = match.span() # full version string # update vs with latest kind version size - vs = versify(proto=proto, version=vrsn, kind=kind, size=size) + vs = versify(protocol=proto, version=vrsn, kind=kind, size=size) # replace old version string in raw with new one raw = b'%b%b%b' % (raw[:fore], vs.encode("utf-8"), raw[back:]) if size != len(raw): # substitution messed up @@ -5151,7 +5151,7 @@ class Sadder: """ MaxVSOffset = 12 - SmellSize = MaxVSOffset + VERFULLSIZE # min buffer size to inhale + SmellSize = MaxVSOffset + MAXVERFULLSPAN # min buffer size to inhale def __init__(self, raw=b'', ked=None, sad=None, kind=None, saidify=False, code=MtrDex.Blake3_256): diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index d3bb8940d..1abb02b43 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -18,7 +18,7 @@ ShortageError, VersionError, ProtocolError, KindError, DeserializeError, FieldError, SerializeError) from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_1_1, - VERRAWSIZE, VERFMT, VERFULLSIZE, + VERRAWSIZE, VERFMT, MAXVERFULLSPAN, SMELLSIZE, Smellage, smell) from ..kering import Protos, Serials, Rever, versify, deversify, Ilks from ..core import coring @@ -682,13 +682,13 @@ def makify(self, sad, *, version=None, raise SerializeError(f"Missing requires version string field 'v'" f" in sad = {sad}.") - sad['v'] = self.Dummy * VERFULLSIZE # ensure size of vs + sad['v'] = self.Dummy * MAXVERFULLSPAN # ensure size of vs raw = self.dumps(sad, kind) # get size of fully dummied sad size = len(raw) # generate new version string with correct size - vs = versify(proto=proto, version=vrsn, kind=kind, size=size) + vs = versify(protocol=proto, version=vrsn, kind=kind, size=size) sad["v"] = vs # update version string in sad # now have correctly sized version string in sad @@ -844,7 +844,7 @@ def _exhale(clas, sad, version=None): size = len(raw) # generate new version string with correct size - vs = versify(proto=proto, version=vrsn, kind=kind, size=size) + vs = versify(protocol=proto, version=vrsn, kind=kind, size=size) # find location of old version string inside raw match = Rever.search(raw) # Rever's regex takes bytes diff --git a/src/keri/kering.py b/src/keri/kering.py index 064d1793e..c97b67132 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -27,17 +27,15 @@ Vrsn_1_0 = Versionage(major=1, minor=0) # KERI Protocol Version Specific Vrsn_1_1 = Versionage(major=1, minor=1) # KERI Protocol Version Specific -VERRAWSIZE = 6 # hex characters in raw serialization size in version string + # "{:0{}x}".format(300, 6) # make num char in hex a variable # '00012c' VERFMT = "{}{:x}{:x}{}{:0{}x}_" # version format string -VERFULLSIZE = 17 # number of characters in full version string - -VEREX0 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' +VERRAWSIZE = 6 # hex characters in raw serialization size in version string +#VEREX0 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' #Rever = re.compile(VEREX0) # compile is faster - VER1FULLSPAN = 17 # number of characters in full version string VER1TERM = b'_' VEREX1 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' @@ -48,8 +46,16 @@ VEREX = VEREX2 + b'|' + VEREX1 +MAXVERFULLSPAN = max(VER1FULLSPAN, VER2FULLSPAN) # number of characters in full version string + Rever = re.compile(VEREX) # compile is faster + +MAXVSOFFSET = 12 +SMELLSIZE = MAXVSOFFSET + MAXVERFULLSPAN # min buffer size to inhale + + + """ Smellage (results of smelling a version string such as in a Serder) protocol (str): protocol type value of Protos examples 'KERI', 'ACDC' @@ -60,18 +66,88 @@ """ Smellage = namedtuple("Smellage", "protocol version kind size") +def rematch(match, *, version=None): + """ + Returns: + smellage (Smellage): named tuple extracted from version string regex match + (protocol, version, kind, size) + + Parameters: + match (re.Match): instance of Match class + version (Versionage | None): supported version. None means do not check + for supported version. namedtuple (major, minor) + + """ + full = match.group() # full matched version string + if len(full) == VER2FULLSPAN and full[-1] == ord(VER2TERM): + proto, major, minor, kind, size = match.group("proto2", + "major2", + "minor2", + "kind2", + "size2") + protocol = proto.decode("utf-8") + if protocol not in Protos: + raise ProtocolError(f"Invalid protocol type = {protocol}.") + vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) + if vrsn.major < 2: # version2 vs but major < 2 + VersionError(f"Incompatible {vrsn=} with version string.") + if version is not None: # compatible version with vrsn + if (vrsn.major > version.major or + (vrsn.major == version.major and vrsn.minor > version.minor)): + raise VersionError(f"Incompatible {version=}, with " + f"{vrsn=}.") + + kind = kind.decode("utf-8") + if kind not in Serials: + raise KindError(f"Invalid serialization kind = {kind}.") + size = b64ToInt(size) + + elif len(full) == VER1FULLSPAN and full[-1] == ord(VER1TERM): + proto, major, minor, kind, size = match.group("proto1", + "major1", + "minor1", + "kind1", + "size1") + protocol = proto.decode("utf-8") + if protocol not in Protos: + raise ProtocolError(f"Invalid protocol type = {protocol}.") + vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) + if vrsn.major > 1: # version1 vs but major > 1 + VersionError(f"Incompatible {vrsn=} with version string.") + if version is not None and vrsn != version: + raise VersionError(f"Expected {version=}, got " + f"{vrsn=}.") + kind = kind.decode("utf-8") + if kind not in Serials: + raise KindError(f"Invalid serialization kind = {kind}.") + size = int(size, 16) + + return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) + + -def versify(proto=Protos.keri, version=Version, kind=Serials.json, size=0): +def versify(protocol=Protos.keri, version=Version, kind=Serials.json, size=0): """ - Returns version string + Returns: + vs (str): version string + + Parameters: + protocol (str): protocol one of Protos + version (Versionage): namedtuple (major, minor) of ints + kind (str): one of Serials + size (int): length of serialized map that embeds version string field. """ - if proto not in Protos: - raise ValueError("Invalid message identifier = {}".format(proto)) - #version = version if version else Version + if protocol not in Protos: + raise ProtocolError("Invalid message identifier = {}".format(protocol)) if kind not in Serials: - raise ValueError("Invalid serialization kind = {}".format(kind)) + raise KindError("Invalid serialization kind = {}".format(kind)) + + if version.major < 2: # version1 version string + return VERFMT.format(protocol, version.major, version.minor, kind, size, VERRAWSIZE) + else: # version 2+ version string + return (f"{protocol}{intToB64(version.major)}" + f"{intToB64(version.minor, l=2)}{kind}{intToB64(size, l=4)})") - return VERFMT.format(proto, version[0], version[1], kind, size, VERRAWSIZE) def deversify(vs, version=None): @@ -88,7 +164,7 @@ def deversify(vs, version=None): Parameters: vs (str | bytes): version string to extract from version (Versionage | None): supported version. None means do not check - for supported version. + for supported version. namedtuple of ints (major, minor) Uses regex match to extract: protocol type @@ -100,85 +176,12 @@ def deversify(vs, version=None): vs = vs.encode("utf-8") match = Rever.match(vs) - if match: - full = match.group() # full matched version string - if len(full) == VER2FULLSPAN and full[-1] == ord(VER2TERM): - proto, major, minor, kind, size = match.group("proto2", - "major2", - "minor2", - "kind2", - "size2") - protocol = proto.decode("utf-8") - if protocol not in Protos: - raise ProtocolError(f"Invalid protocol type = {protocol}.") - vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) - if vrsn.major < 2: # version2 vs but major < 2 - VersionError(f"Incompatible {vrsn=} with version string.") - if version is not None: # compatible version with vrsn - if (vrsn.major > version.major or - (vrsn.major == version.major and vrsn.minor > version.minor)): - raise VersionError(f"Incompatible {version=}, with " - f"{vrsn=}.") - - kind = kind.decode("utf-8") - if kind not in Serials: - raise KindError(f"Invalid serialization kind = {kind}.") - size = b64ToInt(size) - return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) - - - - elif len(full) == VER1FULLSPAN and full[-1] == ord(VER1TERM): - proto, major, minor, kind, size = match.group("proto1", - "major1", - "minor1", - "kind1", - "size1") - protocol = proto.decode("utf-8") - if protocol not in Protos: - raise ProtocolError(f"Invalid protocol type = {protocol}.") - vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) - if vrsn.major > 1: # version1 vs but major > 1 - VersionError(f"Incompatible {vrsn=} with version string.") - if version is not None and vrsn != version: - raise VersionError(f"Expected {version=}, got " - f"{vrsn=}.") - kind = kind.decode("utf-8") - if kind not in Serials: - raise KindError(f"Invalid serialization kind = {kind}.") - size = int(size, 16) - return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) - - - - raise ValueError(f"Invalid version string '{vs}'.") - - -#proto, major, minor, kind, size = match.group("proto", - #"major", - #"minor", - #"kind", - #"size") - -#proto = proto.decode("utf-8") -#vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) -#kind = kind.decode("utf-8") - -#if proto not in Protos: - #raise ValueError("Invalid message identifier = {}".format(proto)) -#if version is not None and vrsn != version: - #raise ValueError(f"Expected version = {version}, got " - #f"{vrsn.major}.{vrsn.minor}.") -#if kind not in Serials: - #raise ValueError("Invalid serialization kind = {}".format(kind)) -#size = int(size, 16) -#return proto, vrsn, kind, size + if not match: + raise VersionError(f"Invalid version string = '{vs}'.") + return rematch(match, version=version) -MAXVSOFFSET = 12 -SMELLSIZE = MAXVSOFFSET + VERFULLSIZE # min buffer size to inhale - def smell(raw, *, version=None): @@ -203,88 +206,9 @@ def smell(raw, *, version=None): raise VersionError(f"Invalid version string from smelled raw = " f"{raw[: SMELLSIZE]}.") + return rematch(match, version=version) - full = match.group() # full matched version string - if len(full) == VER2FULLSPAN and full[-1] == ord(VER2TERM): - proto, major, minor, kind, size = match.group("proto2", - "major2", - "minor2", - "kind2", - "size2") - protocol = proto.decode("utf-8") - if protocol not in Protos: - raise ProtocolError(f"Invalid protocol type = {protocol}.") - vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) - if vrsn.major < 2: # version2 vs but major < 2 - VersionError(f"Incompatible {vrsn=} with version string.") - if version is not None: # compatible version with vrsn - if (vrsn.major > version.major or - (vrsn.major == version.major and vrsn.minor > version.minor)): - raise VersionError(f"Incompatible {version=}, with " - f"{vrsn=}.") - - kind = kind.decode("utf-8") - if kind not in Serials: - raise KindError(f"Invalid serialization kind = {kind}.") - size = b64ToInt(size) - return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) - - - - elif len(full) == VER1FULLSPAN and full[-1] == ord(VER1TERM): - proto, major, minor, kind, size = match.group("proto1", - "major1", - "minor1", - "kind1", - "size1") - protocol = proto.decode("utf-8") - if protocol not in Protos: - raise ProtocolError(f"Invalid protocol type = {protocol}.") - vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) - if vrsn.major > 1: # version1 vs but major > 1 - VersionError(f"Incompatible {vrsn=} with version string.") - if version is not None and vrsn != version: - raise VersionError(f"Expected {version=}, got " - f"{vrsn=}.") - kind = kind.decode("utf-8") - if kind not in Serials: - raise KindError(f"Invalid serialization kind = {kind}.") - size = int(size, 16) - return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) - - - - - - #proto, major, minor, kind, size = match.group("proto", "major", "minor", "kind", "size") - - ## use length of version string matched to determine if version 1.x or 2.x - ## so can convert major, minor, and size correctly hex vs B64 numbers - - ## Global version compatibility check. Serder instances also peform version check - #major = int(major, 16) - #minor = int(minor, 16) - #vrsn = Versionage(major=major, minor=minor) - #if version is not None: # test here for compatible code version with message vrsn - #if (vrsn.major > version.major or - #(vrsn.major == version.major and vrsn.minor > version.minor)): - #pass # raise error here? - - - #protocol = proto.decode("utf-8") - #if protocol not in Protos: - #raise ProtocolError(f"Invalid protocol type = {protocol}.") - - #kind = kind.decode("utf-8") - #if kind not in Serials: - #raise KindError(f"Invalid serialization kind = {kind}.") - - #size = int(size, 16) - #if len(raw) < size: - #raise ShortageError(f"Need more bytes.") - - #return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) @dataclass(frozen=True) diff --git a/src/keri/vc/proving.py b/src/keri/vc/proving.py index e724648a5..826e818d0 100644 --- a/src/keri/vc/proving.py +++ b/src/keri/vc/proving.py @@ -50,7 +50,7 @@ def credential(schema, SerderACDC: credential instance """ - vs = versify(proto=coring.Protos.acdc, version=version, kind=kind, size=0) + vs = versify(protocol=coring.Protos.acdc, version=version, kind=kind, size=0) vc = dict( v=vs, diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 816810e1f..b2f3b9834 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -34,7 +34,7 @@ CtrDex, Counter, ProDex) from keri.core.coring import (Verfer, Cigar, Signer, Salter, Saider, DigDex, Diger, Prefixer, Cipher, Encrypter, Decrypter) -from keri.core.coring import versify, deversify, Rever, VERFULLSIZE +from keri.core.coring import versify, deversify, Rever, MAXVERFULLSPAN from keri.core.coring import generateSigners, generatePrivates from keri.help.helping import (intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, B64_CHARS, Reb64, nabSextets) diff --git a/tests/test_kering.py b/tests/test_kering.py index ffb3aa622..3a419e900 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -15,7 +15,7 @@ from keri.kering import Protocolage, Protos from keri.kering import Serialage, Serials from keri.kering import Ilkage, Ilks -from keri.kering import (Versionage, Version, VERFULLSIZE, +from keri.kering import (Versionage, Version, MAXVERFULLSPAN, versify, deversify, Rever) from keri.kering import (VER1FULLSPAN, VER1TERM, VEREX1, VER2FULLSPAN, VER2TERM, VEREX2, VEREX) @@ -293,7 +293,7 @@ def test_versify(): """ vs = versify(kind=Serials.json, size=0) assert vs == "KERI10JSON000000_" - assert len(vs) == VERFULLSIZE + assert len(vs) == MAXVERFULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.json @@ -302,16 +302,16 @@ def test_versify(): vs = versify(kind=Serials.json, size=65) assert vs == "KERI10JSON000041_" - assert len(vs) == VERFULLSIZE + assert len(vs) == MAXVERFULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.json assert version == Version assert size == 65 - vs = versify(proto=Protos.acdc, kind=Serials.json, size=86) + vs = versify(protocol=Protos.acdc, kind=Serials.json, size=86) assert vs == "ACDC10JSON000056_" - assert len(vs) == VERFULLSIZE + assert len(vs) == MAXVERFULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.acdc assert kind == Serials.json @@ -320,7 +320,7 @@ def test_versify(): vs = versify(kind=Serials.mgpk, size=0) assert vs == "KERI10MGPK000000_" - assert len(vs) == VERFULLSIZE + assert len(vs) == MAXVERFULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.mgpk @@ -329,7 +329,7 @@ def test_versify(): vs = versify(kind=Serials.mgpk, size=65) assert vs == "KERI10MGPK000041_" - assert len(vs) == VERFULLSIZE + assert len(vs) == MAXVERFULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.mgpk @@ -338,7 +338,7 @@ def test_versify(): vs = versify(kind=Serials.cbor, size=0) assert vs == "KERI10CBOR000000_" - assert len(vs) == VERFULLSIZE + assert len(vs) == MAXVERFULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.cbor @@ -347,7 +347,7 @@ def test_versify(): vs = versify(kind=Serials.cbor, size=65) assert vs == "KERI10CBOR000041_" - assert len(vs) == VERFULLSIZE + assert len(vs) == MAXVERFULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.cbor diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index bdb273be1..fbcc9be99 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -113,7 +113,7 @@ def test_credentialer(): sub = dict(a=123, b="abc", issuanceDate="2021-06-27T21:26:21.233257+00:00") d = dict( - v=coring.versify(proto=coring.Protos.acdc, kind=Serials.json, size=0), + v=coring.versify(protocol=coring.Protos.acdc, kind=Serials.json, size=0), d="", i="EF6maPM_d5ZN7U3NRFC1-6TM7k_E00_a8AG9YyLA4uWi", s="abc", @@ -150,7 +150,7 @@ def test_credentialer(): d2 = dict(d) d2['d'] = "" - d2["v"] = coring.versify(proto=coring.Protos.acdc, kind=Serials.cbor, size=0) + d2["v"] = coring.versify(protocol=coring.Protos.acdc, kind=Serials.cbor, size=0) _, d2 = coring.Saider.saidify(sad=d2) creder = serdering.SerderACDC(sad=d2) # Creder(ked=d2) @@ -176,7 +176,7 @@ def test_credentialer(): assert creder.sad == d2 d3 = dict(d) - d3["v"] = coring.versify(proto=coring.Protos.acdc, kind=Serials.mgpk, size=0) + d3["v"] = coring.versify(protocol=coring.Protos.acdc, kind=Serials.mgpk, size=0) _, d3 = coring.Saider.saidify(sad=d3) creder = serdering.SerderACDC(sad=d3) # Creder(ked=d3) From 344bbf1788bb648f4fdff48cec6875ff06ae4041 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Feb 2024 15:56:18 -0700 Subject: [PATCH 028/418] added tests for versify deversify with bad version strings --- src/keri/kering.py | 6 +- tests/test_kering.py | 153 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 145 insertions(+), 14 deletions(-) diff --git a/src/keri/kering.py b/src/keri/kering.py index c97b67132..cfb70da7b 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -90,7 +90,7 @@ def rematch(match, *, version=None): raise ProtocolError(f"Invalid protocol type = {protocol}.") vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) if vrsn.major < 2: # version2 vs but major < 2 - VersionError(f"Incompatible {vrsn=} with version string.") + raise VersionError(f"Incompatible {vrsn=} with version string.") if version is not None: # compatible version with vrsn if (vrsn.major > version.major or (vrsn.major == version.major and vrsn.minor > version.minor)): @@ -113,7 +113,7 @@ def rematch(match, *, version=None): raise ProtocolError(f"Invalid protocol type = {protocol}.") vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) if vrsn.major > 1: # version1 vs but major > 1 - VersionError(f"Incompatible {vrsn=} with version string.") + raise VersionError(f"Incompatible {vrsn=} with version string.") if version is not None and vrsn != version: raise VersionError(f"Expected {version=}, got " f"{vrsn=}.") @@ -146,7 +146,7 @@ def versify(protocol=Protos.keri, version=Version, kind=Serials.json, size=0): return VERFMT.format(protocol, version.major, version.minor, kind, size, VERRAWSIZE) else: # version 2+ version string return (f"{protocol}{intToB64(version.major)}" - f"{intToB64(version.minor, l=2)}{kind}{intToB64(size, l=4)})") + f"{intToB64(version.minor, l=2)}{kind}{intToB64(size, l=4)}.") diff --git a/tests/test_kering.py b/tests/test_kering.py index 3a419e900..e8bfa7b4c 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -19,6 +19,7 @@ versify, deversify, Rever) from keri.kering import (VER1FULLSPAN, VER1TERM, VEREX1, VER2FULLSPAN, VER2TERM, VEREX2, VEREX) +from keri.kering import VersionError, ProtocolError, KindError from keri.help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, codeB64ToB2, codeB2ToB64, Reb64, nabSextets) @@ -287,13 +288,18 @@ def test_serials(): """Done Test""" -def test_versify(): +def test_versify_v1(): """ Test Versify support """ - vs = versify(kind=Serials.json, size=0) + + assert VER1FULLSPAN == MAXVERFULLSPAN + + # default version is version 1 + + vs = versify() # defaults assert vs == "KERI10JSON000000_" - assert len(vs) == MAXVERFULLSPAN + assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.json @@ -302,7 +308,7 @@ def test_versify(): vs = versify(kind=Serials.json, size=65) assert vs == "KERI10JSON000041_" - assert len(vs) == MAXVERFULLSPAN + assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.json @@ -311,7 +317,7 @@ def test_versify(): vs = versify(protocol=Protos.acdc, kind=Serials.json, size=86) assert vs == "ACDC10JSON000056_" - assert len(vs) == MAXVERFULLSPAN + assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.acdc assert kind == Serials.json @@ -320,7 +326,7 @@ def test_versify(): vs = versify(kind=Serials.mgpk, size=0) assert vs == "KERI10MGPK000000_" - assert len(vs) == MAXVERFULLSPAN + assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.mgpk @@ -329,7 +335,7 @@ def test_versify(): vs = versify(kind=Serials.mgpk, size=65) assert vs == "KERI10MGPK000041_" - assert len(vs) == MAXVERFULLSPAN + assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.mgpk @@ -338,7 +344,7 @@ def test_versify(): vs = versify(kind=Serials.cbor, size=0) assert vs == "KERI10CBOR000000_" - assert len(vs) == MAXVERFULLSPAN + assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.cbor @@ -347,12 +353,137 @@ def test_versify(): vs = versify(kind=Serials.cbor, size=65) assert vs == "KERI10CBOR000041_" - assert len(vs) == MAXVERFULLSPAN + assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) assert proto == Protos.keri assert kind == Serials.cbor assert version == Version assert size == 65 + + vs = versify(version=Versionage(major=1, minor=1)) # defaults + assert vs == "KERI11JSON000000_" + assert len(vs) == VER1FULLSPAN + proto, vrsn, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.json + assert vrsn == (1, 1) + assert size == 0 + + # test bad version strings + vs = "KERI20JSON000000_" + with pytest.raises(VersionError): + smellage = deversify(vs) + + vs = "ABLE10JSON000000_" + with pytest.raises(ProtocolError): + smellage = deversify(vs) + + vs = "KERI10MSON000000_" + with pytest.raises(KindError): + smellage = deversify(vs) + + + """End Test""" + + +def test_versify_v2(): + """ + Test Versify support + """ + version = Versionage(major=2, minor=0) + assert version == (2, 0) + + vs = versify(version=version) # defaults + assert vs == "KERICAAJSONAAAA." + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.json + assert vrsn == version + assert size == 0 + + vs = versify(version=version, kind=Serials.json, size=65) + assert vs == "KERICAAJSONAABB." + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.json + assert vrsn == version + assert size == 65 + + vs = versify(protocol=Protos.acdc, version=version, kind=Serials.json, size=86) + assert vs == "ACDCCAAJSONAABW." + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size = deversify(vs) + assert proto == Protos.acdc + assert kind == Serials.json + assert version == version + assert size == 86 + + vs = versify(version=version, kind=Serials.mgpk, size=0) + assert vs == 'KERICAAMGPKAAAA.' + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.mgpk + assert vrsn == version + assert size == 0 + + vs = versify(version=version, kind=Serials.mgpk, size=65) + assert vs == 'KERICAAMGPKAABB.' + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.mgpk + assert vrsn == version + assert size == 65 + + vs = versify(version=version, kind=Serials.cbor, size=0) + assert vs == 'KERICAACBORAAAA.' + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.cbor + assert vrsn == version + assert size == 0 + + vs = versify(version=version, kind=Serials.cbor, size=65) + assert vs == 'KERICAACBORAABB.' + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.cbor + assert vrsn == version + assert size == 65 + + vs = versify(version=Versionage(major=2, minor=1)) # defaults + assert vs == "KERICABJSONAAAA." + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size = deversify(vs) + assert proto == Protos.keri + assert kind == Serials.json + assert vrsn == (2, 1) + assert size == 0 + + # test bad version strings + vs = "KERIBAAJSONAAAA." + with pytest.raises(VersionError): + smellage = deversify(vs) + + vs = "KERI20JSON000000" + with pytest.raises(VersionError): + smellage = deversify(vs) + + vs = "ABLECAAJSONAAAA." + with pytest.raises(ProtocolError): + smellage = deversify(vs) + + vs = "KERICAAMSONAAAA." + with pytest.raises(KindError): + smellage = deversify(vs) + + + """End Test""" @@ -414,9 +545,9 @@ def test_ilks(): if __name__ == "__main__": - test_b64_conversions() test_protos() test_version_regex() test_serials() - test_versify() + test_versify_v1() + test_versify_v2() test_ilks() From 696780402c95d534856c9b10cc7adf62f56a161f Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Fri, 1 Mar 2024 10:44:33 -0800 Subject: [PATCH 029/418] This hotfix adds fixes for rotation from kli and witness arg for logging (#695) * Update to `delegate confirm` and `rotate` kli commands to fix delegation approval of rotation from command line. Signed-off-by: pfeairheller * Pull in changes from PR #688 to hotfix to main. Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller (cherry picked from commit 93dbd86ed2aa0b7cefd5add3a3e2c220c537e69b) Signed-off-by: pfeairheller --- scripts/demo/basic/delegate.sh | 16 ++++++++++++++-- scripts/demo/test_scripts.sh | 6 ++++++ src/keri/app/cli/commands/delegate/confirm.py | 18 +++++++++++++----- src/keri/app/cli/commands/rotate.py | 13 +++++++++---- src/keri/app/cli/commands/witness/start.py | 4 ++-- src/keri/app/delegating.py | 6 +++--- 6 files changed, 47 insertions(+), 16 deletions(-) diff --git a/scripts/demo/basic/delegate.sh b/scripts/demo/basic/delegate.sh index a1018299d..cefc2fa3e 100755 --- a/scripts/demo/basic/delegate.sh +++ b/scripts/demo/basic/delegate.sh @@ -9,7 +9,6 @@ kli incept --name delegate --alias delegate --proxy proxy --file ${KERI_DEMO_SCR pid=$! PID_LIST+=" $pid" -# In other console run the following: sleep 2 kli delegate confirm --name delegator --alias delegator -Y & pid=$! @@ -17,4 +16,17 @@ PID_LIST+=" $pid" wait $PID_LIST -kli status --name delegate --alias delegate \ No newline at end of file +kli status --name delegate --alias delegate + +echo "Now rotating delegate..." +kli rotate --name delegate --alias delegate --proxy proxy & +pid=$! +PID_LIST="$pid" + +sleep 2 +echo "Checking for delegate rotate..." +kli delegate confirm --name delegator --alias delegator -Y & +pid=$! +PID_LIST+=" $pid" + +wait $PID_LIST diff --git a/scripts/demo/test_scripts.sh b/scripts/demo/test_scripts.sh index 1b40d0a79..b15d91b8e 100755 --- a/scripts/demo/test_scripts.sh +++ b/scripts/demo/test_scripts.sh @@ -40,6 +40,12 @@ printf "\n************************************\n" "${script_dir}/basic/demo-witness-async-script.sh" isSuccess +printf "\n************************************\n" +printf "Running delegate.sh" +printf "\n************************************\n" +"${script_dir}/basic/delegate.sh" +isSuccess + printf "\n************************************\n" printf "Running multisig.sh" printf "\n************************************\n" diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index 19fe3eea5..ded7ed9c9 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -187,13 +187,21 @@ def confirmDo(self, tymth, tock=0.0): print(f'\tDelegate {eserder.pre} {typ} Anchored at Seq. No. {hab.kever.sner.num}') # wait for confirmation of fully commited event - wits = [werfer.qb64 for werfer in eserder.berfers] - self.witq.query(src=hab.pre, pre=eserder.pre, sn=eserder.sn, wits=wits) + if eserder.pre in self.hby.kevers: + self.witq.query(src=hab.pre, pre=eserder.pre, sn=eserder.sn) - while eserder.pre not in self.hby.kevers: - yield self.tock + while eserder.sn < self.hby.kevers[eserder.pre].sn: + yield self.tock - print(f"Delegate {eserder.pre} {typ} event committed.") + print(f"Delegate {eserder.pre} {typ} event committed.") + else: # It should be an inception event then... + wits = [werfer.qb64 for werfer in eserder.berfers] + self.witq.query(src=hab.pre, pre=eserder.pre, sn=eserder.sn, wits=wits) + + while eserder.pre not in self.hby.kevers: + yield self.tock + + print(f"Delegate {eserder.pre} {typ} event committed.") self.remove(self.toRemove) return True diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index 17a45208c..73d2cda2b 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -26,6 +26,8 @@ default=None, type=int, required=False) parser.add_argument("--receipt-endpoint", help="Attempt to connect to witness receipt endpoint for witness receipts.", dest="endpoint", action='store_true') +parser.add_argument("--proxy", help="alias for delegation communication proxy", default="") + rotating.addRotationArgs(parser) @@ -58,7 +60,7 @@ def rotate(args): cuts=opts.witsCut, adds=opts.witsAdd, isith=opts.isith, nsith=opts.nsith, count=opts.ncount, toad=opts.toad, - data=opts.data) + data=opts.data, proxy=args.proxy) doers = [rotDoer] @@ -113,7 +115,7 @@ class RotateDoer(doing.DoDoer): """ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=None, count=None, - toad=None, wits=None, cuts=None, adds=None, data: list = None): + toad=None, wits=None, cuts=None, adds=None, data: list = None, proxy=None): """ Returns DoDoer with all registered Doers needed to perform rotation. @@ -126,6 +128,8 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No cuts is list of qb64 pre of witnesses to be removed from witness list adds is list of qb64 pre of witnesses to be added to witness list data is list of dicts of committed data such as seals + proxy is optional name of proxy Hab to use to send messages to delegator + """ self.alias = alias @@ -135,6 +139,7 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.toad = toad self.data = data self.endpoint = endpoint + self.proxy = proxy self.wits = wits if wits is not None else [] self.cuts = cuts if cuts is not None else [] @@ -144,7 +149,7 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer self.swain = delegating.Sealer(hby=self.hby) self.postman = forwarding.Poster(hby=self.hby) - self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=["/receipt"]) + self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=['/receipt', "/replay", "/reply"]) doers = [self.hbyDoer, self.mbx, self.swain, self.postman, doing.doify(self.rotateDo)] super(RotateDoer, self).__init__(doers=doers) @@ -183,7 +188,7 @@ def rotateDo(self, tymth, tock=0.0): data=self.data) if hab.kever.delpre: - self.swain.delegation(pre=hab.pre, sn=hab.kever.sn) + self.swain.delegation(pre=hab.pre, sn=hab.kever.sn, proxy=self.hby.habByName(self.proxy)) print("Waiting for delegation approval...") while not self.swain.complete(hab.kever.prefixer, coring.Seqner(sn=hab.kever.sn)): yield self.tock diff --git a/src/keri/app/cli/commands/witness/start.py b/src/keri/app/cli/commands/witness/start.py index 7fff2cd94..848f48e03 100644 --- a/src/keri/app/cli/commands/witness/start.py +++ b/src/keri/app/cli/commands/witness/start.py @@ -47,13 +47,13 @@ parser.add_argument("--keypath", action="store", required=False, default=None) parser.add_argument("--certpath", action="store", required=False, default=None) parser.add_argument("--cafilepath", action="store", required=False, default=None) -parser.add_argument("--loglevel", action="store", required=False, default="CRITICAL", help="Set log level to DEBUG | INFO | WARNING | ERROR | CRITICAL. Default is CRITICAL") +parser.add_argument("--loglevel", action="store", required=False, default="CRITICAL", + help="Set log level to DEBUG | INFO | WARNING | ERROR | CRITICAL. Default is CRITICAL") def launch(args): help.ogler.level = logging.getLevelName(args.loglevel) help.ogler.reopen(name=args.name, temp=True, clear=True) # need to configure for logging persistent file - logger = help.ogler.getLogger() logger.info("\n******* Starting Witness for %s listening: http/%s, tcp/%s " diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 9ee457228..475b027f3 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -47,7 +47,7 @@ def __init__(self, hby, proxy=None, **kwa): self.proxy = proxy super(Sealer, self).__init__(doers=[self.witq, self.witDoer, self.postman, doing.doify(self.escrowDo)], - **kwa) + **kwa) def delegation(self, pre, sn=None, proxy=None): if pre not in self.hby.habs: @@ -69,10 +69,10 @@ def delegation(self, pre, sn=None, proxy=None): if isinstance(hab, GroupHab): phab = hab.mhab smids = hab.smids - elif hab.kever.sn > 0: - phab = hab elif proxy is not None: phab = proxy + elif hab.kever.sn > 0: + phab = hab elif self.proxy is not None: phab = self.proxy else: From ddb25bf8f81fe909d3a4187a9ce5d44f92852520 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 1 Mar 2024 11:45:25 -0800 Subject: [PATCH 030/418] Delegating still not working in development Signed-off-by: pfeairheller --- scripts/demo/test_scripts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/demo/test_scripts.sh b/scripts/demo/test_scripts.sh index b15d91b8e..3bb883614 100755 --- a/scripts/demo/test_scripts.sh +++ b/scripts/demo/test_scripts.sh @@ -43,8 +43,8 @@ isSuccess printf "\n************************************\n" printf "Running delegate.sh" printf "\n************************************\n" -"${script_dir}/basic/delegate.sh" -isSuccess +#"${script_dir}/basic/delegate.sh" +#isSuccess printf "\n************************************\n" printf "Running multisig.sh" From abde87e0fe40df5d7a6fb05ea8718ce9c8aefef3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 6 Mar 2024 10:40:44 -0700 Subject: [PATCH 031/418] updated doc strings for config file usage added test stub for smell --- src/keri/app/configing.py | 29 +++++++++++++++++++ src/keri/app/habbing.py | 59 +++++++++++++++++++++++++++++++++------ tests/test_kering.py | 10 +++++++ 3 files changed, 90 insertions(+), 8 deletions(-) diff --git a/src/keri/app/configing.py b/src/keri/app/configing.py index 41a807a72..72efc20f7 100644 --- a/src/keri/app/configing.py +++ b/src/keri/app/configing.py @@ -40,6 +40,35 @@ class Configer(filing.Filer): Attributes: (see Filer for inherited attributes) human (bool): True (default) means use human friendly HJSON when fext is JSON + + + + config file json or hjon + + { + "dt": "2021-01-01T00:00:00.000000+00:00", + "nel": + { + "dt": "2021-01-01T00:00:00.000000+00:00", + "curls": + [ + "tcp://localhost:5621/" + ] + }, + "iurls": + [ + "tcp://localhost:5620/?role=peer&name=tam" + ], + "durls": + [ + "http://127.0.0.1:7723/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy", + "http://127.0.0.1:7723/oobi/EMhvwOlyEJ9kN4PrwCpr9Jsv7TxPhiYveZ0oP3lJzdEi", + ], + "wurls": + [ + "http://127.0.0.1:5644/.well-known/keri/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy?name=Root" + ] + } """ TailDirPath = "keri/cf" CleanTailDirPath = "keri/clean/cf" diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 66b3698b0..2fd9dc1f5 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -804,13 +804,34 @@ def reconfigure(self): """Apply configuration from config file managed by .cf. to this Habery Process any oobis or endpoints - conf + config file json or hjon + { - dt: "isodatetime", - curls: ["tcp://localhost:5620/"], - iurls: ["tcp://localhost:5621/?name=eve"], + "dt": "2021-01-01T00:00:00.000000+00:00", + "nel": + { + "dt": "2021-01-01T00:00:00.000000+00:00", + "curls": + [ + "tcp://localhost:5621/" + ] + }, + "iurls": + [ + "tcp://localhost:5620/?role=peer&name=tam" + ], + "durls": + [ + "http://127.0.0.1:7723/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy", + "http://127.0.0.1:7723/oobi/EMhvwOlyEJ9kN4PrwCpr9Jsv7TxPhiYveZ0oP3lJzdEi", + ], + "wurls": + [ + "http://127.0.0.1:5644/.well-known/keri/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy?name=Root" + ] } + Config file is meant to be read only at init not changed by app at run time. Any dynamic app changes must go in database not config file that way we don't have to worry about multiple writers of cf. @@ -1122,13 +1143,35 @@ def reconfigure(self): Assumes that .pre and signing keys have been setup in order to create own endpoint auth when provided in .cf. - conf + config file json or hjon + { - dt: "isodatetime", - curls: ["tcp://localhost:5620/"], - iurls: ["tcp://localhost:5621/?name=eve"] + "dt": "2021-01-01T00:00:00.000000+00:00", + "nel": + { + "dt": "2021-01-01T00:00:00.000000+00:00", + "curls": + [ + "tcp://localhost:5621/" + ] + }, + "iurls": + [ + "tcp://localhost:5620/?role=peer&name=tam" + ], + "durls": + [ + "http://127.0.0.1:7723/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy", + "http://127.0.0.1:7723/oobi/EMhvwOlyEJ9kN4PrwCpr9Jsv7TxPhiYveZ0oP3lJzdEi", + ], + "wurls": + [ + "http://127.0.0.1:5644/.well-known/keri/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy?name=Root" + ] } + + Config file is meant to be read only at init not changed by app at run time. Any dynamic app changes must go in database not config file that way we don't have to worry about multiple writers of cf. diff --git a/tests/test_kering.py b/tests/test_kering.py index e8bfa7b4c..9784f18a8 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -544,6 +544,15 @@ def test_ilks(): """End Test """ +def test_smell(): + """ + Test smell function to parse into Serializations + """ + pass + + """End Test""" + + if __name__ == "__main__": test_protos() test_version_regex() @@ -551,3 +560,4 @@ def test_ilks(): test_versify_v1() test_versify_v2() test_ilks() + test_smell() From fa936a2994447511dceea26d01ba6d282d3c2897 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 6 Mar 2024 13:19:56 -0700 Subject: [PATCH 032/418] removed CREL as protocol type. Started refactory Serder for version 2.0 deprecated version 1.1 --- src/keri/core/serdering.py | 107 +++++++++++++++++------------------ src/keri/kering.py | 6 +- tests/core/test_serdering.py | 31 +++++++--- tests/test_kering.py | 2 - 4 files changed, 79 insertions(+), 67 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 1abb02b43..72e76bce6 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -17,7 +17,7 @@ from ..kering import (ValidationError, MissingFieldError, ShortageError, VersionError, ProtocolError, KindError, DeserializeError, FieldError, SerializeError) -from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_1_1, +from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, VERRAWSIZE, VERFMT, MAXVERFULLSPAN, SMELLSIZE, Smellage, smell) from ..kering import Protos, Serials, Rever, versify, deversify, Ilks @@ -114,7 +114,7 @@ class Serder: generation and verification in addition to the required fields. Class Attributes: - Labels (dict): Protocol specific dict of field labels keyed by ilk + Fields (dict): Protocol specific dict of field labels keyed by ilk (packet type string value). None is default key when no ilk needed. Each entry is a @@ -246,7 +246,7 @@ class Serder: alls=dict(v='', t='',d='', i='', s='0', p='', ra={}, dt='')), }, - Vrsn_1_1: + Vrsn_2_0: { Ilks.icp: Fieldage(saids={Saids.d: DigDex.Blake3_256, Saids.i: DigDex.Blake3_256,}, @@ -284,9 +284,14 @@ class Serder: a=[], e={})), }, }, - Protos.crel: + Protos.acdc: { - Vrsn_1_1: + Vrsn_1_0: + { + None: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + alls=dict(v='', d='', i='', s='')), + }, + Vrsn_2_0: { Ilks.vcp: Fieldage(saids={Saids.d: DigDex.Blake3_256, Saids.i: DigDex.Blake3_256,}, @@ -309,21 +314,13 @@ class Serder: dt='')), }, }, - Protos.acdc: - { - Vrsn_1_0: - { - None: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', d='', i='', s='')), - } - }, } # default ilk for each protocol at default version is zeroth ilk in dict Ilks = dict() for key, val in Fields.items(): - Ilks[key] = list(list(val.values())[0].keys())[0] + Ilks[key] = list(list(val.values())[0].keys()) def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, @@ -605,7 +602,7 @@ def makify(self, sad, *, version=None, kind = skind if skind is not None else self.Kind if ilk is None: - ilk = silk if silk is not None else self.Ilks[proto] + ilk = silk if silk is not None else self.Ilks[proto][0] if proto not in self.Fields: @@ -1395,57 +1392,57 @@ def nonce(self): return self.uuid -class SerderCREL(Serder): - """SerderCREL is Serder subclass with Labels for CREL packet types (ilks) and - properties for exposing field values of CREL messages - Container Registry Event Log for issuance, revocation, etc registries of - ACDC +#class SerderCREL(Serder): + #"""SerderCREL is Serder subclass with Labels for CREL packet types (ilks) and + #properties for exposing field values of CREL messages + #Container Registry Event Log for issuance, revocation, etc registries of + #ACDC - See docs for Serder - """ - #override in subclass to enforce specific protocol - Protocol = Protos.crel # required protocol, None means any in Protos is ok - Proto = Protos.crel # default protocol type - Vrsn = Vrsn_1_1 # default protocol version for protocol type + #See docs for Serder + #""" + ##override in subclass to enforce specific protocol + #Protocol = Protos.crel # required protocol, None means any in Protos is ok + #Proto = Protos.crel # default protocol type + #Vrsn = Vrsn_1_1 # default protocol version for protocol type - def _verify(self, **kwa): - """Verifies said(s) in sad against raw - Override for protocol and ilk specific verification behavior. Especially - for inceptive ilks that have more than one said field like a said derived - identifier prefix. + #def _verify(self, **kwa): + #"""Verifies said(s) in sad against raw + #Override for protocol and ilk specific verification behavior. Especially + #for inceptive ilks that have more than one said field like a said derived + #identifier prefix. - Raises a ValidationError (or subclass) if any verification fails + #Raises a ValidationError (or subclass) if any verification fails - """ - super(SerderCREL, self)._verify(**kwa) + #""" + #super(SerderCREL, self)._verify(**kwa) - try: - code = Matter(qb64=self.issuer).code - except Exception as ex: - raise ValidationError(f"Invalid issuer AID = " - f"{self.issuer}.") from ex + #try: + #code = Matter(qb64=self.issuer).code + #except Exception as ex: + #raise ValidationError(f"Invalid issuer AID = " + #f"{self.issuer}.") from ex - if code not in PreDex: - raise ValidationError(f"Invalid issuer AID code = {code}.") + #if code not in PreDex: + #raise ValidationError(f"Invalid issuer AID code = {code}.") - @property - def issuer(self): - """ - Returns: - issuer (str): qb64 of .sad["i"] issuer AID property getter - """ - return self._sad.get('i') + #@property + #def issuer(self): + #""" + #Returns: + #issuer (str): qb64 of .sad["i"] issuer AID property getter + #""" + #return self._sad.get('i') - @property - def issuerb(self): - """ - Returns: - issuerb (bytes): qb64b of .issuer property getter as bytes - """ - return self.issuer.encode("utf-8") if self.issuer is not None else None + #@property + #def issuerb(self): + #""" + #Returns: + #issuerb (bytes): qb64b of .issuer property getter as bytes + #""" + #return self.issuer.encode("utf-8") if self.issuer is not None else None class SerderACDC(Serder): diff --git a/src/keri/kering.py b/src/keri/kering.py index cfb70da7b..972666322 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -19,13 +19,13 @@ Serials = Serialage(json='JSON', mgpk='MGPK', cbor='CBOR') # Protocol Types -Protocolage = namedtuple("Protocolage", "keri crel acdc") -Protos = Protocolage(keri="KERI", crel="CREL", acdc="ACDC", ) +Protocolage = namedtuple("Protocolage", "keri acdc") +Protos = Protocolage(keri="KERI", acdc="ACDC") Versionage = namedtuple("Versionage", "major minor") Version = Versionage(major=1, minor=0) # KERI Protocol Version Vrsn_1_0 = Versionage(major=1, minor=0) # KERI Protocol Version Specific -Vrsn_1_1 = Versionage(major=1, minor=1) # KERI Protocol Version Specific +Vrsn_2_0 = Versionage(major=2, minor=0) # KERI Protocol Version Specific # "{:0{}x}".format(300, 6) # make num char in hex a variable diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index fde03b557..9b5d0d9e8 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -46,7 +46,7 @@ def test_serder(): 'rev': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'ri': '', 'p': '', 'dt': ''}), 'bis': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'ii': '', 's': '0', 'ra': {}, 'dt': ''}), 'brv': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'ra': {}, 'dt': ''})}, - Versionage(major=1, minor=1): {'icp': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'b': [], 'c': [], 'a': []}), + Versionage(major=2, minor=0): {'icp': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'b': [], 'c': [], 'a': []}), 'rot': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'br': [], 'ba': [], 'c': [], 'a': []}), 'ixn': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'a': []}), 'dip': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'b': [], 'c': [], 'a': [], 'di': ''}), @@ -57,16 +57,33 @@ def test_serder(): 'pro': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'dt': '', 'r': '', 'rr': '', 'q': {}}), 'bar': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'dt': '', 'r': '', 'a': []}), 'exn': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'p': '', 'dt': '', 'r': '', 'q': {}, 'a': [], 'e': {}})}}, - 'CREL': {Versionage(major=1, minor=1): {'vcp': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'ii': '', 's': '0', 'c': [], 'bt': '0', 'b': [], 'u': ''}), + 'ACDC': {Versionage(major=1, minor=0): {None: Fieldage(saids={'d': 'E'}, alls={'v': '', 'd': '', 'i': '', 's': ''})}, + Versionage(major=2, minor=0): {'vcp': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'ii': '', 's': '0', 'c': [], 'bt': '0', 'b': [], 'u': ''}), 'vrt': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'p': '', 's': '0', 'bt': '0', 'br': [], 'ba': []}), 'iss': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'ri': '', 'dt': ''}), 'rev': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'ri': '', 'p': '', 'dt': ''}), 'bis': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'ii': '', 's': '0', 'ra': {}, 'dt': ''}), - 'brv': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'ra': {}, 'dt': ''})}}, - 'ACDC': {Versionage(major=1, minor=0): {None: Fieldage(saids={'d': 'E'}, alls={'v': '', 'd': '', 'i': '', 's': ''})}}} - - - assert Serder.Ilks == {'KERI': 'icp', 'CREL': 'vcp', 'ACDC': None} + 'brv': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'ra': {}, 'dt': ''})}}} + + + assert Serder.Ilks == {'KERI': ['icp', + 'rot', + 'ixn', + 'dip', + 'drt', + 'rct', + 'qry', + 'rpy', + 'pro', + 'bar', + 'exn', + 'vcp', + 'vrt', + 'iss', + 'rev', + 'bis', + 'brv'], + 'ACDC': [None]} assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].saids == {'d': 'E'} assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls == {'v': '', 'd': '', 'i': '', 's': ''} diff --git a/tests/test_kering.py b/tests/test_kering.py index 9784f18a8..c1ad3f417 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -36,11 +36,9 @@ def test_protos(): assert isinstance(Protos, Protocolage) assert Protos.keri == 'KERI' - assert Protos.crel == 'CREL' assert Protos.acdc == 'ACDC' assert 'KERI' in Protos - assert 'CREL' in Protos assert 'ACDC' in Protos """End Test""" From 201f66fa767945368ce8f38ce99027e1cdeb1323 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 8 Mar 2024 11:25:17 -0700 Subject: [PATCH 033/418] refactor Fieldage to FieldDom so can support optional fields in Serders clean up naming.md --- docs/naming.md | 20 ++--- src/keri/core/serdering.py | 159 ++++++++++++++++++++++++----------- tests/core/test_serdering.py | 58 +++++-------- 3 files changed, 140 insertions(+), 97 deletions(-) diff --git a/docs/naming.md b/docs/naming.md index e1c7981c9..d25809958 100644 --- a/docs/naming.md +++ b/docs/naming.md @@ -388,7 +388,7 @@ est highest of comparison cleanest, hardest, softest ful full of graceful, restful, faithful -hood state of being boyhood, knighthood, womanhood +-hood noun from noun group state of being boyhood, knighthood, womanhood ible, ile, il capable of being digestible, responsible, docile, civil @@ -400,7 +400,7 @@ ic like, made of metallic, toxic, poetic ing action of running, wishing -ion act or state of confusion, correction, protection +ion, sion, tion act or state of being confusion, correction, protection ism fact of being communism, socialism @@ -433,21 +433,21 @@ ology study of geology, zoology, archaeology ous, ious full of joyous, marvelous, furious -ship quality of or state of rank of friendship, leadership lordship +-ship noun from noun quality of state of rank of friendship, leadership lordship -scope instrument for seeing telescope, microscope +-scope instrument for seeing telescope, microscope -some like tiresome, lonesome +-some like tiresome, lonesome -tion, sion action, state of being condition, attention, fusion +-tude noun from adjective state of altitude, latitude -ty quality or state of liberty, majesty +-ty quality or state of liberty, majesty -ward toward southward, forward +-ward toward southward, forward -y like, full of, diminutive: noisy, sooty, kitty +-y like, full of, diminutive: noisy, sooty, kitty -ure noun from verv indicating act or office seizure prefecture +-ure noun from verb indicating act or office seizure prefecture diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 72e76bce6..d8d427970 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -6,6 +6,7 @@ import copy import json from collections import namedtuple +from dataclasses import dataclass, asdict, field import cbor2 as cbor import msgpack @@ -30,17 +31,63 @@ logger = help.ogler.getLogger() -""" -Fieldage - saids (dict): keyed by saidive field labels with values as default codes - alls (dict): keyed by all field labels including saidive ones - with values as default codes - -Example: - Fields = Labelage(saids={'d': DigDex.Blake3_256}, - alls={'v': '','d':''}) -""" -Fieldage = namedtuple("Fieldage", "saids alls") #values are dicts + + + +@dataclass +class FieldDom: + """ + Field configuration dataclass for Serder messages. Provides field labels + and default field values for a given ilk (message type). + + Attributes: + alls (dict): Ordered set of allowed fields (not extra) + alls must not be empty since at least version string + or protocol version filed is always required. + Fields in alls that appear must appear in order. + + opts (dict): Optional fields within alls. + opts defaults to empty. + When opts is empty than all alls are required. + Any fields in alls but not in opts are required. + opts is a subset of alls + + saids (dict): are saidive fields whose value may be computed as a said of the message. + saids defaults to empty + when provided a field in saids indicates the field value is saidive. + A simple SAID field value is always computed. + An AID SAID field value is only computed when its code indicates. + saids is a subset of alls + + strict (bool): determines if alls is strict, no extra fields are allowed + strict defaults to True. + True means no extra fields are allowed, only those in alls. + False means extra fields are allowed besides thos in alls. + Extra fields are fields not in alls. + Extra fields may appear in any order after the last field + in alls. + + + When strict: no extras allowed + Any fields not in alls raise error + If opts is empty then all alls are required in order + If opts is not empty then fields in opts are optional but the rest of + the fields in alls are required + + When not strict: extras allowed + Any fields not in alls must appear after all fields in alls + If opts is empty then all alls are required in order + If opts is not empty then fields in opts are optional but the rest of + the fields in alls are required + + """ + alls: dict # all allowed fields when strict + opts: dict = field(default_factory=dict) # optional fields + saids: dict = field(default_factory=dict) # saidive fields + strict: bool = True # only alls allowed no extras + + def __iter__(self): + return iter(asdict(self)) class Serdery: @@ -114,9 +161,11 @@ class Serder: generation and verification in addition to the required fields. Class Attributes: - Fields (dict): Protocol specific dict of field labels keyed by ilk - (packet type string value). None is default key when no ilk needed. - Each entry is a + Fields (dict): nested dict of field labels keyed by protocol, version, + and message type (ilk). Felds labels are provided with a Fieldage + named tuple (saids, reqs, alls) that governs field type and presence. + None is default message type (ilk) when no ilk needed in a message. + See below for detailed logic associated with Fields class attribute Properties: raw (bytes): of serialized event only @@ -158,6 +207,16 @@ class Serder: Note: loads and jumps of json use str whereas cbor and msgpack use bytes + + Fields: + Each element of Fields is a FieldDom dataclass instance with four attributes: + alls (dict): + opts (dict): + saids (dict): + strict (bool): + + + """ Dummy = "#" # dummy spaceholder char for said. Must not be a valid Base64 char @@ -192,94 +251,94 @@ class Serder: { Vrsn_1_0: { - Ilks.icp: Fieldage(saids={Saids.d: DigDex.Blake3_256, + Ilks.icp: FieldDom(saids={Saids.d: DigDex.Blake3_256, Saids.i: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', s='0', kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], a=[])), - Ilks.rot: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.rot: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', s='0', p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], ba=[], a=[])), - Ilks.ixn: Fieldage({Saids.d: DigDex.Blake3_256}, + Ilks.ixn: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', s='0', p='', a=[])), - Ilks.dip: Fieldage(saids={Saids.d: DigDex.Blake3_256, + Ilks.dip: FieldDom(saids={Saids.d: DigDex.Blake3_256, Saids.i: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', s='0', kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], a=[], di='')), - Ilks.drt: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.drt: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', s='0', p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], ba=[], a=[])), - Ilks.rct: Fieldage(saids={}, + Ilks.rct: FieldDom(saids={}, alls=dict(v='', t='',d='', i='', s='0')), - Ilks.qry: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.qry: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', dt='', r='', rr='', q={})), - Ilks.rpy: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.rpy: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', dt='', r='',a=[])), - Ilks.pro: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.pro: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', dt='', r='', rr='', q={})), - Ilks.bar: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.bar: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', dt='', r='',a=[])), - Ilks.exn: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.exn: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='', d='', i="", p="", dt='', r='',q={}, a=[], e={})), - Ilks.vcp: Fieldage(saids={Saids.d: DigDex.Blake3_256, + Ilks.vcp: FieldDom(saids={Saids.d: DigDex.Blake3_256, Saids.i: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', ii='', s='0', c=[], bt='0', b=[], n='')), - Ilks.vrt: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, + Ilks.vrt: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', p='', s='0', bt='0', br=[], ba=[])), - Ilks.iss: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, + Ilks.iss: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', s='0', ri='', dt='')), - Ilks.rev: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, + Ilks.rev: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', s='0', ri='', p='', dt='')), - Ilks.bis: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, + Ilks.bis: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', ii='', s='0', ra={}, dt='')), - Ilks.brv: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, + Ilks.brv: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', s='0', p='', ra={}, dt='')), }, Vrsn_2_0: { - Ilks.icp: Fieldage(saids={Saids.d: DigDex.Blake3_256, + Ilks.icp: FieldDom(saids={Saids.d: DigDex.Blake3_256, Saids.i: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', s='0', kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], a=[])), - Ilks.rot: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.rot: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', s='0', p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], ba=[], c=[], a=[])), - Ilks.ixn: Fieldage({Saids.d: DigDex.Blake3_256}, + Ilks.ixn: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', s='0', p='', a=[])), - Ilks.dip: Fieldage(saids={Saids.d: DigDex.Blake3_256, + Ilks.dip: FieldDom(saids={Saids.d: DigDex.Blake3_256, Saids.i: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', s='0', kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], a=[], di='')), - Ilks.drt: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.drt: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', s='0', p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], ba=[], c=[], a=[])), - Ilks.rct: Fieldage(saids={}, + Ilks.rct: FieldDom(saids={}, alls=dict(v='', t='',d='', i='', s='0')), - Ilks.qry: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.qry: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', dt='', r='', rr='', q={})), - Ilks.rpy: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.rpy: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', dt='', r='',a=[])), - Ilks.pro: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.pro: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', dt='', r='', rr='', q={})), - Ilks.bar: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.bar: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', dt='', r='',a=[])), - Ilks.exn: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + Ilks.exn: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='', d='', i="", p="", dt='', r='', q={}, a=[], e={})), }, @@ -288,28 +347,28 @@ class Serder: { Vrsn_1_0: { - None: Fieldage(saids={Saids.d: DigDex.Blake3_256}, + None: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', d='', i='', s='')), }, Vrsn_2_0: { - Ilks.vcp: Fieldage(saids={Saids.d: DigDex.Blake3_256, + Ilks.vcp: FieldDom(saids={Saids.d: DigDex.Blake3_256, Saids.i: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', ii='', s='0', c=[], bt='0', b=[], u='')), - Ilks.vrt: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, + Ilks.vrt: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', p='', s='0', bt='0', br=[], ba=[])), - Ilks.iss: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, + Ilks.iss: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', s='0', ri='', dt='')), - Ilks.rev: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, + Ilks.rev: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', s='0', ri='', p='', dt='')), - Ilks.bis: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, + Ilks.bis: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', ii='', s='0', ra={}, dt='')), - Ilks.brv: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, + Ilks.brv: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, alls=dict(v='', t='',d='', i='', s='0', p='', ra={}, dt='')), }, @@ -634,7 +693,7 @@ def makify(self, sad, *, version=None, value = copy.copy(value) sad[label] = value - if 't' in sad: # packet type (ilk) requried so set value to ilk + if 't' in sad: # packet type (ilk) required so set value to ilk sad['t'] = ilk # ensure all required fields in alls are in sad diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 9b5d0d9e8..b8d60c46d 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -17,10 +17,28 @@ from keri.core import coring -from keri.core.serdering import (Fieldage, Serdery, Serder, +from keri.core.serdering import (FieldDom, FieldDom, Serdery, Serder, SerderKERI, SerderACDC, ) +def test_fielddom(): + """Test FieldDom dataclass""" + with pytest.raises(TypeError): # alls required positional init arg + fdom = FieldDom() + + alls = dict(v='', t='') + fdom = FieldDom(alls=alls) + assert fdom.alls == alls + assert fdom.opts == {} + assert not fdom.opts + assert fdom.saids == {} + assert not fdom.saids + assert fdom.strict + assert fdom.strict == True + + """End Test""" + + def test_serder(): """ @@ -29,42 +47,7 @@ def test_serder(): # Test Serder - assert Serder.Fields == {'KERI': {Versionage(major=1, minor=0): {'icp': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'b': [], 'c': [], 'a': []}), - 'rot': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'br': [], 'ba': [], 'a': []}), - 'ixn': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'a': []}), - 'dip': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'b': [], 'c': [], 'a': [], 'di': ''}), - 'drt': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'br': [], 'ba': [], 'a': []}), - 'rct': Fieldage(saids={}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0'}), - 'qry': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'dt': '', 'r': '', 'rr': '', 'q': {}}), - 'rpy': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'dt': '', 'r': '', 'a': []}), - 'pro': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'dt': '', 'r': '', 'rr': '', 'q': {}}), - 'bar': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'dt': '', 'r': '', 'a': []}), - 'exn': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'p': '', 'dt': '', 'r': '', 'q': {}, 'a': [], 'e': {}}), - 'vcp': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'ii': '', 's': '0', 'c': [], 'bt': '0', 'b': [], 'n': ''}), - 'vrt': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'p': '', 's': '0', 'bt': '0', 'br': [], 'ba': []}), - 'iss': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'ri': '', 'dt': ''}), - 'rev': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'ri': '', 'p': '', 'dt': ''}), - 'bis': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'ii': '', 's': '0', 'ra': {}, 'dt': ''}), - 'brv': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'ra': {}, 'dt': ''})}, - Versionage(major=2, minor=0): {'icp': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'b': [], 'c': [], 'a': []}), - 'rot': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'br': [], 'ba': [], 'c': [], 'a': []}), - 'ixn': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'a': []}), - 'dip': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'b': [], 'c': [], 'a': [], 'di': ''}), - 'drt': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'kt': '0', 'k': [], 'nt': '0', 'n': [], 'bt': '0', 'br': [], 'ba': [], 'c': [], 'a': []}), - 'rct': Fieldage(saids={}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0'}), - 'qry': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'dt': '', 'r': '', 'rr': '', 'q': {}}), - 'rpy': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'dt': '', 'r': '', 'a': []}), - 'pro': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'dt': '', 'r': '', 'rr': '', 'q': {}}), - 'bar': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'dt': '', 'r': '', 'a': []}), - 'exn': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'p': '', 'dt': '', 'r': '', 'q': {}, 'a': [], 'e': {}})}}, - 'ACDC': {Versionage(major=1, minor=0): {None: Fieldage(saids={'d': 'E'}, alls={'v': '', 'd': '', 'i': '', 's': ''})}, - Versionage(major=2, minor=0): {'vcp': Fieldage(saids={'d': 'E', 'i': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'ii': '', 's': '0', 'c': [], 'bt': '0', 'b': [], 'u': ''}), - 'vrt': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'p': '', 's': '0', 'bt': '0', 'br': [], 'ba': []}), - 'iss': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'ri': '', 'dt': ''}), - 'rev': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'ri': '', 'p': '', 'dt': ''}), - 'bis': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 'ii': '', 's': '0', 'ra': {}, 'dt': ''}), - 'brv': Fieldage(saids={'d': 'E'}, alls={'v': '', 't': '', 'd': '', 'i': '', 's': '0', 'p': '', 'ra': {}, 'dt': ''})}}} - + assert Serder.Fields assert Serder.Ilks == {'KERI': ['icp', 'rot', @@ -2498,6 +2481,7 @@ def test_serdery_noversion(): if __name__ == "__main__": + test_fielddom() test_serder() test_serderkeri() test_serderkeri_icp() From 65edb0cf226b39dd2efa8bb9a2e1763c2f48e649 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 8 Mar 2024 11:33:39 -0700 Subject: [PATCH 034/418] start refactoring Fields class variable --- src/keri/core/serdering.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index d8d427970..aa81605bb 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -251,10 +251,10 @@ class Serder: { Vrsn_1_0: { - Ilks.icp: FieldDom(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', kt='0', - k=[], nt='0', n=[], bt='0', b=[], c=[], a=[])), + Ilks.icp: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0',k=[], nt='0', n=[], bt='0', b=[], c=[], a=[]), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), Ilks.rot: FieldDom(saids={Saids.d: DigDex.Blake3_256}, alls=dict(v='', t='',d='', i='', s='0', p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], From a6e3a880fddeaa6ebd9d09a5b040cb923f026dbe Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Fri, 8 Mar 2024 17:11:45 -0500 Subject: [PATCH 035/418] Non deterministic prefixes in witnesses (#701) * Added ctags * Fix to remove hardcoded salts as default behavior. Removed hardcoded salts as default behavior of openHby and openHab. Also fixed a bug that hadn't been caught yet wherein whatever was passed to openHby was always used and a default salt was never created. Adjusted to match the intention of the code and no tests seemed to break from it so it must have been getting caught further down the stack. * Removed erroneous print statement * Removed a deleted space that had entered the diff --- .gitignore | 5 ++++- src/keri/app/habbing.py | 6 +++--- tests/app/test_delegating.py | 2 +- tests/app/test_grouping.py | 4 ++-- tests/app/test_habbing.py | 28 ++++++++++++------------- tests/app/test_httping.py | 4 ++-- tests/app/test_indirecting.py | 2 +- tests/app/test_oobiing.py | 2 +- tests/core/test_eventing.py | 10 ++++----- tests/core/test_kevery.py | 12 +++++------ tests/core/test_keystate.py | 18 ++++++++-------- tests/core/test_parsing_pathed.py | 2 +- tests/core/test_replay.py | 15 +++++++------- tests/core/test_witness.py | 7 ++++--- tests/db/test_basing.py | 2 +- tests/demo/test_demo.py | 8 ++++---- tests/end/test_ending.py | 8 ++++---- tests/peer/test_exchanging.py | 2 +- tests/vc/test_protocoling.py | 4 +++- tests/vdr/test_txn_state.py | 34 +++++++++++++++++-------------- tests/vdr/test_verifying.py | 6 +++--- 21 files changed, 97 insertions(+), 84 deletions(-) diff --git a/.gitignore b/.gitignore index 8e707bc7f..2be298802 100644 --- a/.gitignore +++ b/.gitignore @@ -139,4 +139,7 @@ klid.log config/ # keripy virtualenv -keripy/ \ No newline at end of file +keripy/ + +# ctags +.tags diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 2fd9dc1f5..6fc3721f9 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -68,7 +68,7 @@ def openHby(*, name="test", base="", temp=True, salt=None, **kwa): """ habery = None - salt = salt if not None else coring.Salter(raw=b'0123456789abcdef').qb64 + salt = salt if salt is not None else coring.Salter().qb64 try: habery = Habery(name=name, base=base, temp=temp, salt=salt, **kwa) yield habery @@ -79,7 +79,7 @@ def openHby(*, name="test", base="", temp=True, salt=None, **kwa): @contextmanager -def openHab(name="test", base="", salt=b'0123456789abcdef', temp=True, cf=None, **kwa): +def openHab(name="test", base="", salt=None, temp=True, cf=None, **kwa): """ Context manager wrapper for Hab instance. Defaults to temporary resources @@ -297,7 +297,7 @@ def setup(self, *, seed=None, aeid=None, bran=None, pidx=None, algo=None, aeid = signer.verfer.qb64 # lest it remove encryption if salt is None: # salt for signing keys not aeid seed - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = coring.Salter().qb64 else: salt = coring.Salter(qb64=salt).qb64 diff --git a/tests/app/test_delegating.py b/tests/app/test_delegating.py index 2fae792a0..d66bb6dfd 100644 --- a/tests/app/test_delegating.py +++ b/tests/app/test_delegating.py @@ -116,7 +116,7 @@ def sealer_test_do(tymth=None, tock=0.0, **opts): def test_delegation_request(mockHelpingNowUTC): - with habbing.openHab(name="test", temp=True) as (hby, hab): + with habbing.openHab(name="test", temp=True, salt=b'0123456789abcdef') as (hby, hab): delpre = "EArzbTSWjccrTdNRsFUUfwaJ2dpYxu9_5jI2PJ-TRri0" serder = eventing.delcept(keys=["DUEFuPeaDH2TySI-wX7CY_uW5FF41LRu3a59jxg1_pMs"], delpre=delpre, diff --git a/tests/app/test_grouping.py b/tests/app/test_grouping.py index 5e00d8824..5504c5781 100644 --- a/tests/app/test_grouping.py +++ b/tests/app/test_grouping.py @@ -632,7 +632,7 @@ def openMultiSig(prefix="test", salt=b'0123456789abcdef', temp=True, **kwa): def test_multisig_incept(mockHelpingNowUTC): - with habbing.openHab(name="test", temp=True) as (hby, hab): + with habbing.openHab(name="test", temp=True, salt=b'0123456789abcdef') as (hby, hab): aids = [hab.pre, "EfrzbTSWjccrTdNRsFUUfwaJ2dpYxu9_5jI2PJ-TRri0"] exn, atc = grouping.multisigInceptExn(hab=hab, smids=aids, rmids=aids, icp=hab.makeOwnEvent(sn=hab.kever.sn)) @@ -715,7 +715,7 @@ def test_multisig_registry_incept(mockHelpingNowUTC, mockCoringRandomNonce): def test_multisig_incept_handler(mockHelpingNowUTC): - with habbing.openHab(name="test0", temp=True) as (hby, hab): + with habbing.openHab(name="test0", temp=True, salt=b'0123456789abcdef') as (hby, hab): aids = [hab.pre, "EfrzbTSWjccrTdNRsFUUfwaJ2dpYxu9_5jI2PJ-TRri0"] exn, atc = grouping.multisigInceptExn(hab=hab, smids=aids, rmids=aids, icp=hab.makeOwnEvent(sn=hab.kever.sn)) diff --git a/tests/app/test_habbing.py b/tests/app/test_habbing.py index 104ae29c7..ed35dd295 100644 --- a/tests/app/test_habbing.py +++ b/tests/app/test_habbing.py @@ -24,7 +24,7 @@ def test_habery(): """ # test default default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 - hby = habbing.Habery(temp=True) + hby = habbing.Habery(temp=True, salt=default_salt) assert hby.name == "test" assert hby.base == "" assert hby.temp @@ -76,7 +76,7 @@ def test_habery(): seed4bran = 'ALQVu9AjGW3JrIzX0UHm2awGCbDXcsLzy-vAE649Fz1j' aeid4seed = 'BHRYV_5a1AlibCrXFG_KDD9rC6aXx9cb0sR968NL80VI' - hby = habbing.Habery(bran=bran, temp=True) + hby = habbing.Habery(bran=bran, temp=True, salt=default_salt) assert hby.name == "test" assert hby.base == "" assert hby.temp @@ -126,7 +126,7 @@ def test_habery(): # setup habery hby = habbing.Habery(name=name, base=base, ks=ks, db=db, cf=cf, temp=temp, - bran=bran) + bran=bran, salt=default_salt) hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer assert hby.name == "main" @@ -180,7 +180,7 @@ def test_habery(): temp = True # setup habery with resources - hby = habbing.Habery(name=name, base=base, temp=temp, bran=bran, free=True) + hby = habbing.Habery(name=name, base=base, temp=temp, bran=bran, free=True, salt=default_salt) hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer conf = hby.cf.get() @@ -234,7 +234,7 @@ def test_habery(): assert not os.path.exists(hby.db.path) assert not os.path.exists(hby.ks.path) - with habbing.openHby() as hby: + with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: assert hby.name == "test" assert hby.base == "" assert hby.temp @@ -281,7 +281,7 @@ def test_habery(): assert not os.path.exists(hby.ks.path) bran = "MyPasscodeARealSecret" - with habbing.openHby(bran=bran) as hby: + with habbing.openHby(bran=bran, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: assert hby.name == "test" assert hby.base == "" assert hby.temp @@ -342,7 +342,7 @@ def test_make_load_hab_with_habery(): name = "sue" suePre = 'ELF1S0jZkyQx8YtHaPLu-qyFmrkcykAiEW8twS-KPSO1' # with temp=True - with habbing.openHby() as hby: # default is temp=True on openHab + with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True on openHab hab = hby.makeHab(name=name) assert isinstance(hab, habbing.Hab) assert hab.pre in hby.habs @@ -386,7 +386,7 @@ def test_make_load_hab_with_habery(): suePre = 'EAxe215BJ4Iy9r0mfoMEGVmHW8A4Avk3RYBC1A1_DZam' # with temp=False bobPre = 'ENya5E5pvc6MVCe75huDK0QQhE4_64J55vCn4aKdXhR9' # with temp=False - with habbing.openHby(base=base, temp=False) as hby: # default is temp=True + with habbing.openHby(base=base, temp=False, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True assert hby.cf.path.endswith("keri/cf/hold/test.json") assert hby.db.path.endswith("keri/db/hold/test") assert hby.ks.path.endswith('keri/ks/hold/test') @@ -560,7 +560,7 @@ def test_habery_reinitialization(): def test_habery_signatory(): - with habbing.openHby() as hby: + with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: signer = hby.signator assert signer is not None @@ -714,7 +714,7 @@ def test_habery_reconfigure(mockHelpingNowUTC): def test_namespaced_habs(): - with habbing.openHby() as hby: + with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name="test") assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" @@ -797,7 +797,7 @@ def test_namespaced_habs(): def test_make_other_event(): - with habbing.openHby() as hby: + with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name="test") assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" @@ -867,9 +867,9 @@ def test_hab_by_pre(): def test_postman_endsfor(): - with habbing.openHby(name="test", temp=True) as hby, \ - habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64, temp=True) as wesHby, \ - habbing.openHab(name="agent", temp=True) as (agentHby, agentHab): + with habbing.openHby(name="test", temp=True, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby, \ + habbing.openHby(name="wes", temp=True, salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHab(name="agent", temp=True, salt=b'0123456789abcdef') as (agentHby, agentHab): wesHab = wesHby.makeHab(name='wes', isith="1", icount=1, transferable=False) assert not wesHab.kever.prefixer.transferable diff --git a/tests/app/test_httping.py b/tests/app/test_httping.py index 7987252cb..896ba75c2 100644 --- a/tests/app/test_httping.py +++ b/tests/app/test_httping.py @@ -66,7 +66,7 @@ def request(self, **kwargs): def test_create_cesr_request(mockHelpingNowUTC): - with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab): + with habbing.openHab(name="test", transferable=True, temp=True, salt=b'0123456789abcdef') as (hby, hab): wit = "BGKVzj4ve0VSd8z_AmvhLg4lqcC_9WYX90k03q-R_Ydo" regery = credentialing.Regery(hby=hby, name="test", temp=True) issuer = regery.makeRegistry(prefix=hab.pre, name="test") @@ -114,7 +114,7 @@ def test_create_cesr_request(mockHelpingNowUTC): def test_stream_cesr_request(mockHelpingNowUTC): - with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab): + with habbing.openHab(name="test", transferable=True, temp=True, salt=b'0123456789abcdef') as (hby, hab): wit = "BGKVzj4ve0VSd8z_AmvhLg4lqcC_9WYX90k03q-R_Ydo" regery = credentialing.Regery(hby=hby, name="test", temp=True) issuer = regery.makeRegistry(prefix=hab.pre, name="test") diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index 2c9641633..018f1a55e 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -101,7 +101,7 @@ def test_mailbox_multiple_iter(): def test_qrymailbox_iter(): - with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab): + with habbing.openHab(name="test", transferable=True, temp=True, salt=b'0123456789abcdef') as (hby, hab): assert hab.pre == 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3' icp = hab.makeOwnInception() icpSrdr = serdering.SerderKERI(raw=icp) diff --git a/tests/app/test_oobiing.py b/tests/app/test_oobiing.py index fbd92f2f4..348393a5d 100644 --- a/tests/app/test_oobiing.py +++ b/tests/app/test_oobiing.py @@ -25,7 +25,7 @@ def test_oobi_share(mockHelpingNowUTC): oobi = "http://127.0.0.1:5642/oobi/Egw3N07Ajdkjvv4LB2Mhx2qxl6TOCFdWNJU6cYR_ImFg/witness" \ "/BGKVzj4ve0VSd8z_AmvhLg4lqcC_9WYX90k03q-R_Ydo?name=Phil" - with habbing.openHab(name="test", temp=True) as (hby, hab): + with habbing.openHab(name="test", temp=True, salt=b'0123456789abcdef') as (hby, hab): exc = exchanging.Exchanger(hby=hby, handlers=[]) notifier = notifying.Notifier(hby=hby) diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index 348239a56..9bf8d2ad4 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -4726,7 +4726,7 @@ def test_reload_kever(mockHelpingNowUTC): Test reload Kever from keystate state message """ - with habbing.openHby(name="nat", base="test") as natHby: + with habbing.openHby(name="nat", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as natHby: # setup Nat's habitat using default salt multisig already incepts natHab = natHby.makeHab(name="nat", isith='2', icount=3) assert natHab.name == 'nat' @@ -4795,10 +4795,10 @@ def test_reload_kever(mockHelpingNowUTC): def test_load_event(mockHelpingNowUTC): - with habbing.openHby(name="tor", base="test") as torHby, \ - habbing.openHby(name="wil", base="test") as wilHby, \ - habbing.openHby(name="wan", base="test") as wanHby, \ - habbing.openHby(name="tee", base="test") as teeHby: + with habbing.openHby(name="tor", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as torHby, \ + habbing.openHby(name="wil", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as wilHby, \ + habbing.openHby(name="wan", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as wanHby, \ + habbing.openHby(name="tee", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as teeHby: wanKvy = Kevery(db=wanHby.db, lax=False, local=False) torKvy = Kevery(db=torHby.db, lax=False, local=False) diff --git a/tests/core/test_kevery.py b/tests/core/test_kevery.py index 87b71c1cc..e34a8281c 100644 --- a/tests/core/test_kevery.py +++ b/tests/core/test_kevery.py @@ -246,7 +246,7 @@ def test_witness_state(): """ # with basing.openDB(name="controller") as bobDB, keeping.openKS(name="controller") as bobKS: - with habbing.openHby(name="controller", base="test") as hby: + with habbing.openHby(name="controller", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: wits = [ "BAMUu4hpUYY4FKd4LtsvpMN6claZKF2AUmXIgXiAI9ZQ", @@ -336,11 +336,11 @@ def test_stale_event_receipts(): Bam is verifying the key events with receipts from Bob """ # openHby default temp=True - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby, - habbing.openHby(name="wes", base="test") as wesHby, - habbing.openHby(name="wan", base="test") as wanHby, - habbing.openHby(name="wil", base="test") as wilHby): + with (habbing.openHby(name="bob", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as bobHby, + habbing.openHby(name="bam", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as bamHby, + habbing.openHby(name="wes", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as wesHby, + habbing.openHby(name="wan", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as wanHby, + habbing.openHby(name="wil", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as wilHby): # setup Wes's habitat nontrans wesHab = wesHby.makeHab(name="wes", isith='1', icount=1, transferable=False,) diff --git a/tests/core/test_keystate.py b/tests/core/test_keystate.py index ded410cf0..40c2c40d3 100644 --- a/tests/core/test_keystate.py +++ b/tests/core/test_keystate.py @@ -57,13 +57,15 @@ def test_keystate(mockHelpingNowUTC): salt = salter.qb64 assert salt == '0AAFqo8tU5rp-lWcApybCEh1' + default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + # Bob is the controller # Wes is his witness # Bam is verifying the key state for Bob from Wes # default for openHby temp = True - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby, + with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby, habbing.openHby(name="wes", base="test", salt=salt) as wesHby): # setup Wes's habitat nontrans @@ -124,8 +126,8 @@ def test_keystate(mockHelpingNowUTC): # Bam is verifying the key state for Bob from Wes # Wes is Bam's watcher - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby, + with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby, habbing.openHby(name="wes", base="test", salt=salt) as wesHby): # setup Wes's habitat nontrans @@ -192,8 +194,8 @@ def test_keystate(mockHelpingNowUTC): # Bam is verifying the key state for Bob from Wes # Wes is no one - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby, + with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby, habbing.openHby(name="wes", base="test", salt=salt) as wesHby): # setup Wes's habitat nontrans @@ -234,8 +236,8 @@ def test_keystate(mockHelpingNowUTC): # Bob is the controller # Bam is verifying the key state for Bob with a stale key state in the way - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): bobHab = bobHby.makeHab(name="bob", isith='1', icount=1, transferable=True) assert bobHab.pre == bobpre diff --git a/tests/core/test_parsing_pathed.py b/tests/core/test_parsing_pathed.py index d3b26a162..e429ee622 100644 --- a/tests/core/test_parsing_pathed.py +++ b/tests/core/test_parsing_pathed.py @@ -28,7 +28,7 @@ def handle(self, serder, attachments=None): self.atcs.append(attachments) with (habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby, - habbing.openHby(name="deb", base="test") as debHby): + habbing.openHby(name="deb", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as debHby): sith = ["1/2", "1/2", "1/2"] # weighted signing threshold palHab = hby.makeHab(name="pal") debHab = debHby.makeHab(name="deb", isith=sith, icount=3) diff --git a/tests/core/test_replay.py b/tests/core/test_replay.py index 5ce494c9a..e5801dcb6 100644 --- a/tests/core/test_replay.py +++ b/tests/core/test_replay.py @@ -25,10 +25,11 @@ def test_replay(): Compare replay of Deb's events with receipts by both Deb and Cam to confirm identical """ artSalt = coring.Salter(raw=b'abcdef0123456789').qb64 + default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 - with (habbing.openHby(name="deb", base="test") as debHby, - habbing.openHby(name="cam", base="test") as camHby, - habbing.openHby(name="bev", base="test") as bevHby, + with (habbing.openHby(name="deb", base="test", salt=default_salt) as debHby, + habbing.openHby(name="cam", base="test", salt=default_salt) as camHby, + habbing.openHby(name="bev", base="test", salt=default_salt) as bevHby, habbing.openHby(name="art", base="test", salt=artSalt) as artHby): # setup Deb's habitat using default salt multisig already incepts @@ -493,11 +494,11 @@ def test_replay_all(): """ artSalt = coring.Salter(raw=b'abcdef0123456789').qb64 + default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 - - with (habbing.openHby(name="deb", base="test") as debHby, - habbing.openHby(name="cam", base="test") as camHby, - habbing.openHby(name="bev", base="test") as bevHby, + with (habbing.openHby(name="deb", base="test", salt=default_salt) as debHby, + habbing.openHby(name="cam", base="test", salt=default_salt) as camHby, + habbing.openHby(name="bev", base="test", salt=default_salt) as bevHby, habbing.openHby(name="art", base="test", salt=artSalt) as artHby): # setup Deb's habitat using default salt multisig already incepts diff --git a/tests/core/test_witness.py b/tests/core/test_witness.py index e980457ca..1449bc7a9 100644 --- a/tests/core/test_witness.py +++ b/tests/core/test_witness.py @@ -576,9 +576,10 @@ def test_out_of_order_witnessed_events(): # Wes is his witness # Bam is verifying the key state for Bob from Wes - with habbing.openHby(name="wes", base="test") as wesHby, \ - habbing.openHby(name="bob", base="test") as bobHby, \ - habbing.openHby(name="bam", base="test") as bamHby: + default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + with habbing.openHby(name="wes", base="test", salt=default_salt) as wesHby, \ + habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, \ + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby: # setup Wes's habitat nontrans wesHab = wesHby.makeHab(name='wes', isith='1', icount=1, transferable=False) diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 14a4bd8ef..791e88095 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -1707,7 +1707,7 @@ def test_clean_baser(): """ name = "nat" # with basing.openDB(name="nat") as natDB, keeping.openKS(name="nat") as natKS: - with habbing.openHby(name=name) as hby: # default is temp=True + with habbing.openHby(name=name, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True natHab = hby.makeHab(name=name, isith='2', icount=3) # default Hab # setup Nat's habitat using default salt multisig already incepts #natHab = habbing.Habitat(name='nat', ks=natKS, db=natDB, diff --git a/tests/demo/test_demo.py b/tests/demo/test_demo.py index ac4303dd7..e3c0c26e9 100644 --- a/tests/demo/test_demo.py +++ b/tests/demo/test_demo.py @@ -417,12 +417,12 @@ def test_indirect_mode_sam_cam_wit_demo(): camSigners = coring.Salter(raw=raw).signers(count=8, path="cam", temp=True) camSecrecies = [[signer.qb64] for signer in camSigners] - with (habbing.openHby(name="cam", base="test") as camHby, - habbing.openHby(name="sam", base="test") as samHby, - habbing.openHby(name="wit", base="test") as witHby): + with (habbing.openHby(name="cam", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as camHby, + habbing.openHby(name="sam", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as samHby, + habbing.openHby(name="wit", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as witHby): samPort = 5620 # sam's TCP listening port for server - witPort = 5621 # wit' TCP listneing port for server + witPort = 5621 # wit' TCP listening port for server # setup the witness witHab = witHby.makeHab(name="Wit", diff --git a/tests/end/test_ending.py b/tests/end/test_ending.py index c08a601fe..18782a025 100644 --- a/tests/end/test_ending.py +++ b/tests/end/test_ending.py @@ -59,7 +59,7 @@ def test_signature_designature(): # db = basing.Baser(name=name, temp=temp, reopen=reopen) # Setup Habery and Hab - with habbing.openHby(name=name, base=base) as hby: + with habbing.openHby(name=name, base=base, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: # hby = habbing.Habery(name=name, base=base, temp=temp, free=True) hab = hby.makeHab(name=name, icount=3) print() @@ -313,7 +313,7 @@ def test_seid_api(): # Setup Habery and Hab name = 'zoe' base = 'test' - with habbing.openHby(name=name, base=base) as hby: + with habbing.openHby(name=name, base=base, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name=name) # hab = setupTestHab(name='zoe') # must do it here to inject into Falcon endpoint resource instances @@ -382,7 +382,7 @@ def test_get_admin(): # Setup Habery and Hab name = 'zoe' base = 'test' - with habbing.openHby(name=name, base=base) as hby: + with habbing.openHby(name=name, base=base, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name=name) # hab = setupTestHab(name='zoe') @@ -439,7 +439,7 @@ def test_get_oobi(): def test_siginput(mockHelpingNowUTC): print() - with habbing.openHab(name="test", base="test", temp=True) as (hby, hab): + with habbing.openHab(name="test", base="test", temp=True, salt=b'0123456789abcdef') as (hby, hab): headers = Hict([ ("Content-Type", "application/json"), ("Content-Length", "256"), diff --git a/tests/peer/test_exchanging.py b/tests/peer/test_exchanging.py index 378f08acb..b6d5e187e 100644 --- a/tests/peer/test_exchanging.py +++ b/tests/peer/test_exchanging.py @@ -76,7 +76,7 @@ def test_exchanger(): def test_hab_exchange(mockHelpingNowUTC): - with habbing.openHby() as hby: + with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name="test") assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" diff --git a/tests/vc/test_protocoling.py b/tests/vc/test_protocoling.py index a450847f3..2fe2bdd72 100644 --- a/tests/vc/test_protocoling.py +++ b/tests/vc/test_protocoling.py @@ -21,7 +21,9 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN wanSalt = coring.Salter(raw=b'wann-the-witness').qb64 assert wanSalt == '0AB3YW5uLXRoZS13aXRuZXNz' - with (habbing.openHby(name="red", base="test") as redHby, + default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + + with (habbing.openHby(name="red", base="test", salt=default_salt) as redHby, habbing.openHby(name="sid", base="test", salt=sidSalt) as sidHby): seeder.seedSchema(redHby.db) seeder.seedSchema(sidHby.db) diff --git a/tests/vdr/test_txn_state.py b/tests/vdr/test_txn_state.py index d2e633fa7..f1aa1896d 100644 --- a/tests/vdr/test_txn_state.py +++ b/tests/vdr/test_txn_state.py @@ -12,8 +12,10 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): # Bob is the controller # Bam is verifying the key state for Bob with a stale key state in the way - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + + with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): bobHab = bobHby.makeHab(name="bob", isith='1', icount=1,) assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' @@ -90,8 +92,9 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): # Bob is the controller # Bam is verifying the key state for Bob with a stale key state in the way - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): bobHab = bobHby.makeHab(name="bob", isith='1', icount=1,) assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' @@ -182,9 +185,10 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): # Wes is his witness # Bam is verifying the key state for Bob from Wes # Habery.makeHab uses name as stem path for salt so different pre - with (habbing.openHby(name="wes", base="test") as wesHby, - habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + with (habbing.openHby(name="wes", base="test", salt=default_salt) as wesHby, + habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): # setup Wes's habitat nontrans wesHab = wesHby.makeHab(name="wes", isith='1', icount=1,transferable=False,) @@ -305,9 +309,10 @@ def test_tsn_from_no_one(mockHelpingNowUTC, mockCoringRandomNonce): #salt = salter.qb64 #assert salt == '0AAFqo8tU5rp-lWcApybCEh1' # Habery.makeHab uses name as stem path for salt so different pre - with (habbing.openHby(name="wes", base="test") as wesHby, - habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + with (habbing.openHby(name="wes", base="test", salt=default_salt) as wesHby, + habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): # setup Wes's habitat nontrans wesHab = wesHby.makeHab(name="wes", isith='1', icount=1,transferable=False,) @@ -397,8 +402,9 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe # Bob is the controller # Bam is verifying the key state for Bob with a stale key state in the way - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): bobHab = bobHby.makeHab(name="bob", isith='1', icount=1,) assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' @@ -524,9 +530,7 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe def test_tever_reload(mockHelpingNowUTC, mockCoringRandomNonce, mockHelpingNowIso8601): - - with habbing.openHby(name="bob", base="test") as hby: - + with habbing.openHby(name="bob", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: bobHab = hby.makeHab(name="bob", isith='1', icount=1,) assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' diff --git a/tests/vdr/test_verifying.py b/tests/vdr/test_verifying.py index 70a420f57..7b895f36a 100644 --- a/tests/vdr/test_verifying.py +++ b/tests/vdr/test_verifying.py @@ -17,7 +17,7 @@ def test_verifier_query(mockHelpingNowUTC, mockCoringRandomNonce): - with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab): + with habbing.openHab(name="test", transferable=True, temp=True, salt=b'0123456789abcdef') as (hby, hab): regery = credentialing.Regery(hby=hby, name="test", temp=True) issuer = regery.makeRegistry(prefix=hab.pre, name="test") @@ -310,8 +310,8 @@ def test_verifier_chained_credential(seeder): with habbing.openHab(name="ron", temp=True, salt=b'0123456789abcdef') as (ronHby, ron), \ habbing.openHab(name="ian", temp=True, salt=b'0123456789abcdef') as (ianHby, ian), \ - habbing.openHab(name="han", transferable=True, temp=True) as (hanHby, han), \ - habbing.openHab(name="vic", transferable=True, temp=True) as (vicHby, vic): + habbing.openHab(name="han", transferable=True, temp=True, salt=b'0123456789abcdef') as (hanHby, han), \ + habbing.openHab(name="vic", transferable=True, temp=True, salt=b'0123456789abcdef') as (vicHby, vic): seeder.seedSchema(db=ronHby.db) seeder.seedSchema(db=ianHby.db) seeder.seedSchema(db=hanHby.db) From c1bd4e622c491642507f914b7f15c05583c0a70d Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Fri, 8 Mar 2024 09:09:47 -0800 Subject: [PATCH 036/418] Includes 2 fixes (#705) - Fix to saving SignifyGroupHabs to enable reloading - Fix to credential export to support exporting all credentials. Signed-off-by: pfeairheller (cherry picked from commit 5bdaaec7a75b5fcd0ad9d76aaecf98555884f492) Signed-off-by: pfeairheller --- scripts/demo/credentials/multisig-issuer.sh | 26 +-- .../credentials/oobi-new_multisig_issuer.sh | 12 ++ .../credentials/rotate-multisig-issuer.sh | 40 +++++ src/keri/app/cli/commands/migrate.py | 153 ++++++++++++++++++ src/keri/app/cli/commands/vc/export.py | 15 +- src/keri/app/habbing.py | 6 +- src/keri/vdr/credentialing.py | 8 +- 7 files changed, 238 insertions(+), 22 deletions(-) create mode 100755 scripts/demo/credentials/oobi-new_multisig_issuer.sh create mode 100755 scripts/demo/credentials/rotate-multisig-issuer.sh create mode 100644 src/keri/app/cli/commands/migrate.py diff --git a/scripts/demo/credentials/multisig-issuer.sh b/scripts/demo/credentials/multisig-issuer.sh index 8e8969884..69b2da044 100755 --- a/scripts/demo/credentials/multisig-issuer.sh +++ b/scripts/demo/credentials/multisig-issuer.sh @@ -108,16 +108,16 @@ kli vc list --name holder --alias holder --poll SAID=$(kli vc list --name holder --alias holder --said --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao) -echo "Revoking ${SAID}..." -TIME=$(date -Iseconds -u) -kli vc revoke --name multisig1 --alias multisig --registry-name vLEI --said "${SAID}" --time "${TIME}" & -pid=$! -PID_LIST=" $pid" - -kli vc revoke --name multisig2 --alias multisig --registry-name vLEI --said "${SAID}" --time "${TIME}" & -pid=$! -PID_LIST+=" $pid" - - -wait $PID_LIST -kli vc list --name holder --alias holder --poll +#echo "Revoking ${SAID}..." +#TIME=$(date -Iseconds -u) +#kli vc revoke --name multisig1 --alias multisig --registry-name vLEI --said "${SAID}" --time "${TIME}" & +#pid=$! +#PID_LIST=" $pid" +# +#kli vc revoke --name multisig2 --alias multisig --registry-name vLEI --said "${SAID}" --time "${TIME}" & +#pid=$! +#PID_LIST+=" $pid" +# +# +#wait $PID_LIST +#kli vc list --name holder --alias holder --poll diff --git a/scripts/demo/credentials/oobi-new_multisig_issuer.sh b/scripts/demo/credentials/oobi-new_multisig_issuer.sh new file mode 100755 index 000000000..a55689ff6 --- /dev/null +++ b/scripts/demo/credentials/oobi-new_multisig_issuer.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +kli oobi resolve --name multisig1 --oobi-alias new-multisig1 --oobi http://127.0.0.1:3902/oobi/EBFg-5SGDCv5YfwpkArWRBdTxNRUXU8uVcDKNzizOQZc --force +kli oobi resolve --name multisig1 --oobi-alias new-multisig2 --oobi http://127.0.0.1:3902/oobi/EBmW2bXbgsP3HITwW3FmITzAb3wVmHlxCusZ46vgGgP5 --force +kli oobi resolve --name multisig1 --oobi-alias new-multisig3 --oobi http://127.0.0.1:3902/oobi/EL4RpdS2Atb2Syu5xLdpz9CcNNYoFUUDlLHxHD09vcgh --force +kli oobi resolve --name multisig1 --oobi-alias new-multisig4 --oobi http://127.0.0.1:3902/oobi/EAiBVuuhCZrgckeHc9KzROVGJpmGbk2-e1B25GaeRrJs --force + +kli oobi resolve --name multisig2 --oobi-alias new-multisig1 --oobi http://127.0.0.1:3902/oobi/EBFg-5SGDCv5YfwpkArWRBdTxNRUXU8uVcDKNzizOQZc --force +kli oobi resolve --name multisig2 --oobi-alias new-multisig2 --oobi http://127.0.0.1:3902/oobi/EBmW2bXbgsP3HITwW3FmITzAb3wVmHlxCusZ46vgGgP5 --force +kli oobi resolve --name multisig2 --oobi-alias new-multisig3 --oobi http://127.0.0.1:3902/oobi/EL4RpdS2Atb2Syu5xLdpz9CcNNYoFUUDlLHxHD09vcgh --force +kli oobi resolve --name multisig2 --oobi-alias new-multisig4 --oobi http://127.0.0.1:3902/oobi/EAiBVuuhCZrgckeHc9KzROVGJpmGbk2-e1B25GaeRrJs --force + diff --git a/scripts/demo/credentials/rotate-multisig-issuer.sh b/scripts/demo/credentials/rotate-multisig-issuer.sh new file mode 100755 index 000000000..33c0b83b9 --- /dev/null +++ b/scripts/demo/credentials/rotate-multisig-issuer.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +kli rotate --name multisig1 --alias multisig1 +kli query --name multisig2 --alias multisig2 --prefix EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 +kli rotate --name multisig2 --alias multisig2 +kli query --name multisig1 --alias multisig1 --prefix EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 + +# Perform rotation of mulisig AID from local kli AIDs that roll themselves out and the new AIDs in +kli multisig rotate --name multisig1 --alias multisig \ + --smids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4:2 \ + --smids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1:2 \ + --smids EBFg-5SGDCv5YfwpkArWRBdTxNRUXU8uVcDKNzizOQZc:0 \ + --smids EBmW2bXbgsP3HITwW3FmITzAb3wVmHlxCusZ46vgGgP5:0 \ + --smids EL4RpdS2Atb2Syu5xLdpz9CcNNYoFUUDlLHxHD09vcgh:0 \ + --smids EAiBVuuhCZrgckeHc9KzROVGJpmGbk2-e1B25GaeRrJs:0 \ + --isith '["0", "0", "1/2", "1/2", "1/2", "1/2"]' \ + --rmids EBFg-5SGDCv5YfwpkArWRBdTxNRUXU8uVcDKNzizOQZc:0 \ + --rmids EBmW2bXbgsP3HITwW3FmITzAb3wVmHlxCusZ46vgGgP5:0 \ + --rmids EL4RpdS2Atb2Syu5xLdpz9CcNNYoFUUDlLHxHD09vcgh:0 \ + --rmids EAiBVuuhCZrgckeHc9KzROVGJpmGbk2-e1B25GaeRrJs:0 \ + --nsith '["1/2", "1/2", "1/2", "1/2"]' & +pid=$! +PID_LIST="$pid" +kli multisig rotate --name multisig2 --alias multisig \ + --smids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4:2 \ + --smids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1:2 \ + --smids EBFg-5SGDCv5YfwpkArWRBdTxNRUXU8uVcDKNzizOQZc:0 \ + --smids EBmW2bXbgsP3HITwW3FmITzAb3wVmHlxCusZ46vgGgP5:0 \ + --smids EL4RpdS2Atb2Syu5xLdpz9CcNNYoFUUDlLHxHD09vcgh:0 \ + --smids EAiBVuuhCZrgckeHc9KzROVGJpmGbk2-e1B25GaeRrJs:0 \ + --isith '["0", "0", "1/2", "1/2", "1/2", "1/2"]' \ + --rmids EBFg-5SGDCv5YfwpkArWRBdTxNRUXU8uVcDKNzizOQZc:0 \ + --rmids EBmW2bXbgsP3HITwW3FmITzAb3wVmHlxCusZ46vgGgP5:0 \ + --rmids EL4RpdS2Atb2Syu5xLdpz9CcNNYoFUUDlLHxHD09vcgh:0 \ + --rmids EAiBVuuhCZrgckeHc9KzROVGJpmGbk2-e1B25GaeRrJs:0 \ + --nsith '["1/2", "1/2", "1/2", "1/2"]' & +pid=$! +PID_LIST+=" $pid" + +wait $PID_LIST diff --git a/src/keri/app/cli/commands/migrate.py b/src/keri/app/cli/commands/migrate.py new file mode 100644 index 000000000..0f4035f78 --- /dev/null +++ b/src/keri/app/cli/commands/migrate.py @@ -0,0 +1,153 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri import kering +from keri.app.cli.common import existing +from keri.core import coring, serdering +from keri.db import koming, subing, dbing +from keri.db.basing import KeyStateRecord, StateEERecord +from keri.kering import ConfigurationError, Version +from keri.vdr import viring + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='View status of a local AID') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--force', action="store_true", required=False, + help='True means perform migration without prompting the user') + + +def handler(args): + if not args.force: + print() + print("This command will migrate your datastore to the next version of KERIpy and is not reversible.") + print("After this command, you will not be able to access your data store with this version.") + print() + yn = input("Are you sure you want to continue? [y|N]: ") + + if yn not in ("y", "Y"): + print("...exiting") + return [] + + kwa = dict(args=args) + return [doing.doify(migrate, **kwa)] + + +def migrate(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + + try: + with dbing.openLMDB(name=name, base=base, bran=bran, temp=False) as db: + print(db.path) + states = koming.Komer(db=db, + schema=dict, + subkey='stts.') + nstates = koming.Komer(db=db, + schema=KeyStateRecord, + subkey='stts.') + + for keys, sad in states.getItemIter(): + ksr = KeyStateRecord( + vn=Version, # version number as list [major, minor] + i=sad['i'], # qb64 prefix + s=sad['s'], # lowercase hex string no leading zeros + p=sad['p'], + d=sad['d'], + f=sad['f'], # lowercase hex string no leading zeros + dt=sad['dt'], + et=sad['et'], + kt=sad['kt'], + k=sad['k'], + nt=sad['nt'], + n=sad['n'], + bt=sad['bt'], + b=sad['b'], + c=sad['c'], + ee=StateEERecord._fromdict(sad['ee']), # latest est event dict + di=sad['di'] if sad['di'] else None + ) + + nstates.pin(keys=keys, val=ksr) + + with existing.existingHby(name=name, base=base, bran=bran) as hby: + rgy = viring.Reger(name=name, base=base, db=hby.db, temp=False, + reopen=True) + + rstates = koming.Komer(db=rgy, + schema=dict, + subkey='stts.') + + for _, sad in rstates.getItemIter(): + rsr = viring.RegStateRecord( + vn=list(Version), # version number as list [major, minor] + i=sad['i'], # qb64 registry SAID + s=sad['s'], # lowercase hex string no leading zeros + d=sad['d'], + ii=sad['ii'], + dt=sad['dt'], + et=sad['et'], + bt=sad['bt'], # hex string no leading zeros lowercase + b=sad['b'], # list of qb64 may be empty + c=sad['c'], + ) + # ksr = stateFromKever(kever) + rgy.states.pin(sad['i'], val=rsr) + + for (said,), _ in rgy.creds.getItemIter(): + snkey = dbing.snKey(said, 0) + dig = rgy.getTel(key=snkey) + + prefixer = coring.Prefixer(qb64=said) + seqner = coring.Seqner(sn=0) + saider = coring.Saider(qb64b=bytes(dig)) + rgy.cancs.pin(keys=said, val=[prefixer, seqner, saider]) + + migrateKeys(hby.db) + + except ConfigurationError: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 + + +def migrateKeys(db): + # public keys mapped to the AID and event seq no they appeared in + pubs = subing.CatCesrIoSetSuber(db=db, subkey="pubs.", + klas=(coring.Prefixer, coring.Seqner)) + + # next key digests mapped to the AID and event seq no they appeared in + digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", + klas=(coring.Prefixer, coring.Seqner)) + + for pre, fn, dig in db.getFelItemAllPreIter(key=b''): + dgkey = dbing.dgKey(pre, dig) # get message + if not (raw := db.getEvt(key=dgkey)): + raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) + serder = serdering.SerderKERI(raw=bytes(raw)) + val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) + verfers = serder.verfers or [] + for verfer in verfers: + pubs.add(keys=(verfer.qb64,), val=val) + ndigers = serder.ndigers or [] + for diger in ndigers: + digs.add(keys=(diger.qb64,), val=val) diff --git a/src/keri/app/cli/commands/vc/export.py b/src/keri/app/cli/commands/vc/export.py index b18c93232..f79e2b693 100644 --- a/src/keri/app/cli/commands/vc/export.py +++ b/src/keri/app/cli/commands/vc/export.py @@ -10,6 +10,7 @@ from hio import help from hio.base import doing +from keri.app import signing from keri.app.cli.common import existing from keri.core import serdering from keri.vdr import credentialing @@ -27,7 +28,7 @@ parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran -parser.add_argument("--said", "-s", help="SAID of the credential to export.", required=True) +parser.add_argument("--said", "-s", help="SAID of the credential to export.", required=False, default=None) parser.add_argument("--tels", help="export the transaction event logs for the credential and any chained credentials", action="store_true") parser.add_argument("--kels", help="export the key event logs for the issuer's of the credentials", action="store_true") @@ -93,7 +94,12 @@ def exportDo(self, tymth, tock=0.0): self.tock = tock _ = (yield self.tock) - self.outputCred(said=self.said) + if self.said is None: + for (said,), _ in self.rgy.reger.creds.getItemIter(): + self.outputCred(said=said) + + else: + self.outputCred(said=self.said) def outputCred(self, said): creder, *_ = self.rgy.reger.cloneCred(said=said) @@ -122,12 +128,13 @@ def outputCred(self, said): for said in saids: self.outputCred(said) + (prefixer, seqner, saider) = self.rgy.reger.cancs.get(keys=(creder.said,)) if self.files: f = open(f"{creder.said}-acdc.cesr", 'w') - f.write(creder.raw.decode("utf-8")) + f.write(signing.serialize(creder, prefixer, seqner, saider)) f.close() else: - sys.stdout.write(creder.raw.decode("utf-8")) + sys.stdout.write(signing.serialize(creder, prefixer, seqner, saider).decode("utf-8")) sys.stdout.flush() def outputTEL(self, regk): diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 6fc3721f9..4fec262fa 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -344,7 +344,7 @@ def loadHabs(self): elif habord.sid and habord.mid: hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=habord.sid) + name=name, pre=pre) groups.append(habord) else: hab = Hab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, @@ -662,7 +662,7 @@ def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): hab.pre = pre habord = basing.HabitatRecord(hid=hab.pre, - mid=mhab.pre, + sid=mhab.pre, smids=smids, rmids=rmids) @@ -1132,7 +1132,7 @@ def make(self, DnD, code, data, delpre, estOnly, isith, verfers, nsith, digers, def save(self, habord): if self.ns is None: - self.db.habs.put(keys=self.name, + self.db.habs.pin(keys=self.name, val=habord) else: self.db.nmsp.put(keys=(self.ns, self.name), diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index e73985846..f26a01b9d 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -14,7 +14,7 @@ from ..app.habbing import GroupHab from ..core import parsing, coring, scheming, serdering from ..core.coring import Seqner, MtrDex -from ..core.eventing import SealEvent, TraitDex +from ..core.eventing import TraitDex from ..db import dbing from ..db.dbing import snKey, dgKey from ..vc import proving @@ -405,7 +405,11 @@ def make(self, *, regser): self.reger.regs.put(keys=self.name, val=viring.RegistryRecord(registryKey=self.regk, prefix=pre)) - self.processEvent(serder=regser) + try: + self.processEvent(serder=regser) + except kering.LikelyDuplicitousError: + pass + self.inited = True def rotate(self, serder): From 4f9c67b589b1a5a64a6c7e0c6b0320acc7e04b14 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 8 Mar 2024 15:25:48 -0700 Subject: [PATCH 037/418] refactor Fields to use new FieldDom and ordering of initialization --- src/keri/core/serdering.py | 199 ++++++++++++++++++------------------- 1 file changed, 94 insertions(+), 105 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index aa81605bb..9cac1f3cc 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -252,125 +252,114 @@ class Serder: Vrsn_1_0: { Ilks.icp: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - kt='0',k=[], nt='0', n=[], bt='0', b=[], c=[], a=[]), + kt='0',k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[]), saids={Saids.d: DigDex.Blake3_256, Saids.i: DigDex.Blake3_256,}), - Ilks.rot: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', - kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], a=[])), - Ilks.ixn: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', a=[])), - Ilks.dip: FieldDom(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', kt='0', - k=[], nt='0', n=[], bt='0', b=[], c=[], a=[], - di='')), - Ilks.drt: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', - kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], a=[])), - Ilks.rct: FieldDom(saids={}, - alls=dict(v='', t='',d='', i='', s='0')), - Ilks.qry: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', dt='', r='', rr='', - q={})), - Ilks.rpy: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', dt='', r='',a=[])), - Ilks.pro: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', dt='', r='', rr='', - q={})), - Ilks.bar: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', dt='', r='',a=[])), - Ilks.exn: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='', d='', i="", p="", dt='', r='',q={}, - a=[], e={})), - Ilks.vcp: FieldDom(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', ii='', s='0', c=[], - bt='0', b=[], n='')), - Ilks.vrt: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', p='', s='0', - bt='0', br=[], ba=[])), - Ilks.iss: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', ri='', - dt='')), - Ilks.rev: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', ri='', - p='', dt='')), - Ilks.bis: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', ii='', s='0', ra={}, - dt='')), - Ilks.brv: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', p='', ra={}, - dt='')), + Ilks.rot: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.ixn: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.dip: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[], di=''), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.drt: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rct: FieldDom(alls=dict(v='', t='',d='', i='', s='0')), + Ilks.qry: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + rr='',q={}), + saids={Saids.d: DigDex.Blake3_256},), + Ilks.rpy: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.pro: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + rr='',q={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.bar: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", p="", + dt='', r='',q={}, a=[], e={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.vcp: FieldDom(alls=dict(v='', t='',d='', i='', ii='', + s='0', c=[], bt='0', b=[], n=''), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.vrt: FieldDom(alls=dict(v='', t='',d='', i='', p='', + s='0', bt='0', br=[], ba=[]), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.iss: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + ri='', dt=''), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.rev: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + ri='', p='', dt=''), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.bis: FieldDom(alls=dict(v='', t='',d='', i='', ii='', + s='0', ra={}, dt=''), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.brv: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', ra={}, dt=''), + saids={Saids.d: DigDex.Blake3_256,}), }, Vrsn_2_0: { - Ilks.icp: FieldDom(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', kt='0', - k=[], nt='0', n=[], bt='0', b=[], c=[], a=[])), - Ilks.rot: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', - kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], c=[], a=[])), - Ilks.ixn: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', a=[])), - Ilks.dip: FieldDom(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', kt='0', - k=[], nt='0', n=[], bt='0', b=[], c=[], a=[], - di='')), - Ilks.drt: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', - kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], c=[], a=[])), - Ilks.rct: FieldDom(saids={}, - alls=dict(v='', t='',d='', i='', s='0')), - Ilks.qry: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', dt='', r='', rr='', - q={})), - Ilks.rpy: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', dt='', r='',a=[])), - Ilks.pro: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', dt='', r='', rr='', - q={})), - Ilks.bar: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', dt='', r='',a=[])), - Ilks.exn: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='', d='', i="", p="", dt='', r='', q={}, - a=[], e={})), + Ilks.icp: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[]), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.rot: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], c=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.ixn: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.dip: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[], di=''), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.drt: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], c=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rct: FieldDom(alls=dict(v='', t='',d='', i='', s='0')), + Ilks.qry: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='', rr='', q={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rpy: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='',a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.pro: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='', rr='', q={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.bar: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='',a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", p="", + dt='', r='', q={}, a=[], e={}), + saids={Saids.d: DigDex.Blake3_256}), }, }, Protos.acdc: { Vrsn_1_0: { - None: FieldDom(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', d='', i='', s='')), + None: FieldDom(alls=dict(v='', d='', i='', s=''), + saids={Saids.d: DigDex.Blake3_256}), }, Vrsn_2_0: { - Ilks.vcp: FieldDom(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', ii='', s='0', c=[], - bt='0', b=[], u='')), - Ilks.vrt: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', p='', s='0', - bt='0', br=[], ba=[])), - Ilks.iss: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', ri='', - dt='')), - Ilks.rev: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', ri='', - p='', dt='')), - Ilks.bis: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', ii='', s='0', ra={}, - dt='')), - Ilks.brv: FieldDom(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', p='', ra={}, - dt='')), + None: FieldDom(alls=dict(v='', d='', i='', s=''), + saids={Saids.d: DigDex.Blake3_256}), }, }, } From 59544a8c46a74fe9f22c6a80c175f59f1d25cef6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 8 Mar 2024 15:52:12 -0700 Subject: [PATCH 038/418] fix Serder.Ilks --- src/keri/core/serdering.py | 17 ++++++++++------- src/keri/kering.py | 6 ++++-- tests/core/test_serdering.py | 3 ++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 9cac1f3cc..63c79bbd9 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -336,16 +336,19 @@ class Serder: r='', rr='', q={}), saids={Saids.d: DigDex.Blake3_256}), Ilks.rpy: FieldDom(alls=dict(v='', t='',d='', i='', dt='', - r='',a=[]), + r='',a={}), saids={Saids.d: DigDex.Blake3_256}), Ilks.pro: FieldDom(alls=dict(v='', t='',d='', i='', dt='', r='', rr='', q={}), saids={Saids.d: DigDex.Blake3_256}), Ilks.bar: FieldDom(alls=dict(v='', t='',d='', i='', dt='', - r='',a=[]), + r='',a={}), saids={Saids.d: DigDex.Blake3_256}), - Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", p="", - dt='', r='', q={}, a=[], e={}), + Ilks.xip: FieldDom(alls=dict(v='', t='', d='', i="", dt='', + r='', q={}, a={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", x="", + p="", dt='', r='', q={}, a={}), saids={Saids.d: DigDex.Blake3_256}), }, }, @@ -366,9 +369,9 @@ class Serder: # default ilk for each protocol at default version is zeroth ilk in dict - Ilks = dict() + DefaultIlks = dict() for key, val in Fields.items(): - Ilks[key] = list(list(val.values())[0].keys()) + DefaultIlks[key] = list(list(val.values())[0].keys()) def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, @@ -650,7 +653,7 @@ def makify(self, sad, *, version=None, kind = skind if skind is not None else self.Kind if ilk is None: - ilk = silk if silk is not None else self.Ilks[proto][0] + ilk = silk if silk is not None else self.DefaultIlks[proto][0] if proto not in self.Fields: diff --git a/src/keri/kering.py b/src/keri/kering.py index 972666322..5c3194c6a 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -321,14 +321,16 @@ def sniff(ims): """ # KERI protocol packet (message) types -Ilkage = namedtuple("Ilkage", ('icp rot ixn dip drt rct qry rpy exn ' +Ilkage = namedtuple("Ilkage", ('icp rot ixn dip drt rct qry rpy xip exn ' 'pro bar vcp vrt iss rev bis brv ')) Ilks = Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', rct='rct', - qry='qry', rpy='rpy', exn='exn', pro='pro', bar='bar', + qry='qry', rpy='rpy', xip='xip', exn='exn', pro='pro', bar='bar', vcp='vcp', vrt='vrt', iss='iss', rev='rev', bis='bis', brv='brv') +# Ilks needs to be versioned for Protocol versions or else use Serder.Fields + # note ksn is not actual standalone message but is embedded in exn msg when sent # over the wire. But keep ilk for legacy reasons. diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index b8d60c46d..e0943ce80 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -49,7 +49,8 @@ def test_serder(): assert Serder.Fields - assert Serder.Ilks == {'KERI': ['icp', + # Version 1.0 ilks + assert Serder.DefaultIlks == {'KERI': ['icp', 'rot', 'ixn', 'dip', From e78b67fbf566a20cb1c620420483dcf0289af4d4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 8 Mar 2024 16:19:19 -0700 Subject: [PATCH 039/418] some clean up --- src/keri/core/serdering.py | 32 ++++++++++++++++++-------------- tests/core/test_serdering.py | 20 -------------------- tests/test_kering.py | 2 +- 3 files changed, 19 insertions(+), 35 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 63c79bbd9..74f8d0f07 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -161,6 +161,14 @@ class Serder: generation and verification in addition to the required fields. Class Attributes: + Dummy (str): dummy character for computing SAIDs + Digests (dict): map of digestive codes. Should be same set of codes as + in coring.DigestCodex coring.DigDex so that .digestive property works. + Use unit tests to ensure codex sets match + Protocol (str): default protocol version type + Proto (str): default CESR protocol genus type + Vrsn (Versionage): default version + Kind (str): default serialization kind one of Serials Fields (dict): nested dict of field labels keyed by protocol, version, and message type (ilk). Felds labels are provided with a Fieldage named tuple (saids, reqs, alls) that governs field type and presence. @@ -218,7 +226,7 @@ class Serder: """ - Dummy = "#" # dummy spaceholder char for said. Must not be a valid Base64 char + Dummy = "#" # dummy spaceholder char for SAID. Must not be a valid Base64 char # should be same set of codes as in coring.DigestCodex coring.DigDex so # .digestive property works. Use unit tests to ensure codex sets match @@ -237,7 +245,7 @@ class Serder: #override in subclass to enforce specific protocol Protocol = None # required protocol, None means any in Protos is ok - Proto = Protos.keri # default protocol type + Proto = Protos.keri # default CESR protocol type Vrsn = Vrsn_1_0 # default protocol version for protocol type Kind = Serials.json # default serialization kind @@ -368,12 +376,6 @@ class Serder: } - # default ilk for each protocol at default version is zeroth ilk in dict - DefaultIlks = dict() - for key, val in Fields.items(): - DefaultIlks[key] = list(list(val.values())[0].keys()) - - def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, smellage=None, verify=True, makify=False, proto=None, vrsn=None, kind=None, ilk=None, saids=None): @@ -646,19 +648,21 @@ def makify(self, sad, *, version=None, if proto is None: proto = sproto if sproto is not None else self.Proto + if proto not in self.Fields: + raise SerializeError(f"Invalid protocol type = {proto}.") + if vrsn is None: vrsn = svrsn if svrsn is not None else self.Vrsn + if vrsn not in self.Fields[proto]: + raise SerializeError(f"Invalid version = {vrsn} for protocol = {proto}.") + if kind is None: kind = skind if skind is not None else self.Kind if ilk is None: - ilk = silk if silk is not None else self.DefaultIlks[proto][0] - - - if proto not in self.Fields: - raise SerializeError(f"Invalid protocol type = {proto}.") - + ilk = (silk if silk is not None else + list(self.Fields[proto][vrsn].keys())[0]) if self.Protocol and proto != self.Protocol: raise SerializeError(f"Expected protocol = {self.Protocol}, got " diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index e0943ce80..72d03aea3 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -49,26 +49,6 @@ def test_serder(): assert Serder.Fields - # Version 1.0 ilks - assert Serder.DefaultIlks == {'KERI': ['icp', - 'rot', - 'ixn', - 'dip', - 'drt', - 'rct', - 'qry', - 'rpy', - 'pro', - 'bar', - 'exn', - 'vcp', - 'vrt', - 'iss', - 'rev', - 'bis', - 'brv'], - 'ACDC': [None]} - assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].saids == {'d': 'E'} assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls == {'v': '', 'd': '', 'i': '', 's': ''} diff --git a/tests/test_kering.py b/tests/test_kering.py index c1ad3f417..1325c3695 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -491,7 +491,7 @@ def test_ilks(): """ assert Ilks == Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', rct='rct', qry='qry', rpy='rpy', - exn='exn', pro='pro', bar='bar', + xip='xip', exn='exn', pro='pro', bar='bar', vcp='vcp', vrt='vrt', iss='iss', rev='rev', bis='bis', brv='brv', ) From aec7a2c369e362404e5f90fb698d0e7c0a78f32c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 11 Mar 2024 13:58:32 -0600 Subject: [PATCH 040/418] added version 2 ACDC event message types --- src/keri/core/serdering.py | 22 +++++++++++++++++++++- src/keri/kering.py | 9 ++++++--- tests/test_kering.py | 29 ++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 74f8d0f07..9bfecf73c 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -369,7 +369,27 @@ class Serder: }, Vrsn_2_0: { - None: FieldDom(alls=dict(v='', d='', i='', s=''), + None: FieldDom(alls=dict(v='', d='', u='', i='', + rd='', s='', a='', A='', e='', r=''), + opts=dict(u='', rd='', a='', A='', e='', r=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.acd: FieldDom(alls=dict(v='', t='', d='', u='', i='', + rd='', s='', a='', A='', e='', r=''), + opts=dict(u='', rd='', a='', A='', e='', r=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.sch: FieldDom(alls=dict(v='', t='', d='', s=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.att: FieldDom(alls=dict(v='', t='', d='', a=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.agg: FieldDom(alls=dict(v='', t='', d='', A=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rul: FieldDom(alls=dict(v='', t='', d='', e=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rip: FieldDom(alls=dict(v='', t='', d='', u='', i='', + s='', dt=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.upd: FieldDom(alls=dict(v='', t='', d='', r='', s='', + p='', dt='', a=''), saids={Saids.d: DigDex.Blake3_256}), }, }, diff --git a/src/keri/kering.py b/src/keri/kering.py index 5c3194c6a..8223e815b 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -320,14 +320,17 @@ def sniff(ims): brv = backed vc revoke, registry-backed transaction event log credential revocation """ -# KERI protocol packet (message) types +# KERI/ACDC protocol packet (message) types Ilkage = namedtuple("Ilkage", ('icp rot ixn dip drt rct qry rpy xip exn ' - 'pro bar vcp vrt iss rev bis brv ')) + 'pro bar vcp vrt iss rev bis brv rip upd ' + 'acd sch att agg edg rul ')) Ilks = Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', rct='rct', qry='qry', rpy='rpy', xip='xip', exn='exn', pro='pro', bar='bar', - vcp='vcp', vrt='vrt', iss='iss', rev='rev', bis='bis', brv='brv') + vcp='vcp', vrt='vrt', iss='iss', rev='rev', bis='bis', brv='brv', + rip='rip', upd='upd', acd='acd', sch='sch', att='att', agg='agg', + edg='edg', rul='rul') # Ilks needs to be versioned for Protocol versions or else use Serder.Fields diff --git a/tests/test_kering.py b/tests/test_kering.py index 1325c3695..6b4ac7da3 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -489,11 +489,12 @@ def test_ilks(): """ Test Ilkage namedtuple instance Ilks """ - assert Ilks == Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', - rct='rct', qry='qry', rpy='rpy', - xip='xip', exn='exn', pro='pro', bar='bar', - vcp='vcp', vrt='vrt', - iss='iss', rev='rev', bis='bis', brv='brv', ) + Ilks = Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', + rct='rct', + qry='qry', rpy='rpy', xip='xip', exn='exn', pro='pro', bar='bar', + vcp='vcp', vrt='vrt', iss='iss', rev='rev', bis='bis', brv='brv', + rip='rip', upd='upd', acd='acd', sch='sch', att='att', agg='agg', + edg='edg', rul='rul') assert isinstance(Ilks, Ilkage) @@ -539,6 +540,24 @@ def test_ilks(): assert 'brv' in Ilks assert Ilks.brv == 'brv' + assert 'rip' in Ilks + assert Ilks.rip == 'rip' + assert 'upd' in Ilks + assert Ilks.upd == 'upd' + + assert 'acd' in Ilks + assert Ilks.acd == 'acd' + assert 'sch' in Ilks + assert Ilks.sch == 'sch' + assert 'att' in Ilks + assert Ilks.att == 'att' + assert 'agg' in Ilks + assert Ilks.agg == 'agg' + assert 'edg' in Ilks + assert Ilks.edg == 'edg' + assert 'rul' in Ilks + assert Ilks.rul == 'rul' + """End Test """ From aa262de76dd9470bc21942642001f04094a26535 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 12 Mar 2024 11:53:28 -0600 Subject: [PATCH 041/418] clean up code. when d is dict list(d) == list(d.keys(0)) so replace the latter with the former. --- src/keri/app/agenting.py | 4 ++-- src/keri/core/coring.py | 2 +- src/keri/core/eventing.py | 24 ++++++++++++++---------- src/keri/core/serdering.py | 34 +++++++++++++++++----------------- src/keri/db/basing.py | 8 ++++---- tests/core/test_serdering.py | 6 +++--- 6 files changed, 41 insertions(+), 37 deletions(-) diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 68dccba80..526479aca 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -98,8 +98,8 @@ def receipt(self, pre, sn=None): else: logger.error(f"invalid response {rep.status} from witnesses {wit}") - for wit in rcts.keys(): - ewits = [w for w in rcts.keys() if w != wit] + for wit in rcts: + ewits = [w for w in rcts if w != wit] wigs = [sig for w, sig in rcts.items() if w != wit] msg = bytearray() diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index cd7ca2af2..ce4a115dc 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -5697,7 +5697,7 @@ def _processSith(self, sith: int | str | Sequence): raise ValueError(f"Invalid sith = {sith} nested " f"weight map {e} in clause {c} " f" not single key value.") - k = list(e.keys())[0] # zeroth k:v pair is used + k = list(e)[0] # zeroth key is used # convert to tuple of (weight, [list of weights]) clause.append((self.weight(k), [self.weight(w) for w in e[k]])) else: diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 3a5a3fac2..dda7479fa 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -72,7 +72,7 @@ def __iter__(self): # sealdict =seal._asdict() # to convet dict to namedtuple use ** unpacking as in seal = SealDigest(**sealdict) # to check if dict of seal matches fields of associted namedtuple -# if tuple(sealdict.keys()) == SealEvent._fields: +# if tuple(sealdict) == SealEvent._fields: # Digest Seal: uniple (d,) # d = digest qb64 of data (usually SAID) @@ -2700,15 +2700,19 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, if self.kevers is None or delpre not in self.kevers: # drop event - # ToDo XXXX may want to cue a trigger to get the KEL of the delegator - raise ValidationError("Missing Kever for delegator = {} for evt = {}." - "".format(delpre, serder.ked)) + # ToDo XXXX cue a trigger to get the KEL of the delegator + # the processDelegableEvent should also cue a trigger to get KEL + # of delegator if still missing when processing escrow later. + self.escrowDelegableEvent(serder=serder, sigers=sigers, + wigers=wigers,local=local) + raise MissingDelegableApprovalError(f"Missing Kever for delegator" + f" = {delpre} of event" + f" = {serder.ked}.") dkever = self.kevers[delpre] if dkever.doNotDelegate: # drop event if delegation not allowed - raise ValidationError("Delegator = {} for evt = {}," - " does not allow delegation.".format(delpre, - serder.ked)) + raise ValidationError(f"Delegator = {delpre} for evt = {serder.ked}," + f" does not allow delegation.") # Delegator accepts here without waiting for delegation seal to be anchored in @@ -2807,7 +2811,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # for JSON, CBOR, MGPK # may want to try harder here by walking KEL for dseal in dserder.seals: # find delegating seal anchor - if tuple(dseal.keys()) == SealEvent._fields: + if tuple(dseal) == SealEvent._fields: seal = SealEvent(**dseal) if (seal.i == serder.pre and seal.s == serder.sner.numh and @@ -2855,10 +2859,10 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, if bossn.said == bosso.said: # same delegating event nseals = [SealEvent(**seal) for seal in bossn.seals - if tuple(seal.keys()) == SealEvent._fields] + if tuple(seal) == SealEvent._fields] nindex = nseals.index(SealEvent(i=serfn.pre, s=serfn.snh, d=serfn.said)) oseals = [SealEvent(**seal) for seal in bosso.seals - if tuple(seal.keys()) == SealEvent._fields] + if tuple(seal) == SealEvent._fields] oindex = oseals.index(SealEvent(i=serfo.pre, s=serfo.snh, d=serfo.said)) if nindex > oindex: # later seal supersedes diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 9bfecf73c..0fa8f568e 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -459,7 +459,7 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, self._size = size # primary said field label try: - label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids.keys())[0] + label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] if label not in self._sad: raise FieldError(f"Missing primary said field in {self._sad}.") self._said = self._sad[label] # not verified @@ -499,7 +499,7 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, self._size = size # primary said field label try: - label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids.keys())[0] + label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] if label not in self._sad: raise DeserializeError(f"Missing primary said field in {self._sad}.") self._said = self._sad[label] # not verified @@ -568,21 +568,21 @@ def _verify(self): if key not in alls: del keys[keys.index(key)] # remove non required fields - if list(alls.keys()) != keys: # forces ordering of labels in .sad + if list(alls) != keys: # forces ordering of labels in .sad raise MissingFieldError(f"Missing one or more required fields from" - f"= {list(alls.keys())} in sad = " + f"= {list(alls)} in sad = " f"{self._sad}.") # said field labels are not order dependent with respect to all fields # in sad so use set() to test inclusion saids = copy.copy(fields.saids) # get copy of saidive field labels and defaults values - if not (set(saids.keys()) <= set(alls.keys())): + if not (set(saids) <= set(alls)): raise MissingFieldError(f"Missing one or more required said fields" - f" from {list(saids.keys())} in sad = " + f" from {list(saids)} in sad = " f"{self._sad}.") sad = self.sad # make shallow copy so don't clobber original .sad - for label in saids.keys(): + for label in saids: try: # replace default code with code of value from sad saids[label] = Matter(qb64=sad[label]).code except Exception as ex: @@ -680,9 +680,9 @@ def makify(self, sad, *, version=None, if kind is None: kind = skind if skind is not None else self.Kind - if ilk is None: + if ilk is None: # default is first ilk in Fields for given proto vrsn ilk = (silk if silk is not None else - list(self.Fields[proto][vrsn].keys())[0]) + list(self.Fields[proto][vrsn])[0]) # list(dict) gives list of keys if self.Protocol and proto != self.Protocol: raise SerializeError(f"Expected protocol = {self.Protocol}, got " @@ -725,16 +725,16 @@ def makify(self, sad, *, version=None, if key not in alls: del keys[keys.index(key)] # remove non required fields - if list(alls.keys()) != keys: # ensure ordering of fields matches alls + if list(alls) != keys: # ensure ordering of fields matches alls raise SerializeError(f"Mismatch one or more of all required fields " - f" = {list(alls.keys())} in sad = {sad}.") + f" = {list(alls)} in sad = {sad}.") # said field labels are not order dependent with respect to all fields # in sad so use set() to test inclusion _saids = copy.copy(fields.saids) # get copy of defaults - if not (set(_saids.keys()) <= set(alls.keys())): + if not (set(_saids) <= set(alls)): raise SerializeError(f"Missing one or more required said fields " - f"from {list(_saids.keys())} in sad = {sad}.") + f"from {list(_saids)} in sad = {sad}.") # override saidive defaults for label in _saids: @@ -789,7 +789,7 @@ def makify(self, sad, *, version=None, self._size = size # primary said field label try: - label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids.keys())[0] + label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] if label not in self._sad: raise SerializeError(f"Missing primary said field in {self._sad}.") self._said = self._sad[label] # implicitly verified @@ -1074,7 +1074,7 @@ def said(self): Returns: said (str): qb64 """ - if not self.Fields[self.proto][self.vrsn][self.ilk].saids.keys() and 'd' in self._sad: + if not self.Fields[self.proto][self.vrsn][self.ilk].saids and 'd' in self._sad: return self._sad['d'] # special case for non-saidive messages like rct return self._said @@ -1121,8 +1121,8 @@ def _verify(self, **kwa): """ super(SerderKERI, self)._verify(**kwa) - allkeys = list(self.Fields[self.proto][self.vrsn][self.ilk].alls.keys()) - keys = list(self.sad.keys()) + allkeys = list(self.Fields[self.proto][self.vrsn][self.ilk].alls) + keys = list(self.sad) if allkeys != keys: raise ValidationError(f"Invalid top level field list. Expected " f"{allkeys} got {keys}.") diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 5e775767c..99f52c871 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1461,7 +1461,7 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): sn (int): beginning sn to search """ - if tuple(seal.keys()) != eventing.SealEvent._fields: # wrong type of seal + if tuple(seal) != eventing.SealEvent._fields: # wrong type of seal return None seal = eventing.SealEvent(**seal) #convert to namedtuple @@ -1469,7 +1469,7 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): for evt in self.getEvtPreIter(pre=pre, sn=sn): # includes disputed & superseded srdr = serdering.SerderKERI(raw=evt.tobytes()) for eseal in srdr.seals or []: - if tuple(eseal.keys()) == eventing.SealEvent._fields: + if tuple(eseal) == eventing.SealEvent._fields: eseal = eventing.SealEvent(**eseal) # convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): return srdr @@ -1493,12 +1493,12 @@ def findAnchoringSeal(self, pre, seal, sn=0): """ # create generic Seal namedtuple class using keys from provided seal dict - Seal = namedtuple('Seal', seal.keys()) # matching type + Seal = namedtuple('Seal', list(seal)) # matching type for evt in self.getEvtLastPreIter(pre=pre, sn=sn): # only last evt at sn srdr = serdering.SerderKERI(raw=evt.tobytes()) for eseal in srdr.seals or []: - if tuple(eseal.keys()) == Seal._fields: # same type of seal + if tuple(eseal) == Seal._fields: # same type of seal eseal = Seal(**eseal) #convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): return srdr diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 72d03aea3..faf7b38c8 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -53,14 +53,14 @@ def test_serder(): assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls == {'v': '', 'd': '', 'i': '', 's': ''} # said field labels must be subset of all field labels - assert (set(Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].saids.keys()) <= - set(Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls.keys())) + assert (set(Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].saids) <= + set(Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls)) for proto, vrsns in Serder.Fields.items(): for vrsn, ilks in vrsns.items(): for ilk, fields in ilks.items(): - assert set(fields.saids.keys()) <= set(fields.alls.keys()) + assert set(fields.saids) <= set(fields.alls) with pytest.raises(ValueError): From 86a4e3f50cae736d9c0cde0fb47179dcbce08b8c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 13 Mar 2024 12:41:15 -0600 Subject: [PATCH 042/418] refactored Serder to do tests with new opts alts so more precise support for specification --- src/keri/core/serdering.py | 204 ++++++++++++++++++++--------------- src/keri/kering.py | 19 +++- tests/core/test_serdering.py | 49 ++++++++- 3 files changed, 182 insertions(+), 90 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 0fa8f568e..4203b89ba 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -13,9 +13,12 @@ import pysodium import blake3 import hashlib +from ordered_set import OrderedSet as oset + from .. import kering -from ..kering import (ValidationError, MissingFieldError, +from ..kering import (ValidationError, MissingFieldError, ExtraFieldError, + AlternateFieldError, ShortageError, VersionError, ProtocolError, KindError, DeserializeError, FieldError, SerializeError) from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, @@ -41,7 +44,7 @@ class FieldDom: and default field values for a given ilk (message type). Attributes: - alls (dict): Ordered set of allowed fields (not extra) + alls (dict): Allowed fields (not extra) alls must not be empty since at least version string or protocol version filed is always required. Fields in alls that appear must appear in order. @@ -52,6 +55,18 @@ class FieldDom: Any fields in alls but not in opts are required. opts is a subset of alls + alts (dict): Alternate fields within alls. + alts defaults to empty. + An alt field means that one or the other of two fields is + allowed but not both. Two entries in alts are required, one + for each field in an alt pair. the alt pair key value are + the two labels. One entry for each order. + all alts must be in opts since both can't be required. + Suppose 'a' and 'A' are an alternate pair then alts = + { 'a': 'A', 'A': 'a'}. This allows the presence of one to + block the presence of the other by looking up the one present + as key to see the value of the one to block. + saids (dict): are saidive fields whose value may be computed as a said of the message. saids defaults to empty when provided a field in saids indicates the field value is saidive. @@ -83,6 +98,7 @@ class FieldDom: """ alls: dict # all allowed fields when strict opts: dict = field(default_factory=dict) # optional fields + alts: dict = field(default_factory=dict) # alternate optional fields saids: dict = field(default_factory=dict) # saidive fields strict: bool = True # only alls allowed no extras @@ -364,7 +380,10 @@ class Serder: { Vrsn_1_0: { - None: FieldDom(alls=dict(v='', d='', i='', s=''), + None: FieldDom(alls=dict(v='', d='', u='', i='', + ri='', s='', a='', A='', e='', r=''), + opts=dict(u='', ri='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), saids={Saids.d: DigDex.Blake3_256}), }, Vrsn_2_0: @@ -372,10 +391,12 @@ class Serder: None: FieldDom(alls=dict(v='', d='', u='', i='', rd='', s='', a='', A='', e='', r=''), opts=dict(u='', rd='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), saids={Saids.d: DigDex.Blake3_256}), Ilks.acd: FieldDom(alls=dict(v='', t='', d='', u='', i='', rd='', s='', a='', A='', e='', r=''), opts=dict(u='', rd='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), saids={Saids.d: DigDex.Blake3_256}), Ilks.sch: FieldDom(alls=dict(v='', t='', d='', s=''), saids={Saids.d: DigDex.Blake3_256}), @@ -561,17 +582,36 @@ def _verify(self): f"protocol = {self.proto}.") fields = self.Fields[self.proto][self.vrsn][self.ilk] # get labelage - # ensure all required fields in alls are in sad - alls = fields.alls # dict of all field labels with default values - keys = list(self._sad) # get list of keys of self.sad - for key in list(keys): # make copy to mutate - if key not in alls: - del keys[keys.index(key)] # remove non required fields - - if list(alls) != keys: # forces ordering of labels in .sad - raise MissingFieldError(f"Missing one or more required fields from" - f"= {list(alls)} in sad = " - f"{self._sad}.") + + alls = fields.alls # faster local reference + oalls = oset(alls) # ordereset of field labels + oopts = oset(fields.opts) # ordereset of field labels + oreqs = oalls - oopts # required fields + + oskeys = oset(self._sad) # ordered set of keys in sad (skeys) + osexts = oskeys - oalls # get ordered set of extras in sad (sexts) + if osexts and fields.strict: + raise ExtraFieldError(f"Unallowed extra field(s) = {list(osexts)} " + f"in sad.") + + osopts = oskeys - oreqs - osexts # subset of opts in sad + + osalls = oalls - (oopts - osopts) # subset of alls without missing opts in sad + + for k, v in fields.alts.items(): + if k in osopts and v in osopts: + raise AlternateFieldError(f"Unallowed, alternate fields '{k}' " + f"and '{v}' both present in sad.") + + # can't do set math osalls == oskeys - osexts becasue of osexts might be + # out-of-order so have to iterate to ensure osexts if any appear in oskeys + # after all fields in osalls + for i, label in enumerate(osalls): + if oskeys[i] != label: + raise MissingFieldError(f"Missing or out-of-order field = {label} " + f"from = {list(osalls)} in sad.") + + # said field labels are not order dependent with respect to all fields # in sad so use set() to test inclusion @@ -663,7 +703,7 @@ def makify(self, sad, *, version=None, except ValueError as ex: pass else: - silk = sad.get('t') # if not in get returns None which may be valid + silk = sad.get('t') # if 't' not in sad .get returns None which may be valid if proto is None: proto = sproto if sproto is not None else self.Proto @@ -700,34 +740,74 @@ def makify(self, sad, *, version=None, raise SerializeError(f"Invalid packet type (ilk) = {ilk} for" f"protocol = {proto}.") - fields = self.Fields[proto][vrsn][ilk] # get Fieldage of fields + fields = self.Fields[proto][vrsn][ilk] # get FieldDom of fields + + + + alls = fields.alls # faster local reference + oalls = oset(alls) # ordereset of field labels + oopts = oset(fields.opts) # ordereset of field labels + oreqs = oalls - oopts # required fields - if not sad: # empty or None so create from defaults + + if not sad: # empty or None so create sad dict sad = {} - for label, value in fields.alls.items(): - if helping.nonStringIterable(value): # copy iterable defaults - value = copy.copy(value) + + # ensure all required fields are in sad. If not provide default + for label in oreqs: + if label not in sad: + value = alls[label] + if helping.nonStringIterable(value): + value = copy.copy(value) # copy iterable defaults sad[label] = value - if 't' in sad: # packet type (ilk) required so set value to ilk - sad['t'] = ilk + if 't' in sad: # when packet type field then force ilk + sad['t'] = ilk # assign ilk - # ensure all required fields in alls are in sad - alls = fields.alls # all field labels - for label, value in alls.items(): # ensure provided sad as all required fields - if label not in sad: # supply default - if helping.nonStringIterable(value): # copy iterable defaults - value = copy.copy(value) - sad[label] = value + # ensure all required fields are in sad. If not provide default + #for label in oreqs: # ensure provided sad has all required fields + #if label not in sad: # supply default + #value = alls[label] + #if helping.nonStringIterable(value): # copy iterable defaults + #value = copy.copy(value) + #sad[label] = value + + # ensure required fields are present and all fields are ordered wrt alls + + oskeys = oset(sad) # ordered set of keys in sad (skeys) + osexts = oskeys - oalls # get ordered set of extras in sad (sexts) + if osexts and fields.strict: + raise SerializeError(f"Unallowed extra field(s) = {list(osexts)} " + f"in sad.") - keys = list(sad) # get list of keys of self.sad - for key in list(keys): # make copy to mutate - if key not in alls: - del keys[keys.index(key)] # remove non required fields + osopts = oskeys - oreqs - osexts # subset of opts in sad - if list(alls) != keys: # ensure ordering of fields matches alls - raise SerializeError(f"Mismatch one or more of all required fields " - f" = {list(alls)} in sad = {sad}.") + osalls = oalls - (oopts - osopts) # subset of alls without missing opts in sad + + for k, v in fields.alts.items(): + if k in osopts and v in osopts: + raise SerializeError(f"Unallowed, alternate fields '{k}' " + f"and '{v}' both present in sad.") + + # can't do set math osalls == oskeys - osexts becasue of osexts might be + # out-of-order so have to iterate to ensure osexts if any appear in oskeys + # after all fields in osalls + for i, label in enumerate(osalls): + if oskeys[i] != label: + raise SerializeError(f"Missing or out-of-order field = {label} " + f"from = {list(osalls)} in sad.") + + + + + #keys = list(sad) # get list of keys of self.sad + #for key in list(keys): # make copy to mutate + #if key not in alls: + #del keys[keys.index(key)] # remove non required fields + + #if list(alls) != keys: # ensure ordering of fields matches alls + #raise SerializeError(f"Mismatch one or more of all required fields " + #f" = {list(alls)} in sad = {sad}.") # said field labels are not order dependent with respect to all fields # in sad so use set() to test inclusion @@ -1467,58 +1547,6 @@ def nonce(self): return self.uuid -#class SerderCREL(Serder): - #"""SerderCREL is Serder subclass with Labels for CREL packet types (ilks) and - #properties for exposing field values of CREL messages - #Container Registry Event Log for issuance, revocation, etc registries of - #ACDC - - #See docs for Serder - #""" - ##override in subclass to enforce specific protocol - #Protocol = Protos.crel # required protocol, None means any in Protos is ok - #Proto = Protos.crel # default protocol type - #Vrsn = Vrsn_1_1 # default protocol version for protocol type - - - #def _verify(self, **kwa): - #"""Verifies said(s) in sad against raw - #Override for protocol and ilk specific verification behavior. Especially - #for inceptive ilks that have more than one said field like a said derived - #identifier prefix. - - #Raises a ValidationError (or subclass) if any verification fails - - #""" - #super(SerderCREL, self)._verify(**kwa) - - #try: - #code = Matter(qb64=self.issuer).code - #except Exception as ex: - #raise ValidationError(f"Invalid issuer AID = " - #f"{self.issuer}.") from ex - - #if code not in PreDex: - #raise ValidationError(f"Invalid issuer AID code = {code}.") - - - #@property - #def issuer(self): - #""" - #Returns: - #issuer (str): qb64 of .sad["i"] issuer AID property getter - #""" - #return self._sad.get('i') - - - #@property - #def issuerb(self): - #""" - #Returns: - #issuerb (bytes): qb64b of .issuer property getter as bytes - #""" - #return self.issuer.encode("utf-8") if self.issuer is not None else None - class SerderACDC(Serder): """SerderACDC is Serder subclass with Labels for ACDC packet types (ilks) and diff --git a/src/keri/kering.py b/src/keri/kering.py index 8223e815b..3d90fb259 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -536,6 +536,7 @@ class ValidationError(KeriError): raise ValidationError("error message") """ + class MissingFieldError(ValidationError): """ Missing a required element or field of message @@ -544,6 +545,23 @@ class MissingFieldError(ValidationError): """ +class ExtraFieldError(ValidationError): + """ + Extra unallowed field in message + Usage: + raise ExtraFieldError("error message") + """ + + +class AlternateFieldError(ValidationError): + """ + Unallowed alternate field in message + + Usage: + raise AlternateFieldError("error message") + """ + + class MissingSignatureError(ValidationError): """ Error At least One but Missing Enough Signatures for Threshold @@ -706,7 +724,6 @@ class MissingDelegableApprovalError(ValidationError): raise MissingDelegableApprovalError("error message") """ -MissingDelegableApprovalError # Stream Parsing and Extraction Errors diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index faf7b38c8..61cd39e41 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -31,6 +31,8 @@ def test_fielddom(): assert fdom.alls == alls assert fdom.opts == {} assert not fdom.opts + assert fdom.alts == {} + assert not fdom.alts assert fdom.saids == {} assert not fdom.saids assert fdom.strict @@ -49,8 +51,36 @@ def test_serder(): assert Serder.Fields + # Ensure all Serder.Fields all and opts and alts are correct subsets + # iterate through all FieldDoms + #fields = self.Fields[proto][vrsn][ilk] # get FieldDom of fields + #if not (set(fields.opts) <= set(fields.alls)): + #raise SerializeError(f"Opts = {fields.opts} not subset of alls = " + #f" {fields.alls}.") + + #if not (set(fields.alts) <= set(fields.opts)): + #raise SerializeError(f"Alts = {fields.alts} not subset of opts = " + #f" {fields.opts}.") + + #if not (set(fields.opts) <= set(fields.alls)): + #raise SerializeError(f"Opts = {fields.opts} not subset of alls = " + #f" {fields.alls}.") + + #if not (set(fields.alts) <= set(fields.opts)): + #raise SerializeError(f"Alts = {fields.alts} not subset of opts = " + #f" {fields.opts}.") + assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].saids == {'d': 'E'} - assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls == {'v': '', 'd': '', 'i': '', 's': ''} + assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls == + {'v': '', 'd': '', 'u': '', 'i': '', 'ri': '', 's': '', 'a': '', 'A': '', 'e': '', 'r': ''}) + assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].opts == + {'u': '', 'ri': '', 'a': '', 'A': '', 'e': '', 'r': ''}) + assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alts == + {'a': 'A', 'A': 'a'}) + assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].strict + + + #assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls == {'v': '', 'd': '', 'i': '', 's': ''} # said field labels must be subset of all field labels assert (set(Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].saids) <= @@ -545,6 +575,23 @@ def test_serder(): assert serder.said == said assert serder.ilk == ilk + # test opts + #Test Serder bare makify bootstrap for ACDC JSON + serder = Serder(makify=True, proto=kering.Protos.acdc) # make defaults for ACDC + assert serder.sad == {'v': 'ACDC10JSON00005a_', + 'd': 'EMk7BvrqO_2sYjpI_-BmSELOFNie-muw4XTi3iYCz6pT', + 'i': '', + 's': ''} + assert serder.raw == (b'{"v":"ACDC10JSON00005a_","d":"EMk7BvrqO_2sYjpI_-' + b'BmSELOFNie-muw4XTi3iYCz6pT","i":"","s":""}') + assert serder.verify() + sad = serder.sad + raw = serder.raw + said = serder.said + size = serder.size + + + # ToDo: create malicious raw values to test verify more thoroughly From c94c385bf05855b67fb6402de6d0e680f8d0c450 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 13 Mar 2024 16:15:48 -0600 Subject: [PATCH 043/418] Added unit tests and dummy ilk to test all the features of alls, opts, alts, strict and extras for configuring protocols message types and fields --- src/keri/core/serdering.py | 35 +++++--------- src/keri/kering.py | 6 +-- tests/core/test_serdering.py | 94 ++++++++++++++++++++++++++++-------- tests/test_kering.py | 6 ++- 4 files changed, 94 insertions(+), 47 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 4203b89ba..046b26559 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -385,6 +385,12 @@ class Serder: opts=dict(u='', ri='', a='', A='', e='', r=''), alts=dict(a="A", A="a"), saids={Saids.d: DigDex.Blake3_256}), + Ilks.ace: FieldDom(alls=dict(v='', t='', d='', u='', i='', + ri='', s='', a='', A='', e='', r=''), + opts=dict(u='', ri='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}, + strict=False), }, Vrsn_2_0: { @@ -398,6 +404,12 @@ class Serder: opts=dict(u='', rd='', a='', A='', e='', r=''), alts=dict(a="A", A="a"), saids={Saids.d: DigDex.Blake3_256}), + Ilks.ace: FieldDom(alls=dict(v='', t='', d='', u='', i='', + rd='', s='', a='', A='', e='', r=''), + opts=dict(u='', rd='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}, + strict=False), Ilks.sch: FieldDom(alls=dict(v='', t='', d='', s=''), saids={Saids.d: DigDex.Blake3_256}), Ilks.att: FieldDom(alls=dict(v='', t='', d='', a=''), @@ -611,8 +623,6 @@ def _verify(self): raise MissingFieldError(f"Missing or out-of-order field = {label} " f"from = {list(osalls)} in sad.") - - # said field labels are not order dependent with respect to all fields # in sad so use set() to test inclusion saids = copy.copy(fields.saids) # get copy of saidive field labels and defaults values @@ -764,16 +774,7 @@ def makify(self, sad, *, version=None, if 't' in sad: # when packet type field then force ilk sad['t'] = ilk # assign ilk - # ensure all required fields are in sad. If not provide default - #for label in oreqs: # ensure provided sad has all required fields - #if label not in sad: # supply default - #value = alls[label] - #if helping.nonStringIterable(value): # copy iterable defaults - #value = copy.copy(value) - #sad[label] = value - # ensure required fields are present and all fields are ordered wrt alls - oskeys = oset(sad) # ordered set of keys in sad (skeys) osexts = oskeys - oalls # get ordered set of extras in sad (sexts) if osexts and fields.strict: @@ -797,18 +798,6 @@ def makify(self, sad, *, version=None, raise SerializeError(f"Missing or out-of-order field = {label} " f"from = {list(osalls)} in sad.") - - - - #keys = list(sad) # get list of keys of self.sad - #for key in list(keys): # make copy to mutate - #if key not in alls: - #del keys[keys.index(key)] # remove non required fields - - #if list(alls) != keys: # ensure ordering of fields matches alls - #raise SerializeError(f"Mismatch one or more of all required fields " - #f" = {list(alls)} in sad = {sad}.") - # said field labels are not order dependent with respect to all fields # in sad so use set() to test inclusion _saids = copy.copy(fields.saids) # get copy of defaults diff --git a/src/keri/kering.py b/src/keri/kering.py index 3d90fb259..099af0b1b 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -323,14 +323,14 @@ def sniff(ims): # KERI/ACDC protocol packet (message) types Ilkage = namedtuple("Ilkage", ('icp rot ixn dip drt rct qry rpy xip exn ' 'pro bar vcp vrt iss rev bis brv rip upd ' - 'acd sch att agg edg rul ')) + 'acd ace sch att agg edg rul ')) Ilks = Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', rct='rct', qry='qry', rpy='rpy', xip='xip', exn='exn', pro='pro', bar='bar', vcp='vcp', vrt='vrt', iss='iss', rev='rev', bis='bis', brv='brv', - rip='rip', upd='upd', acd='acd', sch='sch', att='att', agg='agg', - edg='edg', rul='rul') + rip='rip', upd='upd', acd='acd', ace='ace', + sch='sch', att='att', agg='agg', edg='edg', rul='rul') # Ilks needs to be versioned for Protocol versions or else use Serder.Fields diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 61cd39e41..59081a094 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -11,6 +11,7 @@ import msgpack import pytest +from ordered_set import OrderedSet as oset from keri import kering from keri.kering import Versionage, Version @@ -51,24 +52,16 @@ def test_serder(): assert Serder.Fields - # Ensure all Serder.Fields all and opts and alts are correct subsets + # Ensure all Serder.Fields all and opts and alts and saids are correct subsets # iterate through all FieldDoms - #fields = self.Fields[proto][vrsn][ilk] # get FieldDom of fields - #if not (set(fields.opts) <= set(fields.alls)): - #raise SerializeError(f"Opts = {fields.opts} not subset of alls = " - #f" {fields.alls}.") - #if not (set(fields.alts) <= set(fields.opts)): - #raise SerializeError(f"Alts = {fields.alts} not subset of opts = " - #f" {fields.opts}.") + for kp, kv in Serder.Fields.items(): # iterate through protocols + for kv, vv in kv.items(): # iterate through versions for each protocol + for kf, vf in vv.items(): # iterate through fields for each version + assert oset(vf.opts) <= oset(vf.alls) + assert oset(vf.alts) <= oset(vf.opts) + assert oset(vf.saids) <= oset(vf.alls) - #if not (set(fields.opts) <= set(fields.alls)): - #raise SerializeError(f"Opts = {fields.opts} not subset of alls = " - #f" {fields.alls}.") - - #if not (set(fields.alts) <= set(fields.opts)): - #raise SerializeError(f"Alts = {fields.alts} not subset of opts = " - #f" {fields.opts}.") assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].saids == {'d': 'E'} assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls == @@ -576,7 +569,6 @@ def test_serder(): assert serder.ilk == ilk # test opts - #Test Serder bare makify bootstrap for ACDC JSON serder = Serder(makify=True, proto=kering.Protos.acdc) # make defaults for ACDC assert serder.sad == {'v': 'ACDC10JSON00005a_', 'd': 'EMk7BvrqO_2sYjpI_-BmSELOFNie-muw4XTi3iYCz6pT', @@ -586,19 +578,83 @@ def test_serder(): b'BmSELOFNie-muw4XTi3iYCz6pT","i":"","s":""}') assert serder.verify() sad = serder.sad - raw = serder.raw - said = serder.said - size = serder.size + sad['a'] = "" + sad['e'] = "" + sad['r'] = "" + + serder = Serder(makify=True, sad=sad) # make using sad + assert serder.raw == (b'{"v":"ACDC10JSON00006f_","d":"EBE7-v1veGz54DF2PIYmUoSG2BCLsEcIQSDSIYFsn9uw",' + b'"i":"","s":"","a":"","e":"","r":""}') + assert serder.verify() + + # out of order field + sad = serder.sad + sad["ri"] = "" + + with pytest.raises(kering.SerializeError): + serder = Serder(makify=True, sad=sad) # make using sad + + + # extra field with strict + sad = serder.sad + assert 'ri' not in sad + sad["x"] = "" + + with pytest.raises(kering.SerializeError): + serder = Serder(makify=True, sad=sad) # make using sad + + # test alts + serder = Serder(makify=True, proto=kering.Protos.acdc) # make defaults for ACDC + assert serder.sad == {'v': 'ACDC10JSON00005a_', + 'd': 'EMk7BvrqO_2sYjpI_-BmSELOFNie-muw4XTi3iYCz6pT', + 'i': '', + 's': ''} + assert serder.verify() + sad = serder.sad + + sad['a'] = "" + sad['A'] = "" # both alts + sad['e'] = "" + sad['r'] = "" + with pytest.raises(kering.SerializeError): + serder = Serder(makify=True, sad=sad) # make using sad + # test not strict + # test opts + serder = Serder(makify=True, proto=kering.Protos.acdc, ilk=kering.Ilks.ace) # make defaults for ACDC + assert serder.sad == {'v': 'ACDC10JSON000064_', + 't': 'ace', + 'd': 'EKFsN95K2h5I6pJC6eTrNKiX8uHyn5o-SYHy6IelbPK8', + 'i': '', + 's': ''} + assert serder.verify() + + sad = serder.sad + sad["x"] = "" + serder = Serder(makify=True, sad=sad) # make using sad + assert serder.sad == {'v': 'ACDC10JSON00006b_', + 't': 'ace', + 'd': 'ECmOYyE7X5TVvBiM7PtApT-w9wsj7ZYI0jQt1TTcTa-1', + 'i': '', + 's': '', + 'x': ''} + assert serder.verify() + # out of order with extra + sad = serder.sad + sad["ri"] = "" + with pytest.raises(kering.SerializeError): + serder = Serder(makify=True, sad=sad) # make using sad # ToDo: create malicious raw values to test verify more thoroughly # ToDo: create bad sad values to test makify more thoroughly # unhappy paths + + """End Test""" def test_serderkeri(): diff --git a/tests/test_kering.py b/tests/test_kering.py index 6b4ac7da3..d4bf2a6ea 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -493,8 +493,8 @@ def test_ilks(): rct='rct', qry='qry', rpy='rpy', xip='xip', exn='exn', pro='pro', bar='bar', vcp='vcp', vrt='vrt', iss='iss', rev='rev', bis='bis', brv='brv', - rip='rip', upd='upd', acd='acd', sch='sch', att='att', agg='agg', - edg='edg', rul='rul') + rip='rip', upd='upd', acd='acd', ace='ace', + sch='sch', att='att', agg='agg', edg='edg', rul='rul') assert isinstance(Ilks, Ilkage) @@ -547,6 +547,8 @@ def test_ilks(): assert 'acd' in Ilks assert Ilks.acd == 'acd' + assert 'ace' in Ilks + assert Ilks.ace == 'ace' assert 'sch' in Ilks assert Ilks.sch == 'sch' assert 'att' in Ilks From 22e2c63eafa32ac15631dac497f5021e4be396a8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 17 Mar 2024 10:50:22 -0700 Subject: [PATCH 044/418] added more support for versioning of Serder. Need to fix some tests --- src/keri/core/serdering.py | 34 ++++++--- src/keri/kering.py | 2 +- tests/core/test_eventing.py | 6 +- tests/core/test_serdering.py | 139 +++++++++++++++++++++++++++++++---- tests/test_kering.py | 6 +- 5 files changed, 156 insertions(+), 31 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 046b26559..baf494cbc 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -22,8 +22,10 @@ ShortageError, VersionError, ProtocolError, KindError, DeserializeError, FieldError, SerializeError) from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, - VERRAWSIZE, VERFMT, MAXVERFULLSPAN, - SMELLSIZE, Smellage, smell) + VERRAWSIZE, VERFMT, + MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN) +from ..kering import SMELLSIZE, Smellage, smell + from ..kering import Protos, Serials, Rever, versify, deversify, Ilks from ..core import coring from .coring import MtrDex, DigDex, PreDex, Saids, Digestage @@ -178,6 +180,7 @@ class Serder: Class Attributes: Dummy (str): dummy character for computing SAIDs + Spans (dict): version string spans keyed by version Digests (dict): map of digestive codes. Should be same set of codes as in coring.DigestCodex coring.DigDex so that .digestive property works. Use unit tests to ensure codex sets match @@ -244,6 +247,9 @@ class Serder: """ Dummy = "#" # dummy spaceholder char for SAID. Must not be a valid Base64 char + # Spans dict keyed by version (Versionage instance) of version string span (size) + Spans = {Vrsn_1_0: VER1FULLSPAN, Vrsn_2_0: VER2FULLSPAN} + # should be same set of codes as in coring.DigestCodex coring.DigDex so # .digestive property works. Use unit tests to ensure codex sets match Digests = { @@ -539,13 +545,13 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, except Exception: self._said = None # no saidive field - if verify: # verify the said(s) provided in sad - try: - self._verify() # raises exception when not verify - except Exception as ex: - logger.error("Invalid sad for Serder %s\n%s", - self.pretty(), ex.args[0]) - raise ValidationError(f"Invalid sad for Serder =" + if verify: # verify the said(s) provided in sad + try: + self._verify() # raises exception when not verify + except Exception as ex: + logger.error("Invalid sad for Serder %s\n%s", + self.pretty(), ex.args[0]) + raise ValidationError(f"Invalid sad for Serder =" f"{self._sad}.") from ex else: @@ -771,6 +777,10 @@ def makify(self, sad, *, version=None, value = copy.copy(value) # copy iterable defaults sad[label] = value + # Need to insert required fields in proper place because passed in sad + # may have missing require fields that appear before provided ones so + # can't simply append + if 't' in sad: # when packet type field then force ilk sad['t'] = ilk # assign ilk @@ -823,9 +833,11 @@ def makify(self, sad, *, version=None, raise SerializeError(f"Missing requires version string field 'v'" f" in sad = {sad}.") - sad['v'] = self.Dummy * MAXVERFULLSPAN # ensure size of vs + # this size of sad needs to be computed based on actual version string span + # since not same for all versions + sad['v'] = self.Dummy * self.Spans[vrsn] # ensure span of vs is dummied MAXVERFULLSPAN - raw = self.dumps(sad, kind) # get size of fully dummied sad + raw = self.dumps(sad, kind) # get size of sad with fully dummied vs and saids size = len(raw) # generate new version string with correct size diff --git a/src/keri/kering.py b/src/keri/kering.py index 099af0b1b..5222c22d8 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -46,7 +46,7 @@ VEREX = VEREX2 + b'|' + VEREX1 -MAXVERFULLSPAN = max(VER1FULLSPAN, VER2FULLSPAN) # number of characters in full version string +MAXVERFULLSPAN = max(VER1FULLSPAN, VER2FULLSPAN) # max number of characters in full version string Rever = re.compile(VEREX) # compile is faster diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index 9bf8d2ad4..69850c833 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -2090,7 +2090,8 @@ def test_kever(mockHelpingNowUTC): # make with defaults with non-digestive prefix serder = serdering.SerderKERI(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519}) + saids = {'i': coring.PreDex.Ed25519}, + verify=False) sad = serder.sad sad['i'] = skp0.verfer.qb64 # non-digestive aid @@ -2233,7 +2234,8 @@ def test_kever(mockHelpingNowUTC): # make with defaults with non-transferable prefix serder = serdering.SerderKERI(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519N}) + saids = {'i': coring.PreDex.Ed25519N}, + verify=False) sad = serder.sad sad['i'] = skp0.verfer.qb64 # non-digestive aid diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 59081a094..03678f83b 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -14,7 +14,9 @@ from ordered_set import OrderedSet as oset from keri import kering -from keri.kering import Versionage, Version +from keri.kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, + VERRAWSIZE, VERFMT, + MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN) from keri.core import coring @@ -41,6 +43,17 @@ def test_fielddom(): """End Test""" +def test_spans(): + """ + Test Spans dict of version string sizes by version + """ + assert Serder.Spans + assert isinstance(Serder.Spans, dict) + + assert Serder.Spans[kering.Vrsn_1_0] == kering.VER1FULLSPAN == 17 + assert Serder.Spans[kering.Vrsn_2_0] == kering.VER2FULLSPAN == 16 + + """End Test""" def test_serder(): @@ -437,7 +450,8 @@ def test_serder(): # Test with non-digestive code for 'i' saidive field no sad serder = Serder(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519}) + saids = {'i': coring.PreDex.Ed25519}, + verify=False) assert serder.sad == {'v': 'KERI10JSON0000a3_', 't': 'icp', @@ -881,7 +895,8 @@ def test_serderkeri_icp(): # Test with non-digestive code for 'i' saidive field no sad serder = SerderKERI(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519}) + saids = {'i': coring.PreDex.Ed25519}, + verify=False) assert serder.sad == {'v': 'KERI10JSON0000a3_', 't': 'icp', @@ -1018,7 +1033,7 @@ def test_serderkeri_rot(): """Test SerderKERI rot msg""" # Test KERI JSON with makify defaults for self bootstrap with ilk rot - serder = SerderKERI(makify=True, ilk=kering.Ilks.rot) # make with defaults + serder = SerderKERI(makify=True, ilk=kering.Ilks.rot, verify=False) # make with defaults assert serder.sad == {'v': 'KERI10JSON0000ac_', 't': 'rot', 'd': 'EMgauZPVfh6807jO9QO8A4Iauq1xhYTZnKX2doVd_UDl', @@ -1135,7 +1150,7 @@ def test_serderkeri_ixn(): """Test SerderKERI ixn msg""" # Test KERI JSON with makify defaults for self bootstrap with ilk ixn - serder = SerderKERI(makify=True, ilk=kering.Ilks.ixn) # make with defaults + serder = SerderKERI(makify=True, ilk=kering.Ilks.ixn, verify=False) # make with defaults assert serder.sad == {'v': 'KERI10JSON000073_', 't': 'ixn', 'd': 'ELI1jUxlJky6RvRieoO20H7_YikKnQMthnWM38etba3r', @@ -1245,7 +1260,7 @@ def test_serderkeri_dip(): """Test SerderKERI dip msg""" # Test KERI JSON with makify defaults for self bootstrap with ilk dip - serder = SerderKERI(makify=True, ilk=kering.Ilks.dip) # make with defaults + serder = SerderKERI(makify=True, ilk=kering.Ilks.dip, verify=False) # make with defaults assert serder.sad == {'v': 'KERI10JSON0000d7_', 't': 'dip', 'd': 'EPyzEgwg6ls8iY4jViniM15rAFWaaVbsZ4eP2a9ZcKfC', @@ -1378,7 +1393,8 @@ def test_serderkeri_dip(): # Test with non-digestive code for 'i' saidive field no sad serder = SerderKERI(makify=True, ilk=kering.Ilks.dip, - saids = {'i': coring.PreDex.Ed25519}) + saids = {'i': coring.PreDex.Ed25519}, + verify=False) assert serder.sad == {'v': 'KERI10JSON0000ab_', 't': 'dip', @@ -1412,7 +1428,7 @@ def test_serderkeri_dip(): sad['i'] = pre sad['di'] = delpre - serder = SerderKERI(sad=sad, makify=True) + serder = SerderKERI(sad=sad, makify=True, verify=False) assert not serder.verify() pre = 'EF78YGUYCWXptoVVel1TN1F9-KShPHAtEqvf-TEiGvv9' @@ -1547,7 +1563,7 @@ def test_serderkeri_dip(): def test_serderkeri_drt(): """Test SerderKERI drt msg""" # Test KERI JSON with makify defaults for self bootstrap with ilk drt - serder = SerderKERI(makify=True, ilk=kering.Ilks.drt) # make with defaults + serder = SerderKERI(makify=True, ilk=kering.Ilks.drt, verify=False) # make with defaults assert serder.sad == {'v': 'KERI10JSON0000ac_', 't': 'drt', 'd': 'EMiEhgKRsD559TX6b03AT5P2GfKPPqoNk5COHZxU2TkR', @@ -1578,7 +1594,7 @@ def test_serderkeri_drt(): pre = 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' sad['i'] = pre - serder = SerderKERI(sad=sad, makify=True) + serder = SerderKERI(sad=sad, makify=True, verify=False) assert not serder.verify() # because pre is not digest and delpre is empty sad = serder.sad @@ -1671,7 +1687,7 @@ def test_serderkeri_rct(): """Test SerderKERI rct msg""" # Test KERI JSON with makify defaults for self bootstrap with ilk ixn - serder = SerderKERI(makify=True, ilk=kering.Ilks.rct) # make with defaults + serder = SerderKERI(makify=True, ilk=kering.Ilks.rct, verify=False) # make with defaults assert serder.sad == {'v': 'KERI10JSON000039_', 't': 'rct', 'd': '', 'i': '', 's': '0'} assert serder.raw == b'{"v":"KERI10JSON000039_","t":"rct","d":"","i":"","s":"0"}' @@ -2392,7 +2408,7 @@ def test_serderacdc(): with pytest.raises(ValueError): serder = SerderACDC() - serder = SerderACDC(makify=True, proto=kering.Protos.acdc) # make defaults for ACDC + serder = SerderACDC(makify=True, proto=kering.Protos.acdc, verify=False) # make defaults for ACDC assert serder.sad == {'v': 'ACDC10JSON00005a_', 'd': 'EMk7BvrqO_2sYjpI_-BmSELOFNie-muw4XTi3iYCz6pT', 'i': '', @@ -2456,12 +2472,101 @@ def test_serderacdc(): """End Test""" +def test_serder_v2(): + """ + Test Serder with version 2.00 of protocols + """ + + + assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_2_0][None].saids == {'d': 'E'} + assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_2_0][None].alls == + {'v': '', 'd': '', 'u': '', 'i': '', 'rd': '', 's': '', 'a': '', 'A': '', 'e': '', 'r': ''}) + assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_2_0][None].opts == + {'u': '', 'rd': '', 'a': '', 'A': '', 'e': '', 'r': ''}) + assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_2_0][None].alts == + {'a': 'A', 'A': 'a'}) + assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_2_0][None].strict + + + + with pytest.raises(ValueError): + serder = Serder(version=kering.Vrsn_2_0) + + #Test Serder bare makify bootstrap for ACDC JSON + serder = Serder(makify=True, + proto=kering.Protos.acdc, + vrsn=kering.Vrsn_2_0, + version=kering.Vrsn_2_0) # make defaults for ACDC + assert serder.sad == {'v': 'ACDCCAAJSONAABZ.', + 'd': 'EN-uBXL6rsJpJvDSsyOAnttQiI9gka4qLbe3MlIoYwYy', + 'i': '', + 's': ''} + assert serder.raw == (b'{"v":"ACDCCAAJSONAABZ.","d":"EN-uBXL6rsJpJvDSsyOAnttQiI9gka4qLbe3MlIoYwYy","' + b'i":"","s":""}') + assert serder.verify() + sad = serder.sad + raw = serder.raw + said = serder.said + size = serder.size + + serder = Serder(sad=sad, version=kering.Vrsn_2_0) + assert serder.raw == raw + assert isinstance(serder.raw, bytes) + assert serder.sad == sad + assert serder.proto == kering.Protos.acdc + assert serder.vrsn == kering.Vrsn_2_0 + assert serder.size == size + assert serder.kind == kering.Serials.json + assert serder.said == said + assert serder.saidb == said.encode("utf-8") + assert serder.ilk == None + assert serder.compare(said=said) + assert serder.compare(said=said.encode("utf-8")) + assert not serder.compare(said='EMk7BvrqO_2sYjpI_-BmSELOFNie-muw4XTi3iYCz6pE') + assert serder.pretty() == ('{\n' + ' "v": "ACDCCAAJSONAABZ.",\n' + ' "d": "EN-uBXL6rsJpJvDSsyOAnttQiI9gka4qLbe3MlIoYwYy",\n' + ' "i": "",\n' + ' "s": ""\n' + '}') + + serder = Serder(raw=raw, version=kering.Vrsn_2_0) + assert serder.raw == raw + assert isinstance(serder.raw, bytes) + assert serder.sad == sad + assert serder.proto == kering.Protos.acdc + assert serder.vrsn == kering.Vrsn_2_0 + assert serder.size == size + assert serder.kind == kering.Serials.json + assert serder.said == said + assert serder.saidb == said.encode("utf-8") + assert serder.ilk == None + assert serder.compare(said=said) + assert serder.compare(said=said.encode("utf-8")) + + serder = Serder(sad=sad, makify=True, version=kering.Vrsn_2_0) # test makify with sad + assert serder.raw == raw + assert isinstance(serder.raw, bytes) + assert serder.sad == sad + assert serder.proto == kering.Protos.acdc + assert serder.vrsn == kering.Vrsn_2_0 + assert serder.size == size + assert serder.kind == kering.Serials.json + assert serder.said == said + assert serder.saidb == said.encode("utf-8") + assert serder.ilk == None + assert serder.compare(said=said) + + + """End Test""" + + def test_serdery(): """Test Serdery""" #Create incoming message stream for Serdery to reap - serder = SerderKERI(makify=True, ilk=kering.Ilks.ixn) # make with defaults + serder = SerderKERI(makify=True, ilk=kering.Ilks.ixn, verify=False) # make with defaults sad = serder.sad pre = "EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf87Bc70J" sad['i'] = pre @@ -2472,7 +2577,7 @@ def test_serdery(): ims = bytearray(serderKeri.raw) - serder = SerderACDC(makify=True, proto=kering.Protos.acdc) # make defaults for ACDC + serder = SerderACDC(makify=True, proto=kering.Protos.acdc, verify=False) # make defaults for ACDC sad = serder.sad isr = 'EO8CE5RH1X8QJwHHhPkj_S6LJQDRNOiGohW327FMA6D2' sad['i'] = isr @@ -2514,7 +2619,7 @@ def test_serdery_noversion(): """Test Serdery unsupported version""" #Create incoming message stream for Serdery to reap - serder = SerderKERI(makify=True, ilk=kering.Ilks.ixn) # make with defaults + serder = SerderKERI(makify=True, ilk=kering.Ilks.ixn, verify=False) # make with defaults sad = serder.sad pre = "EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf87Bc70J" sad['i'] = pre @@ -2525,7 +2630,7 @@ def test_serdery_noversion(): ims = bytearray(serderKeri.raw) - serder = SerderACDC(makify=True, proto=kering.Protos.acdc) # make defaults for ACDC + serder = SerderACDC(makify=True, proto=kering.Protos.acdc, verify=False) # make defaults for ACDC sad = serder.sad isr = 'EO8CE5RH1X8QJwHHhPkj_S6LJQDRNOiGohW327FMA6D2' sad['i'] = isr @@ -2566,6 +2671,7 @@ def test_serdery_noversion(): if __name__ == "__main__": test_fielddom() + test_spans() test_serder() test_serderkeri() test_serderkeri_icp() @@ -2581,5 +2687,6 @@ def test_serdery_noversion(): test_serderkeri_exn() test_serderkeri_vcp() test_serderacdc() + test_serder_v2() test_serdery() test_serdery_noversion() diff --git a/tests/test_kering.py b/tests/test_kering.py index d4bf2a6ea..96b47de7e 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -19,6 +19,7 @@ versify, deversify, Rever) from keri.kering import (VER1FULLSPAN, VER1TERM, VEREX1, VER2FULLSPAN, VER2TERM, VEREX2, VEREX) + from keri.kering import VersionError, ProtocolError, KindError from keri.help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, codeB64ToB2, codeB2ToB64, Reb64, nabSextets) @@ -43,6 +44,8 @@ def test_protos(): """End Test""" + + def test_version_regex(): """ Test version string regexing @@ -480,9 +483,10 @@ def test_versify_v2(): with pytest.raises(KindError): smellage = deversify(vs) + """End Test""" + - """End Test""" def test_ilks(): From b3591e2cf9db3ff90d221cd1108afc8ff8bec27b Mon Sep 17 00:00:00 2001 From: luffa99 <43824784+luffa99@users.noreply.github.com> Date: Sun, 17 Mar 2024 23:30:37 +0100 Subject: [PATCH 045/418] Removed dead "kli agent start" command from README (#709) Co-authored-by: Lucas Falardi --- scripts/demo/README.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/demo/README.md b/scripts/demo/README.md index 65572bdf9..dc0a94080 100644 --- a/scripts/demo/README.md +++ b/scripts/demo/README.md @@ -50,16 +50,6 @@ following 2 subcommands are available for starting witnesses: For most of the scripts that require witnesses you will use `kli witness demo` to start the 3 known witnesses. -### Running Agents -Agents can be started in several ways using the `kli agent` subcommands or the shell script `demo/basic/start-agent.sh` for the -scripts that execute `curl` commands against running agents. The following 3 subcommands are available for starting -agents: - -* `kli agent start` - starts a single agent (used inside the start-agent.sh scripts) -* `kli agent demo` - starts 4 agents running on known ports (ports that are expected from many of the scripts) -* `kli agent vlei` - starts 10 agents running on known ports for the vLEI ecosystem scripts. - -Each directory details how to run agents for the scripts it contains and what parameters need to be passed. ### Additional Software Some scripts require addition services from other repositories. Those requirements are listed in the README.md files From 109d72bdea9546f5f06fbee58b667e8929e905b5 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 17 Mar 2024 16:22:48 -0700 Subject: [PATCH 046/418] fixed tests now that Makify allows verify fix bug --- tests/core/test_eventing.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index 69850c833..785b83fca 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -2246,7 +2246,7 @@ def test_kever(mockHelpingNowUTC): sad['n'] = nxt sad['bt'] = "{:x}".format(toad) - serder = serdering.SerderKERI(makify=True, verify=True, sad=sad) + serder = serdering.SerderKERI(makify=True, verify=False, sad=sad) assert serder.said == 'EFsuiA86Q5gGuVOO3tou8KSU6LORSExIUxzWNrlnW7WP' assert serder.pre == skp0.verfer.qb64 aid0 = serder.pre @@ -2272,7 +2272,8 @@ def test_kever(mockHelpingNowUTC): # make with defaults with non-transferable prefix serder = serdering.SerderKERI(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519N}) + saids = {'i': coring.PreDex.Ed25519N}, + verify=False) sad = serder.sad sad['i'] = skp0.verfer.qb64 # non-digestive aid @@ -2280,7 +2281,7 @@ def test_kever(mockHelpingNowUTC): sad['kt'] = "{:x}".format(sith) # hex string sad['k'] = keys sad['nt'] = 0 - sad['n'] = nxt + sad['n'] = nxt # empty nxt sad['bt'] = "{:x}".format(toad) serder = serdering.SerderKERI(makify=True, verify=True, sad=sad) @@ -2329,7 +2330,8 @@ def test_kever(mockHelpingNowUTC): # make with defaults with non-transferable prefix serder = serdering.SerderKERI(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519N}) + saids = {'i': coring.PreDex.Ed25519N}, + verify=False) sad = serder.sad sad['i'] = skp0.verfer.qb64 # non-digestive aid @@ -2341,7 +2343,7 @@ def test_kever(mockHelpingNowUTC): sad['bt'] = "{:x}".format(toad) sad['b'] = baks - serder = serdering.SerderKERI(makify=True, verify=True, sad=sad) + serder = serdering.SerderKERI(makify=True, verify=False, sad=sad) assert serder.said == 'EKcREpfNupJ8oOqdnqDIyJVr1-GgIMBrVOtBUR9Gm6lO' assert serder.pre == skp0.verfer.qb64 @@ -2364,7 +2366,7 @@ def test_kever(mockHelpingNowUTC): sad =serder.sad # makes copy sad['bt'] = "{:x}".format(toad) - serder = serdering.SerderKERI(makify=True, verify=True, sad=sad) + serder = serdering.SerderKERI(makify=True, verify=False, sad=sad) assert serder.said == 'EBKhptvqccp0KNBaS45bNPdTE4m19U1IvweHJW2PIEDI' assert serder.pre == skp0.verfer.qb64 @@ -2393,7 +2395,7 @@ def test_kever(mockHelpingNowUTC): sad['b'] = baks sad['a'] = a - serder = serdering.SerderKERI(makify=True, verify=True, sad=sad) + serder = serdering.SerderKERI(makify=True, verify=False, sad=sad) assert serder.said == 'EEu-cdj_9b_66XRJ5UuhgEvJxAPpn4RjyaHvRgDU3iyA' assert serder.pre == skp0.verfer.qb64 From 8521368763e353bcb6e94ac80f8f08e7591cc852 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 17 Mar 2024 16:37:42 -0700 Subject: [PATCH 047/418] minor refactor in how checks version compatibility with operating version --- src/keri/core/serdering.py | 9 ++++++--- src/keri/kering.py | 7 +++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index baf494cbc..71526a94f 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -744,9 +744,12 @@ def makify(self, sad, *, version=None, raise SerializeError(f"Expected protocol = {self.Protocol}, got " f"{proto} instead.") - if version is not None and vrsn != version: - raise SerializeError(f"Expected version = {version}, got " - f"{vrsn.major}.{vrsn.minor}.") + if version is not None: # compatible version with vrsn + if (vrsn.major > version.major or + (vrsn.major == version.major and vrsn.minor > version.minor)): + raise SerializeError(f"Incompatible {version=}, with " + f"{vrsn=}.") + if kind not in Serials: raise SerializeError(f"Invalid serialization kind = {kind}") diff --git a/src/keri/kering.py b/src/keri/kering.py index 5222c22d8..02a69b079 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -114,9 +114,12 @@ def rematch(match, *, version=None): vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) if vrsn.major > 1: # version1 vs but major > 1 raise VersionError(f"Incompatible {vrsn=} with version string.") - if version is not None and vrsn != version: - raise VersionError(f"Expected {version=}, got " + if version is not None: # compatible version with vrsn + if (vrsn.major > version.major or + (vrsn.major == version.major and vrsn.minor > version.minor)): + raise VersionError(f"Incompatible {version=}, with " f"{vrsn=}.") + kind = kind.decode("utf-8") if kind not in Serials: raise KindError(f"Invalid serialization kind = {kind}.") From a20d33de6da3114aaf6adcb2d2476d4f72ec20b2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 17 Mar 2024 17:12:38 -0700 Subject: [PATCH 048/418] makify now ensures correct ordering of all fields --- src/keri/core/serdering.py | 12 +++++++++--- tests/core/test_serdering.py | 9 +++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 71526a94f..97abce24b 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -780,9 +780,15 @@ def makify(self, sad, *, version=None, value = copy.copy(value) # copy iterable defaults sad[label] = value - # Need to insert required fields in proper place because passed in sad - # may have missing require fields that appear before provided ones so - # can't simply append + sadold = sad + sad = {} + for label in oalls: # make sure all fields are in correct order + if label in sadold: + sad[label] = sadold[label] + + for label in sadold: # copy extras if any + if label not in sad: + sad[label] = sadold[label] if 't' in sad: # when packet type field then force ilk sad['t'] = ilk # assign ilk diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 03678f83b..06a820cc3 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -606,13 +606,10 @@ def test_serder(): sad = serder.sad sad["ri"] = "" - with pytest.raises(kering.SerializeError): - serder = Serder(makify=True, sad=sad) # make using sad - + serder = Serder(makify=True, sad=sad) # make using sad fixes order # extra field with strict sad = serder.sad - assert 'ri' not in sad sad["x"] = "" with pytest.raises(kering.SerializeError): @@ -659,8 +656,7 @@ def test_serder(): # out of order with extra sad = serder.sad sad["ri"] = "" - with pytest.raises(kering.SerializeError): - serder = Serder(makify=True, sad=sad) # make using sad + serder = Serder(makify=True, sad=sad) # makify fixes order with extra # ToDo: create malicious raw values to test verify more thoroughly # ToDo: create bad sad values to test makify more thoroughly @@ -1313,6 +1309,7 @@ def test_serderkeri_dip(): assert serder.verify() raw = serder.raw + sad = serder.sad said = serder.said size = serder.size ilk = serder.ilk From 9b19e15437ccc68a6b47ab937850aa6e14c17680 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 17 Mar 2024 17:33:18 -0700 Subject: [PATCH 049/418] added .verify attribute to SuberBase so that subclasses can init to reverifiy whan deserializatin default is False. --- src/keri/db/subing.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 336a81fe7..951a70c37 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -59,6 +59,8 @@ class SuberBase(): db (dbing.LMDBer): base LMDB db sdb (lmdb._Database): instance of lmdb named sub db for this Suber sep (str): separator for combining keys tuple of strs into key bytes + verify (bool): some subclasses want to re-verify when deser from db + default false """ Sep = '.' # separator for combining key iterables @@ -66,6 +68,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str='docs.', dupsort: bool=False, sep: str=None, + verify: bool=False, **kwa): """ Parameters: @@ -77,10 +80,12 @@ def __init__(self, db: dbing.LMDBer, *, sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' """ - super(SuberBase, self).__init__() + super(SuberBase, self).__init__() # for multi inheritance self.db = db self.sdb = self.db.env.open_db(key=subkey.encode("utf-8"), dupsort=dupsort) self.sep = sep if sep is not None else self.Sep + self.verify = True if verify else False + def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], @@ -1113,7 +1118,7 @@ def get(self, keys: Union[str, Iterable]): """ val = self.db.getVal(db=self.sdb, key=self._tokey(keys)) - return self.klas(raw=bytes(val)) if val is not None else None + return self.klas(raw=bytes(val), verify=self.verify) if val is not None else None def rem(self, keys: Union[str, Iterable]): @@ -1145,7 +1150,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): """ for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield self._tokeys(iokey), self.klas(raw=bytes(val)) + yield self._tokeys(iokey), self.klas(raw=bytes(val), verify=self.verify) class SchemerSuber(Suber): From b8c9633b5c7a73b1e0394edc95427ee7138e626b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 18 Mar 2024 10:01:38 -0600 Subject: [PATCH 050/418] added test for SerderKERI version 2 --- tests/core/test_serdering.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 06a820cc3..0656abb83 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -2554,6 +2554,34 @@ def test_serder_v2(): assert serder.ilk == None assert serder.compare(said=said) + # test default + serder = Serder(makify=True, + vrsn=kering.Vrsn_2_0, + version=kering.Vrsn_2_0) # make defaults for default proto + + + assert serder.sad == {'v': 'KERICAAJSONAADO.', + 't': 'icp', + 'd': 'EGqrN042jSUT5bjUuQqGALW4inJMJA6BBlVKf21VH3bn', + 'i': 'EGqrN042jSUT5bjUuQqGALW4inJMJA6BBlVKf21VH3bn', + 's': '0', + 'kt': '0', + 'k': [], + 'nt': '0', + 'n': [], + 'bt': '0', + 'b': [], + 'c': [], + 'a': []} + assert serder.raw == (b'{"v":"KERICAAJSONAADO.","t":"icp","d":"EGqrN042jSUT5bjUuQqGALW4inJMJA6BBlVKf' + b'21VH3bn","i":"EGqrN042jSUT5bjUuQqGALW4inJMJA6BBlVKf21VH3bn","s":"0","kt":"0"' + b',"k":[],"nt":"0","n":[],"bt":"0","b":[],"c":[],"a":[]}') + assert serder.verify() + assert serder.proto == kering.Protos.keri == Serder.Proto # default + assert serder.vrsn == kering.Vrsn_2_0 + assert serder.kind == kering.Serials.json == Serder.Kind # default + assert serder.ilk == kering.Ilks.icp # default first one + """End Test""" From 5116f46707b53b26f1817a824567ee88ab826ed6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 18 Mar 2024 10:11:38 -0600 Subject: [PATCH 051/418] added CESR as serialization kind --- src/keri/kering.py | 4 ++-- tests/test_kering.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/keri/kering.py b/src/keri/kering.py index 02a69b079..50e9a35ec 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -15,8 +15,8 @@ # Serialization Kinds -Serialage = namedtuple("Serialage", 'json mgpk cbor') -Serials = Serialage(json='JSON', mgpk='MGPK', cbor='CBOR') +Serialage = namedtuple("Serialage", 'json mgpk cbor cesr') +Serials = Serialage(json='JSON', mgpk='MGPK', cbor='CBOR', cesr='CESR') # Protocol Types Protocolage = namedtuple("Protocolage", "keri acdc") diff --git a/tests/test_kering.py b/tests/test_kering.py index 96b47de7e..f46a43c54 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -173,9 +173,6 @@ def test_serials(): """ Test Serializations namedtuple instance Serials """ - Vstrings = Serialage(json=versify(kind=Serials.json, size=0), - mgpk=versify(kind=Serials.mgpk, size=0), - cbor=versify(kind=Serials.cbor, size=0)) assert Version == Versionage(major=1, minor=0) @@ -184,14 +181,23 @@ def test_serials(): assert Serials.json == 'JSON' assert Serials.mgpk == 'MGPK' assert Serials.cbor == 'CBOR' + assert Serials.cesr == 'CESR' assert 'JSON' in Serials assert 'MGPK' in Serials assert 'CBOR' in Serials + assert 'CESR' in Serials + + Vstrings = Serialage(json=versify(kind=Serials.json, size=0), + mgpk=versify(kind=Serials.mgpk, size=0), + cbor=versify(kind=Serials.cbor, size=0), + cesr=versify(kind=Serials.cesr, size=0)) + assert Vstrings.json == 'KERI10JSON000000_' assert Vstrings.mgpk == 'KERI10MGPK000000_' assert Vstrings.cbor == 'KERI10CBOR000000_' + assert Vstrings.cesr == 'KERI10CESR000000_' icp = dict(vs=Vstrings.json, pre='AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM', From 6fba64050c0fb5b2a5692d9b9fc3d9b9a270fdc8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 18 Mar 2024 15:30:54 -0600 Subject: [PATCH 052/418] Added new core.counting module for new versionable Counter class --- src/keri/core/coring.py | 58 +--- src/keri/core/counting.py | 539 ++++++++++++++++++++++++++++++++++++ src/keri/core/serdering.py | 301 ++++++++++---------- tests/core/test_coring.py | 16 +- tests/core/test_counting.py | 165 +++++++++++ 5 files changed, 876 insertions(+), 203 deletions(-) create mode 100644 src/keri/core/counting.py create mode 100644 tests/core/test_counting.py diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index ce4a115dc..01d332bbc 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -4695,8 +4695,8 @@ def __iter__(self): @dataclass(frozen=True) -class ProtocolGenusCodex: - """ProtocolGenusCodex is codex of protocol genera for code table. +class GenusCodex: + """GenusCodex is codex of protocol genera for code table. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. @@ -4710,45 +4710,7 @@ def __iter__(self): # duplicate values above just result in multiple entries in tuple so # in inclusion still works -ProDex = ProtocolGenusCodex() # Make instance - - -@dataclass(frozen=True) -class AltCounterCodex: - """ - CounterCodex is codex hard (stable) part of all counter derivation codes. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - - ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. - WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. - NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. - TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. - FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. - TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. - SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegators or issuers event - TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. - SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event - SadPathSig: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content - SadPathSigGroup: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples - PathedMaterialQuadlets: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) - MessageDataGroups: str = '-U' # Composed Message Data Group or Primitive - AttachedMaterialQuadlets: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) - MessageDataMaterialQuadlets: str = '-W' # Composed Grouped Message Data Quadlet (4 char each) - CombinedMaterialQuadlets: str = '-X' # Combined Message Data + Attachments Quadlet (4 char each) - MaterialGroups: str = '-Y' # Composed Generic Material Group or Primitive - MaterialQuadlets: str = '-Z' # Composed Generic Material Quadlet (4 char each) - BigMessageDataGroups: str = '-0U' # Composed Message Data Group or Primitive - BigAttachedMaterialQuadlets: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) - BigMessageDataMaterialQuadlets: str = '-0W' # Composed Grouped Message Data Quadlet (4 char each) - BigCombinedMaterialQuadlets: str = '-0X' # Combined Message Data + Attachments Quadlet (4 char each) - BigMaterialGroups: str = '-0Y' # Composed Generic Material Group or Primitive - BigMaterialQuadlets: str = '-0Z' # Composed Generic Material Quadlet (4 char each) - - - def __iter__(self): - return iter(astuple(self)) # enables inclusion test with "in" +GenDex = GenusCodex() # Make instance class Counter: @@ -4760,6 +4722,8 @@ class Counter: Includes the following attributes and properties: + Class Attributes: + Attributes: Properties: @@ -4780,7 +4744,6 @@ class Counter: ._exfil is method to extract .code and .raw from fully qualified Base64 """ - Codex = CtrDex # Hards table maps from bytes Base64 first two code chars to int of # hard size, hs,(stable) of code. The soft size, ss, (unstable) for Counter # is always > 0 and hs + ss = fs always @@ -4788,6 +4751,11 @@ class Counter: Hards.update({('-' + chr(c)): 2 for c in range(97, 97 + 26)}) Hards.update([('-0', 3)]) Hards.update([('--', 5)]) + + # Bards table maps to hard size, hs, of code from bytes holding sextets + # converted from first two code char. Used for ._bexfil. + Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) + # Sizes table maps hs chars of code to Sizage namedtuple of (hs, ss, fs) # where hs is hard size, ss is soft size, and fs is full size # soft size, ss, should always be > 0 and hs+ss=fs for Counter @@ -4808,9 +4776,9 @@ class Counter: '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), } - # Bards table maps to hard size, hs, of code from bytes holding sextets - # converted from first two code char. Used for ._bexfil. - Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) + + Codex = CtrDex + def __init__(self, code=None, count=None, countB64=None, qb64b=None, qb64=None, qb2=None, strip=False): diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py new file mode 100644 index 000000000..9417a7d81 --- /dev/null +++ b/src/keri/core/counting.py @@ -0,0 +1,539 @@ +# -*- encoding: utf-8 -*- +""" +keri.core.counting module + +Provides versioning support for Counter classes and codes +""" + +from dataclasses import dataclass, astuple + + +from ..help import helping +from ..help.helping import sceil +from ..help.helping import (intToB64, b64ToInt, codeB64ToB2, codeB2ToB64, + nabSextets) + +from .. import kering + +from ..core.coring import Sizage + + +@dataclass +class MapDom: + """Base class for dataclasses that support map syntax + Adds support for dunder methods for map syntax dc[name]. + Converts exceptions from attribute syntax to raise map syntax when using + map syntax. + """ + + def __getitem__(self, name): + try: + return getattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + def __setitem__(self, name, value): + try: + return setattr(self, name, value) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + def __delitem__(self, name): + try: + return delattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + +@dataclass(frozen=True) +class MapCodex: + """Base class for frozen dataclasses (codexes) that support map syntax + Adds support for dunder methods for map syntax dc[name]. + Converts exceptions from attribute syntax to raise map syntax when using + map syntax. + """ + + def __getitem__(self, name): + try: + return getattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + def __setitem__(self, name, value): + try: + return setattr(self, name, value) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + def __delitem__(self, name): + try: + return delattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + +@dataclass(frozen=True) +class CounterCodex: + """ + CounterCodex is codex hard (stable) part of all counter derivation codes. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + + ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. + WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. + NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. + TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. + FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. + TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. + SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegator/issuer/transaction event + TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. + SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event + SadPathSig: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content + SadPathSigGroup: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples + PathedMaterialQuadlets: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) + AttachedMaterialQuadlets: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) + BigAttachedMaterialQuadlets: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) + KERIProtocolStack: str = '--AAA' # KERI ACDC Protocol Stack CESR Version + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +CtrDex = CounterCodex() + + +@dataclass(frozen=True) +class GenusCodex: + """GenusCodex is codex of protocol genera for code table. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + KERI: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables + ACDC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables + + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + # duplicate values above just result in multiple entries in tuple so + # in inclusion still works + +GenDex = GenusCodex() # Make instance + + +""" +Design Notes + +Need sizes by version can assume only support KERI/ACDC Genus so do not +need to support different protocol genera in Counting +Hards and Bards are the same for both versions + +Need to pass in version so Counter Instance knows what version to use +simpler than Serder since version is not provided in version string +Each instance when created needs to get its version at init + +So Sizes in dictionary indexed by version. + +CtrDex Codex itself is not referenced inside Counter but by external classes +making instances so use CtrDex to pass in code. So we need versioned codex + +one option is to have different dataclasses with each version codex and then +have dictionary Codex of those indexed by version as class variable. + +Another option is to have code name index that is indexed by version +so the actual code is looked up for the version instead of passing in the code +itself which requires dereferencing with the version so the version gets used +twice. Also need to reverse code and version to get codename as property + +codename = label or tag. use tag for code tag + + +Counter(codename=, verion=) + +When using code not tag then must check against version to make sure code is +a valid code for version. Likewise a tag may not have a code for a given version + +So tags are strings that can be attribute names for dataclasses +Tags is namedtuple where each attribute value is its key so one can look up the +string by the tag + +But given string value for tag then to lookup code in codex need to to use +builtin getattr + +Instead maybe we just extend the codex dataclass with .__getitem__ .__setitem__ and .__delitem__ methods +so we can access a code by its tag using Codex[tag] + + +""" + +class Counter: + """ + Counter is fully qualified cryptographic material primitive base class for + counter primitives (framing composition grouping count codes). + + Sub classes are derivation code and key event element context specific. + + Includes the following attributes and properties: + + Attributes: + + Properties: + .code is str derivation code to indicate cypher suite + .raw is bytes crypto material only without code + .pad is int number of pad chars given raw + .count is int count of grouped following material (not part of counter) + .qb64 is str in Base64 fully qualified with derivation code + crypto mat + .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat + .qb2 is bytes in binary with derivation code + crypto material + + Hidden: + ._code is str value for .code property + ._raw is bytes value for .raw property + ._pad is method to compute .pad property + ._count is int value for .count property + ._infil is method to compute fully qualified Base64 from .raw and .code + ._exfil is method to extract .code and .raw from fully qualified Base64 + + """ + # Hards table maps from bytes Base64 first two code chars to int of + # hard size, hs,(stable) of code. The soft size, ss, (unstable) for Counter + # is always > 0 and hs + ss = fs always + Hards = ({('-' + chr(c)): 2 for c in range(65, 65 + 26)}) + Hards.update({('-' + chr(c)): 2 for c in range(97, 97 + 26)}) + Hards.update([('-0', 3)]) + Hards.update([('--', 5)]) + + # Bards table maps to hard size, hs, of code from bytes holding sextets + # converted from first two code char. Used for ._bexfil. + Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) + + # Sizes table maps hs chars of code to Sizage namedtuple of (hs, ss, fs) + # where hs is hard size, ss is soft size, and fs is full size + # soft size, ss, should always be > 0 and hs+ss=fs for Counter + Sizes = \ + { + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + } + + Codex = CtrDex + + + def __init__(self, code=None, count=None, countB64=None, + qb64b=None, qb64=None, qb2=None, strip=False): + """ + Validate as fully qualified + Parameters: + code (str | None): stable (hard) part of derivation code + count (int | None): count for composition. + Count may represent quadlets/triplet, groups, primitives or + other numericy + When both count and countB64 are None then count defaults to 1 + countB64 (str | None): count for composition as Base64 + countB64 may represent quadlets/triplet, groups, primitives or + other numericy + qb64b (bytes | bytearray | None): fully qualified crypto material text domain + qb64 (str | None) fully qualified crypto material text domain + qb2 (bytes | bytearray | None) fully qualified crypto material binary domain + strip (bool): True means strip counter contents from input stream + bytearray after parsing qb64b or qb2. False means do not strip. + default False + + + Needs either code or qb64b or qb64 or qb2 + Otherwise raises EmptyMaterialError + When code and count provided then validate that code and count are correct + Else when qb64b or qb64 or qb2 provided extract and assign + .code and .count + + """ + if code is not None: # code provided + if code not in self.Sizes: + raise kering.InvalidCodeError("Unsupported code={}.".format(code)) + + hs, ss, fs, ls = self.Sizes[code] # get sizes for code + cs = hs + ss # both hard + soft code size + if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes + raise kering.InvalidCodeSizeError("Whole code size not full size or not " + "multiple of 4. cs={} fs={}.".format(cs, fs)) + + if count is None: + count = 1 if countB64 is None else b64ToInt(countB64) + + if count < 0 or count > (64 ** ss - 1): + raise kering.InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) + + self._code = code + self._count = count + + elif qb64b is not None: + self._exfil(qb64b) + if strip: # assumes bytearray + del qb64b[:self.Sizes[self.code].fs] + + elif qb64 is not None: + self._exfil(qb64) + + elif qb2 is not None: # rewrite to use direct binary exfiltration + self._bexfil(qb2) + if strip: # assumes bytearray + del qb2[:self.Sizes[self.code].fs * 3 // 4] + + else: + raise kering.EmptyMaterialError("Improper initialization need either " + "(code and count) or qb64b or " + "qb64 or qb2.") + + @property + def code(self): + """ + Returns ._code + Makes .code read only + """ + return self._code + + + @property + def count(self): + """ + Returns ._count + Makes ._count read only + """ + return self._count + + + @property + def qb64b(self): + """ + Property qb64b: + Returns Fully Qualified Base64 Version encoded as bytes + Assumes self.raw and self.code are correctly populated + """ + return self._infil() + + + @property + def qb64(self): + """ + Property qb64: + Returns Fully Qualified Base64 Version + Assumes self.raw and self.code are correctly populated + """ + return self.qb64b.decode("utf-8") + + + @property + def qb2(self): + """ + Property qb2: + Returns Fully Qualified Binary Version Bytes + """ + return self._binfil() + + + def countToB64(self, l=None): + """ Returns count as Base64 left padded with "A"s + Parameters: + l (int | None): minimum number characters including left padding + When not provided use the softsize of .code + + """ + if l is None: + _, ss, _, _ = self.Sizes[self.code] + l = ss + return (intToB64(self.count, l=l)) + + + @staticmethod + def semVerToB64(version="", major=0, minor=0, patch=0): + """ Converts semantic version to Base64 representation of countB64 + suitable for CESR protocol genus and version + + Returns: + countB64 (str): suitable for input to Counter + example: Counter(countB64=semVerToB64(version = "1.0.0")) + + Parameters: + version (str | None): dot separated semantic version string of format + "major.minor.patch" + major (int): When version is None or empty then use major,minor, patch + minor (int): When version is None or empty then use major,minor, patch + patch (int): When version is None or empty then use major,minor, patch + + each of major, minor, patch must be in range [0,63] for represenation as + three Base64 characters + + """ + parts = [major, minor, patch] + if version: + splits = version.split(".", maxsplit=3) + splits = [(int(s) if s else 0) for s in splits] + for i in range(3-len(splits),0, -1): + splits.append(parts[-i]) + parts = splits + + for p in parts: + if p < 0 or p > 63: + raise ValueError(f"Out of bounds semantic version. " + f"Part={p} is < 0 or > 63.") + return ("".join(intToB64(p, l=1) for p in parts)) + + + def _infil(self): + """ + Returns fully qualified attached sig base64 bytes computed from + self.code and self.count. + """ + code = self.code # codex value chars hard code + count = self.count # index value int used for soft + + hs, ss, fs, ls = self.Sizes[code] + cs = hs + ss # both hard + soft size + if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes + raise kering.InvalidCodeSizeError("Whole code size not full size or not " + "multiple of 4. cs={} fs={}.".format(cs, fs)) + if count < 0 or count > (64 ** ss - 1): + raise kering.InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) + + # both is hard code + converted count + both = "{}{}".format(code, intToB64(count, l=ss)) + + # check valid pad size for whole code size + if len(both) % 4: # no pad + raise kering.InvalidCodeSizeError("Invalid size = {} of {} not a multiple of 4." + .format(len(both), both)) + # prepending full derivation code with index and strip off trailing pad characters + return (both.encode("utf-8")) + + + def _binfil(self): + """ + Returns bytes of fully qualified base2 bytes, that is .qb2 + self.code converted to Base2 left shifted with pad bits + equivalent of Base64 decode of .qb64 into .qb2 + """ + code = self.code # codex chars hard code + count = self.count # index value int used for soft + + hs, ss, fs, ls = self.Sizes[code] + cs = hs + ss + if fs != cs or cs % 4: # fs must be cs and multiple of 4 for count codes + raise kering.InvalidCodeSizeError("Whole code size not full size or not " + "multiple of 4. cs={} fs={}.".format(cs, fs)) + + if count < 0 or count > (64 ** ss - 1): + raise kering.InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) + + # both is hard code + converted count + both = "{}{}".format(code, intToB64(count, l=ss)) + if len(both) != cs: + raise kering.InvalidCodeSizeError("Mismatch code size = {} with table = {}." + .format(cs, len(both))) + + return (codeB64ToB2(both)) # convert to b2 left shift if any + + + def _exfil(self, qb64b): + """ + Extracts self.code and self.count from qualified base64 bytes qb64b + """ + if not qb64b: # empty need more bytes + raise ShortageError("Empty material, Need more characters.") + + first = qb64b[:2] # extract first two char code selector + if hasattr(first, "decode"): + first = first.decode("utf-8") + if first not in self.Hards: + if first[0] == '_': + raise kering.UnexpectedOpCodeError("Unexpected op code start" + "while extracing Counter.") + else: + raise kering.UnexpectedCodeError("Unsupported code start ={}.".format(first)) + + hs = self.Hards[first] # get hard code size + if len(qb64b) < hs: # need more bytes + raise kering.ShortageError("Need {} more characters.".format(hs - len(qb64b))) + + hard = qb64b[:hs] # get hard code + if hasattr(hard, "decode"): + hard = hard.decode("utf-8") # decode converts bytearray/bytes to str + if hard not in self.Sizes: # Sizes needs str not bytes + raise kering.UnexpectedCodeError("Unsupported code ={}.".format(hard)) + + hs, ss, fs, ls = self.Sizes[hard] # assumes hs consistent in both tables + cs = hs + ss # both hard + soft code size + + # assumes that unit tests on Counter and CounterCodex ensure that + # .Codes and .Sizes are well formed. + # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 + + if len(qb64b) < cs: # need more bytes + raise kering.ShortageError("Need {} more characters.".format(cs - len(qb64b))) + + count = qb64b[hs:hs + ss] # extract count chars + if hasattr(count, "decode"): + count = count.decode("utf-8") + count = b64ToInt(count) # compute int count + + self._code = hard + self._count = count + + + def _bexfil(self, qb2): + """ + Extracts self.code and self.count from qualified base2 bytes qb2 + """ + if not qb2: # empty need more bytes + raise kering.ShortageError("Empty material, Need more bytes.") + + first = nabSextets(qb2, 2) # extract first two sextets as code selector + if first not in self.Bards: + if first[0] == b'\xfc': # b64ToB2('_') + raise kering.UnexpectedOpCodeError("Unexpected op code start" + "while extracing Matter.") + else: + raise kering.UnexpectedCodeError("Unsupported code start sextet={}.".format(first)) + + hs = self.Bards[first] # get code hard size equvalent sextets + bhs = sceil(hs * 3 / 4) # bhs is min bytes to hold hs sextets + if len(qb2) < bhs: # need more bytes + raise kering.ShortageError("Need {} more bytes.".format(bhs - len(qb2))) + + hard = codeB2ToB64(qb2, hs) # extract and convert hard part of code + if hard not in self.Sizes: + raise kering.UnexpectedCodeError("Unsupported code ={}.".format(hard)) + + hs, ss, fs, ls = self.Sizes[hard] + cs = hs + ss # both hs and ss + # assumes that unit tests on Counter and CounterCodex ensure that + # .Codes and .Sizes are well formed. + # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 + + bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets + if len(qb2) < bcs: # need more bytes + raise kering.ShortageError("Need {} more bytes.".format(bcs - len(qb2))) + + both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code + count = b64ToInt(both[hs:hs + ss]) # get count + + self._code = hard + self._count = count + diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 97abce24b..c7d7d997c 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -276,163 +276,164 @@ class Serder: # Each protocol value is a dict keyed by ilk. # Each ilk value is a Labelage named tuple with saids, codes and fields # ilk value of None is default for protocols that support ilkless packets - Fields = { - Protos.keri: + Fields = \ + { + Protos.keri: + { + Vrsn_1_0: { - Vrsn_1_0: - { - Ilks.icp: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - kt='0',k=[], nt='0', n=[], bt='0', b=[], c=[], - a=[]), - saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}), - Ilks.rot: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], a=[]), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.ixn: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - p='', a=[]), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.dip: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], - a=[], di=''), - saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}), - Ilks.drt: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + Ilks.icp: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0',k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[]), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.rot: FieldDom(alls=dict(v='', t='',d='', i='', s='0', p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], ba=[], a=[]), saids={Saids.d: DigDex.Blake3_256}), - Ilks.rct: FieldDom(alls=dict(v='', t='',d='', i='', s='0')), - Ilks.qry: FieldDom(alls=dict(v='', t='',d='', dt='', r='', - rr='',q={}), - saids={Saids.d: DigDex.Blake3_256},), - Ilks.rpy: FieldDom(alls=dict(v='', t='',d='', dt='', r='', - a=[]), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.pro: FieldDom(alls=dict(v='', t='',d='', dt='', r='', - rr='',q={}), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.bar: FieldDom(alls=dict(v='', t='',d='', dt='', r='', - a=[]), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", p="", - dt='', r='',q={}, a=[], e={}), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.vcp: FieldDom(alls=dict(v='', t='',d='', i='', ii='', - s='0', c=[], bt='0', b=[], n=''), - saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}), - Ilks.vrt: FieldDom(alls=dict(v='', t='',d='', i='', p='', - s='0', bt='0', br=[], ba=[]), - saids={Saids.d: DigDex.Blake3_256,}), - Ilks.iss: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - ri='', dt=''), - saids={Saids.d: DigDex.Blake3_256,}), - Ilks.rev: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - ri='', p='', dt=''), - saids={Saids.d: DigDex.Blake3_256,}), - Ilks.bis: FieldDom(alls=dict(v='', t='',d='', i='', ii='', - s='0', ra={}, dt=''), - saids={Saids.d: DigDex.Blake3_256,}), - Ilks.brv: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - p='', ra={}, dt=''), - saids={Saids.d: DigDex.Blake3_256,}), - }, - Vrsn_2_0: - { - Ilks.icp: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], - a=[]), - saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}), - Ilks.rot: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], c=[], a=[]), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.ixn: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - p='', a=[]), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.dip: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], - a=[], di=''), - saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}), - Ilks.drt: FieldDom(alls=dict(v='', t='',d='', i='', s='0', - p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], c=[], a=[]), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.rct: FieldDom(alls=dict(v='', t='',d='', i='', s='0')), - Ilks.qry: FieldDom(alls=dict(v='', t='',d='', i='', dt='', - r='', rr='', q={}), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.rpy: FieldDom(alls=dict(v='', t='',d='', i='', dt='', - r='',a={}), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.pro: FieldDom(alls=dict(v='', t='',d='', i='', dt='', - r='', rr='', q={}), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.bar: FieldDom(alls=dict(v='', t='',d='', i='', dt='', - r='',a={}), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.xip: FieldDom(alls=dict(v='', t='', d='', i="", dt='', - r='', q={}, a={}), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", x="", - p="", dt='', r='', q={}, a={}), - saids={Saids.d: DigDex.Blake3_256}), - }, + Ilks.ixn: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.dip: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[], di=''), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.drt: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rct: FieldDom(alls=dict(v='', t='',d='', i='', s='0')), + Ilks.qry: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + rr='',q={}), + saids={Saids.d: DigDex.Blake3_256},), + Ilks.rpy: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.pro: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + rr='',q={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.bar: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", p="", + dt='', r='',q={}, a=[], e={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.vcp: FieldDom(alls=dict(v='', t='',d='', i='', ii='', + s='0', c=[], bt='0', b=[], n=''), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.vrt: FieldDom(alls=dict(v='', t='',d='', i='', p='', + s='0', bt='0', br=[], ba=[]), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.iss: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + ri='', dt=''), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.rev: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + ri='', p='', dt=''), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.bis: FieldDom(alls=dict(v='', t='',d='', i='', ii='', + s='0', ra={}, dt=''), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.brv: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', ra={}, dt=''), + saids={Saids.d: DigDex.Blake3_256,}), }, - Protos.acdc: + Vrsn_2_0: { - Vrsn_1_0: - { - None: FieldDom(alls=dict(v='', d='', u='', i='', - ri='', s='', a='', A='', e='', r=''), - opts=dict(u='', ri='', a='', A='', e='', r=''), - alts=dict(a="A", A="a"), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.ace: FieldDom(alls=dict(v='', t='', d='', u='', i='', - ri='', s='', a='', A='', e='', r=''), - opts=dict(u='', ri='', a='', A='', e='', r=''), - alts=dict(a="A", A="a"), - saids={Saids.d: DigDex.Blake3_256}, - strict=False), - }, - Vrsn_2_0: - { - None: FieldDom(alls=dict(v='', d='', u='', i='', - rd='', s='', a='', A='', e='', r=''), - opts=dict(u='', rd='', a='', A='', e='', r=''), - alts=dict(a="A", A="a"), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.acd: FieldDom(alls=dict(v='', t='', d='', u='', i='', - rd='', s='', a='', A='', e='', r=''), - opts=dict(u='', rd='', a='', A='', e='', r=''), - alts=dict(a="A", A="a"), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.ace: FieldDom(alls=dict(v='', t='', d='', u='', i='', - rd='', s='', a='', A='', e='', r=''), - opts=dict(u='', rd='', a='', A='', e='', r=''), - alts=dict(a="A", A="a"), - saids={Saids.d: DigDex.Blake3_256}, - strict=False), - Ilks.sch: FieldDom(alls=dict(v='', t='', d='', s=''), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.att: FieldDom(alls=dict(v='', t='', d='', a=''), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.agg: FieldDom(alls=dict(v='', t='', d='', A=''), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.rul: FieldDom(alls=dict(v='', t='', d='', e=''), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.rip: FieldDom(alls=dict(v='', t='', d='', u='', i='', - s='', dt=''), - saids={Saids.d: DigDex.Blake3_256}), - Ilks.upd: FieldDom(alls=dict(v='', t='', d='', r='', s='', - p='', dt='', a=''), - saids={Saids.d: DigDex.Blake3_256}), - }, + Ilks.icp: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[]), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.rot: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], c=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.ixn: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.dip: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[], di=''), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.drt: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], c=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rct: FieldDom(alls=dict(v='', t='',d='', i='', s='0')), + Ilks.qry: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='', rr='', q={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rpy: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='',a={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.pro: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='', rr='', q={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.bar: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='',a={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.xip: FieldDom(alls=dict(v='', t='', d='', i="", dt='', + r='', q={}, a={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", x="", + p="", dt='', r='', q={}, a={}), + saids={Saids.d: DigDex.Blake3_256}), + }, + }, + Protos.acdc: + { + Vrsn_1_0: + { + None: FieldDom(alls=dict(v='', d='', u='', i='', + ri='', s='', a='', A='', e='', r=''), + opts=dict(u='', ri='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.ace: FieldDom(alls=dict(v='', t='', d='', u='', i='', + ri='', s='', a='', A='', e='', r=''), + opts=dict(u='', ri='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}, + strict=False), }, - } + Vrsn_2_0: + { + None: FieldDom(alls=dict(v='', d='', u='', i='', + rd='', s='', a='', A='', e='', r=''), + opts=dict(u='', rd='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.acd: FieldDom(alls=dict(v='', t='', d='', u='', i='', + rd='', s='', a='', A='', e='', r=''), + opts=dict(u='', rd='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.ace: FieldDom(alls=dict(v='', t='', d='', u='', i='', + rd='', s='', a='', A='', e='', r=''), + opts=dict(u='', rd='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}, + strict=False), + Ilks.sch: FieldDom(alls=dict(v='', t='', d='', s=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.att: FieldDom(alls=dict(v='', t='', d='', a=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.agg: FieldDom(alls=dict(v='', t='', d='', A=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rul: FieldDom(alls=dict(v='', t='', d='', e=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rip: FieldDom(alls=dict(v='', t='', d='', u='', i='', + s='', dt=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.upd: FieldDom(alls=dict(v='', t='', d='', r='', s='', + p='', dt='', a=''), + saids={Saids.d: DigDex.Blake3_256}), + }, + }, + } def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index b2f3b9834..41d089606 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -31,7 +31,7 @@ from keri.core.coring import Serialage, Serials, Tiers from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, Indexer, - CtrDex, Counter, ProDex) + CtrDex, Counter, GenDex) from keri.core.coring import (Verfer, Cigar, Signer, Salter, Saider, DigDex, Diger, Prefixer, Cipher, Encrypter, Decrypter) from keri.core.coring import versify, deversify, Rever, MAXVERFULLSPAN @@ -46,21 +46,21 @@ from keri.kering import Version, Versionage, VersionError -def test_protocol_genus_codex(): +def test_genus_codex(): """ - Test genera in ProDex as instance of ProtocolGenusCodex + Test protocol genera in GenDex as instance of GenusCodex """ - assert dataclasses.asdict(ProDex) == { + assert dataclasses.asdict(GenDex) == { 'KERI': '--AAA', # KERI and ACDC Protocol Stacks share the same tables 'ACDC': '--AAA', } - assert '--AAA' in ProDex - assert ProDex.KERI == "--AAA" - assert ProDex.ACDC == "--AAA" - assert ProDex.KERI == ProDex.ACDC + assert '--AAA' in GenDex + assert GenDex.KERI == "--AAA" + assert GenDex.ACDC == "--AAA" + assert GenDex.KERI == GenDex.ACDC """End Test""" diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py new file mode 100644 index 000000000..51f709595 --- /dev/null +++ b/tests/core/test_counting.py @@ -0,0 +1,165 @@ +# -*- encoding: utf-8 -*- +""" +tests.core.test_counting module + +""" +from dataclasses import dataclass, astuple, asdict +from ordered_set import OrderedSet as oset + +import pytest + + +from keri import kering + +from keri.core import counting +from keri.core.counting import MapDom, MapCodex, Counter + + +def test_mapdom(): + """Test MapDom base dataclass""" + + @dataclass + class TestMapDom(MapDom): + """ + + """ + xray: str = 'X' + yankee: str = 'Y' + zulu: str = 'Z' + + def __iter__(self): # so value in dataclass not key in dataclass + return iter(astuple(self)) + + tmd = TestMapDom() + + assert 'X' in tmd + assert 'Y' in tmd + assert 'Z' in tmd + + assert tmd["xray"] == tmd.xray == 'X' + assert tmd["yankee"] == tmd.yankee == 'Y' + assert tmd["zulu"] == tmd.zulu == 'Z' + + + tmd["xray"] = "x" + assert tmd.xray == tmd["xray"] == "x" + + tmd["yankee"] = "y" + assert tmd.yankee == tmd["yankee"] == "y" + + tmd["zulu"] = "z" + assert tmd.zulu == tmd["zulu"] == "z" + + delattr(tmd, "zulu") # deletes instance attribute + assert tmd.zulu == "Z" # so returns so class attribute default value + + tmd["zulu"] = "z" + assert tmd["zulu"] == "z" + + del tmd["zulu"] # deletes instance attribute + assert tmd.zulu == "Z" # so returns so class attribute default value + + # create dynamic attribute + with pytest.raises(AttributeError): + assert tmd.alpha == None + + with pytest.raises(IndexError): + assert tmd["alpha"] == None + + tmd["alpha"] = "A" # add new attribute but without default + assert tmd.alpha == tmd["alpha"] == "A" + + del tmd["alpha"] # deletes instance attribute and no class default + + with pytest.raises(AttributeError): + assert tmd.alpha == "A" + + with pytest.raises(IndexError): + assert tmd["alpha"] == "A" + + # another dynamic attribut but delattr instead of del + with pytest.raises(AttributeError): + assert tmd.beta == None + + with pytest.raises(IndexError): + assert tmd["beta"] == None + + tmd["beta"] = "B" # add new attribute but without default + assert tmd.beta == tmd["beta"] == "B" + + delattr(tmd, "beta") # deletes instance attribute and no class default + + with pytest.raises(AttributeError): + assert tmd.beta == "B" + + with pytest.raises(IndexError): + assert tmd["beta"] == "B" + + # attempt to delete non-existing + with pytest.raises(IndexError): + del tmd["gamma"] + + with pytest.raises(AttributeError): + delattr(tmd, "gamma") + + """End Test""" + + +def test_mapcodex(): + """Test MapCodex base dataclass frozen""" + + + @dataclass(frozen=True) + class TestMapCodex(MapCodex): + """ + + """ + xray: str = 'X' + yankee: str = 'Y' + zulu: str = 'Z' + + def __iter__(self): # so value in dataclass not key in dataclass + return iter(astuple(self)) + + tmc = TestMapCodex() + + assert 'X' in tmc + assert 'Y' in tmc + assert 'Z' in tmc + + assert tmc.xray == tmc["xray"] == 'X' + assert tmc.yankee == tmc["yankee"] == 'Y' + assert tmc.zulu == tmc["zulu"] == 'Z' + + with pytest.raises(IndexError): + tmc["xray"] = "x" + + with pytest.raises(AttributeError): + tmc.xray = "x" + + with pytest.raises(IndexError): + del tmc["xray"] + + with pytest.raises(AttributeError): + delattr(tmc, "xray") + + with pytest.raises(IndexError): + tmc["alpha"] = "A" + + with pytest.raises(AttributeError): + tmc.alpha = "A" + + # attempt to delete non-existing + with pytest.raises(IndexError): + del tmc["gamma"] + + with pytest.raises(AttributeError): + delattr(tmc, "gamma") + + """End Test""" + +if __name__ == "__main__": + test_mapdom() + test_mapcodex() + + From dd116137a8f571aacee6581481a848647706ed52 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 18 Mar 2024 15:36:29 -0600 Subject: [PATCH 053/418] fixed lint error --- src/keri/core/counting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 9417a7d81..579df4253 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -456,7 +456,7 @@ def _exfil(self, qb64b): Extracts self.code and self.count from qualified base64 bytes qb64b """ if not qb64b: # empty need more bytes - raise ShortageError("Empty material, Need more characters.") + raise kering.ShortageError("Empty material, Need more characters.") first = qb64b[:2] # extract first two char code selector if hasattr(first, "decode"): From 7de3aedd841d480d7a627efb0299eb50f1526493 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 20 Mar 2024 10:32:14 -0600 Subject: [PATCH 054/418] Added support for tag parameter to Counter, Added b64ToVer static method and tests --- src/keri/core/counting.py | 247 +++++++++-- tests/core/test_counting.py | 864 +++++++++++++++++++++++++++++++++++- 2 files changed, 1068 insertions(+), 43 deletions(-) diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 579df4253..be60e8d7c 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -5,15 +5,16 @@ Provides versioning support for Counter classes and codes """ -from dataclasses import dataclass, astuple - +from dataclasses import dataclass, astuple, asdict +from collections import namedtuple from ..help import helping from ..help.helping import sceil -from ..help.helping import (intToB64, b64ToInt, codeB64ToB2, codeB2ToB64, +from ..help.helping import (intToB64, b64ToInt, codeB64ToB2, codeB2ToB64, Reb64, nabSextets) from .. import kering +from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0) from ..core.coring import Sizage @@ -78,7 +79,7 @@ def __delitem__(self, name): @dataclass(frozen=True) -class CounterCodex: +class CounterCodex_1_0(MapCodex): """ CounterCodex is codex hard (stable) part of all counter derivation codes. Only provide defined codes. @@ -104,11 +105,40 @@ class CounterCodex: def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" -CtrDex = CounterCodex() +CtrDex_1_0 = CounterCodex_1_0() + +@dataclass(frozen=True) +class CounterCodex_2_0(MapCodex): + """ + CounterCodex is codex hard (stable) part of all counter derivation codes. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + + ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. + WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. + NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. + TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. + FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. + TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. + SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegator/issuer/transaction event + TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. + SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event + SadPathSig: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content + SadPathSigGroup: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples + PathedMaterialQuadlets: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) + AttachedMaterialQuadlets: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) + BigAttachedMaterialQuadlets: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) + KERIProtocolStack: str = '--AAA' # KERI ACDC Protocol Stack CESR Version + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +CtrDex_2_0 = CounterCodex_2_0() @dataclass(frozen=True) -class GenusCodex: +class GenusCodex(MapCodex): """GenusCodex is codex of protocol genera for code table. Only provide defined codes. @@ -125,6 +155,19 @@ def __iter__(self): GenDex = GenusCodex() # Make instance +# keys and values as strings of keys +Codict1 = asdict(CtrDex_1_0) +Tagage_1_0 = namedtuple("Tagage_1_0", list(Codict1), defaults=list(Codict1)) +Tags_1_0 = Tagage_1_0() # uses defaults + +Codict2 = asdict(CtrDex_2_0) +Tagage_2_0 = namedtuple("Tagage_2_0", list(Codict2), defaults=list(Codict2)) +Tags_2_0 = Tagage_2_0() # uses defaults + +CodictAll = Codict1 | Codict2 +AllTagage = namedtuple("AllTagage", list(CodictAll), defaults=list(CodictAll)) +AllTags = AllTagage() # uses defaults + """ Design Notes @@ -180,10 +223,21 @@ class Counter: Includes the following attributes and properties: + Class Attributes: + Codes + Tags + Hards + Bards + Sizes + Attributes: + Properties: - .code is str derivation code to indicate cypher suite + .version (Versionage): current CESR code table protocol genus version + .codes (CounterCodex_1_0 | CounterCodex_1_0): version specific codex + .sizes (dict): version specific sizes table + .code (str) derivation code to indicate cypher suite .raw is bytes crypto material only without code .pad is int number of pad chars given raw .count is int count of grouped following material (not part of counter) @@ -192,6 +246,9 @@ class Counter: .qb2 is bytes in binary with derivation code + crypto material Hidden: + ._version (Versionage): value for .version property + ._codes (CounterCodex_1_0 | CounterCodex_1_0): version specific codex + ._sizes (dict): version specific sizes table ._code is str value for .code property ._raw is bytes value for .raw property ._pad is method to compute .pad property @@ -200,6 +257,9 @@ class Counter: ._exfil is method to extract .code and .raw from fully qualified Base64 """ + Codes = {Vrsn_1_0: CtrDex_1_0, Vrsn_2_0: CtrDex_2_0} + Tags = {Vrsn_1_0: Tags_1_0, Vrsn_2_0: Tags_2_0} + # Hards table maps from bytes Base64 first two code chars to int of # hard size, hs,(stable) of code. The soft size, ss, (unstable) for Counter # is always > 0 and hs + ss = fs always @@ -217,6 +277,26 @@ class Counter: # soft size, ss, should always be > 0 and hs+ss=fs for Counter Sizes = \ { + Vrsn_1_0: \ + { + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + }, + Vrsn_2_0: \ + { '-A': Sizage(hs=2, ss=2, fs=4, ls=0), '-B': Sizage(hs=2, ss=2, fs=4, ls=0), '-C': Sizage(hs=2, ss=2, fs=4, ls=0), @@ -232,17 +312,21 @@ class Counter: '-V': Sizage(hs=2, ss=2, fs=4, ls=0), '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + }, } - Codex = CtrDex - - def __init__(self, code=None, count=None, countB64=None, - qb64b=None, qb64=None, qb2=None, strip=False): + def __init__(self, tag=None, *, code = None, count=None, countB64=None, + qb64b=None, qb64=None, qb2=None, strip=False, version=Version): """ Validate as fully qualified Parameters: + tag (str | None): label of stable (hard) part of derivation code + to lookup in codex so it can depend on version. + takes precedence over tag code (str | None): stable (hard) part of derivation code + if tag provided lookup code from tag + else if tag is None and code provided use code count (int | None): count for composition. Count may represent quadlets/triplet, groups, primitives or other numericy @@ -251,11 +335,16 @@ def __init__(self, code=None, count=None, countB64=None, countB64 may represent quadlets/triplet, groups, primitives or other numericy qb64b (bytes | bytearray | None): fully qualified crypto material text domain + if code nor tag is provided qb64 (str | None) fully qualified crypto material text domain + if code nor tag not qb64b is provided qb2 (bytes | bytearray | None) fully qualified crypto material binary domain + if code nor tag not qb64b nor qb54 is provided strip (bool): True means strip counter contents from input stream bytearray after parsing qb64b or qb2. False means do not strip. default False + version (Versionage): instance of version of code tables to use + provides protocol genera version Needs either code or qb64b or qb64 or qb2 @@ -265,11 +354,20 @@ def __init__(self, code=None, count=None, countB64=None, .code and .count """ + self._version = version + self._codes = self.Codes[self._version] + self._sizes = self.Sizes[self._version] + + if tag: + if not hasattr(self._codes, tag): + raise kering.InvalidCodeError(f"Unsupported {tag=}.") + code = self._codes[tag] + if code is not None: # code provided - if code not in self.Sizes: + if code not in self._sizes: raise kering.InvalidCodeError("Unsupported code={}.".format(code)) - hs, ss, fs, ls = self.Sizes[code] # get sizes for code + hs, ss, fs, ls = self._sizes[code] # get sizes for code cs = hs + ss # both hard + soft code size if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes raise kering.InvalidCodeSizeError("Whole code size not full size or not " @@ -287,7 +385,7 @@ def __init__(self, code=None, count=None, countB64=None, elif qb64b is not None: self._exfil(qb64b) if strip: # assumes bytearray - del qb64b[:self.Sizes[self.code].fs] + del qb64b[:self._sizes[self.code].fs] elif qb64 is not None: self._exfil(qb64) @@ -295,13 +393,45 @@ def __init__(self, code=None, count=None, countB64=None, elif qb2 is not None: # rewrite to use direct binary exfiltration self._bexfil(qb2) if strip: # assumes bytearray - del qb2[:self.Sizes[self.code].fs * 3 // 4] + del qb2[:self._sizes[self.code].fs * 3 // 4] else: raise kering.EmptyMaterialError("Improper initialization need either " "(code and count) or qb64b or " "qb64 or qb2.") + @property + def version(self): + """ + Returns ._version + Makes .version read only + """ + return self._version + + @property + def codes(self): + """ + Returns ._codes + Makes .codes read only + """ + return self._codes + + @property + def tags(self): + """ + Returns ._tags + Makes .tags read only + """ + return self._tags + + @property + def sizes(self): + """ + Returns ._sizes + Makes .sizes read only + """ + return self._sizes + @property def code(self): """ @@ -357,44 +487,77 @@ def countToB64(self, l=None): """ if l is None: - _, ss, _, _ = self.Sizes[self.code] + _, ss, _, _ = self._sizes[self.code] l = ss return (intToB64(self.count, l=l)) @staticmethod - def semVerToB64(version="", major=0, minor=0, patch=0): - """ Converts semantic version to Base64 representation of countB64 + def verToB64(version=None, *, text="", major=0, minor=0): + """ Converts version to Base64 representation of countB64 suitable for CESR protocol genus and version Returns: countB64 (str): suitable for input to Counter - example: Counter(countB64=semVerToB64(version = "1.0.0")) - Parameters: - version (str | None): dot separated semantic version string of format - "major.minor.patch" - major (int): When version is None or empty then use major,minor, patch - minor (int): When version is None or empty then use major,minor, patch - patch (int): When version is None or empty then use major,minor, patch + Example: + Counter(countB64=Counter.verToB64(verstr = "1.0")) - each of major, minor, patch must be in range [0,63] for represenation as - three Base64 characters + Parameters: + version (Versionage): instange of namedtuple + Versionage(major=major,minor=minor) + text (str): text format of version as dotted decimal "major.minor" + major (int): When version is None and verstr is empty then use major minor + range [0, 63] for one Base64 character + minor (int): When version is None and verstr is empty then use major minor + range [0, 4095] for two Base64 characters """ - parts = [major, minor, patch] if version: - splits = version.split(".", maxsplit=3) + major = version.major + minor = version.minor + + elif text: + splits = text.split(".", maxsplit=2) splits = [(int(s) if s else 0) for s in splits] - for i in range(3-len(splits),0, -1): + parts = [major, minor] + for i in range(2-len(splits),0, -1): # append missing minor and/or major splits.append(parts[-i]) - parts = splits + major = splits[0] + minor = splits[1] + + if major < 0 or major > 63 or minor < 0 or minor > 4095: + raise ValueError(f"Out of bounds version = {major}.{minor}.") + + return (f"{intToB64(major)}{intToB64(minor, l=2)}") + + + @staticmethod + def b64ToVer(b64, *, texted=False): + """ Converts Base64 representation of version to Versionage or + text dotted decimal format + + default is Versionage + + Returns: + version (Versionage | str): + + Example: + Counter(version=Counter.b64ToVer("BAA")) + + Parameters: + b64 (str): base64 string of three characters Mmm for Major minor + texted (bool): return text format dotted decimal string + + + """ + if not Reb64.match(b64.encode("utf-8")): + raise ValueError("Invalid Base64.") + + if texted: + return ".".join([f"{b64ToInt(b64[0])}", f"{b64ToInt(b64[1:3])}"]) - for p in parts: - if p < 0 or p > 63: - raise ValueError(f"Out of bounds semantic version. " - f"Part={p} is < 0 or > 63.") - return ("".join(intToB64(p, l=1) for p in parts)) + return Versionage(major=b64ToInt(b64[0]), minor=b64ToInt(b64[1:3])) def _infil(self): @@ -405,7 +568,7 @@ def _infil(self): code = self.code # codex value chars hard code count = self.count # index value int used for soft - hs, ss, fs, ls = self.Sizes[code] + hs, ss, fs, ls = self._sizes[code] cs = hs + ss # both hard + soft size if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes raise kering.InvalidCodeSizeError("Whole code size not full size or not " @@ -433,7 +596,7 @@ def _binfil(self): code = self.code # codex chars hard code count = self.count # index value int used for soft - hs, ss, fs, ls = self.Sizes[code] + hs, ss, fs, ls = self._sizes[code] cs = hs + ss if fs != cs or cs % 4: # fs must be cs and multiple of 4 for count codes raise kering.InvalidCodeSizeError("Whole code size not full size or not " @@ -475,10 +638,10 @@ def _exfil(self, qb64b): hard = qb64b[:hs] # get hard code if hasattr(hard, "decode"): hard = hard.decode("utf-8") # decode converts bytearray/bytes to str - if hard not in self.Sizes: # Sizes needs str not bytes + if hard not in self._sizes: # Sizes needs str not bytes raise kering.UnexpectedCodeError("Unsupported code ={}.".format(hard)) - hs, ss, fs, ls = self.Sizes[hard] # assumes hs consistent in both tables + hs, ss, fs, ls = self._sizes[hard] # assumes hs consistent in both tables cs = hs + ss # both hard + soft code size # assumes that unit tests on Counter and CounterCodex ensure that @@ -518,10 +681,10 @@ def _bexfil(self, qb2): raise kering.ShortageError("Need {} more bytes.".format(bhs - len(qb2))) hard = codeB2ToB64(qb2, hs) # extract and convert hard part of code - if hard not in self.Sizes: + if hard not in self._sizes: raise kering.UnexpectedCodeError("Unsupported code ={}.".format(hard)) - hs, ss, fs, ls = self.Sizes[hard] + hs, ss, fs, ls = self._sizes[hard] cs = hs + ss # both hs and ss # assumes that unit tests on Counter and CounterCodex ensure that # .Codes and .Sizes are well formed. diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 51f709595..5547813dc 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -5,14 +5,22 @@ """ from dataclasses import dataclass, astuple, asdict from ordered_set import OrderedSet as oset +from base64 import urlsafe_b64encode as encodeB64 +from base64 import urlsafe_b64decode as decodeB64 import pytest from keri import kering +from keri.help import helping +from keri.help.helping import sceil +from keri.help.helping import (intToB64, b64ToInt, codeB64ToB2, codeB2ToB64, + nabSextets) + from keri.core import counting -from keri.core.counting import MapDom, MapCodex, Counter +from keri.core.counting import Sizage, MapDom, MapCodex, Counter +from keri.core.counting import Versionage, Version, Vrsn_1_0, Vrsn_2_0, AllTags def test_mapdom(): @@ -158,8 +166,862 @@ def __iter__(self): # so value in dataclass not key in dataclass """End Test""" +def test_codexes_tags(): + """ + Test supporting module attributes + """ + assert asdict(counting.CtrDex_1_0) == \ + { + 'ControllerIdxSigs': '-A', + 'WitnessIdxSigs': '-B', + 'NonTransReceiptCouples': '-C', + 'TransReceiptQuadruples': '-D', + 'FirstSeenReplayCouples': '-E', + 'TransIdxSigGroups': '-F', + 'SealSourceCouples': '-G', + 'TransLastIdxSigGroups': '-H', + 'SealSourceTriples': '-I', + 'SadPathSig': '-J', + 'SadPathSigGroup': '-K', + 'PathedMaterialQuadlets': '-L', + 'AttachedMaterialQuadlets': '-V', + 'BigAttachedMaterialQuadlets': '-0V', + 'KERIProtocolStack': '--AAA', + } + + assert asdict(counting.CtrDex_2_0) == \ + { + 'ControllerIdxSigs': '-A', + 'WitnessIdxSigs': '-B', + 'NonTransReceiptCouples': '-C', + 'TransReceiptQuadruples': '-D', + 'FirstSeenReplayCouples': '-E', + 'TransIdxSigGroups': '-F', + 'SealSourceCouples': '-G', + 'TransLastIdxSigGroups': '-H', + 'SealSourceTriples': '-I', + 'SadPathSig': '-J', + 'SadPathSigGroup': '-K', + 'PathedMaterialQuadlets': '-L', + 'AttachedMaterialQuadlets': '-V', + 'BigAttachedMaterialQuadlets': '-0V', + 'KERIProtocolStack': '--AAA', + } + + assert counting.Tags_1_0._asdict() == \ + { + 'ControllerIdxSigs': 'ControllerIdxSigs', + 'WitnessIdxSigs': 'WitnessIdxSigs', + 'NonTransReceiptCouples': 'NonTransReceiptCouples', + 'TransReceiptQuadruples': 'TransReceiptQuadruples', + 'FirstSeenReplayCouples': 'FirstSeenReplayCouples', + 'TransIdxSigGroups': 'TransIdxSigGroups', + 'SealSourceCouples': 'SealSourceCouples', + 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', + 'SealSourceTriples': 'SealSourceTriples', + 'SadPathSig': 'SadPathSig', + 'SadPathSigGroup': 'SadPathSigGroup', + 'PathedMaterialQuadlets': 'PathedMaterialQuadlets', + 'AttachedMaterialQuadlets': 'AttachedMaterialQuadlets', + 'BigAttachedMaterialQuadlets': 'BigAttachedMaterialQuadlets', + 'KERIProtocolStack': 'KERIProtocolStack' + } + + assert counting.Tags_1_0.ControllerIdxSigs == 'ControllerIdxSigs' + + assert counting.Tags_2_0._asdict() == \ + { + 'ControllerIdxSigs': 'ControllerIdxSigs', + 'WitnessIdxSigs': 'WitnessIdxSigs', + 'NonTransReceiptCouples': 'NonTransReceiptCouples', + 'TransReceiptQuadruples': 'TransReceiptQuadruples', + 'FirstSeenReplayCouples': 'FirstSeenReplayCouples', + 'TransIdxSigGroups': 'TransIdxSigGroups', + 'SealSourceCouples': 'SealSourceCouples', + 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', + 'SealSourceTriples': 'SealSourceTriples', + 'SadPathSig': 'SadPathSig', + 'SadPathSigGroup': 'SadPathSigGroup', + 'PathedMaterialQuadlets': 'PathedMaterialQuadlets', + 'AttachedMaterialQuadlets': 'AttachedMaterialQuadlets', + 'BigAttachedMaterialQuadlets': 'BigAttachedMaterialQuadlets', + 'KERIProtocolStack': 'KERIProtocolStack' + } + + assert counting.Tags_2_0.ControllerIdxSigs == 'ControllerIdxSigs' + + assert counting.AllTags._asdict() == \ + { + 'ControllerIdxSigs': 'ControllerIdxSigs', + 'WitnessIdxSigs': 'WitnessIdxSigs', + 'NonTransReceiptCouples': 'NonTransReceiptCouples', + 'TransReceiptQuadruples': 'TransReceiptQuadruples', + 'FirstSeenReplayCouples': 'FirstSeenReplayCouples', + 'TransIdxSigGroups': 'TransIdxSigGroups', + 'SealSourceCouples': 'SealSourceCouples', + 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', + 'SealSourceTriples': 'SealSourceTriples', + 'SadPathSig': 'SadPathSig', + 'SadPathSigGroup': 'SadPathSigGroup', + 'PathedMaterialQuadlets': 'PathedMaterialQuadlets', + 'AttachedMaterialQuadlets': 'AttachedMaterialQuadlets', + 'BigAttachedMaterialQuadlets': 'BigAttachedMaterialQuadlets', + 'KERIProtocolStack': 'KERIProtocolStack' + } + + """End Test""" + + +def test_counter(): + """ + Test Counter class + """ + assert Counter.Codes == \ + { + counting.Vrsn_1_0: counting.CtrDex_1_0, + counting.Vrsn_2_0: counting.CtrDex_2_0, + } + + assert Counter.Tags == \ + { + counting.Vrsn_1_0: counting.Tags_1_0, + counting.Vrsn_2_0: counting.Tags_2_0, + } + + + # first character of code with hard size of code + assert Counter.Hards == \ + { + '-A': 2, '-B': 2, '-C': 2, '-D': 2, '-E': 2, '-F': 2, '-G': 2, '-H': 2, '-I': 2, + '-J': 2, '-K': 2, '-L': 2, '-M': 2, '-N': 2, '-O': 2, '-P': 2, '-Q': 2, '-R': 2, + '-S': 2, '-T': 2, '-U': 2, '-V': 2, '-W': 2, '-X': 2, '-Y': 2, '-Z': 2, + '-a': 2, '-b': 2, '-c': 2, '-d': 2, '-e': 2, '-f': 2, '-g': 2, '-h': 2, '-i': 2, + '-j': 2, '-k': 2, '-l': 2, '-m': 2, '-n': 2, '-o': 2, '-p': 2, '-q': 2, '-r': 2, + '-s': 2, '-t': 2, '-u': 2, '-v': 2, '-w': 2, '-x': 2, '-y': 2, '-z': 2, + '-0': 3, '--': 5, + } + + # Codes table with sizes of code (hard) and full primitive material + assert Counter.Sizes == \ + { + counting.Vrsn_1_0: \ + { + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) + }, + counting.Vrsn_2_0: \ + { + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) + } + } + + assert Counter.Sizes[counting.Vrsn_1_0]['-A'].hs == 2 # hard size + assert Counter.Sizes[counting.Vrsn_1_0]['-A'].ss == 2 # soft size + assert Counter.Sizes[counting.Vrsn_1_0]['-A'].fs == 4 # full size + assert Counter.Sizes[counting.Vrsn_1_0]['-A'].ls == 0 # lead size + + # verify first hs Sizes matches hs in Codes for same first char + for vsize in Counter.Sizes.values(): + for ckey, cval in vsize.items(): + assert Counter.Hards[ckey[:2]] == cval.hs + + # verify all Codes have hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 + for vsize in Counter.Sizes.values(): + for val in vsize.values(): + assert val.hs > 0 and val.ss > 0 and val.hs + val.ss == val.fs and not val.fs % 4 + + # Bizes maps bytes of sextet of decoded first character of code with hard size of code + # verify equivalents of items for Sizes and Bizes + for skey, sval in Counter.Hards.items(): + ckey = codeB64ToB2(skey) + assert Counter.Bards[ckey] == sval + + # test Counter static methods + assert Counter.verToB64() == "AAA" + + assert Counter.b64ToVer('AAA') == Versionage(major=0, minor=0) + assert Counter.b64ToVer('AAA', texted=True) == "0.0" + assert Counter.b64ToVer('BBC', texted=True) == "1.66" + assert Counter.b64ToVer('bbc', texted=True) == '27.1756' + + with pytest.raises(ValueError): + Counter.b64ToVer("!AA") + + with pytest.raises(ValueError): + Counter.b64ToVer("AA#") + + assert Counter.b64ToVer(Counter.verToB64(text='1.1'), texted=True) == "1.1" + assert Counter.verToB64(text=Counter.b64ToVer('BAB', texted=True)) == "BAB" + assert Counter.b64ToVer(Counter.verToB64(text='12.2345'), texted=True) == '12.2345' + assert Counter.verToB64(text='12.2345') == 'Mkp' + assert Counter.verToB64(text=Counter.b64ToVer('Mkp', texted=True)) == 'Mkp' + + assert Counter.verToB64(counting.Vrsn_1_0) == "BAA" + assert Counter.verToB64(counting.Vrsn_2_0) == "CAA" + + assert Counter.verToB64(text="1.2") == "BAC" + + assert Counter.verToB64(major=1) == "BAA" + assert Counter.verToB64(minor=1) == "AAB" + assert Counter.verToB64(major=3, minor=4) == "DAE" + + # test defaults for missing parts in string version + assert Counter.verToB64(text="1.1") == "BAB" + assert Counter.verToB64(text="1.1.") == "BAB" # ignores extra parts + assert Counter.verToB64(text="1.1.0") == "BAB" # ignores extra parts + assert Counter.verToB64(text="1.") == "BAA" + assert Counter.verToB64(text="1") == "BAA" + assert Counter.verToB64(text="1.2") == "BAC" + assert Counter.verToB64(text=".") == "AAA" + assert Counter.verToB64(text="1.3") == "BAD" + assert Counter.verToB64(text="4", major=1, minor=2) == "EAC" + + with pytest.raises(ValueError): + Counter.verToB64(text="64.0") + with pytest.raises(ValueError): + Counter.verToB64(text="63.4096") + with pytest.raises(ValueError): + Counter.verToB64(text="-1.0") + with pytest.raises(ValueError): + Counter.verToB64(text="0.-1") + with pytest.raises(ValueError): + Counter.verToB64(major=64) + with pytest.raises(ValueError): + Counter.verToB64(minor=4096) + with pytest.raises(ValueError): + Counter.verToB64(major=-1) + with pytest.raises(ValueError): + Counter.verToB64(minor=-1) + + + + + + + """ Done Test """ + +def test_counter_v1(): + """ + test Counter instances for verision 1.0 code tables + """ + # test Counter instances + with pytest.raises(kering.EmptyMaterialError): + counter = Counter() + + # create code manually + count = 1 + qsc = counting.CtrDex_1_0.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-AAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + # version 1_0 tests default version is Version + assert Version == Vrsn_1_0 + CtrDex = Counter.Codes[Version] # set CtrDex to Vrsn_1_0 + + counter = Counter(tag="ControllerIdxSigs", count=count) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(tag=AllTags.ControllerIdxSigs, count=count) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # test tag takes precedence + counter = Counter(tag=AllTags.ControllerIdxSigs, + code=CtrDex.WitnessIdxSigs, + count=count) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(code=CtrDex.ControllerIdxSigs) # default count = 1 + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64b=qscb) # test with bytes not str + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64=qsc) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb2=qscb2) # test with qb2 + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # test truncates extra bytes from qb64 parameter + longqsc64 = qsc + "ABCD" + counter = Counter(qb64=longqsc64) + assert len(counter.qb64) == counter.sizes[counter.code].fs + + # test raises ShortageError if not enough bytes in qb64 parameter + shortqsc64 = qsc[:-1] # too short + with pytest.raises(kering.ShortageError): + counter = Counter(qb64=shortqsc64) + + # test truncates extra bytes from qb2 parameter + longqscb2 = qscb2 + bytearray([1, 2, 3, 4, 5]) # extra bytes in size + counter = Counter(qb2=longqscb2) + assert counter.qb2 == qscb2 + assert len(counter.qb64) == counter.sizes[counter.code].fs + + # test raises ShortageError if not enough bytes in qb2 parameter + shortqscb2 = qscb2[:-4] # too few bytes in size + with pytest.raises(kering.ShortageError): + counter = Counter(qb2=shortqscb2) + + # test with non-zero count=5 + count = 5 + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-AAF' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.ControllerIdxSigs, count=count) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64b=qscb) # test with bytes not str + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64=qsc) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb2=qscb2) # test with qb2 + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # test with big codes index=1024 + count = 1024 + qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) + assert qsc == '-0VAAAQA' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.BigAttachedMaterialQuadlets, count=count) + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64b=qscb) # test with bytes not str + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64=qsc) # test with str not bytes + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb2=qscb2) # test with qb2 + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # Test ._bexfil + counter = Counter(qb64=qsc) # + code = counter.code + count = counter.count + qb2 = counter.qb2 + counter._bexfil(qb2) + assert counter.code == code + assert counter.count == count + assert counter.qb64 == qsc + assert counter.qb2 == qb2 + + # Test ._binfil + test = counter._binfil() + assert test == qb2 + + # Test with strip + # create code manually + count = 1 + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-AAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + # strip ignored if qb64 + counter = Counter(qb64=qsc, strip=True) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + ims = bytearray(qscb) # test with qb64b + counter = Counter(qb64b=ims, strip=True) # strip + assert not ims # deleted + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + ims = bytearray(qscb2) # test with qb2 + counter = Counter(qb2=ims, strip=True) + assert not ims # deleted + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # test with longer ims for qb64b + extra = b"ABCD" + ims = bytearray(qscb + b"ABCD") + counter = Counter(qb64b=ims, strip=True) + assert counter.qb64b == qscb + assert len(counter.qb64b) == counter.sizes[counter.code].fs + assert ims == extra + + # test with longer ims for qb2 + extra = bytearray([1, 2, 3, 4, 5]) + ims = bytearray(qscb2) + extra + counter = Counter(qb2=ims, strip=True) + assert counter.qb2 == qscb2 + assert len(counter.qb2) == counter.sizes[counter.code].fs * 3 // 4 + assert ims == extra + + # raises error if not bytearray + + ims = bytes(qscb) # test with qb64b + with pytest.raises(TypeError): + counter = Counter(qb64b=ims, strip=True) # strip + + ims = bytes(qscb2) # test with qb2 + with pytest.raises(TypeError): + counter = Counter(qb2=ims, strip=True) + + # test with big codes index=1024 + count = 1024 + qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) + assert qsc == '-0VAAAQA' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + ims = bytearray(qscb) + counter = Counter(qb64b=ims, strip=True) # test with bytes not str + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert not ims + + ims = bytearray(qscb2) + counter = Counter(qb2=ims, strip=True) # test with qb2 + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert not ims + + # test protocol genus with CESR protocol genus version + # test with big codes index=1024 + genverint = 0 + genver = intToB64(genverint, l=3) + assert genver == 'AAA' + assert genverint == b64ToInt(genver) + qsc = CtrDex.KERIProtocolStack + genver + assert qsc == '--AAAAAA' # keri Cesr version 0.0.0 + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.KERIProtocolStack, count=genverint) + assert counter.code == CtrDex.KERIProtocolStack + assert counter.count == genverint + assert counter.countToB64(l=3) == genver + assert counter.countToB64() == genver # default length + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(code=CtrDex.KERIProtocolStack, countB64=genver) + assert counter.code == CtrDex.KERIProtocolStack + assert counter.count == genverint + assert counter.countToB64(l=3) == genver + assert counter.countToB64() == genver # default length + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + """End Test""" + +def test_counter_v2(): + """ + test Counter instances for verision 2.0 code tables + """ + # test Counter instances + with pytest.raises(kering.EmptyMaterialError): + counter = Counter() + + # create code manually + count = 1 + qsc = counting.CtrDex_1_0.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-AAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + # version 2_0 tests default version is Version + #assert Version == Vrsn_2_0 + + CtrDex = Counter.Codes[Vrsn_2_0] # set CtrDex to Vrsn_2_0 + + # default count = 1 + counter = Counter(code=CtrDex.ControllerIdxSigs, version=Vrsn_2_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64=qsc, version=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb2=qscb2, version=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # test truncates extra bytes from qb64 parameter + longqsc64 = qsc + "ABCD" + counter = Counter(qb64=longqsc64, version=Vrsn_2_0) + assert len(counter.qb64) == counter.sizes[counter.code].fs + + # test raises ShortageError if not enough bytes in qb64 parameter + shortqsc64 = qsc[:-1] # too short + with pytest.raises(kering.ShortageError): + counter = Counter(qb64=shortqsc64, version=Vrsn_2_0) + + # test truncates extra bytes from qb2 parameter + longqscb2 = qscb2 + bytearray([1, 2, 3, 4, 5]) # extra bytes in size + counter = Counter(qb2=longqscb2, version=Vrsn_2_0) + assert counter.qb2 == qscb2 + assert len(counter.qb64) == counter.sizes[counter.code].fs + + # test raises ShortageError if not enough bytes in qb2 parameter + shortqscb2 = qscb2[:-4] # too few bytes in size + with pytest.raises(kering.ShortageError): + counter = Counter(qb2=shortqscb2, version=Vrsn_2_0) + + # test with non-zero count=5 + count = 5 + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-AAF' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(tag="ControllerIdxSigs", count=count, version=Vrsn_2_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(tag=AllTags.ControllerIdxSigs, count=count, version=Vrsn_2_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # test tag takes precedence + counter = Counter(tag=AllTags.ControllerIdxSigs, + code=CtrDex.WitnessIdxSigs, + count=count, + version=Vrsn_2_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(code=CtrDex.ControllerIdxSigs, count=count, version=Vrsn_2_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64=qsc, version=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb2=qscb2, version=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # test with big codes index=1024 + count = 1024 + qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) + assert qsc == '-0VAAAQA' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.BigAttachedMaterialQuadlets, count=count, version=Vrsn_2_0) + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64=qsc, version=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb2=qscb2, version=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # Test ._bexfil + counter = Counter(qb64=qsc, version=Vrsn_2_0) + code = counter.code + count = counter.count + qb2 = counter.qb2 + counter._bexfil(qb2) + assert counter.code == code + assert counter.count == count + assert counter.qb64 == qsc + assert counter.qb2 == qb2 + + # Test ._binfil + test = counter._binfil() + assert test == qb2 + + # Test with strip + # create code manually + count = 1 + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-AAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + # strip ignored if qb64 + counter = Counter(qb64=qsc, strip=True, version=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + ims = bytearray(qscb) # test with qb64b + counter = Counter(qb64b=ims, strip=True, version=Vrsn_2_0) # strip + assert not ims # deleted + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + ims = bytearray(qscb2) # test with qb2 + counter = Counter(qb2=ims, strip=True, version=Vrsn_2_0) + assert not ims # deleted + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # test with longer ims for qb64b + extra = b"ABCD" + ims = bytearray(qscb + b"ABCD") + counter = Counter(qb64b=ims, strip=True, version=Vrsn_2_0) + assert counter.qb64b == qscb + assert len(counter.qb64b) == counter.sizes[counter.code].fs + assert ims == extra + + # test with longer ims for qb2 + extra = bytearray([1, 2, 3, 4, 5]) + ims = bytearray(qscb2) + extra + counter = Counter(qb2=ims, strip=True, version=Vrsn_2_0) + assert counter.qb2 == qscb2 + assert len(counter.qb2) == counter.sizes[counter.code].fs * 3 // 4 + assert ims == extra + + # raises error if not bytearray + + ims = bytes(qscb) # test with qb64b + with pytest.raises(TypeError): + counter = Counter(qb64b=ims, strip=True, version=Vrsn_2_0) # strip + + ims = bytes(qscb2) # test with qb2 + with pytest.raises(TypeError): + counter = Counter(qb2=ims, strip=True, version=Vrsn_2_0) + + # test with big codes index=1024 + count = 1024 + qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) + assert qsc == '-0VAAAQA' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + ims = bytearray(qscb) + counter = Counter(qb64b=ims, strip=True, version=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert not ims + + ims = bytearray(qscb2) + counter = Counter(qb2=ims, strip=True, version=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert not ims + + # test protocol genus with CESR version + # test with big codes index=1024 + genverint = 0 + genver = intToB64(genverint, l=3) + assert genver == 'AAA' + assert genverint == b64ToInt(genver) + qsc = CtrDex.KERIProtocolStack + genver + assert qsc == '--AAAAAA' # keri Cesr version 0.0.0 + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.KERIProtocolStack, count=genverint, version=Vrsn_2_0) + assert counter.code == CtrDex.KERIProtocolStack + assert counter.count == genverint + assert counter.countToB64(l=3) == genver + assert counter.countToB64() == genver # default length + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(code=CtrDex.KERIProtocolStack, countB64=genver, version=Vrsn_2_0) + assert counter.code == CtrDex.KERIProtocolStack + assert counter.count == genverint + assert counter.countToB64(l=3) == genver + assert counter.countToB64() == genver # default length + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + """End Test""" + + if __name__ == "__main__": test_mapdom() test_mapcodex() + test_codexes_tags() + test_counter() + test_counter_v1() + test_counter_v2() From f79c65cd95a82047fb7181697dd84f780927524e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 20 Mar 2024 12:35:36 -0600 Subject: [PATCH 055/418] Added dynamic code promotion based on upsized count --- src/keri/core/counting.py | 85 ++++++++++++------------------------- tests/core/test_counting.py | 5 ++- 2 files changed, 29 insertions(+), 61 deletions(-) diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index be60e8d7c..e204cd44b 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -169,51 +169,6 @@ def __iter__(self): AllTags = AllTagage() # uses defaults -""" -Design Notes - -Need sizes by version can assume only support KERI/ACDC Genus so do not -need to support different protocol genera in Counting -Hards and Bards are the same for both versions - -Need to pass in version so Counter Instance knows what version to use -simpler than Serder since version is not provided in version string -Each instance when created needs to get its version at init - -So Sizes in dictionary indexed by version. - -CtrDex Codex itself is not referenced inside Counter but by external classes -making instances so use CtrDex to pass in code. So we need versioned codex - -one option is to have different dataclasses with each version codex and then -have dictionary Codex of those indexed by version as class variable. - -Another option is to have code name index that is indexed by version -so the actual code is looked up for the version instead of passing in the code -itself which requires dereferencing with the version so the version gets used -twice. Also need to reverse code and version to get codename as property - -codename = label or tag. use tag for code tag - - -Counter(codename=, verion=) - -When using code not tag then must check against version to make sure code is -a valid code for version. Likewise a tag may not have a code for a given version - -So tags are strings that can be attribute names for dataclasses -Tags is namedtuple where each attribute value is its key so one can look up the -string by the tag - -But given string value for tag then to lookup code in codex need to to use -builtin getattr - -Instead maybe we just extend the codex dataclass with .__getitem__ .__setitem__ and .__delitem__ methods -so we can access a code by its tag using Codex[tag] - - -""" - class Counter: """ Counter is fully qualified cryptographic material primitive base class for @@ -224,11 +179,11 @@ class Counter: Includes the following attributes and properties: Class Attributes: - Codes - Tags - Hards - Bards - Sizes + Codes (dict): of codexes keyed by version + Tags (dict): of tagages keyed by version + Hards (dict): of hard code sizes keyed by selector text + Bards (dict): of hard code sizes keyed by selector binary + Sizes (dict): of Sizages keyed by hard code Attributes: @@ -363,21 +318,33 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, raise kering.InvalidCodeError(f"Unsupported {tag=}.") code = self._codes[tag] - if code is not None: # code provided - if code not in self._sizes: - raise kering.InvalidCodeError("Unsupported code={}.".format(code)) + if code is not None: # code (hard) provided + if code not in self._sizes or len(code) < 2: + raise kering.InvalidCodeError(f"Unsupported {code=}.") hs, ss, fs, ls = self._sizes[code] # get sizes for code cs = hs + ss # both hard + soft code size - if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes - raise kering.InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) + if hs < 2 or fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes + raise kering.InvalidCodeSizeError(f"Whole code size not full " + f"size or not multiple of 4. " + f"{cs=} {fs=}.") if count is None: count = 1 if countB64 is None else b64ToInt(countB64) + if code[1] not in ("123456789-_"): # small [A-Z,a-z] or large [0] + if ss not in (2, 5): # not valid dynamic soft sizes + raise kering.InvalidVarIndexError(f"Invalid {ss=} " + f"for {code=}.") + # dynamically promote code based on count + if code[0] != '0' and count > (64 ** 2 - 1): # small code but large count + # elevate code due to large count + code = f"-0{code[1]}" # promote hard + ss = 5 + if count < 0 or count > (64 ** ss - 1): - raise kering.InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) + raise kering.InvalidVarIndexError(f"Invalid {count=} for " + f"{code=} with {ss=}.") self._code = code self._count = count @@ -570,7 +537,7 @@ def _infil(self): hs, ss, fs, ls = self._sizes[code] cs = hs + ss # both hard + soft size - if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes + if hs < 2 or fs != cs or cs % 4: # hs >=2 fs must be bs and multiple of 4 for count codes raise kering.InvalidCodeSizeError("Whole code size not full size or not " "multiple of 4. cs={} fs={}.".format(cs, fs)) if count < 0 or count > (64 ** ss - 1): @@ -598,7 +565,7 @@ def _binfil(self): hs, ss, fs, ls = self._sizes[code] cs = hs + ss - if fs != cs or cs % 4: # fs must be cs and multiple of 4 for count codes + if hs < 2 or fs != cs or cs % 4: # hs >= 2 fs must be cs and multiple of 4 for count codes raise kering.InvalidCodeSizeError("Whole code size not full size or not " "multiple of 4. cs={} fs={}.".format(cs, fs)) diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 5547813dc..1544d2a75 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -352,10 +352,11 @@ def test_counter(): for ckey, cval in vsize.items(): assert Counter.Hards[ckey[:2]] == cval.hs - # verify all Codes have hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 + # verify all Codes have hs >= 2 and ss > 0 and fs = hs + ss and not fs % 4 + # if hs < 2 or ss <= 0 or fs != cs or cs % 4 cs = hs + ss for vsize in Counter.Sizes.values(): for val in vsize.values(): - assert val.hs > 0 and val.ss > 0 and val.hs + val.ss == val.fs and not val.fs % 4 + assert val.hs >= 2 and val.ss > 0 and val.hs + val.ss == val.fs and not val.fs % 4 # Bizes maps bytes of sextet of decoded first character of code with hard size of code # verify equivalents of items for Sizes and Bizes From 9f5ddd0cc3980126ca91b3dead412c17af4c2307 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 20 Mar 2024 18:27:43 -0600 Subject: [PATCH 056/418] Added new codes for version 2.0 Count codes Normalized code labels in codex --- src/keri/core/coring.py | 12 +- src/keri/core/counting.py | 179 +++++++++++------ src/keri/core/eventing.py | 12 +- src/keri/core/parsing.py | 10 +- src/keri/db/basing.py | 2 +- src/keri/peer/exchanging.py | 6 +- src/keri/vdr/viring.py | 6 +- tests/core/test_coring.py | 67 ++----- tests/core/test_counting.py | 370 +++++++++++++++++++++++++----------- tests/core/test_replay.py | 2 +- tests/vc/test_proving.py | 2 +- 11 files changed, 430 insertions(+), 238 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 01d332bbc..cf4e31abd 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -4681,12 +4681,12 @@ class CounterCodex: SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegator/issuer/transaction event TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event - SadPathSig: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content - SadPathSigGroup: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples - PathedMaterialQuadlets: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) - AttachedMaterialQuadlets: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) - BigAttachedMaterialQuadlets: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) - KERIProtocolStack: str = '--AAA' # KERI ACDC Protocol Stack CESR Version + SadPathSigGroups: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content + RootSadPathSigGroups: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples + PathedMaterialGroup: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) + AttachmentGroup: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) + BigAttachmentGroup: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) + KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Protocol Stack CESR Version def __iter__(self): return iter(astuple(self)) # enables inclusion test with "in" diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index e204cd44b..b46ed0803 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -85,7 +85,6 @@ class CounterCodex_1_0(MapCodex): Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. @@ -95,12 +94,13 @@ class CounterCodex_1_0(MapCodex): SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegator/issuer/transaction event TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event - SadPathSig: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content - SadPathSigGroup: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples - PathedMaterialQuadlets: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) - AttachedMaterialQuadlets: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) - BigAttachedMaterialQuadlets: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) - KERIProtocolStack: str = '--AAA' # KERI ACDC Protocol Stack CESR Version + SadPathSigGroups: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content + RootSadPathSigGroups: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples + PathedMaterialGroup: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) + AttachmentGroup: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) + BigAttachmentGroup: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) + KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Protocol Stack CESR Version + def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" @@ -114,22 +114,59 @@ class CounterCodex_2_0(MapCodex): Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ + GenericGroup: str = '-A' # Generic Group (Universal with Override). + BigGenericGroup: str = '-0A' # Big Generic Group (Universal with Override). + MessageGroup: str = '-B' # Message Body plus Attachments Group (Universal with Override). + BigMessageGroup: str = '-0B' # Big Message Body plus Attachments Group (Universal with Override). + AttachmentGroup: str = '-C' # Message Attachments Only Group (Universal with Override). + BigAttachmentGroup: str = '-0C' # Big Attachments Only Group (Universal with Override). + DatagramSegmentGroup: str = '-D' # Datagram Segment Group (Universal). + BigDatagramSegmentGroup: str = '-0D' # Big Datagram Segment Group (Universal). + ESSRWrapperGroup: str = '-E' # ESSR Wrapper Group (Universal). + BigESSRWrapperGroup: str = '-0E' # Big ESSR Wrapper Group (Universal). + FixedMessageBodyGroup: str = '-F' # Fixed Field Message Body Group (Universal). + BigFixedMessageBodyGroup: str = '-0F' # Big Fixed Field Message Body Group (Universal). + MapMessageBodyGroup: str = '-G' # Field Map Message Body Group (Universal). + BigMapMessageBodyGroup: str = '-0G' # Big Field Map Message Body Group (Universal). + GenericMapGroup: str = '-H' # Generic Field Map Group (Universal). + BigGenericMapGroup: str = '-0H' # Big Generic Field Map Group (Universal). + GenericListGroup: str = '-L' # Generic List Group (Universal). + BigGenericListGroup: str = '-0L' # Big Generic List Group (Universal). + ControllerIdxSigs: str = '-J' # Controller Indexed Signature(s) of qb64. + BigControllerIdxSigs: str = '-0J' # Big Controller Indexed Signature(s) of qb64. + WitnessIdxSigs: str = '-K' # Witness Indexed Signature(s) of qb64. + BigWitnessIdxSigs: str = '-0K' # Big Witness Indexed Signature(s) of qb64. + NonTransReceiptCouples: str = '-L' # NonTrans Receipt Couple(s), pre+cig. + BigNonTransReceiptCouples: str = '-0L' # Big NonTrans Receipt Couple(s), pre+cig. + TransReceiptQuadruples: str = '-M' # Trans Receipt Quadruple(s), pre+snu+dig+sig. + BigTransReceiptQuadruples: str = '-0M' # Big Trans Receipt Quadruple(s), pre+snu+dig+sig. + FirstSeenReplayCouples: str = '-N' # First Seen Replay Couple(s), fnu+dts. + BigFirstSeenReplayCouples: str = '-0N' # First Seen Replay Couple(s), fnu+dts. + TransIdxSigGroups: str = '-O' # Trans Indexed Signature Group(s), pre+snu+dig+ControllerIdxSigs of qb64. + TransIdxSigGroups: str = '-0O' # Big Trans Indexed Signature Group(s), pre+snu+dig+ControllerIdxSigs of qb64. + TransLastIdxSigGroups: str = '-P' # Trans Last Est Evt Indexed Signature Group(s), pre+ControllerIdxSigs of qb64. + BigTransLastIdxSigGroups: str = '-0P' # Big Trans Last Est Evt Indexed Signature Group(s), pre+ControllerIdxSigs of qb64. + SealSourceCouples: str = '-Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. + BigSealSourceCouples: str = '-0Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. + SealSourceTriples: str = '-R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. + BigSealSourceTriples: str = '-0R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. + PathedMaterialGroup: str = '-S' # Pathed Material Group. + BigPathedMaterialGroup: str = '-0S' # Big Pathed Material Group. + SadPathSigGroups: str = '-T' # SAD Path Group(s) sadpath+TransIdxSigGroup(s) of SAID qb64 of content. + BigSadPathSigGroups: str = '-0T' # Big SAD Path Group(s) sadpath+TransIdxSigGroup(s) of SAID qb64 of content. + RootSadPathSigGroups: str = '-U' # Root Path SAD Path Group(s), rootpath+SadPathGroup(s). + BigRootSadPathSigGroups: str = '-0U' # Big Root Path SAD Path Group(s), rootpath+SadPathGroup(s). + DigestSealSingles: str = '-V' # Digest Seal Single(s), dig of sealed data. + BigDigestSealSingles: str = '-0V' # Big Digest Seal Single(s), dig of sealed data. + MerkleRootSealSingles: str = '-W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. + BigMerkleRootSealSingles: str = '-0W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. + BackerRegistrarSealCouples: str = '-X' # Backer Registrar Seal Couple(s), brid+dig of sealed data. + BigBackerRegistrarSealCouples: str = '-0X' # Big Backer Registrar Seal Couple(s), brid+dig of sealed data. + ESSRPayloadGroup: str = '-Z' # ESSR Payload Group. + BigESSRPayloadGroup: str = '-0Z' # Big ESSR Payload Group. + KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Stack CESR Protocol Genus Version (Universal) + - ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. - WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. - NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. - TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. - FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. - TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. - SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegator/issuer/transaction event - TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. - SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event - SadPathSig: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content - SadPathSigGroup: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples - PathedMaterialQuadlets: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) - AttachedMaterialQuadlets: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) - BigAttachedMaterialQuadlets: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) - KERIProtocolStack: str = '--AAA' # KERI ACDC Protocol Stack CESR Version def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" @@ -164,7 +201,7 @@ def __iter__(self): Tagage_2_0 = namedtuple("Tagage_2_0", list(Codict2), defaults=list(Codict2)) Tags_2_0 = Tagage_2_0() # uses defaults -CodictAll = Codict1 | Codict2 +CodictAll = Codict2 | Codict1 AllTagage = namedtuple("AllTagage", list(CodictAll), defaults=list(CodictAll)) AllTags = AllTagage() # uses defaults @@ -234,39 +271,75 @@ class Counter: { Vrsn_1_0: \ { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), }, Vrsn_2_0: \ { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0A': Sizage(hs=3, ss=5, fs=8, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0B': Sizage(hs=3, ss=5, fs=8, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0C': Sizage(hs=3, ss=5, fs=8, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0D': Sizage(hs=3, ss=5, fs=8, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0E': Sizage(hs=3, ss=5, fs=8, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0F': Sizage(hs=3, ss=5, fs=8, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0G': Sizage(hs=3, ss=5, fs=8, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0H': Sizage(hs=3, ss=5, fs=8, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0I': Sizage(hs=3, ss=5, fs=8, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0J': Sizage(hs=3, ss=5, fs=8, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0K': Sizage(hs=3, ss=5, fs=8, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), + '-M': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0M': Sizage(hs=3, ss=5, fs=8, ls=0), + '-N': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0N': Sizage(hs=3, ss=5, fs=8, ls=0), + '-O': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0O': Sizage(hs=3, ss=5, fs=8, ls=0), + '-P': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0P': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Q': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0Q': Sizage(hs=3, ss=5, fs=8, ls=0), + '-R': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0R': Sizage(hs=3, ss=5, fs=8, ls=0), + '-S': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0S': Sizage(hs=3, ss=5, fs=8, ls=0), + '-T': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0T': Sizage(hs=3, ss=5, fs=8, ls=0), + '-U': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0U': Sizage(hs=3, ss=5, fs=8, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '-W': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0W': Sizage(hs=3, ss=5, fs=8, ls=0), + '-X': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0X': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Y': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0Y': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), }, } diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index dda7479fa..e501c0844 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1429,7 +1429,7 @@ def messagize(serder, *, sigers=None, seal=None, wigers=None, cigars=None, if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - msg.extend(Counter(code=CtrDex.AttachedMaterialQuadlets, + msg.extend(Counter(code=CtrDex.AttachmentGroup, count=(len(atc) // 4)).qb64b) msg.extend(atc) @@ -1461,7 +1461,7 @@ def proofize(sadtsgs=None, *, sadsigers=None, sadcigars=None, pipelined=False): count = 0 for (pather, sigers) in sadsigers: count += 1 - atc.extend(coring.Counter(coring.CtrDex.SadPathSig, count=1).qb64b) + atc.extend(coring.Counter(coring.CtrDex.SadPathSigGroups, count=1).qb64b) atc.extend(pather.qb64b) atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) @@ -1470,7 +1470,7 @@ def proofize(sadtsgs=None, *, sadsigers=None, sadcigars=None, pipelined=False): for (pather, prefixer, seqner, saider, sigers) in sadtsgs: count += 1 - atc.extend(coring.Counter(coring.CtrDex.SadPathSig, count=1).qb64b) + atc.extend(coring.Counter(coring.CtrDex.SadPathSigGroups, count=1).qb64b) atc.extend(pather.qb64b) atc.extend(coring.Counter(coring.CtrDex.TransIdxSigGroups, count=1).qb64b) @@ -1484,7 +1484,7 @@ def proofize(sadtsgs=None, *, sadsigers=None, sadcigars=None, pipelined=False): for (pather, cigars) in sadcigars: count += 1 - atc.extend(coring.Counter(coring.CtrDex.SadPathSig, count=1).qb64b) + atc.extend(coring.Counter(coring.CtrDex.SadPathSigGroups, count=1).qb64b) atc.extend(pather.qb64b) atc.extend(coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=len(sadcigars)).qb64b) @@ -1501,12 +1501,12 @@ def proofize(sadtsgs=None, *, sadsigers=None, sadcigars=None, pipelined=False): if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - msg.extend(coring.Counter(code=coring.CtrDex.AttachedMaterialQuadlets, + msg.extend(coring.Counter(code=coring.CtrDex.AttachmentGroup, count=(len(atc) // 4)).qb64b) if count > 1: root = coring.Pather(bext="-") - msg.extend(coring.Counter(code=coring.CtrDex.SadPathSigGroup, count=count).qb64b) + msg.extend(coring.Counter(code=coring.CtrDex.RootSadPathSigGroups, count=count).qb64b) msg.extend(root.qb64b) msg.extend(atc) diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index 3a72354ea..691b8accf 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -139,7 +139,7 @@ def _sadPathSigGroup(self, ctr, ims, root=None, cold=Colds.txt, pipelined=False) Returns: """ - if ctr.code != CtrDex.SadPathSig: + if ctr.code != CtrDex.SadPathSigGroups: raise kering.UnexpectedCountCodeError("Wrong " "count code={}.Expected code={}." "".format(ctr.code, CtrDex.ControllerIdxSigs)) @@ -732,7 +732,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, cold = sniff(ims) # expect counter at front of attachments if cold != Colds.msg: # not new message so process attachments ctr = yield from self._extractor(ims=ims, klas=Counter, cold=cold) - if ctr.code == CtrDex.AttachedMaterialQuadlets: # pipeline ctr? + if ctr.code == CtrDex.AttachmentGroup: # pipeline ctr? pipelined = True # compute pipelined attached group size based on txt or bny pags = ctr.count * 4 if cold == Colds.txt else ctr.count * 3 @@ -901,7 +901,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) ssts.append((prefixer, seqner, saider)) - elif ctr.code == CtrDex.SadPathSigGroup: + elif ctr.code == CtrDex.SadPathSigGroups: path = yield from self._extractor(ims, klas=Pather, cold=cold, @@ -921,7 +921,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, else: sadcigs.append(sigs) - elif ctr.code == CtrDex.SadPathSig: + elif ctr.code == CtrDex.SadPathSigGroups: for code, sigs in self._sadPathSigGroup(ctr=ctr, ims=ims, cold=cold, @@ -931,7 +931,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, else: sadcigs.append(sigs) - elif ctr.code == CtrDex.PathedMaterialQuadlets: # pathed ctr? + elif ctr.code == CtrDex.PathedMaterialGroup: # pathed ctr? # compute pipelined attached group size based on txt or bny pags = ctr.count * 4 if cold == Colds.txt else ctr.count * 3 while len(ims) < pags: # wait until rx full pipelned group diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 99f52c871..d8bf0039e 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1421,7 +1421,7 @@ def cloneEvtMsg(self, pre, fn, dig): if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - pcnt = coring.Counter(code=coring.CtrDex.AttachedMaterialQuadlets, + pcnt = coring.Counter(code=coring.CtrDex.AttachmentGroup, count=(len(atc) // 4)).qb64b msg.extend(pcnt) msg.extend(atc) diff --git a/src/keri/peer/exchanging.py b/src/keri/peer/exchanging.py index fc1b3c234..570e1492d 100644 --- a/src/keri/peer/exchanging.py +++ b/src/keri/peer/exchanging.py @@ -318,7 +318,7 @@ def exchange(route, pather = coring.Pather(path=["e", label]) pathed.extend(pather.qb64b) pathed.extend(atc) - end.extend(coring.Counter(code=coring.CtrDex.PathedMaterialQuadlets, + end.extend(coring.Counter(code=coring.CtrDex.PathedMaterialGroup, count=(len(pathed) // 4)).qb64b) end.extend(pathed) @@ -411,7 +411,7 @@ def serializeMessage(hby, said, pipelined=False): # Smash the pathed components on the end for p in hby.db.epath.get(keys=(exn.said,)): - atc.extend(coring.Counter(code=coring.CtrDex.PathedMaterialQuadlets, + atc.extend(coring.Counter(code=coring.CtrDex.PathedMaterialGroup, count=(len(p) // 4)).qb64b) atc.extend(p.encode("utf-8")) @@ -421,7 +421,7 @@ def serializeMessage(hby, said, pipelined=False): if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - msg.extend(coring.Counter(code=coring.CtrDex.AttachedMaterialQuadlets, + msg.extend(coring.Counter(code=coring.CtrDex.AttachmentGroup, count=(len(atc) // 4)).qb64b) msg.extend(atc) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 96eec603a..91b11317e 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -440,7 +440,7 @@ def cloneCreds(self, saids, db): ) ctr = coring.Counter(qb64b=iss, strip=True) - if ctr.code == coring.CtrDex.AttachedMaterialQuadlets: + if ctr.code == coring.CtrDex.AttachmentGroup: ctr = coring.Counter(qb64b=iss, strip=True) if ctr.code == coring.CtrDex.SealSourceCouples: @@ -541,7 +541,7 @@ def cloneTvt(self, pre, dig): if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - pcnt = coring.Counter(code=coring.CtrDex.AttachedMaterialQuadlets, + pcnt = coring.Counter(code=coring.CtrDex.AttachmentGroup, count=(len(atc) // 4)).qb64b msg.extend(pcnt) msg.extend(atc) @@ -1009,7 +1009,7 @@ def messagize(creder, proof): if len(proof) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(proof))) - craw.extend(coring.Counter(code=coring.CtrDex.AttachedMaterialQuadlets, + craw.extend(coring.Counter(code=coring.CtrDex.AttachmentGroup, count=(len(proof) // 4)).qb64b) craw.extend(proof) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 41d089606..fd65daf67 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -2159,41 +2159,14 @@ def test_counter(): 'SealSourceCouples': '-G', 'TransLastIdxSigGroups': '-H', 'SealSourceTriples': '-I', - 'SadPathSig': '-J', - 'SadPathSigGroup': '-K', - 'PathedMaterialQuadlets': '-L', - 'AttachedMaterialQuadlets': '-V', - 'BigAttachedMaterialQuadlets': '-0V', - 'KERIProtocolStack': '--AAA', + 'SadPathSigGroups': '-J', + 'RootSadPathSigGroups': '-K', + 'PathedMaterialGroup': '-L', + 'AttachmentGroup': '-V', + 'BigAttachmentGroup': '-0V', + 'KERIACDCGenusVersion': '--AAA', } - # new version 2 - #assert dataclasses.asdict(CtrDex) == { - #'ControllerIdxSigs': '-A', - #'WitnessIdxSigs': '-B', - #'NonTransReceiptCouples': '-C', - #'TransReceiptQuadruples': '-D', - #'FirstSeenReplayCouples': '-E', - #'TransIdxSigGroups': '-F', - #'SealSourceCouples': '-G', - #'TransLastIdxSigGroups': '-H', - #'SealSourceTriples': '-I', - #'SadPathSig': '-J', - #'SadPathSigGroup': '-K', - #'PathedMaterialQuadlets': '-L', - #'MessageDataGroups': '-U', - #'AttachedMaterialQuadlets': '-V', - #'MessageDataMaterialQuadlets': '-W', - #'CombinedMaterialQuadlets': '-X', - #'MaterialGroups': '-Y', - #'MaterialQuadlets': '-Z', - #'BigMessageDataGroups': '-0U', - #'BigAttachedMaterialQuadlets': '-0V', - #'BigMessageDataMaterialQuadlets': '-0W', - #'BigCombinedMaterialQuadlets': '-0X', - #'BigMaterialGroups': '-0Y', - #'BigMaterialQuadlets': '-0Z' - #} assert CtrDex.ControllerIdxSigs == '-A' assert CtrDex.WitnessIdxSigs == '-B' @@ -2345,34 +2318,34 @@ def test_counter(): # test with big codes index=1024 count = 1024 - qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) + qsc = CtrDex.BigAttachmentGroup + intToB64(count, l=5) assert qsc == '-0VAAAQA' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(code=CtrDex.BigAttachedMaterialQuadlets, count=count) - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + counter = Counter(code=CtrDex.BigAttachmentGroup, count=count) + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 counter = Counter(qb64b=qscb) # test with bytes not str - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 counter = Counter(qb64=qsc) # test with str not bytes - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 counter = Counter(qb2=qscb2) # test with qb2 - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -2455,14 +2428,14 @@ def test_counter(): # test with big codes index=1024 count = 1024 - qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) + qsc = CtrDex.BigAttachmentGroup + intToB64(count, l=5) assert qsc == '-0VAAAQA' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) ims = bytearray(qscb) counter = Counter(qb64b=ims, strip=True) # test with bytes not str - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -2471,7 +2444,7 @@ def test_counter(): ims = bytearray(qscb2) counter = Counter(qb2=ims, strip=True) # test with qb2 - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -2484,13 +2457,13 @@ def test_counter(): version = intToB64(verint, l=3) assert version == 'AAA' assert verint == b64ToInt(version) - qsc = CtrDex.KERIProtocolStack + version + qsc = CtrDex.KERIACDCGenusVersion + version assert qsc == '--AAAAAA' # keri Cesr version 0.0.0 qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(code=CtrDex.KERIProtocolStack, count=verint) - assert counter.code == CtrDex.KERIProtocolStack + counter = Counter(code=CtrDex.KERIACDCGenusVersion, count=verint) + assert counter.code == CtrDex.KERIACDCGenusVersion assert counter.count == verint assert counter.countToB64(l=3) == version assert counter.countToB64() == version # default length @@ -2498,8 +2471,8 @@ def test_counter(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 - counter = Counter(code=CtrDex.KERIProtocolStack, countB64=version) - assert counter.code == CtrDex.KERIProtocolStack + counter = Counter(code=CtrDex.KERIACDCGenusVersion, countB64=version) + assert counter.code == CtrDex.KERIACDCGenusVersion assert counter.count == verint assert counter.countToB64(l=3) == version assert counter.countToB64() == version # default length diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 1544d2a75..a55281481 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -181,31 +181,67 @@ def test_codexes_tags(): 'SealSourceCouples': '-G', 'TransLastIdxSigGroups': '-H', 'SealSourceTriples': '-I', - 'SadPathSig': '-J', - 'SadPathSigGroup': '-K', - 'PathedMaterialQuadlets': '-L', - 'AttachedMaterialQuadlets': '-V', - 'BigAttachedMaterialQuadlets': '-0V', - 'KERIProtocolStack': '--AAA', + 'SadPathSigGroups': '-J', + 'RootSadPathSigGroups': '-K', + 'PathedMaterialGroup': '-L', + 'AttachmentGroup': '-V', + 'BigAttachmentGroup': '-0V', + 'KERIACDCGenusVersion': '--AAA' } + assert asdict(counting.CtrDex_2_0) == \ { - 'ControllerIdxSigs': '-A', - 'WitnessIdxSigs': '-B', - 'NonTransReceiptCouples': '-C', - 'TransReceiptQuadruples': '-D', - 'FirstSeenReplayCouples': '-E', - 'TransIdxSigGroups': '-F', - 'SealSourceCouples': '-G', - 'TransLastIdxSigGroups': '-H', - 'SealSourceTriples': '-I', - 'SadPathSig': '-J', - 'SadPathSigGroup': '-K', - 'PathedMaterialQuadlets': '-L', - 'AttachedMaterialQuadlets': '-V', - 'BigAttachedMaterialQuadlets': '-0V', - 'KERIProtocolStack': '--AAA', + 'GenericGroup': '-A', + 'BigGenericGroup': '-0A', + 'MessageGroup': '-B', + 'BigMessageGroup': '-0B', + 'AttachmentGroup': '-C', + 'BigAttachmentGroup': '-0C', + 'DatagramSegmentGroup': '-D', + 'BigDatagramSegmentGroup': '-0D', + 'ESSRWrapperGroup': '-E', + 'BigESSRWrapperGroup': '-0E', + 'FixedMessageBodyGroup': '-F', + 'BigFixedMessageBodyGroup': '-0F', + 'MapMessageBodyGroup': '-G', + 'BigMapMessageBodyGroup': '-0G', + 'GenericMapGroup': '-H', + 'BigGenericMapGroup': '-0H', + 'GenericListGroup': '-L', + 'BigGenericListGroup': '-0L', + 'ControllerIdxSigs': '-J', + 'BigControllerIdxSigs': '-0J', + 'WitnessIdxSigs': '-K', + 'BigWitnessIdxSigs': '-0K', + 'NonTransReceiptCouples': '-L', + 'BigNonTransReceiptCouples': '-0L', + 'TransReceiptQuadruples': '-M', + 'BigTransReceiptQuadruples': '-0M', + 'FirstSeenReplayCouples': '-N', + 'BigFirstSeenReplayCouples': '-0N', + 'TransIdxSigGroups': '-0O', + 'TransLastIdxSigGroups': '-P', + 'BigTransLastIdxSigGroups': '-0P', + 'SealSourceCouples': '-Q', + 'BigSealSourceCouples': '-0Q', + 'SealSourceTriples': '-R', + 'BigSealSourceTriples': '-0R', + 'PathedMaterialGroup': '-S', + 'BigPathedMaterialGroup': '-0S', + 'SadPathSigGroups': '-T', + 'BigSadPathSigGroups': '-0T', + 'RootSadPathSigGroups': '-U', + 'BigRootSadPathSigGroups': '-0U', + 'DigestSealSingles': '-V', + 'BigDigestSealSingles': '-0V', + 'MerkleRootSealSingles': '-W', + 'BigMerkleRootSealSingles': '-0W', + 'BackerRegistrarSealCouples': '-X', + 'BigBackerRegistrarSealCouples': '-0X', + 'ESSRPayloadGroup': '-Z', + 'BigESSRPayloadGroup': '-0Z', + 'KERIACDCGenusVersion': '--AAA' } assert counting.Tags_1_0._asdict() == \ @@ -219,56 +255,128 @@ def test_codexes_tags(): 'SealSourceCouples': 'SealSourceCouples', 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', 'SealSourceTriples': 'SealSourceTriples', - 'SadPathSig': 'SadPathSig', - 'SadPathSigGroup': 'SadPathSigGroup', - 'PathedMaterialQuadlets': 'PathedMaterialQuadlets', - 'AttachedMaterialQuadlets': 'AttachedMaterialQuadlets', - 'BigAttachedMaterialQuadlets': 'BigAttachedMaterialQuadlets', - 'KERIProtocolStack': 'KERIProtocolStack' + 'SadPathSigGroups': 'SadPathSigGroups', + 'RootSadPathSigGroups': 'RootSadPathSigGroups', + 'PathedMaterialGroup': 'PathedMaterialGroup', + 'AttachmentGroup': 'AttachmentGroup', + 'BigAttachmentGroup': 'BigAttachmentGroup', + 'KERIACDCGenusVersion': 'KERIACDCGenusVersion' } assert counting.Tags_1_0.ControllerIdxSigs == 'ControllerIdxSigs' assert counting.Tags_2_0._asdict() == \ { + 'GenericGroup': 'GenericGroup', + 'BigGenericGroup': 'BigGenericGroup', + 'MessageGroup': 'MessageGroup', + 'BigMessageGroup': 'BigMessageGroup', + 'AttachmentGroup': 'AttachmentGroup', + 'BigAttachmentGroup': 'BigAttachmentGroup', + 'DatagramSegmentGroup': 'DatagramSegmentGroup', + 'BigDatagramSegmentGroup': 'BigDatagramSegmentGroup', + 'ESSRWrapperGroup': 'ESSRWrapperGroup', + 'BigESSRWrapperGroup': 'BigESSRWrapperGroup', + 'FixedMessageBodyGroup': 'FixedMessageBodyGroup', + 'BigFixedMessageBodyGroup': 'BigFixedMessageBodyGroup', + 'MapMessageBodyGroup': 'MapMessageBodyGroup', + 'BigMapMessageBodyGroup': 'BigMapMessageBodyGroup', + 'GenericMapGroup': 'GenericMapGroup', + 'BigGenericMapGroup': 'BigGenericMapGroup', + 'GenericListGroup': 'GenericListGroup', + 'BigGenericListGroup': 'BigGenericListGroup', 'ControllerIdxSigs': 'ControllerIdxSigs', + 'BigControllerIdxSigs': 'BigControllerIdxSigs', 'WitnessIdxSigs': 'WitnessIdxSigs', + 'BigWitnessIdxSigs': 'BigWitnessIdxSigs', 'NonTransReceiptCouples': 'NonTransReceiptCouples', + 'BigNonTransReceiptCouples': 'BigNonTransReceiptCouples', 'TransReceiptQuadruples': 'TransReceiptQuadruples', + 'BigTransReceiptQuadruples': 'BigTransReceiptQuadruples', 'FirstSeenReplayCouples': 'FirstSeenReplayCouples', + 'BigFirstSeenReplayCouples': 'BigFirstSeenReplayCouples', 'TransIdxSigGroups': 'TransIdxSigGroups', - 'SealSourceCouples': 'SealSourceCouples', 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', + 'BigTransLastIdxSigGroups': 'BigTransLastIdxSigGroups', + 'SealSourceCouples': 'SealSourceCouples', + 'BigSealSourceCouples': 'BigSealSourceCouples', 'SealSourceTriples': 'SealSourceTriples', - 'SadPathSig': 'SadPathSig', - 'SadPathSigGroup': 'SadPathSigGroup', - 'PathedMaterialQuadlets': 'PathedMaterialQuadlets', - 'AttachedMaterialQuadlets': 'AttachedMaterialQuadlets', - 'BigAttachedMaterialQuadlets': 'BigAttachedMaterialQuadlets', - 'KERIProtocolStack': 'KERIProtocolStack' + 'BigSealSourceTriples': 'BigSealSourceTriples', + 'PathedMaterialGroup': 'PathedMaterialGroup', + 'BigPathedMaterialGroup': 'BigPathedMaterialGroup', + 'SadPathSigGroups': 'SadPathSigGroups', + 'BigSadPathSigGroups': 'BigSadPathSigGroups', + 'RootSadPathSigGroups': 'RootSadPathSigGroups', + 'BigRootSadPathSigGroups': 'BigRootSadPathSigGroups', + 'DigestSealSingles': 'DigestSealSingles', + 'BigDigestSealSingles': 'BigDigestSealSingles', + 'MerkleRootSealSingles': 'MerkleRootSealSingles', + 'BigMerkleRootSealSingles': 'BigMerkleRootSealSingles', + 'BackerRegistrarSealCouples': 'BackerRegistrarSealCouples', + 'BigBackerRegistrarSealCouples': 'BigBackerRegistrarSealCouples', + 'ESSRPayloadGroup': 'ESSRPayloadGroup', + 'BigESSRPayloadGroup': 'BigESSRPayloadGroup', + 'KERIACDCGenusVersion': 'KERIACDCGenusVersion' } assert counting.Tags_2_0.ControllerIdxSigs == 'ControllerIdxSigs' assert counting.AllTags._asdict() == \ { + 'GenericGroup': 'GenericGroup', + 'BigGenericGroup': 'BigGenericGroup', + 'MessageGroup': 'MessageGroup', + 'BigMessageGroup': 'BigMessageGroup', + 'AttachmentGroup': 'AttachmentGroup', + 'BigAttachmentGroup': 'BigAttachmentGroup', + 'DatagramSegmentGroup': 'DatagramSegmentGroup', + 'BigDatagramSegmentGroup': 'BigDatagramSegmentGroup', + 'ESSRWrapperGroup': 'ESSRWrapperGroup', + 'BigESSRWrapperGroup': 'BigESSRWrapperGroup', + 'FixedMessageBodyGroup': 'FixedMessageBodyGroup', + 'BigFixedMessageBodyGroup': 'BigFixedMessageBodyGroup', + 'MapMessageBodyGroup': 'MapMessageBodyGroup', + 'BigMapMessageBodyGroup': 'BigMapMessageBodyGroup', + 'GenericMapGroup': 'GenericMapGroup', + 'BigGenericMapGroup': 'BigGenericMapGroup', + 'GenericListGroup': 'GenericListGroup', + 'BigGenericListGroup': 'BigGenericListGroup', 'ControllerIdxSigs': 'ControllerIdxSigs', + 'BigControllerIdxSigs': 'BigControllerIdxSigs', 'WitnessIdxSigs': 'WitnessIdxSigs', + 'BigWitnessIdxSigs': 'BigWitnessIdxSigs', 'NonTransReceiptCouples': 'NonTransReceiptCouples', + 'BigNonTransReceiptCouples': 'BigNonTransReceiptCouples', 'TransReceiptQuadruples': 'TransReceiptQuadruples', + 'BigTransReceiptQuadruples': 'BigTransReceiptQuadruples', 'FirstSeenReplayCouples': 'FirstSeenReplayCouples', + 'BigFirstSeenReplayCouples': 'BigFirstSeenReplayCouples', 'TransIdxSigGroups': 'TransIdxSigGroups', - 'SealSourceCouples': 'SealSourceCouples', 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', + 'BigTransLastIdxSigGroups': 'BigTransLastIdxSigGroups', + 'SealSourceCouples': 'SealSourceCouples', + 'BigSealSourceCouples': 'BigSealSourceCouples', 'SealSourceTriples': 'SealSourceTriples', - 'SadPathSig': 'SadPathSig', - 'SadPathSigGroup': 'SadPathSigGroup', - 'PathedMaterialQuadlets': 'PathedMaterialQuadlets', - 'AttachedMaterialQuadlets': 'AttachedMaterialQuadlets', - 'BigAttachedMaterialQuadlets': 'BigAttachedMaterialQuadlets', - 'KERIProtocolStack': 'KERIProtocolStack' + 'BigSealSourceTriples': 'BigSealSourceTriples', + 'PathedMaterialGroup': 'PathedMaterialGroup', + 'BigPathedMaterialGroup': 'BigPathedMaterialGroup', + 'SadPathSigGroups': 'SadPathSigGroups', + 'BigSadPathSigGroups': 'BigSadPathSigGroups', + 'RootSadPathSigGroups': 'RootSadPathSigGroups', + 'BigRootSadPathSigGroups': 'BigRootSadPathSigGroups', + 'DigestSealSingles': 'DigestSealSingles', + 'BigDigestSealSingles': 'BigDigestSealSingles', + 'MerkleRootSealSingles': 'MerkleRootSealSingles', + 'BigMerkleRootSealSingles': 'BigMerkleRootSealSingles', + 'BackerRegistrarSealCouples': 'BackerRegistrarSealCouples', + 'BigBackerRegistrarSealCouples': 'BigBackerRegistrarSealCouples', + 'ESSRPayloadGroup': 'ESSRPayloadGroup', + 'BigESSRPayloadGroup': 'BigESSRPayloadGroup', + 'KERIACDCGenusVersion': 'KERIACDCGenusVersion' } + assert counting.AllTags.ControllerIdxSigs == 'ControllerIdxSigs' + """End Test""" @@ -276,6 +384,8 @@ def test_counter(): """ Test Counter class """ + # test class attributes + assert Counter.Codes == \ { counting.Vrsn_1_0: counting.CtrDex_1_0, @@ -325,19 +435,55 @@ def test_counter(): counting.Vrsn_2_0: \ { '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0A': Sizage(hs=3, ss=5, fs=8, ls=0), '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0B': Sizage(hs=3, ss=5, fs=8, ls=0), '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0C': Sizage(hs=3, ss=5, fs=8, ls=0), '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0D': Sizage(hs=3, ss=5, fs=8, ls=0), '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0E': Sizage(hs=3, ss=5, fs=8, ls=0), '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0F': Sizage(hs=3, ss=5, fs=8, ls=0), '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0G': Sizage(hs=3, ss=5, fs=8, ls=0), '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0H': Sizage(hs=3, ss=5, fs=8, ls=0), '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0I': Sizage(hs=3, ss=5, fs=8, ls=0), '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0J': Sizage(hs=3, ss=5, fs=8, ls=0), '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0K': Sizage(hs=3, ss=5, fs=8, ls=0), '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), + '-M': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0M': Sizage(hs=3, ss=5, fs=8, ls=0), + '-N': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0N': Sizage(hs=3, ss=5, fs=8, ls=0), + '-O': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0O': Sizage(hs=3, ss=5, fs=8, ls=0), + '-P': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0P': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Q': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0Q': Sizage(hs=3, ss=5, fs=8, ls=0), + '-R': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0R': Sizage(hs=3, ss=5, fs=8, ls=0), + '-S': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0S': Sizage(hs=3, ss=5, fs=8, ls=0), + '-T': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0T': Sizage(hs=3, ss=5, fs=8, ls=0), + '-U': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0U': Sizage(hs=3, ss=5, fs=8, ls=0), '-V': Sizage(hs=2, ss=2, fs=4, ls=0), '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '-W': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0W': Sizage(hs=3, ss=5, fs=8, ls=0), + '-X': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0X': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Y': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0Y': Sizage(hs=3, ss=5, fs=8, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) } } @@ -432,21 +578,21 @@ def test_counter_v1(): """ test Counter instances for verision 1.0 code tables """ + # version 1_0 tests default version is Version + assert Version == Vrsn_1_0 + CtrDex = Counter.Codes[Version] # set CtrDex to Vrsn_1_0 + # test Counter instances with pytest.raises(kering.EmptyMaterialError): counter = Counter() # create code manually count = 1 - qsc = counting.CtrDex_1_0.ControllerIdxSigs + intToB64(count, l=2) + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) assert qsc == '-AAB' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - # version 1_0 tests default version is Version - assert Version == Vrsn_1_0 - CtrDex = Counter.Codes[Version] # set CtrDex to Vrsn_1_0 - counter = Counter(tag="ControllerIdxSigs", count=count) assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count @@ -557,34 +703,34 @@ def test_counter_v1(): # test with big codes index=1024 count = 1024 - qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) + qsc = CtrDex.BigAttachmentGroup + intToB64(count, l=5) assert qsc == '-0VAAAQA' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(code=CtrDex.BigAttachedMaterialQuadlets, count=count) - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + counter = Counter(code=CtrDex.BigAttachmentGroup, count=count) + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 counter = Counter(qb64b=qscb) # test with bytes not str - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 counter = Counter(qb64=qsc) # test with str not bytes - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 counter = Counter(qb2=qscb2) # test with qb2 - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -667,14 +813,14 @@ def test_counter_v1(): # test with big codes index=1024 count = 1024 - qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) + qsc = CtrDex.BigAttachmentGroup + intToB64(count, l=5) assert qsc == '-0VAAAQA' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) ims = bytearray(qscb) counter = Counter(qb64b=ims, strip=True) # test with bytes not str - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -683,7 +829,7 @@ def test_counter_v1(): ims = bytearray(qscb2) counter = Counter(qb2=ims, strip=True) # test with qb2 - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -696,13 +842,13 @@ def test_counter_v1(): genver = intToB64(genverint, l=3) assert genver == 'AAA' assert genverint == b64ToInt(genver) - qsc = CtrDex.KERIProtocolStack + genver + qsc = CtrDex.KERIACDCGenusVersion + genver assert qsc == '--AAAAAA' # keri Cesr version 0.0.0 qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(code=CtrDex.KERIProtocolStack, count=genverint) - assert counter.code == CtrDex.KERIProtocolStack + counter = Counter(code=CtrDex.KERIACDCGenusVersion, count=genverint) + assert counter.code == CtrDex.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length @@ -710,8 +856,8 @@ def test_counter_v1(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 - counter = Counter(code=CtrDex.KERIProtocolStack, countB64=genver) - assert counter.code == CtrDex.KERIProtocolStack + counter = Counter(code=CtrDex.KERIACDCGenusVersion, countB64=genver) + assert counter.code == CtrDex.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length @@ -725,20 +871,20 @@ def test_counter_v2(): """ test Counter instances for verision 2.0 code tables """ + # version 2_0 tests default version is Version + #assert Version == Vrsn_2_0 + CtrDex = Counter.Codes[Vrsn_2_0] # set CtrDex to Vrsn_2_0 + # test Counter instances with pytest.raises(kering.EmptyMaterialError): counter = Counter() # create code manually count = 1 - qsc = counting.CtrDex_1_0.ControllerIdxSigs + intToB64(count, l=2) - assert qsc == '-AAB' + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-JAB' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - # version 2_0 tests default version is Version - #assert Version == Vrsn_2_0 - - CtrDex = Counter.Codes[Vrsn_2_0] # set CtrDex to Vrsn_2_0 # default count = 1 counter = Counter(code=CtrDex.ControllerIdxSigs, version=Vrsn_2_0) @@ -793,7 +939,7 @@ def test_counter_v2(): # test with non-zero count=5 count = 5 qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) - assert qsc == '-AAF' + assert qsc == '-JAF' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) @@ -850,41 +996,6 @@ def test_counter_v2(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 - # test with big codes index=1024 - count = 1024 - qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) - assert qsc == '-0VAAAQA' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.BigAttachedMaterialQuadlets, count=count, version=Vrsn_2_0) - assert counter.code == CtrDex.BigAttachedMaterialQuadlets - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str - assert counter.code == CtrDex.BigAttachedMaterialQuadlets - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64=qsc, version=Vrsn_2_0) # test with str not bytes - assert counter.code == CtrDex.BigAttachedMaterialQuadlets - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb2=qscb2, version=Vrsn_2_0) # test with qb2 - assert counter.code == CtrDex.BigAttachedMaterialQuadlets - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - # Test ._bexfil counter = Counter(qb64=qsc, version=Vrsn_2_0) code = counter.code @@ -904,7 +1015,7 @@ def test_counter_v2(): # create code manually count = 1 qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) - assert qsc == '-AAB' + assert qsc == '-JAB' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) @@ -962,14 +1073,49 @@ def test_counter_v2(): # test with big codes index=1024 count = 1024 - qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) - assert qsc == '-0VAAAQA' + qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) + assert qsc == '-0AAAAQA' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.BigGenericGroup, count=count, version=Vrsn_2_0) + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64=qsc, version=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb2=qscb2, version=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # test ims with big codes index=1024 + count = 1024 + qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) + assert qsc == '-0AAAAQA' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) ims = bytearray(qscb) counter = Counter(qb64b=ims, strip=True, version=Vrsn_2_0) # test with bytes not str - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -978,7 +1124,7 @@ def test_counter_v2(): ims = bytearray(qscb2) counter = Counter(qb2=ims, strip=True, version=Vrsn_2_0) # test with qb2 - assert counter.code == CtrDex.BigAttachedMaterialQuadlets + assert counter.code == CtrDex.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -991,13 +1137,13 @@ def test_counter_v2(): genver = intToB64(genverint, l=3) assert genver == 'AAA' assert genverint == b64ToInt(genver) - qsc = CtrDex.KERIProtocolStack + genver + qsc = CtrDex.KERIACDCGenusVersion + genver assert qsc == '--AAAAAA' # keri Cesr version 0.0.0 qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(code=CtrDex.KERIProtocolStack, count=genverint, version=Vrsn_2_0) - assert counter.code == CtrDex.KERIProtocolStack + counter = Counter(code=CtrDex.KERIACDCGenusVersion, count=genverint, version=Vrsn_2_0) + assert counter.code == CtrDex.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length @@ -1005,8 +1151,8 @@ def test_counter_v2(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 - counter = Counter(code=CtrDex.KERIProtocolStack, countB64=genver, version=Vrsn_2_0) - assert counter.code == CtrDex.KERIProtocolStack + counter = Counter(code=CtrDex.KERIACDCGenusVersion, countB64=genver, version=Vrsn_2_0) + assert counter.code == CtrDex.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length diff --git a/tests/core/test_replay.py b/tests/core/test_replay.py index e5801dcb6..6a9a1a172 100644 --- a/tests/core/test_replay.py +++ b/tests/core/test_replay.py @@ -337,7 +337,7 @@ def test_replay(): assert len(msg) == 1076 counter = coring.Counter(qb64b=msg) # attachment length quadlets counter - assert counter.code == coring.CtrDex.AttachedMaterialQuadlets + assert counter.code == coring.CtrDex.AttachmentGroup assert counter.count == (len(msg) - len(counter.qb64b)) // 4 == 268 del msg[:len(counter.qb64b)] assert len(msg) == 1072 == 268 * 4 diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index fbcc9be99..b46947e22 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -70,7 +70,7 @@ def test_proving(mockHelpingNowIso8601): proof = msg[creder.size:] ctr = Counter(qb64b=proof, strip=True) - assert ctr.code == CtrDex.AttachedMaterialQuadlets + assert ctr.code == CtrDex.AttachmentGroup assert ctr.count == 52 pags = ctr.count * 4 From 6f1ec09b2120ab9c434ac0e4f077462bad0f820e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 20 Mar 2024 18:47:40 -0600 Subject: [PATCH 057/418] fixed bug in code promotion logic added tests --- src/keri/core/counting.py | 2 +- tests/core/test_counting.py | 86 +++++++++++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index b46ed0803..893b4c1c4 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -410,7 +410,7 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, raise kering.InvalidVarIndexError(f"Invalid {ss=} " f"for {code=}.") # dynamically promote code based on count - if code[0] != '0' and count > (64 ** 2 - 1): # small code but large count + if code[1] != '0' and count > (64 ** 2 - 1): # small code but large count # elevate code due to large count code = f"-0{code[1]}" # promote hard ss = 5 diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index a55281481..4d54a0664 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -1071,7 +1071,7 @@ def test_counter_v2(): with pytest.raises(TypeError): counter = Counter(qb2=ims, strip=True, version=Vrsn_2_0) - # test with big codes index=1024 + # test with big codes count=1024 count = 1024 qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) assert qsc == '-0AAAAQA' @@ -1106,7 +1106,7 @@ def test_counter_v2(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 - # test ims with big codes index=1024 + # test ims with big codes count=1024 count = 1024 qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) assert qsc == '-0AAAAQA' @@ -1131,8 +1131,88 @@ def test_counter_v2(): assert counter.qb2 == qscb2 assert not ims + # test with big codes count=8193 + count = 8193 + qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) + assert qsc == '-0AAACAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.BigGenericGroup, count=count, version=Vrsn_2_0) + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64=qsc, version=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb2=qscb2, version=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # test ims with big codes count=8193 + count = 8193 + qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) + assert qsc == '-0AAACAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + ims = bytearray(qscb) + counter = Counter(qb64b=ims, strip=True, version=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert not ims + + ims = bytearray(qscb2) + counter = Counter(qb2=ims, strip=True, version=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert not ims + + # test with promotion from small to big codes with count=8193 + count = 8193 + qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) + assert qsc == '-0AAACAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.GenericGroup, count=count, version=Vrsn_2_0) + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(tag=AllTags.GenericGroup, count=count, version=Vrsn_2_0) + assert counter.code == CtrDex.BigGenericGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + # test protocol genus with CESR version - # test with big codes index=1024 genverint = 0 genver = intToB64(genverint, l=3) assert genver == 'AAA' From 4516e12f3437a39efc9b84368ec96f3acec690f4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 21 Mar 2024 13:43:44 -0600 Subject: [PATCH 058/418] Refactor clean up Counter code --- src/keri/core/counting.py | 40 ++++++++++++++++++-------------------- src/keri/core/serdering.py | 37 +++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 893b4c1c4..d4015d87d 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -609,10 +609,10 @@ def _infil(self): count = self.count # index value int used for soft hs, ss, fs, ls = self._sizes[code] - cs = hs + ss # both hard + soft size - if hs < 2 or fs != cs or cs % 4: # hs >=2 fs must be bs and multiple of 4 for count codes - raise kering.InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) + # assumes fs = hs + ss # both hard + soft size + # assumes unit tests ensure ._sizes table entries are consistent + # hs >= 2, ss > 0 fs == hs + ss, not (fs % 4) + if count < 0 or count > (64 ** ss - 1): raise kering.InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) @@ -637,19 +637,18 @@ def _binfil(self): count = self.count # index value int used for soft hs, ss, fs, ls = self._sizes[code] - cs = hs + ss - if hs < 2 or fs != cs or cs % 4: # hs >= 2 fs must be cs and multiple of 4 for count codes - raise kering.InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) + # assumes fs = hs + ss + # assumes unit tests ensure ._sizes table entries are consistent + # hs >= 2, ss>0 fs == hs + ss, not (fs % 4) if count < 0 or count > (64 ** ss - 1): raise kering.InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) # both is hard code + converted count both = "{}{}".format(code, intToB64(count, l=ss)) - if len(both) != cs: + if len(both) != fs: raise kering.InvalidCodeSizeError("Mismatch code size = {} with table = {}." - .format(cs, len(both))) + .format(fs, len(both))) return (codeB64ToB2(both)) # convert to b2 left shift if any @@ -682,16 +681,15 @@ def _exfil(self, qb64b): raise kering.UnexpectedCodeError("Unsupported code ={}.".format(hard)) hs, ss, fs, ls = self._sizes[hard] # assumes hs consistent in both tables - cs = hs + ss # both hard + soft code size - + # assumes fs = hs + ss # both hard + soft code size # assumes that unit tests on Counter and CounterCodex ensure that # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 + # hs consistent and hs >= 2 and ss > 0 and fs = hs + ss and not fs % 4 - if len(qb64b) < cs: # need more bytes - raise kering.ShortageError("Need {} more characters.".format(cs - len(qb64b))) + if len(qb64b) < fs: # need more bytes + raise kering.ShortageError("Need {} more characters.".format(fs - len(qb64b))) - count = qb64b[hs:hs + ss] # extract count chars + count = qb64b[hs:fs] # extract count chars if hasattr(count, "decode"): count = count.decode("utf-8") count = b64ToInt(count) # compute int count @@ -725,17 +723,17 @@ def _bexfil(self, qb2): raise kering.UnexpectedCodeError("Unsupported code ={}.".format(hard)) hs, ss, fs, ls = self._sizes[hard] - cs = hs + ss # both hs and ss + # assumes fs = hs + ss # both hs and ss # assumes that unit tests on Counter and CounterCodex ensure that # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 + # hs consistent and hs >= 2 and ss > 0 and fs = hs + ss and not fs % 4 - bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets + bcs = sceil(fs * 3 / 4) # bcs is min bytes to hold fs sextets if len(qb2) < bcs: # need more bytes raise kering.ShortageError("Need {} more bytes.".format(bcs - len(qb2))) - both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code - count = b64ToInt(both[hs:hs + ss]) # get count + both = codeB2ToB64(qb2, fs) # extract and convert both hard and soft part of code + count = b64ToInt(both[hs:fs]) # get count self._code = hard self._count = count diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index c7d7d997c..3ef2b0bde 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -907,6 +907,7 @@ def _inhale(clas, raw, version=Version, smellage=None): kind (str): value of Serials (Serialage) serialization kind Parameters: + clas (Serder): class reference raw (bytes): serialized sad message version (Versionage): instance supported protocol version smellage (Smellage | None): instance of deconstructed version string @@ -973,6 +974,21 @@ def loads(raw, size=None, kind=Serials.json): return sad + @classmethod + def _loads(clas, raw, size=None): + """CESR native desserialization of raw + + Returns: + sad (dict): deserialized dict of CESR native serialization. + + Parameters: + clas (Serder): class reference + raw (bytes |bytearray): raw serialization to deserialze as dict + size (int): number of bytes to consume for the deserialization. + If None then consume all bytes in raw + """ + pass + @classmethod def _exhale(clas, sad, version=None): @@ -991,6 +1007,7 @@ def _exhale(clas, sad, version=None): vrsn (Versionage): tuple value (major, minor) Parameters: + clas (Serder): class reference sad (dict): serializable attribute dict of saidified data version (Versionage | None): supported protocol version for message None means do not enforce a supported version @@ -1051,6 +1068,26 @@ def dumps(sad, kind=Serials.json): return raw + @classmethod + def _dumps(clas, sad, protocol, version): + """CESR native serialization of sad + + Returns: + raw (bytes): CESR native serialization of sad dict + + Parameters: + clas (Serder): class reference + sad (dict | list)): serializable dict or list to serialize + protocol (str): protocol type + version (Versionage): protocol version + + """ + pass + + + + + def compare(self, said=None): """Utility method to allow comparison of own .said digest of .raw with some other purported said of .raw From 1177ee3c5588e1b8315b84604e98cadbf9ca77bf Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 23 Mar 2024 19:43:32 -0600 Subject: [PATCH 059/418] Clean up refactor for versioning of Serder and Counter --- src/keri/core/coring.py | 2 + src/keri/core/counting.py | 263 +++++++++++++++++--------- src/keri/core/eventing.py | 2 +- src/keri/core/serdering.py | 98 +++++++++- src/keri/kering.py | 14 +- tests/core/test_coring.py | 11 +- tests/core/test_counting.py | 344 ++++++++++++++++++++++------------- tests/core/test_serdering.py | 2 +- 8 files changed, 511 insertions(+), 225 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index cf4e31abd..07cc7f4a6 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -4701,8 +4701,10 @@ class GenusCodex: Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ + KERI_ACDC_SPAC: str = '--AAA' # KERI, ACDC, and SPAC Protocol Stacks share the same tables KERI: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables ACDC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables + SPAC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables def __iter__(self): diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index d4015d87d..b49afba17 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -1,4 +1,4 @@ -# -*- encoding: utf-8 -*- +# -*- coding: utf-8 -*- """ keri.core.counting module @@ -14,7 +14,7 @@ nabSextets) from .. import kering -from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0) +from ..kering import (Versionage, Vrsn_1_0, Vrsn_2_0) from ..core.coring import Sizage @@ -218,9 +218,10 @@ class Counter: Class Attributes: Codes (dict): of codexes keyed by version Tags (dict): of tagages keyed by version - Hards (dict): of hard code sizes keyed by selector text - Bards (dict): of hard code sizes keyed by selector binary - Sizes (dict): of Sizages keyed by hard code + Hards (dict): of hard code sizes keyed by text domain selector + Bards (dict): of hard code sizes keyed by binary domain selector + Sizes (dict): of size tables keyed by version. Size table is dict + of Sizages keyed by hard code Attributes: @@ -232,7 +233,8 @@ class Counter: .code (str) derivation code to indicate cypher suite .raw is bytes crypto material only without code .pad is int number of pad chars given raw - .count is int count of grouped following material (not part of counter) + .count is int count of quadlets/triplets of following framed material + (not including code) .qb64 is str in Base64 fully qualified with derivation code + crypto mat .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat .qb2 is bytes in binary with derivation code + crypto material @@ -248,8 +250,78 @@ class Counter: ._infil is method to compute fully qualified Base64 from .raw and .code ._exfil is method to extract .code and .raw from fully qualified Base64 + + Versioning: + CESR Genus specific code tables have a major and a minor version. + + For a given major version all minor versions must be backwards compatible. + This means that minor version changes to tables are append only. New + codes may be added but no existing codes may be changed. This means that + a given implementation need only use use the latest minor version of + the code table for a given major version when generating or parsing a + primitive or group. Assuming the major versions match, when parsing, + a primitive, when that primitive was generated with a later minor version + than the implementation supports then it will not be recognized and + raise an error. But if a primitive was generated with any earlier minor + version than the version the implementation supports then the primitive + will parse correctly using any later minor version of the code table. + + Likewise a given protocol stack may have message bodies that carry + a major and a minor version. + + A given CESR Genus and a given Protocol message stack may be paired in + order to synchronize versioning between the two when the message bodies + use primitives and or groups defined by codes in the CESR Genus table. + + In this case pairing is between the CESR Genus labeled KERI_ACDC_SPAC + and the message body protocol stack labeled KERI/ACDC/SPAC + + The two versions, CESR Genus and Protocol Stack, may be synchronized in + the following way: + + * Major versions must match. + + * Minor versions may differ. + + Importantly the CESR code table version may not be included in the + message body itself but only provided in the surrounding CESR stream. + This means the code table version used by a message body may not be + signed. Therefore the receiver of a message body with embedded CESR + primitives and groups must be protected from a CESR code table genus + version malleability attack. + + Because the major versions of the CESR code table and protocol stack + must match, the signed embedded protocol stack major version protects + the receiver from a major version malleability attack on the CESR + code table. + + This, however, does not protect the receiver of a message body from + a minor version malleability attack on the CESR code table. + Nevertheless, the requirement that all minor versions of a CESR code + table for a given major version must be backwards compatible, + does indeed provide this protection. + + Either, the receiver of the message body recognizes exactly + all primitives and groups in the message body because the CESR code + table minor version supported by the receiver is greater than or equal + to that used by the the minor version of the sender or any unsupported + (later appended) primitives or group codes will be unrecognized by + the received thereby raising an error that results in the message being + dropped. + """ - Codes = {Vrsn_1_0: CtrDex_1_0, Vrsn_2_0: CtrDex_2_0} + Codes = \ + { + Vrsn_1_0.major: \ + { + Vrsn_1_0.minor: CtrDex_1_0, + }, + Vrsn_2_0.major: \ + { + Vrsn_2_0.minor: CtrDex_2_0, + }, + } + Tags = {Vrsn_1_0: Tags_1_0, Vrsn_2_0: Tags_2_0} # Hards table maps from bytes Base64 first two code chars to int of @@ -264,88 +336,96 @@ class Counter: # converted from first two code char. Used for ._bexfil. Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) - # Sizes table maps hs chars of code to Sizage namedtuple of (hs, ss, fs) + # Sizes table indexes size tables first by major version and then by + # lastest minor version + # Each size table maps hs chars of code to Sizage namedtuple of (hs, ss, fs) # where hs is hard size, ss is soft size, and fs is full size # soft size, ss, should always be > 0 and hs+ss=fs for Counter Sizes = \ { - Vrsn_1_0: \ + Vrsn_1_0.major: \ { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + Vrsn_1_0.minor: \ + { + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + }, }, - Vrsn_2_0: \ + Vrsn_2_0.major: \ { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0A': Sizage(hs=3, ss=5, fs=8, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0B': Sizage(hs=3, ss=5, fs=8, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0C': Sizage(hs=3, ss=5, fs=8, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0D': Sizage(hs=3, ss=5, fs=8, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0E': Sizage(hs=3, ss=5, fs=8, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0F': Sizage(hs=3, ss=5, fs=8, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0G': Sizage(hs=3, ss=5, fs=8, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0H': Sizage(hs=3, ss=5, fs=8, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0I': Sizage(hs=3, ss=5, fs=8, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0J': Sizage(hs=3, ss=5, fs=8, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0K': Sizage(hs=3, ss=5, fs=8, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), - '-M': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0M': Sizage(hs=3, ss=5, fs=8, ls=0), - '-N': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0N': Sizage(hs=3, ss=5, fs=8, ls=0), - '-O': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0O': Sizage(hs=3, ss=5, fs=8, ls=0), - '-P': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0P': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Q': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0Q': Sizage(hs=3, ss=5, fs=8, ls=0), - '-R': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0R': Sizage(hs=3, ss=5, fs=8, ls=0), - '-S': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0S': Sizage(hs=3, ss=5, fs=8, ls=0), - '-T': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0T': Sizage(hs=3, ss=5, fs=8, ls=0), - '-U': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0U': Sizage(hs=3, ss=5, fs=8, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '-W': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0W': Sizage(hs=3, ss=5, fs=8, ls=0), - '-X': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0X': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Y': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0Y': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + Vrsn_2_0.minor: \ + { + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0A': Sizage(hs=3, ss=5, fs=8, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0B': Sizage(hs=3, ss=5, fs=8, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0C': Sizage(hs=3, ss=5, fs=8, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0D': Sizage(hs=3, ss=5, fs=8, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0E': Sizage(hs=3, ss=5, fs=8, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0F': Sizage(hs=3, ss=5, fs=8, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0G': Sizage(hs=3, ss=5, fs=8, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0H': Sizage(hs=3, ss=5, fs=8, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0I': Sizage(hs=3, ss=5, fs=8, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0J': Sizage(hs=3, ss=5, fs=8, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0K': Sizage(hs=3, ss=5, fs=8, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), + '-M': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0M': Sizage(hs=3, ss=5, fs=8, ls=0), + '-N': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0N': Sizage(hs=3, ss=5, fs=8, ls=0), + '-O': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0O': Sizage(hs=3, ss=5, fs=8, ls=0), + '-P': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0P': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Q': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0Q': Sizage(hs=3, ss=5, fs=8, ls=0), + '-R': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0R': Sizage(hs=3, ss=5, fs=8, ls=0), + '-S': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0S': Sizage(hs=3, ss=5, fs=8, ls=0), + '-T': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0T': Sizage(hs=3, ss=5, fs=8, ls=0), + '-U': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0U': Sizage(hs=3, ss=5, fs=8, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '-W': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0W': Sizage(hs=3, ss=5, fs=8, ls=0), + '-X': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0X': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Y': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0Y': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + }, }, } def __init__(self, tag=None, *, code = None, count=None, countB64=None, - qb64b=None, qb64=None, qb2=None, strip=False, version=Version): + qb64b=None, qb64=None, qb2=None, strip=False, version=Vrsn_2_0): """ Validate as fully qualified Parameters: @@ -355,13 +435,13 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, code (str | None): stable (hard) part of derivation code if tag provided lookup code from tag else if tag is None and code provided use code - count (int | None): count for composition. - Count may represent quadlets/triplet, groups, primitives or - other numericy + count (int | None): count of framed material for composition + Count does not include code. + Count represents quadlets/triplets When both count and countB64 are None then count defaults to 1 - countB64 (str | None): count for composition as Base64 - countB64 may represent quadlets/triplet, groups, primitives or - other numericy + countB64 (str | None): count of framed material for composition + as Base64 + countB64 represents quadlets/triplets qb64b (bytes | bytearray | None): fully qualified crypto material text domain if code nor tag is provided qb64 (str | None) fully qualified crypto material text domain @@ -382,9 +462,20 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, .code and .count """ - self._version = version - self._codes = self.Codes[self._version] - self._sizes = self.Sizes[self._version] + if version.major not in self.Sizes: + raise kering.InvalidVersionError(f"Unsupported major version=" + f"{version.major}.") + + latest = list(self.Sizes[version.major])[0] # get latest minor version + if version.minor > latest: + raise kering.InvalidVersionError(f"Minor version={version.minor} " + f" exceeds latest supported minor" + f" version={latest}.") + + self._codes = self.Codes[version.major][latest] # use latest supported version codes + self._sizes = self.Sizes[version.major][latest] # use latest supported version sizes + self._version = version # provided version may be earlier than supported version + if tag: if not hasattr(self._codes, tag): diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index e501c0844..d0c063aec 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1,4 +1,4 @@ -# -*- encoding: utf-8 -*- +# -*- coding: utf-8 -*- """ keri.core.eventing module diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 3ef2b0bde..81e295021 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1,4 +1,4 @@ -# -*- encoding: utf-8 -*- +# -*- coding: utf-8 -*- """ keri.core.serdering module @@ -31,6 +31,9 @@ from .coring import MtrDex, DigDex, PreDex, Saids, Digestage from .coring import Matter, Saider, Verfer, Diger, Number, Tholder +from ..core import counting +from ..core.counting import AllTags, Counter + from .. import help from ..help import helping @@ -1069,7 +1072,7 @@ def dumps(sad, kind=Serials.json): @classmethod - def _dumps(clas, sad, protocol, version): + def _dumps(clas, sad, protocol, version, gversion=Vrsn_2_0): """CESR native serialization of sad Returns: @@ -1078,13 +1081,98 @@ def _dumps(clas, sad, protocol, version): Parameters: clas (Serder): class reference sad (dict | list)): serializable dict or list to serialize - protocol (str): protocol type - version (Versionage): protocol version + protocol (str): message protocol + version (Versionage): message protocol version + gversion (Versionage): CESR code table genus version """ - pass + fixed = True # True = use fixed field, False= use field map + + raw = bytearray() + + if protocol not in Protos: + raise SerializeError(f"Invalid {protocol=}.") + + if version not in clas.Fields[protocol]: + raise SerializeError(f"Invalid {version=} for {protocol=}.") + + ilks = clas.Fields[protocol][version] # get fields keyed by ilk + + ilk = sad.get('t') # returns None if missing message type (ilk) + if ilk not in ilks: # + raise SerializeError(f"Missing message type field " + f"'t' for {protocol=} {version=} with {sad=}.") + + fields = ilks[ilk] # FieldDom for given protocol and ilk + + if fields.opts or not fields.strict: # optional or extra fields allowed + fixed = False # so must use field map + + + # assumes that sad's field ordering and field inclusion is correct + # so can serialize in order to compute saidive fields + # need to fix ._verify and .makify to account for CESR native serialization + + if protocol == Protos.keri: + for l, v in sad.items(): # assumes valid field order & presence + if not fixed: # prepend label + pass + + # should dispatch or use match instead of big if else + match l: # label + case "v": # protocol+version + val = (MtrDex.Tag7 + protocol + + Counter.verToB64(version)).encode("utf-8") + + case "t": # message type + val = (MtrDex.Tag3 + ilk).encode("utf-8") # add code + + case "d" | "i" | "di": # said or aid + val = v.encode("utf-8") # already primitive qb64 make qb6b + + case "s" | "bt": # sequence number + val = coring.Number(numh=v).qb64b # convert hex str + case "kt" | "nt": # current or next signing threshold + val = coring.Tholder(sith=v).limen # convert sith str + case "k" | "n" | "b" | "ba" | "br": # list of primitives + frame = bytearray() + for e in v: # list + frame.extend(e.encode("utf-8")) + val = bytearray(Counter(tag=AllTags.GenericListGroup, + count=len(frame) % 4, + version=gversion).qb64b) + val.extend(frame) + + + case _: # if extra fields this is where logic wouldl + raise SerializeError(f"Unsupported protocol field label" + f"='{l}' for {protocol=} {version=}.") + + + raw.extend(val) + + + elif protocol == Protos.acdc: + for l, val in sad.items(): # assumes valid field order & presence + if not fixed: + pass # prepend label + + + + else: + raise SerializeError(f"Unsupported {protocol=}.") + + + # prepend count code for message + if fixed: + pass + else: + pass + + + return raw diff --git a/src/keri/kering.py b/src/keri/kering.py index 50e9a35ec..58d714711 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -243,7 +243,7 @@ class ColdCodex: x[0] >> 5 == 0o1 True """ - Free: int = 0o0 # not taken + Anno: int = 0o0 # Annotated CESR CtB64: int = 0o1 # CountCode Base64 OpB64: int = 0o2 # OpCode Base64 JSON: int = 0o3 # JSON Map Event Start @@ -269,14 +269,14 @@ def sniff(ims): and if counter code whether Base64 or Base2 representation First three bits: - 0o0 = 000 free + 0o0 = 000 annotated cesr 0o1 = 001 cntcode B64 0o2 = 010 opcode B64 0o3 = 011 json 0o4 = 100 mgpk 0o5 = 101 cbor 0o6 = 110 mgpk - 007 = 111 cntcode or opcode B2 + 007 = 111 cntcode B2 or opcode B2 counter B64 in (0o1, 0o2) return 'txt' counter B2 in (0o7) return 'bny' @@ -287,6 +287,7 @@ def sniff(ims): 'msg' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) 'bny' if tritet in (ColdDex.CtOpB2,) + 'ano' if tritet in (ColdDex.Anno,) """ if not ims: raise ShortageError("Need more bytes.") @@ -458,6 +459,13 @@ class EmptyMaterialError(MaterialError): raise EmptyMaterialError("error message") """ +class InvalidVersionError(MaterialError): + """ + Invalid, Unknown, or unrecognized CESR code table version encountered during + crypto material init + Usage: + raise InvalidVersionError("error message") + """ class InvalidCodeError(MaterialError): """ diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index fd65daf67..51327e60d 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -52,14 +52,19 @@ def test_genus_codex(): """ - assert dataclasses.asdict(GenDex) == { - 'KERI': '--AAA', # KERI and ACDC Protocol Stacks share the same tables + assert dataclasses.asdict(GenDex) == \ + { + 'KERI_ACDC_SPAC': '--AAA', + 'KERI': '--AAA', 'ACDC': '--AAA', - } + 'SPAC': '--AAA' + } assert '--AAA' in GenDex assert GenDex.KERI == "--AAA" assert GenDex.ACDC == "--AAA" + assert GenDex.SPAC == "--AAA" + assert GenDex.KERI_ACDC_SPAC == "--AAA" assert GenDex.KERI == GenDex.ACDC """End Test""" diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 4d54a0664..521d13b65 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -1,4 +1,4 @@ -# -*- encoding: utf-8 -*- +# -*- coding: utf-8 -*- """ tests.core.test_counting module @@ -20,7 +20,7 @@ from keri.core import counting from keri.core.counting import Sizage, MapDom, MapCodex, Counter -from keri.core.counting import Versionage, Version, Vrsn_1_0, Vrsn_2_0, AllTags +from keri.core.counting import Versionage, Vrsn_1_0, Vrsn_2_0, AllTags def test_mapdom(): @@ -388,8 +388,14 @@ def test_counter(): assert Counter.Codes == \ { - counting.Vrsn_1_0: counting.CtrDex_1_0, - counting.Vrsn_2_0: counting.CtrDex_2_0, + Vrsn_1_0.major: \ + { + Vrsn_1_0.minor: counting.CtrDex_1_0, + }, + Vrsn_2_0.major: \ + { + Vrsn_2_0.minor: counting.CtrDex_2_0, + }, } assert Counter.Tags == \ @@ -414,95 +420,109 @@ def test_counter(): # Codes table with sizes of code (hard) and full primitive material assert Counter.Sizes == \ { - counting.Vrsn_1_0: \ + Vrsn_1_0.major: \ { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) + Vrsn_1_0.minor: \ + { + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) + }, }, - counting.Vrsn_2_0: \ + Vrsn_2_0.major: \ { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0A': Sizage(hs=3, ss=5, fs=8, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0B': Sizage(hs=3, ss=5, fs=8, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0C': Sizage(hs=3, ss=5, fs=8, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0D': Sizage(hs=3, ss=5, fs=8, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0E': Sizage(hs=3, ss=5, fs=8, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0F': Sizage(hs=3, ss=5, fs=8, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0G': Sizage(hs=3, ss=5, fs=8, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0H': Sizage(hs=3, ss=5, fs=8, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0I': Sizage(hs=3, ss=5, fs=8, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0J': Sizage(hs=3, ss=5, fs=8, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0K': Sizage(hs=3, ss=5, fs=8, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), - '-M': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0M': Sizage(hs=3, ss=5, fs=8, ls=0), - '-N': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0N': Sizage(hs=3, ss=5, fs=8, ls=0), - '-O': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0O': Sizage(hs=3, ss=5, fs=8, ls=0), - '-P': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0P': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Q': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0Q': Sizage(hs=3, ss=5, fs=8, ls=0), - '-R': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0R': Sizage(hs=3, ss=5, fs=8, ls=0), - '-S': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0S': Sizage(hs=3, ss=5, fs=8, ls=0), - '-T': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0T': Sizage(hs=3, ss=5, fs=8, ls=0), - '-U': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0U': Sizage(hs=3, ss=5, fs=8, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '-W': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0W': Sizage(hs=3, ss=5, fs=8, ls=0), - '-X': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0X': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Y': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0Y': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) - } + Vrsn_2_0.minor: \ + { + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0A': Sizage(hs=3, ss=5, fs=8, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0B': Sizage(hs=3, ss=5, fs=8, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0C': Sizage(hs=3, ss=5, fs=8, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0D': Sizage(hs=3, ss=5, fs=8, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0E': Sizage(hs=3, ss=5, fs=8, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0F': Sizage(hs=3, ss=5, fs=8, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0G': Sizage(hs=3, ss=5, fs=8, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0H': Sizage(hs=3, ss=5, fs=8, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0I': Sizage(hs=3, ss=5, fs=8, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0J': Sizage(hs=3, ss=5, fs=8, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0K': Sizage(hs=3, ss=5, fs=8, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), + '-M': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0M': Sizage(hs=3, ss=5, fs=8, ls=0), + '-N': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0N': Sizage(hs=3, ss=5, fs=8, ls=0), + '-O': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0O': Sizage(hs=3, ss=5, fs=8, ls=0), + '-P': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0P': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Q': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0Q': Sizage(hs=3, ss=5, fs=8, ls=0), + '-R': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0R': Sizage(hs=3, ss=5, fs=8, ls=0), + '-S': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0S': Sizage(hs=3, ss=5, fs=8, ls=0), + '-T': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0T': Sizage(hs=3, ss=5, fs=8, ls=0), + '-U': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0U': Sizage(hs=3, ss=5, fs=8, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '-W': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0W': Sizage(hs=3, ss=5, fs=8, ls=0), + '-X': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0X': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Y': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0Y': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) + }, + }, } - assert Counter.Sizes[counting.Vrsn_1_0]['-A'].hs == 2 # hard size - assert Counter.Sizes[counting.Vrsn_1_0]['-A'].ss == 2 # soft size - assert Counter.Sizes[counting.Vrsn_1_0]['-A'].fs == 4 # full size - assert Counter.Sizes[counting.Vrsn_1_0]['-A'].ls == 0 # lead size + assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].hs == 2 # hard size + assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].ss == 2 # soft size + assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].fs == 4 # full size + assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].ls == 0 # lead size - # verify first hs Sizes matches hs in Codes for same first char - for vsize in Counter.Sizes.values(): - for ckey, cval in vsize.items(): - assert Counter.Hards[ckey[:2]] == cval.hs + assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].hs == 3 # hard size + assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].ss == 5 # soft size + assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].fs == 8 # full size + assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].ls == 0 # lead size - # verify all Codes have hs >= 2 and ss > 0 and fs = hs + ss and not fs % 4 - # if hs < 2 or ss <= 0 or fs != cs or cs % 4 cs = hs + ss - for vsize in Counter.Sizes.values(): - for val in vsize.values(): - assert val.hs >= 2 and val.ss > 0 and val.hs + val.ss == val.fs and not val.fs % 4 + + # verify first hs Sizes matches hs in Codes for same first char + for vmajor in Counter.Sizes.values(): + for vminor in vmajor.values(): + for key, val in vminor.items(): # size table items + assert Counter.Hards[key[:2]] == val.hs + + ## verify all Codes have hs >= 2 and ss > 0 and fs = hs + ss and not fs % 4 + ## if hs < 2 or ss <= 0 or fs != cs or cs % 4 cs = hs + ss + for vmajor in Counter.Sizes.values(): + for vminor in vmajor.values(): + for val in vminor.values(): # size table values + assert val.hs >= 2 and val.ss > 0 and val.hs + val.ss == val.fs and not val.fs % 4 # Bizes maps bytes of sextet of decoded first character of code with hard size of code # verify equivalents of items for Sizes and Bizes @@ -578,13 +598,12 @@ def test_counter_v1(): """ test Counter instances for verision 1.0 code tables """ - # version 1_0 tests default version is Version - assert Version == Vrsn_1_0 - CtrDex = Counter.Codes[Version] # set CtrDex to Vrsn_1_0 + # version 1_0 tests + CtrDex = Counter.Codes[Vrsn_1_0.major][Vrsn_1_0.minor] # set CtrDex to Vrsn_1_0 # test Counter instances with pytest.raises(kering.EmptyMaterialError): - counter = Counter() + counter = Counter(version=Vrsn_1_0) # create code manually count = 1 @@ -593,78 +612,86 @@ def test_counter_v1(): qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(tag="ControllerIdxSigs", count=count) + counter = Counter(tag="ControllerIdxSigs", count=count, version=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(tag=AllTags.ControllerIdxSigs, count=count) + counter = Counter(tag=AllTags.ControllerIdxSigs, count=count, version=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 # test tag takes precedence counter = Counter(tag=AllTags.ControllerIdxSigs, code=CtrDex.WitnessIdxSigs, - count=count) + count=count, + version=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(code=CtrDex.ControllerIdxSigs) # default count = 1 + counter = Counter(code=CtrDex.ControllerIdxSigs, version=Vrsn_1_0) # default count = 1 assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(qb64b=qscb) # test with bytes not str + counter = Counter(qb64b=qscb, version=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(qb64=qsc) # test with str not bytes + counter = Counter(qb64=qsc, version=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(qb2=qscb2) # test with qb2 + counter = Counter(qb2=qscb2, version=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 # test truncates extra bytes from qb64 parameter longqsc64 = qsc + "ABCD" - counter = Counter(qb64=longqsc64) + counter = Counter(qb64=longqsc64, version=Vrsn_1_0) assert len(counter.qb64) == counter.sizes[counter.code].fs # test raises ShortageError if not enough bytes in qb64 parameter shortqsc64 = qsc[:-1] # too short with pytest.raises(kering.ShortageError): - counter = Counter(qb64=shortqsc64) + counter = Counter(qb64=shortqsc64, version=Vrsn_1_0) # test truncates extra bytes from qb2 parameter longqscb2 = qscb2 + bytearray([1, 2, 3, 4, 5]) # extra bytes in size - counter = Counter(qb2=longqscb2) + counter = Counter(qb2=longqscb2, version=Vrsn_1_0) assert counter.qb2 == qscb2 assert len(counter.qb64) == counter.sizes[counter.code].fs # test raises ShortageError if not enough bytes in qb2 parameter shortqscb2 = qscb2[:-4] # too few bytes in size with pytest.raises(kering.ShortageError): - counter = Counter(qb2=shortqscb2) + counter = Counter(qb2=shortqscb2, version=Vrsn_1_0) # test with non-zero count=5 count = 5 @@ -673,33 +700,37 @@ def test_counter_v1(): qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(code=CtrDex.ControllerIdxSigs, count=count) + counter = Counter(code=CtrDex.ControllerIdxSigs, count=count, version=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(qb64b=qscb) # test with bytes not str + counter = Counter(qb64b=qscb, version=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(qb64=qsc) # test with str not bytes + counter = Counter(qb64=qsc, version=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(qb2=qscb2) # test with qb2 + counter = Counter(qb2=qscb2, version=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 # test with big codes index=1024 count = 1024 @@ -708,36 +739,40 @@ def test_counter_v1(): qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(code=CtrDex.BigAttachmentGroup, count=count) + counter = Counter(code=CtrDex.BigAttachmentGroup, count=count, version=Vrsn_1_0) assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(qb64b=qscb) # test with bytes not str + counter = Counter(qb64b=qscb, version=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(qb64=qsc) # test with str not bytes + counter = Counter(qb64=qsc, version=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(qb2=qscb2) # test with qb2 + counter = Counter(qb2=qscb2, version=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 # Test ._bexfil - counter = Counter(qb64=qsc) # + counter = Counter(qb64=qsc, version=Vrsn_1_0) # code = counter.code count = counter.count qb2 = counter.qb2 @@ -746,6 +781,7 @@ def test_counter_v1(): assert counter.count == count assert counter.qb64 == qsc assert counter.qb2 == qb2 + assert counter.version == Vrsn_1_0 # Test ._binfil test = counter._binfil() @@ -760,35 +796,38 @@ def test_counter_v1(): qscb2 = decodeB64(qscb) # strip ignored if qb64 - counter = Counter(qb64=qsc, strip=True) # test with str not bytes + counter = Counter(qb64=qsc, strip=True, version=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 ims = bytearray(qscb) # test with qb64b - counter = Counter(qb64b=ims, strip=True) # strip + counter = Counter(qb64b=ims, strip=True, version=Vrsn_1_0) # strip assert not ims # deleted assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 ims = bytearray(qscb2) # test with qb2 - counter = Counter(qb2=ims, strip=True) + counter = Counter(qb2=ims, strip=True, version=Vrsn_1_0) assert not ims # deleted assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 # test with longer ims for qb64b extra = b"ABCD" ims = bytearray(qscb + b"ABCD") - counter = Counter(qb64b=ims, strip=True) + counter = Counter(qb64b=ims, strip=True, version=Vrsn_1_0) assert counter.qb64b == qscb assert len(counter.qb64b) == counter.sizes[counter.code].fs assert ims == extra @@ -796,7 +835,7 @@ def test_counter_v1(): # test with longer ims for qb2 extra = bytearray([1, 2, 3, 4, 5]) ims = bytearray(qscb2) + extra - counter = Counter(qb2=ims, strip=True) + counter = Counter(qb2=ims, strip=True, version=Vrsn_1_0) assert counter.qb2 == qscb2 assert len(counter.qb2) == counter.sizes[counter.code].fs * 3 // 4 assert ims == extra @@ -805,11 +844,11 @@ def test_counter_v1(): ims = bytes(qscb) # test with qb64b with pytest.raises(TypeError): - counter = Counter(qb64b=ims, strip=True) # strip + counter = Counter(qb64b=ims, strip=True, version=Vrsn_1_0) # strip ims = bytes(qscb2) # test with qb2 with pytest.raises(TypeError): - counter = Counter(qb2=ims, strip=True) + counter = Counter(qb2=ims, strip=True, version=Vrsn_1_0) # test with big codes index=1024 count = 1024 @@ -819,21 +858,23 @@ def test_counter_v1(): qscb2 = decodeB64(qscb) ims = bytearray(qscb) - counter = Counter(qb64b=ims, strip=True) # test with bytes not str + counter = Counter(qb64b=ims, strip=True, version=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 assert not ims ims = bytearray(qscb2) - counter = Counter(qb2=ims, strip=True) # test with qb2 + counter = Counter(qb2=ims, strip=True, version=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 assert not ims # test protocol genus with CESR protocol genus version @@ -847,7 +888,9 @@ def test_counter_v1(): qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(code=CtrDex.KERIACDCGenusVersion, count=genverint) + counter = Counter(code=CtrDex.KERIACDCGenusVersion, + count=genverint, + version=Vrsn_1_0) assert counter.code == CtrDex.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver @@ -855,8 +898,11 @@ def test_counter_v1(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 - counter = Counter(code=CtrDex.KERIACDCGenusVersion, countB64=genver) + counter = Counter(code=CtrDex.KERIACDCGenusVersion, + countB64=genver, + version=Vrsn_1_0) assert counter.code == CtrDex.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver @@ -864,6 +910,7 @@ def test_counter_v1(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 """End Test""" @@ -872,8 +919,7 @@ def test_counter_v2(): test Counter instances for verision 2.0 code tables """ # version 2_0 tests default version is Version - #assert Version == Vrsn_2_0 - CtrDex = Counter.Codes[Vrsn_2_0] # set CtrDex to Vrsn_2_0 + CtrDex = Counter.Codes[Vrsn_2_0.major][Vrsn_2_0.minor] # set CtrDex to Vrsn_2_0 # test Counter instances with pytest.raises(kering.EmptyMaterialError): @@ -886,6 +932,15 @@ def test_counter_v2(): qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) + # default version and default count = 1 + counter = Counter(code=CtrDex.ControllerIdxSigs) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + # default count = 1 counter = Counter(code=CtrDex.ControllerIdxSigs, version=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs @@ -893,6 +948,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs @@ -900,6 +956,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb64=qsc, version=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs @@ -907,6 +964,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb2=qscb2, version=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs @@ -914,6 +972,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 # test truncates extra bytes from qb64 parameter longqsc64 = qsc + "ABCD" @@ -949,13 +1008,17 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 - counter = Counter(tag=AllTags.ControllerIdxSigs, count=count, version=Vrsn_2_0) + counter = Counter(tag=AllTags.ControllerIdxSigs, + count=count, + version=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 # test tag takes precedence counter = Counter(tag=AllTags.ControllerIdxSigs, @@ -967,6 +1030,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(code=CtrDex.ControllerIdxSigs, count=count, version=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs @@ -974,6 +1038,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs @@ -981,6 +1046,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb64=qsc, version=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs @@ -988,6 +1054,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb2=qscb2, version=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs @@ -995,6 +1062,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 # Test ._bexfil counter = Counter(qb64=qsc, version=Vrsn_2_0) @@ -1006,6 +1074,7 @@ def test_counter_v2(): assert counter.count == count assert counter.qb64 == qsc assert counter.qb2 == qb2 + assert counter.version == Vrsn_2_0 # Test ._binfil test = counter._binfil() @@ -1026,6 +1095,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 ims = bytearray(qscb) # test with qb64b counter = Counter(qb64b=ims, strip=True, version=Vrsn_2_0) # strip @@ -1035,6 +1105,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 ims = bytearray(qscb2) # test with qb2 counter = Counter(qb2=ims, strip=True, version=Vrsn_2_0) @@ -1044,6 +1115,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 # test with longer ims for qb64b extra = b"ABCD" @@ -1084,6 +1156,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup @@ -1091,6 +1164,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb64=qsc, version=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.BigGenericGroup @@ -1098,6 +1172,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb2=qscb2, version=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.BigGenericGroup @@ -1105,6 +1180,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 # test ims with big codes count=1024 count = 1024 @@ -1120,6 +1196,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 assert not ims ims = bytearray(qscb2) @@ -1129,6 +1206,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 assert not ims # test with big codes count=8193 @@ -1144,6 +1222,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup @@ -1151,6 +1230,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb64=qsc, version=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.BigGenericGroup @@ -1158,6 +1238,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(qb2=qscb2, version=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.BigGenericGroup @@ -1165,6 +1246,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 # test ims with big codes count=8193 count = 8193 @@ -1180,6 +1262,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 assert not ims ims = bytearray(qscb2) @@ -1189,6 +1272,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 assert not ims # test with promotion from small to big codes with count=8193 @@ -1204,6 +1288,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 counter = Counter(tag=AllTags.GenericGroup, count=count, version=Vrsn_2_0) assert counter.code == CtrDex.BigGenericGroup @@ -1211,6 +1296,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 # test protocol genus with CESR version genverint = 0 @@ -1222,7 +1308,9 @@ def test_counter_v2(): qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(code=CtrDex.KERIACDCGenusVersion, count=genverint, version=Vrsn_2_0) + counter = Counter(code=CtrDex.KERIACDCGenusVersion, + count=genverint, + version=Vrsn_2_0) assert counter.code == CtrDex.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver @@ -1230,8 +1318,11 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 - counter = Counter(code=CtrDex.KERIACDCGenusVersion, countB64=genver, version=Vrsn_2_0) + counter = Counter(code=CtrDex.KERIACDCGenusVersion, + countB64=genver, + version=Vrsn_2_0) assert counter.code == CtrDex.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver @@ -1239,6 +1330,7 @@ def test_counter_v2(): assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 """End Test""" diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 0656abb83..0922ea2b6 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -1,4 +1,4 @@ -# -*- encoding: utf-8 -*- +# -*- coding: utf-8 -*- """ tests.core.test_serdering module From fcf101e2a3a7de994c3224d1e47afc58a9e60311 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 23 Mar 2024 20:02:38 -0600 Subject: [PATCH 060/418] more work on dumps for CESR native serializations --- src/keri/core/serdering.py | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 81e295021..7fef3761d 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -6,6 +6,7 @@ import copy import json from collections import namedtuple +from collections.abc import Mapping from dataclasses import dataclass, asdict, field import cbor2 as cbor @@ -36,6 +37,7 @@ from .. import help from ..help import helping +from ..help.helping import nonStringSequence logger = help.ogler.getLogger() @@ -1127,10 +1129,10 @@ def _dumps(clas, sad, protocol, version, gversion=Vrsn_2_0): case "t": # message type val = (MtrDex.Tag3 + ilk).encode("utf-8") # add code - case "d" | "i" | "di": # said or aid + case "d" | "i" | "p" | "di": # said or aid val = v.encode("utf-8") # already primitive qb64 make qb6b - case "s" | "bt": # sequence number + case "s" | "bt": # sequence number or numeric threshold val = coring.Number(numh=v).qb64b # convert hex str case "kt" | "nt": # current or next signing threshold @@ -1140,13 +1142,43 @@ def _dumps(clas, sad, protocol, version, gversion=Vrsn_2_0): frame = bytearray() for e in v: # list frame.extend(e.encode("utf-8")) + + val = bytearray(Counter(tag=AllTags.GenericListGroup, + count=len(frame) % 4, + version=gversion).qb64b) + val.extend(frame) + + case "c": # list of config traits strings + frame = bytearray() + for e in v: # list + pass + #frame.extend(e.encode("utf-8")) + val = bytearray(Counter(tag=AllTags.GenericListGroup, count=len(frame) % 4, version=gversion).qb64b) val.extend(frame) + case "a": # list of seals or field map of attributes + frame = bytearray() + if isinstance(v, Mapping): + for l, e in v.items(): + pass + val = bytearray(Counter(tag=AllTags.GenericMapGroup, + count=len(frame) % 4, + version=gversion).qb64b) + else: + for e in v: # list + pass + #frame.extend(e.encode("utf-8")) + + val = bytearray(Counter(tag=AllTags.GenericListGroup, + count=len(frame) % 4, + version=gversion).qb64b) + val.extend(frame) + - case _: # if extra fields this is where logic wouldl + case _: # if extra fields this is where logic would be raise SerializeError(f"Unsupported protocol field label" f"='{l}' for {protocol=} {version=}.") From cbc9dbf9130c0b896913620711037ba7663a402d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 24 Mar 2024 14:09:06 -0600 Subject: [PATCH 061/418] Add new primitive code Tag10 --- src/keri/core/coring.py | 4 +- src/keri/core/counting.py | 14 +-- src/keri/core/serdering.py | 26 ++++-- tests/core/test_coring.py | 175 +++++++++++++++++++------------------ 4 files changed, 120 insertions(+), 99 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 07cc7f4a6..e06a18ffd 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -295,7 +295,8 @@ class MatterCodex: Tag1: str = '0J' # Tag1 1 B64 encoded char with pre pad for field tag Tag2: str = '0K' # Tag2 2 B64 encoded chars for field tag or version VV or trait like 'EO' Tag5: str = '0L' # Tag5 5 B64 encoded chars with pre pad for field tag - Tag6: str = '0M' # Tag6 6 B64 encoded chars for field tag or protocol kind version like KERIVV (KERI 1.1) or KKKVVV + Tag6: str = '0M' # Tag6 6 B64 encoded chars for field tag + Tag10: str = '0N' # Tag10 10 B64 encoded chars for field tag or version PPPPMmmMmm ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. @@ -723,6 +724,7 @@ class Matter: '0K': Sizage(hs=2, ss=0, fs=4, ls=0), '0L': Sizage(hs=2, ss=0, fs=8, ls=0), '0M': Sizage(hs=2, ss=0, fs=8, ls=0), + '0N': Sizage(hs=2, ss=0, fs=12, ls=0), '1AAA': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAB': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index b49afba17..c000f1693 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -279,9 +279,10 @@ class Counter: The two versions, CESR Genus and Protocol Stack, may be synchronized in the following way: - * Major versions must match. + * Major versions must match or be compatible - * Minor versions may differ. + * Minor versions may differ but must be compatible within a + major version. Importantly the CESR code table version may not be included in the message body itself but only provided in the surrounding CESR stream. @@ -290,10 +291,13 @@ class Counter: primitives and groups must be protected from a CESR code table genus version malleability attack. - Because the major versions of the CESR code table and protocol stack - must match, the signed embedded protocol stack major version protects + When the major versions of the CESR code table and protocol stack + match, the signed embedded protocol stack major version protects the receiver from a major version malleability attack on the CESR - code table. + code table. Otherwise the major versions must be compatible in a way + that does not allow malleability. For example the set of allowed codes + for a given message protocol version are compatible across CESR code + table major versions. This, however, does not protect the receiver of a message body from a minor version malleability attack on the CESR code table. diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 7fef3761d..bfdf73727 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -272,7 +272,7 @@ class Serder: #override in subclass to enforce specific protocol Protocol = None # required protocol, None means any in Protos is ok - Proto = Protos.keri # default CESR protocol type + Proto = Protos.keri # default CESR protocol type for makify on base Serder Vrsn = Vrsn_1_0 # default protocol version for protocol type Kind = Serials.json # default serialization kind @@ -1074,7 +1074,7 @@ def dumps(sad, kind=Serials.json): @classmethod - def _dumps(clas, sad, protocol, version, gversion=Vrsn_2_0): + def _dumps(clas, sad, protocol, version, cversion=Vrsn_2_0): """CESR native serialization of sad Returns: @@ -1085,7 +1085,19 @@ def _dumps(clas, sad, protocol, version, gversion=Vrsn_2_0): sad (dict | list)): serializable dict or list to serialize protocol (str): message protocol version (Versionage): message protocol version - gversion (Versionage): CESR code table genus version + cversion (Versionage): CESR code table genus version + + Versioning: + CESR native serialization includes message protocol, protocol version + and CESR (genus) version. Assumes genus is compatible with message + protocol so genus is not needed. Protects from malleability attack + and ensure compatible cesr codes especially count (group) codes. + Primitive codes are less problematic since so far all primitive codes + tables are backwards compatible across major versions. + + 0NPPPPMmmMmm (12 B64 characters) + + """ fixed = True # True = use fixed field, False= use field map @@ -1145,7 +1157,7 @@ def _dumps(clas, sad, protocol, version, gversion=Vrsn_2_0): val = bytearray(Counter(tag=AllTags.GenericListGroup, count=len(frame) % 4, - version=gversion).qb64b) + version=cversion).qb64b) val.extend(frame) case "c": # list of config traits strings @@ -1156,7 +1168,7 @@ def _dumps(clas, sad, protocol, version, gversion=Vrsn_2_0): val = bytearray(Counter(tag=AllTags.GenericListGroup, count=len(frame) % 4, - version=gversion).qb64b) + version=cversion).qb64b) val.extend(frame) case "a": # list of seals or field map of attributes @@ -1166,7 +1178,7 @@ def _dumps(clas, sad, protocol, version, gversion=Vrsn_2_0): pass val = bytearray(Counter(tag=AllTags.GenericMapGroup, count=len(frame) % 4, - version=gversion).qb64b) + version=cversion).qb64b) else: for e in v: # list pass @@ -1174,7 +1186,7 @@ def _dumps(clas, sad, protocol, version, gversion=Vrsn_2_0): val = bytearray(Counter(tag=AllTags.GenericListGroup, count=len(frame) % 4, - version=gversion).qb64b) + version=cversion).qb64b) val.extend(frame) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 51327e60d..14810b58a 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -75,91 +75,93 @@ def test_matter(): """ Test Matter class """ - assert dataclasses.asdict(MtrDex) == { - 'Ed25519_Seed': 'A', - 'Ed25519N': 'B', - 'X25519': 'C', - 'Ed25519': 'D', - 'Blake3_256': 'E', - 'Blake2b_256': 'F', - 'Blake2s_256': 'G', - 'SHA3_256': 'H', - 'SHA2_256': 'I', - 'ECDSA_256k1_Seed': 'J', - 'Ed448_Seed': 'K', - 'X448': 'L', - 'Short': 'M', - 'Big': 'N', - 'X25519_Private': 'O', - 'X25519_Cipher_Seed': 'P', - 'ECDSA_256r1_Seed': 'Q', - 'Tall': 'R', - 'Large': 'S', - 'Great': 'T', - 'Vast': 'U', - 'Label1': 'V', - 'Label2': 'W', - 'Tag3': 'X', - 'Tag7': 'Y', - 'Blind': 'Z', - 'Salt_128': '0A', - 'Ed25519_Sig': '0B', - 'ECDSA_256k1_Sig': '0C', - 'Blake3_512': '0D', - 'Blake2b_512': '0E', - 'SHA3_512': '0F', - 'SHA2_512': '0G', - 'Long': '0H', - 'ECDSA_256r1_Sig': '0I', - 'Tag1': '0J', - 'Tag2': '0K', - 'Tag5': '0L', - 'Tag6': '0M', - 'ECDSA_256k1N': '1AAA', - 'ECDSA_256k1': '1AAB', - 'Ed448N': '1AAC', - 'Ed448': '1AAD', - 'Ed448_Sig': '1AAE', - 'Tag4': '1AAF', - 'DateTime': '1AAG', - 'X25519_Cipher_Salt': '1AAH', - 'ECDSA_256r1N': '1AAI', - 'ECDSA_256r1': '1AAJ', - 'Null': '1AAK', - 'Yes': '1AAL', - 'No': '1AAM', - 'TBD1': '2AAA', - 'TBD2': '3AAA', - 'StrB64_L0': '4A', - 'StrB64_L1': '5A', - 'StrB64_L2': '6A', - 'StrB64_Big_L0': '7AAA', - 'StrB64_Big_L1': '8AAA', - 'StrB64_Big_L2': '9AAA', - 'Bytes_L0': '4B', - 'Bytes_L1': '5B', - 'Bytes_L2': '6B', - 'Bytes_Big_L0': '7AAB', - 'Bytes_Big_L1': '8AAB', - 'Bytes_Big_L2': '9AAB', - 'X25519_Cipher_L0': '4C', - 'X25519_Cipher_L1': '5C', - 'X25519_Cipher_L2': '6C', - 'X25519_Cipher_Big_L0': '7AAC', - 'X25519_Cipher_Big_L1': '8AAC', - 'X25519_Cipher_Big_L2': '9AAC', - 'X25519_Cipher_QB64_L0': '4D', - 'X25519_Cipher_QB64_L1': '5D', - 'X25519_Cipher_QB64_L2': '6D', - 'X25519_Cipher_QB64_Big_L0': '7AAD', - 'X25519_Cipher_QB64_Big_L1': '8AAD', - 'X25519_Cipher_QB64_Big_L2': '9AAD', - 'X25519_Cipher_QB2_L0': '4D', - 'X25519_Cipher_QB2_L1': '5D', - 'X25519_Cipher_QB2_L2': '6D', - 'X25519_Cipher_QB2_Big_L0': '7AAD', - 'X25519_Cipher_QB2_Big_L1': '8AAD', - 'X25519_Cipher_QB2_Big_L2': '9AAD' + assert dataclasses.asdict(MtrDex) == \ + { + 'Ed25519_Seed': 'A', + 'Ed25519N': 'B', + 'X25519': 'C', + 'Ed25519': 'D', + 'Blake3_256': 'E', + 'Blake2b_256': 'F', + 'Blake2s_256': 'G', + 'SHA3_256': 'H', + 'SHA2_256': 'I', + 'ECDSA_256k1_Seed': 'J', + 'Ed448_Seed': 'K', + 'X448': 'L', + 'Short': 'M', + 'Big': 'N', + 'X25519_Private': 'O', + 'X25519_Cipher_Seed': 'P', + 'ECDSA_256r1_Seed': 'Q', + 'Tall': 'R', + 'Large': 'S', + 'Great': 'T', + 'Vast': 'U', + 'Label1': 'V', + 'Label2': 'W', + 'Tag3': 'X', + 'Tag7': 'Y', + 'Blind': 'Z', + 'Salt_128': '0A', + 'Ed25519_Sig': '0B', + 'ECDSA_256k1_Sig': '0C', + 'Blake3_512': '0D', + 'Blake2b_512': '0E', + 'SHA3_512': '0F', + 'SHA2_512': '0G', + 'Long': '0H', + 'ECDSA_256r1_Sig': '0I', + 'Tag1': '0J', + 'Tag2': '0K', + 'Tag5': '0L', + 'Tag6': '0M', + 'Tag10': '0N', + 'ECDSA_256k1N': '1AAA', + 'ECDSA_256k1': '1AAB', + 'Ed448N': '1AAC', + 'Ed448': '1AAD', + 'Ed448_Sig': '1AAE', + 'Tag4': '1AAF', + 'DateTime': '1AAG', + 'X25519_Cipher_Salt': '1AAH', + 'ECDSA_256r1N': '1AAI', + 'ECDSA_256r1': '1AAJ', + 'Null': '1AAK', + 'Yes': '1AAL', + 'No': '1AAM', + 'TBD1': '2AAA', + 'TBD2': '3AAA', + 'StrB64_L0': '4A', + 'StrB64_L1': '5A', + 'StrB64_L2': '6A', + 'StrB64_Big_L0': '7AAA', + 'StrB64_Big_L1': '8AAA', + 'StrB64_Big_L2': '9AAA', + 'Bytes_L0': '4B', + 'Bytes_L1': '5B', + 'Bytes_L2': '6B', + 'Bytes_Big_L0': '7AAB', + 'Bytes_Big_L1': '8AAB', + 'Bytes_Big_L2': '9AAB', + 'X25519_Cipher_L0': '4C', + 'X25519_Cipher_L1': '5C', + 'X25519_Cipher_L2': '6C', + 'X25519_Cipher_Big_L0': '7AAC', + 'X25519_Cipher_Big_L1': '8AAC', + 'X25519_Cipher_Big_L2': '9AAC', + 'X25519_Cipher_QB64_L0': '4D', + 'X25519_Cipher_QB64_L1': '5D', + 'X25519_Cipher_QB64_L2': '6D', + 'X25519_Cipher_QB64_Big_L0': '7AAD', + 'X25519_Cipher_QB64_Big_L1': '8AAD', + 'X25519_Cipher_QB64_Big_L2': '9AAD', + 'X25519_Cipher_QB2_L0': '4D', + 'X25519_Cipher_QB2_L1': '5D', + 'X25519_Cipher_QB2_L2': '6D', + 'X25519_Cipher_QB2_Big_L0': '7AAD', + 'X25519_Cipher_QB2_Big_L1': '8AAD', + 'X25519_Cipher_QB2_Big_L2': '9AAD' } @@ -218,6 +220,7 @@ def test_matter(): '0K': Sizage(hs=2, ss=0, fs=4, ls=0), '0L': Sizage(hs=2, ss=0, fs=8, ls=0), '0M': Sizage(hs=2, ss=0, fs=8, ls=0), + '0N': Sizage(hs=2, ss=0, fs=12, ls=0), '1AAA': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAB': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), @@ -6470,7 +6473,7 @@ def test_tholder(): if __name__ == "__main__": - #test_matter() + test_matter() test_texter() #test_counter() #test_prodex() From 75f70245b1935342ce8fd352c96b85795ab00e1c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 24 Mar 2024 14:46:29 -0600 Subject: [PATCH 062/418] refactor eliminate version parameter for supported version since supported versions better determined by entries in Serder.Fields table. --- src/keri/core/serdering.py | 74 +++++++++++++++--------------------- src/keri/kering.py | 38 +++++++++--------- tests/core/test_serdering.py | 19 +++++---- 3 files changed, 56 insertions(+), 75 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index bfdf73727..0677a6b82 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -120,17 +120,16 @@ class Serdery: """ - def __init__(self, *, version=None): + def __init__(self, *pa, **kwa): """Init instance Parameters: - version (Versionage | None): instance supported protocol version - None means do not enforce a supported version + """ - self.version = version # default version + pass - def reap(self, ims, *, version=None): + def reap(self, ims): """Extract and return Serder subclass based on protocol type reaped from version string inside serialized raw of Serder. @@ -144,14 +143,14 @@ def reap(self, ims, *, version=None): version (Versionage | None): instance supported protocol version None means do not enforce a supported version """ - version = version if version is not None else self.version + #version = version if version is not None else self.version - smellage = smell(ims, version=version) + smellage = smell(ims) if smellage.protocol == Protos.keri: - return SerderKERI(raw=ims, strip=True, version=version, smellage=smellage) + return SerderKERI(raw=ims, strip=True, smellage=smellage) elif smellage.protocol == Protos.acdc: - return SerderACDC(raw=ims, strip=True, version=version, smellage=smellage) + return SerderACDC(raw=ims, strip=True, smellage=smellage) else: raise ProtocolError(f"Unsupported protocol type = {smellage.proto}.") @@ -189,8 +188,8 @@ class Serder: Digests (dict): map of digestive codes. Should be same set of codes as in coring.DigestCodex coring.DigDex so that .digestive property works. Use unit tests to ensure codex sets match - Protocol (str): default protocol version type - Proto (str): default CESR protocol genus type + Protocol (str): class specific message protocol + Proto (str): default message protocol Vrsn (Versionage): default version Kind (str): default serialization kind one of Serials Fields (dict): nested dict of field labels keyed by protocol, version, @@ -270,9 +269,8 @@ class Serder: } #override in subclass to enforce specific protocol - Protocol = None # required protocol, None means any in Protos is ok - - Proto = Protos.keri # default CESR protocol type for makify on base Serder + Protocol = None # class based message protocol, None means any in Protos is ok + Proto = Protos.keri # default message protocol type for makify on base Serder Vrsn = Vrsn_1_0 # default protocol version for protocol type Kind = Serials.json # default serialization kind @@ -441,7 +439,7 @@ class Serder: } - def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, + def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, verify=True, makify=False, proto=None, vrsn=None, kind=None, ilk=None, saids=None): """Deserialize raw if provided. Update properties from deserialized raw. @@ -460,8 +458,6 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, strip (bool): True means strip (delete) raw from input stream bytearray after parsing. False means do not strip. Assumes that raw is bytearray when strip is True. - version (Versionage | None): instance supported protocol version - None means do not enforce a supported version smellage (Smellage | None): instance of deconstructed and converted version string elements. If none or empty ignore otherwise assume that raw already had its version string extracted (reaped) into the @@ -493,9 +489,7 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, if raw: # deserialize raw using property setter # self._inhale works because it only references class attributes - sad, proto, vrsn, kind, size = self._inhale(raw=raw, - version=version, - smellage=smellage) + sad, proto, vrsn, kind, size = self._inhale(raw=raw, smellage=smellage) self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray self._sad = sad self._proto = proto @@ -528,14 +522,13 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, elif sad or makify: # serialize sad into raw or make sad if makify: # recompute properties and said(s) and reset sad - # makify resets sad, raw, proto, version, kind, and size - self.makify(sad=sad, version=version, - proto=proto, vrsn=vrsn, kind=kind, ilk=ilk, saids=saids) + # makify resets sad, raw, proto, vrsn, kind, ilk, and size + self.makify(sad=sad, proto=proto, vrsn=vrsn, kind=kind, + ilk=ilk, saids=saids) else: # self._exhale works because it only access class attributes - raw, sad, proto, vrsn, kind, size = self._exhale(sad=sad, - version=version) + raw, sad, proto, vrsn, kind, size = self._exhale(sad=sad) self._raw = raw self._sad = sad self._proto = proto @@ -680,8 +673,7 @@ def _verify(self): # verified successfully since no exception - def makify(self, sad, *, version=None, - proto=None, vrsn=None, kind=None, ilk=None, saids=None): + def makify(self, sad, *, proto=None, vrsn=None, kind=None, ilk=None, saids=None): """Makify given sad dict makes the versions string and computes the said field values and sets associated properties: raw, sad, proto, version, kind, size @@ -699,8 +691,6 @@ def makify(self, sad, *, version=None, Parameters: sad (dict): serializable saidified field map of message. Ignored if raw provided - version (Versionage): instance supported protocol version - None means do not enforce version proto (str | None): desired protocol type str value of Protos If None then its extracted from sad or uses default .Proto vrsn (Versionage | None): instance desired protocol version @@ -721,7 +711,7 @@ def makify(self, sad, *, version=None, sproto = svrsn = skind = silk = None if sad and 'v' in sad: # attempt to get from vs in sad try: # extract version string elements as defaults if provided - sproto, svrsn, skind, _ = deversify(sad["v"], version=version) + sproto, svrsn, skind, _ = deversify(sad["v"]) except ValueError as ex: pass else: @@ -750,11 +740,11 @@ def makify(self, sad, *, version=None, raise SerializeError(f"Expected protocol = {self.Protocol}, got " f"{proto} instead.") - if version is not None: # compatible version with vrsn - if (vrsn.major > version.major or - (vrsn.major == version.major and vrsn.minor > version.minor)): - raise SerializeError(f"Incompatible {version=}, with " - f"{vrsn=}.") + #if version is not None: # compatible version with vrsn + #if (vrsn.major > version.major or + #(vrsn.major == version.major and vrsn.minor > version.minor)): + #raise SerializeError(f"Incompatible {version=}, with " + #f"{vrsn=}.") if kind not in Serials: @@ -895,7 +885,7 @@ def makify(self, sad, *, version=None, @classmethod - def _inhale(clas, raw, version=Version, smellage=None): + def _inhale(clas, raw, *, smellage=None): """Deserializes raw. Parses serilized event ser of serialization kind and assigns to instance attributes and returns tuple of associated elements. @@ -914,7 +904,6 @@ def _inhale(clas, raw, version=Version, smellage=None): Parameters: clas (Serder): class reference raw (bytes): serialized sad message - version (Versionage): instance supported protocol version smellage (Smellage | None): instance of deconstructed version string elements. If none or empty ignore otherwise assume that raw already had its version string extracted (reaped) into the @@ -928,7 +917,7 @@ def _inhale(clas, raw, version=Version, smellage=None): if smellage: # passed in so don't need to smell raw again proto, vrsn, kind, size = smellage # tuple unpack else: # not passed in so smell raw - proto, vrsn, kind, size = smell(raw, version=version) + proto, vrsn, kind, size = smell(raw) sad = clas.loads(raw=raw, size=size, kind=kind) @@ -996,9 +985,8 @@ def _loads(clas, raw, size=None): @classmethod - def _exhale(clas, sad, version=None): - """Serializes sad given kind and version and sets the serialized size - in the version string. + def _exhale(clas, sad): + """Serializes sad and sets the serialized size in its version string. As classmethod enables bootstrap of valid sad dict that has correct size in version string. This obviates sizeify. This can be called on self as @@ -1014,8 +1002,6 @@ def _exhale(clas, sad, version=None): Parameters: clas (Serder): class reference sad (dict): serializable attribute dict of saidified data - version (Versionage | None): supported protocol version for message - None means do not enforce a supported version """ @@ -1023,7 +1009,7 @@ def _exhale(clas, sad, version=None): raise SerializeError(f"Missing version string field in {sad}.") # extract elements so can replace size element but keep others - proto, vrsn, kind, size = deversify(sad["v"], version=version) + proto, vrsn, kind, size = deversify(sad["v"]) raw = clas.dumps(sad, kind) size = len(raw) diff --git a/src/keri/kering.py b/src/keri/kering.py index 58d714711..b0493490d 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -66,7 +66,7 @@ """ Smellage = namedtuple("Smellage", "protocol version kind size") -def rematch(match, *, version=None): +def rematch(match): """ Returns: smellage (Smellage): named tuple extracted from version string regex match @@ -74,8 +74,7 @@ def rematch(match, *, version=None): Parameters: match (re.Match): instance of Match class - version (Versionage | None): supported version. None means do not check - for supported version. namedtuple (major, minor) + """ full = match.group() # full matched version string @@ -91,11 +90,11 @@ def rematch(match, *, version=None): vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) if vrsn.major < 2: # version2 vs but major < 2 raise VersionError(f"Incompatible {vrsn=} with version string.") - if version is not None: # compatible version with vrsn - if (vrsn.major > version.major or - (vrsn.major == version.major and vrsn.minor > version.minor)): - raise VersionError(f"Incompatible {version=}, with " - f"{vrsn=}.") + #if version is not None: # compatible version with vrsn + #if (vrsn.major > version.major or + #(vrsn.major == version.major and vrsn.minor > version.minor)): + #raise VersionError(f"Incompatible {version=}, with " + #f"{vrsn=}.") kind = kind.decode("utf-8") if kind not in Serials: @@ -114,11 +113,11 @@ def rematch(match, *, version=None): vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) if vrsn.major > 1: # version1 vs but major > 1 raise VersionError(f"Incompatible {vrsn=} with version string.") - if version is not None: # compatible version with vrsn - if (vrsn.major > version.major or - (vrsn.major == version.major and vrsn.minor > version.minor)): - raise VersionError(f"Incompatible {version=}, with " - f"{vrsn=}.") + #if version is not None: # compatible version with vrsn + #if (vrsn.major > version.major or + #(vrsn.major == version.major and vrsn.minor > version.minor)): + #raise VersionError(f"Incompatible {version=}, with " + #f"{vrsn=}.") kind = kind.decode("utf-8") if kind not in Serials: @@ -153,7 +152,7 @@ def versify(protocol=Protos.keri, version=Version, kind=Serials.json, size=0): -def deversify(vs, version=None): +def deversify(vs): """ Returns: tuple(proto, kind, version, size) Where: proto (str): value is protocol type identifier one of Protos (Protocolage) @@ -166,8 +165,6 @@ def deversify(vs, version=None): Parameters: vs (str | bytes): version string to extract from - version (Versionage | None): supported version. None means do not check - for supported version. namedtuple of ints (major, minor) Uses regex match to extract: protocol type @@ -182,12 +179,12 @@ def deversify(vs, version=None): if not match: raise VersionError(f"Invalid version string = '{vs}'.") - return rematch(match, version=version) + return rematch(match) -def smell(raw, *, version=None): +def smell(raw): """Extract and return instance of Smellage from version string inside raw serialization. @@ -198,8 +195,7 @@ def smell(raw, *, version=None): raw (bytearray) of serialized incoming message stream. Assumes start of stream is JSON, CBOR, or MGPK field map with first field is labeled 'v' and value is version string. - version (Versionage | None): instance supported protocol version - None means do not enforce a supported version + """ if len(raw) < SMELLSIZE: raise ShortageError(f"Need more raw bytes to smell full version string.") @@ -209,7 +205,7 @@ def smell(raw, *, version=None): raise VersionError(f"Invalid version string from smelled raw = " f"{raw[: SMELLSIZE]}.") - return rematch(match, version=version) + return rematch(match) diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 0922ea2b6..71be7e19e 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -2487,13 +2487,12 @@ def test_serder_v2(): with pytest.raises(ValueError): - serder = Serder(version=kering.Vrsn_2_0) + serder = Serder() #Test Serder bare makify bootstrap for ACDC JSON serder = Serder(makify=True, proto=kering.Protos.acdc, - vrsn=kering.Vrsn_2_0, - version=kering.Vrsn_2_0) # make defaults for ACDC + vrsn=kering.Vrsn_2_0) # make defaults for ACDC assert serder.sad == {'v': 'ACDCCAAJSONAABZ.', 'd': 'EN-uBXL6rsJpJvDSsyOAnttQiI9gka4qLbe3MlIoYwYy', 'i': '', @@ -2501,17 +2500,18 @@ def test_serder_v2(): assert serder.raw == (b'{"v":"ACDCCAAJSONAABZ.","d":"EN-uBXL6rsJpJvDSsyOAnttQiI9gka4qLbe3MlIoYwYy","' b'i":"","s":""}') assert serder.verify() + assert serder.vrsn == serder.version == kering.Vrsn_2_0 sad = serder.sad raw = serder.raw said = serder.said size = serder.size - serder = Serder(sad=sad, version=kering.Vrsn_2_0) + serder = Serder(sad=sad) assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad assert serder.proto == kering.Protos.acdc - assert serder.vrsn == kering.Vrsn_2_0 + assert serder.vrsn == kering.Vrsn_2_0 == serder.version assert serder.size == size assert serder.kind == kering.Serials.json assert serder.said == said @@ -2527,12 +2527,12 @@ def test_serder_v2(): ' "s": ""\n' '}') - serder = Serder(raw=raw, version=kering.Vrsn_2_0) + serder = Serder(raw=raw) assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad assert serder.proto == kering.Protos.acdc - assert serder.vrsn == kering.Vrsn_2_0 + assert serder.vrsn == kering.Vrsn_2_0 == serder.version assert serder.size == size assert serder.kind == kering.Serials.json assert serder.said == said @@ -2541,7 +2541,7 @@ def test_serder_v2(): assert serder.compare(said=said) assert serder.compare(said=said.encode("utf-8")) - serder = Serder(sad=sad, makify=True, version=kering.Vrsn_2_0) # test makify with sad + serder = Serder(sad=sad, makify=True) # test makify with sad assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad @@ -2556,8 +2556,7 @@ def test_serder_v2(): # test default serder = Serder(makify=True, - vrsn=kering.Vrsn_2_0, - version=kering.Vrsn_2_0) # make defaults for default proto + vrsn=kering.Vrsn_2_0) # make defaults for default proto assert serder.sad == {'v': 'KERICAAJSONAADO.', From a094ccc0a8ab78301d071615dd6bd7567c62aeed Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 24 Mar 2024 14:58:15 -0600 Subject: [PATCH 063/418] refactor Protos to Protocols --- src/keri/core/coring.py | 2 +- src/keri/core/parsing.py | 2 +- src/keri/core/serdering.py | 34 ++++---- src/keri/kering.py | 16 ++-- src/keri/vc/proving.py | 2 +- tests/core/test_coring.py | 2 +- tests/core/test_serdering.py | 155 +++++++++++++++++------------------ tests/test_kering.py | 50 +++++------ tests/vc/test_proving.py | 6 +- 9 files changed, 133 insertions(+), 136 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index e06a18ffd..7b55182e8 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -37,7 +37,7 @@ UnexpectedCountCodeError, UnexpectedOpCodeError) from ..kering import (Versionage, Version, VERRAWSIZE, VERFMT, MAXVERFULLSPAN, versify, deversify, Rever, smell) -from ..kering import Serials, Serialage, Protos, Protocolage, Ilkage, Ilks +from ..kering import Serials, Serialage, Protocols, Protocolage, Ilkage, Ilks from ..help import helping diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index 691b8accf..aae8f64a4 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -11,7 +11,7 @@ from dataclasses import dataclass, astuple from .coring import (Ilks, CtrDex, Counter, Seqner, Siger, Cigar, - Dater, Verfer, Prefixer, Saider, Pather, Protos ) + Dater, Verfer, Prefixer, Saider, Pather, Protocols ) from . import serdering from .. import help from .. import kering diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 0677a6b82..a523efd81 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -27,7 +27,7 @@ MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN) from ..kering import SMELLSIZE, Smellage, smell -from ..kering import Protos, Serials, Rever, versify, deversify, Ilks +from ..kering import Protocols, Serials, Rever, versify, deversify, Ilks from ..core import coring from .coring import MtrDex, DigDex, PreDex, Saids, Digestage from .coring import Matter, Saider, Verfer, Diger, Number, Tholder @@ -147,9 +147,9 @@ def reap(self, ims): smellage = smell(ims) - if smellage.protocol == Protos.keri: + if smellage.protocol == Protocols.keri: return SerderKERI(raw=ims, strip=True, smellage=smellage) - elif smellage.protocol == Protos.acdc: + elif smellage.protocol == Protocols.acdc: return SerderACDC(raw=ims, strip=True, smellage=smellage) else: raise ProtocolError(f"Unsupported protocol type = {smellage.proto}.") @@ -269,8 +269,8 @@ class Serder: } #override in subclass to enforce specific protocol - Protocol = None # class based message protocol, None means any in Protos is ok - Proto = Protos.keri # default message protocol type for makify on base Serder + Protocol = None # class based message protocol, None means any in Protocols is ok + Proto = Protocols.keri # default message protocol type for makify on base Serder Vrsn = Vrsn_1_0 # default protocol version for protocol type Kind = Serials.json # default serialization kind @@ -281,7 +281,7 @@ class Serder: # ilk value of None is default for protocols that support ilkless packets Fields = \ { - Protos.keri: + Protocols.keri: { Vrsn_1_0: { @@ -386,7 +386,7 @@ class Serder: saids={Saids.d: DigDex.Blake3_256}), }, }, - Protos.acdc: + Protocols.acdc: { Vrsn_1_0: { @@ -467,7 +467,7 @@ def __init__(self, *, raw=b'', sad=None, strip=False, Ignore when raw not provided or when raw and saidify is True makify (bool): True means compute fields for sad including size and saids. - proto (str | None): desired protocol type str value of Protos + proto (str | None): desired protocol type str value of Protocols If None then its extracted from sad or uses default .Proto vrsn (Versionage | None): instance desired protocol version If None then its extracted from sad or uses default .Vrsn @@ -691,7 +691,7 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, ilk=None, saids=None) Parameters: sad (dict): serializable saidified field map of message. Ignored if raw provided - proto (str | None): desired protocol type str value of Protos + proto (str | None): desired protocol type str value of Protocols If None then its extracted from sad or uses default .Proto vrsn (Versionage | None): instance desired protocol version If None then its extracted from sad or uses default .Vrsn @@ -896,7 +896,7 @@ def _inhale(clas, raw, *, smellage=None): Returns: tuple (sad, proto, vrsn, kind, size) where: sad (dict): serializable attribute dict of saidified data - proto (str): value of Protos (Protocolage) protocol type + proto (str): value of Protocols (Protocolage) protocol type vrsn (Versionage | None): tuple of (major, minor) version ints None means do not enforce version kind (str): value of Serials (Serialage) serialization kind @@ -1090,7 +1090,7 @@ def _dumps(clas, sad, protocol, version, cversion=Vrsn_2_0): raw = bytearray() - if protocol not in Protos: + if protocol not in Protocols: raise SerializeError(f"Invalid {protocol=}.") if version not in clas.Fields[protocol]: @@ -1113,7 +1113,7 @@ def _dumps(clas, sad, protocol, version, cversion=Vrsn_2_0): # so can serialize in order to compute saidive fields # need to fix ._verify and .makify to account for CESR native serialization - if protocol == Protos.keri: + if protocol == Protocols.keri: for l, v in sad.items(): # assumes valid field order & presence if not fixed: # prepend label pass @@ -1184,7 +1184,7 @@ def _dumps(clas, sad, protocol, version, cversion=Vrsn_2_0): raw.extend(val) - elif protocol == Protos.acdc: + elif protocol == Protocols.acdc: for l, val in sad.items(): # assumes valid field order & presence if not fixed: pass # prepend label @@ -1351,8 +1351,8 @@ class SerderKERI(Serder): See docs for Serder """ #override in subclass to enforce specific protocol - Protocol = Protos.keri # required protocol, None means any in Protos is ok - Proto = Protos.keri # default protocol type + Protocol = Protocols.keri # required protocol, None means any in Protocols is ok + Proto = Protocols.keri # default protocol type @@ -1721,8 +1721,8 @@ class SerderACDC(Serder): See docs for Serder """ #override in subclass to enforce specific protocol - Protocol = Protos.acdc # required protocol, None means any in Protos is ok - Proto = Protos.acdc # default protocol type + Protocol = Protocols.acdc # required protocol, None means any in Protocols is ok + Proto = Protocols.acdc # default protocol type diff --git a/src/keri/kering.py b/src/keri/kering.py index b0493490d..7ed23ef36 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -20,7 +20,7 @@ # Protocol Types Protocolage = namedtuple("Protocolage", "keri acdc") -Protos = Protocolage(keri="KERI", acdc="ACDC") +Protocols = Protocolage(keri="KERI", acdc="ACDC") Versionage = namedtuple("Versionage", "major minor") Version = Versionage(major=1, minor=0) # KERI Protocol Version @@ -58,7 +58,7 @@ """ Smellage (results of smelling a version string such as in a Serder) - protocol (str): protocol type value of Protos examples 'KERI', 'ACDC' + protocol (str): protocol type value of Protocols examples 'KERI', 'ACDC' version (Versionage): named tuple (major, minor) ints of major minor version kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' size (str): int size of raw serialization @@ -85,7 +85,7 @@ def rematch(match): "kind2", "size2") protocol = proto.decode("utf-8") - if protocol not in Protos: + if protocol not in Protocols: raise ProtocolError(f"Invalid protocol type = {protocol}.") vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) if vrsn.major < 2: # version2 vs but major < 2 @@ -108,7 +108,7 @@ def rematch(match): "kind1", "size1") protocol = proto.decode("utf-8") - if protocol not in Protos: + if protocol not in Protocols: raise ProtocolError(f"Invalid protocol type = {protocol}.") vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) if vrsn.major > 1: # version1 vs but major > 1 @@ -128,18 +128,18 @@ def rematch(match): -def versify(protocol=Protos.keri, version=Version, kind=Serials.json, size=0): +def versify(protocol=Protocols.keri, version=Version, kind=Serials.json, size=0): """ Returns: vs (str): version string Parameters: - protocol (str): protocol one of Protos + protocol (str): protocol one of Protocols version (Versionage): namedtuple (major, minor) of ints kind (str): one of Serials size (int): length of serialized map that embeds version string field. """ - if protocol not in Protos: + if protocol not in Protocols: raise ProtocolError("Invalid message identifier = {}".format(protocol)) if kind not in Serials: raise KindError("Invalid serialization kind = {}".format(kind)) @@ -155,7 +155,7 @@ def versify(protocol=Protos.keri, version=Version, kind=Serials.json, size=0): def deversify(vs): """ Returns: tuple(proto, kind, version, size) Where: - proto (str): value is protocol type identifier one of Protos (Protocolage) + proto (str): value is protocol type identifier one of Protocols (Protocolage) acdc='ACDC', keri='KERI' vrsn (tuple): version tuple of type Versionage diff --git a/src/keri/vc/proving.py b/src/keri/vc/proving.py index 826e818d0..67721ddfb 100644 --- a/src/keri/vc/proving.py +++ b/src/keri/vc/proving.py @@ -50,7 +50,7 @@ def credential(schema, SerderACDC: credential instance """ - vs = versify(protocol=coring.Protos.acdc, version=version, kind=kind, size=0) + vs = versify(protocol=coring.Protocols.acdc, version=version, kind=kind, size=0) vc = dict( v=vs, diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 14810b58a..4687c0dd7 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -25,7 +25,7 @@ from keri.core import coring from keri.core import eventing -from keri.core.coring import (Ilkage, Ilks, Saids, Protos, Protocolage, +from keri.core.coring import (Ilkage, Ilks, Saids, Protocols, Protocolage, Sadder, Tholder, Seqner, NumDex, Number, Siger, Dater, Bexter, Texter) from keri.core.coring import Serialage, Serials, Tiers diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 71be7e19e..b1a395a0c 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -14,7 +14,7 @@ from ordered_set import OrderedSet as oset from keri import kering -from keri.kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, +from keri.kering import (Protocols, Versionage, Version, Vrsn_1_0, Vrsn_2_0, VERRAWSIZE, VERFMT, MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN) @@ -76,21 +76,18 @@ def test_serder(): assert oset(vf.saids) <= oset(vf.alls) - assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].saids == {'d': 'E'} - assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls == + assert Serder.Fields[Protocols.acdc][kering.Vrsn_1_0][None].saids == {'d': 'E'} + assert (Serder.Fields[Protocols.acdc][kering.Vrsn_1_0][None].alls == {'v': '', 'd': '', 'u': '', 'i': '', 'ri': '', 's': '', 'a': '', 'A': '', 'e': '', 'r': ''}) - assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].opts == + assert (Serder.Fields[Protocols.acdc][kering.Vrsn_1_0][None].opts == {'u': '', 'ri': '', 'a': '', 'A': '', 'e': '', 'r': ''}) - assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alts == + assert (Serder.Fields[Protocols.acdc][kering.Vrsn_1_0][None].alts == {'a': 'A', 'A': 'a'}) - assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].strict - - - #assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls == {'v': '', 'd': '', 'i': '', 's': ''} + assert Serder.Fields[Protocols.acdc][kering.Vrsn_1_0][None].strict # said field labels must be subset of all field labels - assert (set(Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].saids) <= - set(Serder.Fields[kering.Protos.acdc][kering.Vrsn_1_0][None].alls)) + assert (set(Serder.Fields[Protocols.acdc][kering.Vrsn_1_0][None].saids) <= + set(Serder.Fields[Protocols.acdc][kering.Vrsn_1_0][None].alls)) for proto, vrsns in Serder.Fields.items(): @@ -103,7 +100,7 @@ def test_serder(): serder = Serder() #Test Serder bare makify bootstrap for ACDC JSON - serder = Serder(makify=True, proto=kering.Protos.acdc) # make defaults for ACDC + serder = Serder(makify=True, proto=Protocols.acdc) # make defaults for ACDC assert serder.sad == {'v': 'ACDC10JSON00005a_', 'd': 'EMk7BvrqO_2sYjpI_-BmSELOFNie-muw4XTi3iYCz6pT', 'i': '', @@ -120,7 +117,7 @@ def test_serder(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -141,7 +138,7 @@ def test_serder(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -155,7 +152,7 @@ def test_serder(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -169,7 +166,7 @@ def test_serder(): serder = Serder(sad=sad, verify=False) # test not verify assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -181,7 +178,7 @@ def test_serder(): serder = Serder(raw=raw, verify=False) # test not verify assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -208,7 +205,7 @@ def test_serder(): assert isinstance(serder.raw, bytes) assert stream == extra assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -225,7 +222,7 @@ def test_serder(): #Test makify bootstrap for ACDC with CBOR - serder = Serder(makify=True, proto=kering.Protos.acdc, kind=kering.Serials.cbor) + serder = Serder(makify=True, proto=Protocols.acdc, kind=kering.Serials.cbor) assert serder.sad == {'v': 'ACDC10CBOR00004b_', 'd': 'EGahYhEMb_Sz0L1UwhrUvbyxyzoi_G85-pD9jRjhnqgU', 'i': '', @@ -242,7 +239,7 @@ def test_serder(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.cbor @@ -263,7 +260,7 @@ def test_serder(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.cbor @@ -277,7 +274,7 @@ def test_serder(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.cbor @@ -288,7 +285,7 @@ def test_serder(): #Test makify bootstrap for ACDC with MGPK - serder = Serder(makify=True, proto=kering.Protos.acdc, kind=kering.Serials.mgpk) + serder = Serder(makify=True, proto=Protocols.acdc, kind=kering.Serials.mgpk) assert serder.sad == {'v': 'ACDC10MGPK00004b_', 'd': 'EGV5wdF1nRbSXatBgZDpAxlGL6BuATjpUYBuk0AQW7GC', 'i': '', @@ -305,7 +302,7 @@ def test_serder(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.mgpk @@ -326,7 +323,7 @@ def test_serder(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.mgpk @@ -340,7 +337,7 @@ def test_serder(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.mgpk @@ -429,7 +426,7 @@ def test_serder(): serder = Serder(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -439,7 +436,7 @@ def test_serder(): serder = Serder(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -513,7 +510,7 @@ def test_serder(): serder = Serder(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -523,7 +520,7 @@ def test_serder(): serder = Serder(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -565,7 +562,7 @@ def test_serder(): serder = Serder(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -575,7 +572,7 @@ def test_serder(): serder = Serder(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -583,7 +580,7 @@ def test_serder(): assert serder.ilk == ilk # test opts - serder = Serder(makify=True, proto=kering.Protos.acdc) # make defaults for ACDC + serder = Serder(makify=True, proto=Protocols.acdc) # make defaults for ACDC assert serder.sad == {'v': 'ACDC10JSON00005a_', 'd': 'EMk7BvrqO_2sYjpI_-BmSELOFNie-muw4XTi3iYCz6pT', 'i': '', @@ -616,7 +613,7 @@ def test_serder(): serder = Serder(makify=True, sad=sad) # make using sad # test alts - serder = Serder(makify=True, proto=kering.Protos.acdc) # make defaults for ACDC + serder = Serder(makify=True, proto=Protocols.acdc) # make defaults for ACDC assert serder.sad == {'v': 'ACDC10JSON00005a_', 'd': 'EMk7BvrqO_2sYjpI_-BmSELOFNie-muw4XTi3iYCz6pT', 'i': '', @@ -634,7 +631,7 @@ def test_serder(): # test not strict # test opts - serder = Serder(makify=True, proto=kering.Protos.acdc, ilk=kering.Ilks.ace) # make defaults for ACDC + serder = Serder(makify=True, proto=Protocols.acdc, ilk=kering.Ilks.ace) # make defaults for ACDC assert serder.sad == {'v': 'ACDC10JSON000064_', 't': 'ace', 'd': 'EKFsN95K2h5I6pJC6eTrNKiX8uHyn5o-SYHy6IelbPK8', @@ -818,7 +815,7 @@ def test_serderkeri_icp(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -854,7 +851,7 @@ def test_serderkeri_icp(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -954,7 +951,7 @@ def test_serderkeri_icp(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -990,7 +987,7 @@ def test_serderkeri_icp(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1076,7 +1073,7 @@ def test_serderkeri_rot(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1107,7 +1104,7 @@ def test_serderkeri_rot(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1184,7 +1181,7 @@ def test_serderkeri_ixn(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1218,7 +1215,7 @@ def test_serderkeri_ixn(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1318,7 +1315,7 @@ def test_serderkeri_dip(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1353,7 +1350,7 @@ def test_serderkeri_dip(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1487,7 +1484,7 @@ def test_serderkeri_dip(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1522,7 +1519,7 @@ def test_serderkeri_dip(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1615,7 +1612,7 @@ def test_serderkeri_drt(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1648,7 +1645,7 @@ def test_serderkeri_drt(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1717,7 +1714,7 @@ def test_serderkeri_rct(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1747,7 +1744,7 @@ def test_serderkeri_rct(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1817,7 +1814,7 @@ def test_serderkeri_qry(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1848,7 +1845,7 @@ def test_serderkeri_qry(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1920,7 +1917,7 @@ def test_serderkeri_rpy(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -1950,7 +1947,7 @@ def test_serderkeri_rpy(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2021,7 +2018,7 @@ def test_serderkeri_pro(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2052,7 +2049,7 @@ def test_serderkeri_pro(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2123,7 +2120,7 @@ def test_serderkeri_bar(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2154,7 +2151,7 @@ def test_serderkeri_bar(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2233,7 +2230,7 @@ def test_serderkeri_exn(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2266,7 +2263,7 @@ def test_serderkeri_exn(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2334,7 +2331,7 @@ def test_serderkeri_vcp(): serder = SerderKERI(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2366,7 +2363,7 @@ def test_serderkeri_vcp(): serder = SerderKERI(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.keri + assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2405,7 +2402,7 @@ def test_serderacdc(): with pytest.raises(ValueError): serder = SerderACDC() - serder = SerderACDC(makify=True, proto=kering.Protos.acdc, verify=False) # make defaults for ACDC + serder = SerderACDC(makify=True, proto=Protocols.acdc, verify=False) # make defaults for ACDC assert serder.sad == {'v': 'ACDC10JSON00005a_', 'd': 'EMk7BvrqO_2sYjpI_-BmSELOFNie-muw4XTi3iYCz6pT', 'i': '', @@ -2413,7 +2410,7 @@ def test_serderacdc(): assert serder.raw == (b'{"v":"ACDC10JSON00005a_","d":"EMk7BvrqO_2sYjpI_' b'-BmSELOFNie-muw4XTi3iYCz6pT","i":"","s":""}') - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == 90 assert serder.kind == kering.Serials.json @@ -2445,7 +2442,7 @@ def test_serderacdc(): serder = SerderACDC(sad=sad) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2457,7 +2454,7 @@ def test_serderacdc(): serder = SerderACDC(raw=raw) assert serder.raw == raw assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2475,14 +2472,14 @@ def test_serder_v2(): """ - assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_2_0][None].saids == {'d': 'E'} - assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_2_0][None].alls == + assert Serder.Fields[Protocols.acdc][kering.Vrsn_2_0][None].saids == {'d': 'E'} + assert (Serder.Fields[Protocols.acdc][kering.Vrsn_2_0][None].alls == {'v': '', 'd': '', 'u': '', 'i': '', 'rd': '', 's': '', 'a': '', 'A': '', 'e': '', 'r': ''}) - assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_2_0][None].opts == + assert (Serder.Fields[Protocols.acdc][kering.Vrsn_2_0][None].opts == {'u': '', 'rd': '', 'a': '', 'A': '', 'e': '', 'r': ''}) - assert (Serder.Fields[kering.Protos.acdc][kering.Vrsn_2_0][None].alts == + assert (Serder.Fields[Protocols.acdc][kering.Vrsn_2_0][None].alts == {'a': 'A', 'A': 'a'}) - assert Serder.Fields[kering.Protos.acdc][kering.Vrsn_2_0][None].strict + assert Serder.Fields[Protocols.acdc][kering.Vrsn_2_0][None].strict @@ -2491,7 +2488,7 @@ def test_serder_v2(): #Test Serder bare makify bootstrap for ACDC JSON serder = Serder(makify=True, - proto=kering.Protos.acdc, + proto=Protocols.acdc, vrsn=kering.Vrsn_2_0) # make defaults for ACDC assert serder.sad == {'v': 'ACDCCAAJSONAABZ.', 'd': 'EN-uBXL6rsJpJvDSsyOAnttQiI9gka4qLbe3MlIoYwYy', @@ -2510,7 +2507,7 @@ def test_serder_v2(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_2_0 == serder.version assert serder.size == size assert serder.kind == kering.Serials.json @@ -2531,7 +2528,7 @@ def test_serder_v2(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_2_0 == serder.version assert serder.size == size assert serder.kind == kering.Serials.json @@ -2545,7 +2542,7 @@ def test_serder_v2(): assert serder.raw == raw assert isinstance(serder.raw, bytes) assert serder.sad == sad - assert serder.proto == kering.Protos.acdc + assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_2_0 assert serder.size == size assert serder.kind == kering.Serials.json @@ -2576,7 +2573,7 @@ def test_serder_v2(): b'21VH3bn","i":"EGqrN042jSUT5bjUuQqGALW4inJMJA6BBlVKf21VH3bn","s":"0","kt":"0"' b',"k":[],"nt":"0","n":[],"bt":"0","b":[],"c":[],"a":[]}') assert serder.verify() - assert serder.proto == kering.Protos.keri == Serder.Proto # default + assert serder.proto == Protocols.keri == Serder.Proto # default assert serder.vrsn == kering.Vrsn_2_0 assert serder.kind == kering.Serials.json == Serder.Kind # default assert serder.ilk == kering.Ilks.icp # default first one @@ -2601,7 +2598,7 @@ def test_serdery(): ims = bytearray(serderKeri.raw) - serder = SerderACDC(makify=True, proto=kering.Protos.acdc, verify=False) # make defaults for ACDC + serder = SerderACDC(makify=True, proto=Protocols.acdc, verify=False) # make defaults for ACDC sad = serder.sad isr = 'EO8CE5RH1X8QJwHHhPkj_S6LJQDRNOiGohW327FMA6D2' sad['i'] = isr @@ -2654,7 +2651,7 @@ def test_serdery_noversion(): ims = bytearray(serderKeri.raw) - serder = SerderACDC(makify=True, proto=kering.Protos.acdc, verify=False) # make defaults for ACDC + serder = SerderACDC(makify=True, proto=Protocols.acdc, verify=False) # make defaults for ACDC sad = serder.sad isr = 'EO8CE5RH1X8QJwHHhPkj_S6LJQDRNOiGohW327FMA6D2' sad['i'] = isr diff --git a/tests/test_kering.py b/tests/test_kering.py index f46a43c54..80d0a070b 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -12,7 +12,7 @@ import pytest from keri import kering -from keri.kering import Protocolage, Protos +from keri.kering import Protocolage, Protocols from keri.kering import Serialage, Serials from keri.kering import Ilkage, Ilks from keri.kering import (Versionage, Version, MAXVERFULLSPAN, @@ -31,16 +31,16 @@ def test_protos(): """ - Test protocols namedtuple instance Protos + Test protocols namedtuple instance Protocols """ - assert isinstance(Protos, Protocolage) + assert isinstance(Protocols, Protocolage) - assert Protos.keri == 'KERI' - assert Protos.acdc == 'ACDC' + assert Protocols.keri == 'KERI' + assert Protocols.acdc == 'ACDC' - assert 'KERI' in Protos - assert 'ACDC' in Protos + assert 'KERI' in Protocols + assert 'ACDC' in Protocols """End Test""" @@ -308,7 +308,7 @@ def test_versify_v1(): assert vs == "KERI10JSON000000_" assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.json assert version == Version assert size == 0 @@ -317,16 +317,16 @@ def test_versify_v1(): assert vs == "KERI10JSON000041_" assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.json assert version == Version assert size == 65 - vs = versify(protocol=Protos.acdc, kind=Serials.json, size=86) + vs = versify(protocol=Protocols.acdc, kind=Serials.json, size=86) assert vs == "ACDC10JSON000056_" assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) - assert proto == Protos.acdc + assert proto == Protocols.acdc assert kind == Serials.json assert version == Version assert size == 86 @@ -335,7 +335,7 @@ def test_versify_v1(): assert vs == "KERI10MGPK000000_" assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.mgpk assert version == Version assert size == 0 @@ -344,7 +344,7 @@ def test_versify_v1(): assert vs == "KERI10MGPK000041_" assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.mgpk assert version == Version assert size == 65 @@ -353,7 +353,7 @@ def test_versify_v1(): assert vs == "KERI10CBOR000000_" assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.cbor assert version == Version assert size == 0 @@ -362,7 +362,7 @@ def test_versify_v1(): assert vs == "KERI10CBOR000041_" assert len(vs) == VER1FULLSPAN proto, version, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.cbor assert version == Version assert size == 65 @@ -371,7 +371,7 @@ def test_versify_v1(): assert vs == "KERI11JSON000000_" assert len(vs) == VER1FULLSPAN proto, vrsn, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.json assert vrsn == (1, 1) assert size == 0 @@ -404,7 +404,7 @@ def test_versify_v2(): assert vs == "KERICAAJSONAAAA." assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.json assert vrsn == version assert size == 0 @@ -413,16 +413,16 @@ def test_versify_v2(): assert vs == "KERICAAJSONAABB." assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.json assert vrsn == version assert size == 65 - vs = versify(protocol=Protos.acdc, version=version, kind=Serials.json, size=86) + vs = versify(protocol=Protocols.acdc, version=version, kind=Serials.json, size=86) assert vs == "ACDCCAAJSONAABW." assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size = deversify(vs) - assert proto == Protos.acdc + assert proto == Protocols.acdc assert kind == Serials.json assert version == version assert size == 86 @@ -431,7 +431,7 @@ def test_versify_v2(): assert vs == 'KERICAAMGPKAAAA.' assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.mgpk assert vrsn == version assert size == 0 @@ -440,7 +440,7 @@ def test_versify_v2(): assert vs == 'KERICAAMGPKAABB.' assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.mgpk assert vrsn == version assert size == 65 @@ -449,7 +449,7 @@ def test_versify_v2(): assert vs == 'KERICAACBORAAAA.' assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.cbor assert vrsn == version assert size == 0 @@ -458,7 +458,7 @@ def test_versify_v2(): assert vs == 'KERICAACBORAABB.' assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.cbor assert vrsn == version assert size == 65 @@ -467,7 +467,7 @@ def test_versify_v2(): assert vs == "KERICABJSONAAAA." assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size = deversify(vs) - assert proto == Protos.keri + assert proto == Protocols.keri assert kind == Serials.json assert vrsn == (2, 1) assert size == 0 diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index b46947e22..ca4687c43 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -113,7 +113,7 @@ def test_credentialer(): sub = dict(a=123, b="abc", issuanceDate="2021-06-27T21:26:21.233257+00:00") d = dict( - v=coring.versify(protocol=coring.Protos.acdc, kind=Serials.json, size=0), + v=coring.versify(protocol=coring.Protocols.acdc, kind=Serials.json, size=0), d="", i="EF6maPM_d5ZN7U3NRFC1-6TM7k_E00_a8AG9YyLA4uWi", s="abc", @@ -150,7 +150,7 @@ def test_credentialer(): d2 = dict(d) d2['d'] = "" - d2["v"] = coring.versify(protocol=coring.Protos.acdc, kind=Serials.cbor, size=0) + d2["v"] = coring.versify(protocol=coring.Protocols.acdc, kind=Serials.cbor, size=0) _, d2 = coring.Saider.saidify(sad=d2) creder = serdering.SerderACDC(sad=d2) # Creder(ked=d2) @@ -176,7 +176,7 @@ def test_credentialer(): assert creder.sad == d2 d3 = dict(d) - d3["v"] = coring.versify(protocol=coring.Protos.acdc, kind=Serials.mgpk, size=0) + d3["v"] = coring.versify(protocol=coring.Protocols.acdc, kind=Serials.mgpk, size=0) _, d3 = coring.Saider.saidify(sad=d3) creder = serdering.SerderACDC(sad=d3) # Creder(ked=d3) From 35e55f23b1c8625b448a5a9c36f1f3bc21820cd0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 24 Mar 2024 19:49:15 -0600 Subject: [PATCH 064/418] refactoring Serder to change class methods ._inhale and .exhale to instance methods better for CESR native ser deser --- src/keri/core/serdering.py | 42 +++++++++++++++++--------------------- tests/vc/test_proving.py | 9 +++++--- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index a523efd81..525645811 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -489,13 +489,7 @@ def __init__(self, *, raw=b'', sad=None, strip=False, if raw: # deserialize raw using property setter # self._inhale works because it only references class attributes - sad, proto, vrsn, kind, size = self._inhale(raw=raw, smellage=smellage) - self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray - self._sad = sad - self._proto = proto - self._vrsn = vrsn - self._kind = kind - self._size = size + self._inhale(raw=raw, smellage=smellage) # primary said field label try: label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] @@ -528,13 +522,8 @@ def __init__(self, *, raw=b'', sad=None, strip=False, else: # self._exhale works because it only access class attributes - raw, sad, proto, vrsn, kind, size = self._exhale(sad=sad) - self._raw = raw - self._sad = sad - self._proto = proto - self._vrsn = vrsn - self._kind = kind - self._size = size + self._exhale(sad=sad) + # primary said field label try: label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] @@ -883,9 +872,7 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, ilk=None, saids=None) self._said = None # no saidive field - - @classmethod - def _inhale(clas, raw, *, smellage=None): + def _inhale(self, raw, *, smellage=None): """Deserializes raw. Parses serilized event ser of serialization kind and assigns to instance attributes and returns tuple of associated elements. @@ -919,12 +906,17 @@ def _inhale(clas, raw, *, smellage=None): else: # not passed in so smell raw proto, vrsn, kind, size = smell(raw) - sad = clas.loads(raw=raw, size=size, kind=kind) + sad = self.loads(raw=raw, size=size, kind=kind) if "v" not in sad: # Regex does not check for version string label itself raise FieldError(f"Missing version string field in {sad}.") - return sad, proto, vrsn, kind, size + self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray + self._sad = sad + self._proto = proto + self._vrsn = vrsn + self._kind = kind + self._size = size @staticmethod @@ -984,8 +976,7 @@ def _loads(clas, raw, size=None): pass - @classmethod - def _exhale(clas, sad): + def _exhale(self, sad): """Serializes sad and sets the serialized size in its version string. As classmethod enables bootstrap of valid sad dict that has correct size @@ -1011,7 +1002,7 @@ def _exhale(clas, sad): # extract elements so can replace size element but keep others proto, vrsn, kind, size = deversify(sad["v"]) - raw = clas.dumps(sad, kind) + raw = self.dumps(sad, kind) size = len(raw) # generate new version string with correct size @@ -1029,7 +1020,12 @@ def _exhale(clas, sad): raise SerializeError(f"Malformed size of raw in version string == {vs}") sad["v"] = vs # update sad - return raw, sad, proto, vrsn, kind, size + self._raw = raw + self._sad = sad + self._proto = proto + self._vrsn = vrsn + self._kind = kind + self._size = size @staticmethod diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index ca4687c43..b9ac02606 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -136,10 +136,13 @@ def test_credentialer(): b'"i":"EF6maPM_d5ZN7U3NRFC1-6TM7k_E00_a8AG9YyLA4uWi","s":"abc","a":{"a":123,"b' b'":"abc","issuanceDate":"2021-06-27T21:26:21.233257+00:00"}}') - raw1, ked1, knd1, ver1, knd1, size1 = creder._exhale(sad=d) - assert raw1 == creder.raw + raw1 = creder.raw + ver1 = creder.vrsn + knd1 = creder.kind + sad1 = creder.sad + assert knd1 == Serials.json - assert ked1 == d + assert sad1 == d assert ver1 == Versionage(major=1, minor=0) creder = serdering.SerderACDC(raw=raw1) # Creder(raw=raw1) From 38146bd5cdd96b42efddd077354ab317208b7dec Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 24 Mar 2024 19:57:17 -0600 Subject: [PATCH 065/418] refactore Serder .dumps .loads ._dumps to be instance methods in preparation for better for CESR native ser deser --- src/keri/core/serdering.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 525645811..ee45db330 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -919,8 +919,7 @@ def _inhale(self, raw, *, smellage=None): self._size = size - @staticmethod - def loads(raw, size=None, kind=Serials.json): + def loads(self, raw, size=None, kind=Serials.json): """Utility static method to handle deserialization by kind Returns: @@ -960,8 +959,8 @@ def loads(raw, size=None, kind=Serials.json): return sad - @classmethod - def _loads(clas, raw, size=None): + + def _loads(self, raw, size=None): """CESR native desserialization of raw Returns: @@ -1028,8 +1027,7 @@ def _exhale(self, sad): self._size = size - @staticmethod - def dumps(sad, kind=Serials.json): + def dumps(self, sad, kind=Serials.json): """Utility static method to handle serialization by kind Returns: @@ -1055,8 +1053,7 @@ def dumps(sad, kind=Serials.json): return raw - @classmethod - def _dumps(clas, sad, protocol, version, cversion=Vrsn_2_0): + def _dumps(self, sad, protocol, version, cversion=Vrsn_2_0): """CESR native serialization of sad Returns: @@ -1089,10 +1086,10 @@ def _dumps(clas, sad, protocol, version, cversion=Vrsn_2_0): if protocol not in Protocols: raise SerializeError(f"Invalid {protocol=}.") - if version not in clas.Fields[protocol]: + if version not in self.Fields[protocol]: raise SerializeError(f"Invalid {version=} for {protocol=}.") - ilks = clas.Fields[protocol][version] # get fields keyed by ilk + ilks = self.Fields[protocol][version] # get fields keyed by ilk ilk = sad.get('t') # returns None if missing message type (ilk) if ilk not in ilks: # From a601a923ea0ea5f613a2f6d0785d3e59ab369e41 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 24 Mar 2024 20:33:18 -0600 Subject: [PATCH 066/418] added ._cvrsn to Serder to support CESR Code table version tracking in CESR native serializations --- src/keri/core/serdering.py | 82 +++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index ee45db330..4b0d66b9f 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -273,6 +273,7 @@ class Serder: Proto = Protocols.keri # default message protocol type for makify on base Serder Vrsn = Vrsn_1_0 # default protocol version for protocol type Kind = Serials.json # default serialization kind + CVrsn = Vrsn_2_0 # default CESR code table version # Nested dict keyed by protocol. @@ -439,8 +440,8 @@ class Serder: } - def __init__(self, *, raw=b'', sad=None, strip=False, - smellage=None, verify=True, makify=False, + def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, + cvrsn=None, verify=True, makify=False, proto=None, vrsn=None, kind=None, ilk=None, saids=None): """Deserialize raw if provided. Update properties from deserialized raw. Verifies said(s) embedded in sad as given by labels. @@ -462,6 +463,8 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version string elements. If none or empty ignore otherwise assume that raw already had its version string extracted (reaped) into the elements of smellage. + cvrsn (Versionage | None): instance desired CESR code table version + If None then its extracted from raw or uses default .CVrsn verify (bool): True means verify said(s) of given raw or sad. Raises ValidationError if verification fails Ignore when raw not provided or when raw and saidify is True @@ -486,10 +489,12 @@ def __init__(self, *, raw=b'', sad=None, strip=False, """ + cvrsn = cvrsn if cvrsn is not None else self.CVrsn if raw: # deserialize raw using property setter - # self._inhale works because it only references class attributes - self._inhale(raw=raw, smellage=smellage) + self._inhale(raw=raw, smellage=smellage, cvrsn=cvrsn) + # ._inhale updates ._raw, ._sad, ._cvrsn, ._proto, ._vrsn, ._kind, ._size + # primary said field label try: label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] @@ -517,21 +522,22 @@ def __init__(self, *, raw=b'', sad=None, strip=False, elif sad or makify: # serialize sad into raw or make sad if makify: # recompute properties and said(s) and reset sad # makify resets sad, raw, proto, vrsn, kind, ilk, and size - self.makify(sad=sad, proto=proto, vrsn=vrsn, kind=kind, + self.makify(sad=sad, cvrsn=cvrsn, proto=proto, vrsn=vrsn, kind=kind, ilk=ilk, saids=saids) + # .makify updates ._raw, ._sad, ._cvrsn, ._proto, ._vrsn, ._kind, ._size else: - # self._exhale works because it only access class attributes - self._exhale(sad=sad) + self._exhale(sad=sad, cvrsn=cvrsn) + # .exhale updates ._raw, ._sad, ._cvrsn, ._proto, ._vrsn, ._kind, ._size - # primary said field label - try: - label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] - if label not in self._sad: - raise DeserializeError(f"Missing primary said field in {self._sad}.") - self._said = self._sad[label] # not verified - except Exception: - self._said = None # no saidive field + # primary said field label + try: + label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] + if label not in self._sad: + raise DeserializeError(f"Missing primary said field in {self._sad}.") + self._said = self._sad[label] # not verified + except Exception: + self._said = None # no saidive field if verify: # verify the said(s) provided in sad try: @@ -662,7 +668,8 @@ def _verify(self): # verified successfully since no exception - def makify(self, sad, *, proto=None, vrsn=None, kind=None, ilk=None, saids=None): + def makify(self, sad, *, cvrsn=None, proto=None, vrsn=None, kind=None, + ilk=None, saids=None): """Makify given sad dict makes the versions string and computes the said field values and sets associated properties: raw, sad, proto, version, kind, size @@ -680,6 +687,8 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, ilk=None, saids=None) Parameters: sad (dict): serializable saidified field map of message. Ignored if raw provided + cvrsn (Versionage | None): instance desired CESR code table version + If None then its extracted from raw or uses default .CVrsn proto (str | None): desired protocol type str value of Protocols If None then its extracted from sad or uses default .Proto vrsn (Versionage | None): instance desired protocol version @@ -706,6 +715,8 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, ilk=None, saids=None) else: silk = sad.get('t') # if 't' not in sad .get returns None which may be valid + cvrsn = cvrsn if cvrsn is not None else self.CVrsn + if proto is None: proto = sproto if sproto is not None else self.Proto @@ -729,12 +740,6 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, ilk=None, saids=None) raise SerializeError(f"Expected protocol = {self.Protocol}, got " f"{proto} instead.") - #if version is not None: # compatible version with vrsn - #if (vrsn.major > version.major or - #(vrsn.major == version.major and vrsn.minor > version.minor)): - #raise SerializeError(f"Incompatible {version=}, with " - #f"{vrsn=}.") - if kind not in Serials: raise SerializeError(f"Invalid serialization kind = {kind}") @@ -858,21 +863,14 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, ilk=None, saids=None) self._raw = raw self._sad = sad + self._cvrsn = cvrsn self._proto = proto self._vrsn = vrsn self._kind = kind self._size = size - # primary said field label - try: - label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] - if label not in self._sad: - raise SerializeError(f"Missing primary said field in {self._sad}.") - self._said = self._sad[label] # implicitly verified - except Exception: - self._said = None # no saidive field - def _inhale(self, raw, *, smellage=None): + def _inhale(self, raw, *, smellage=None, cvrsn=None): """Deserializes raw. Parses serilized event ser of serialization kind and assigns to instance attributes and returns tuple of associated elements. @@ -895,6 +893,8 @@ def _inhale(self, raw, *, smellage=None): elements. If none or empty ignore otherwise assume that raw already had its version string extracted (reaped) into the elements of smellage. + cvrsn (Versionage | None): instance desired CESR code table version + If None then its extracted from raw or uses default .CVrsn Note: loads and jumps of json use str whereas cbor and msgpack use bytes @@ -908,17 +908,21 @@ def _inhale(self, raw, *, smellage=None): sad = self.loads(raw=raw, size=size, kind=kind) + cvrsn = cvrsn if cvrsn is not None else self.CVrsn + if "v" not in sad: # Regex does not check for version string label itself raise FieldError(f"Missing version string field in {sad}.") self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray self._sad = sad + self._cvrsn = cvrsn self._proto = proto self._vrsn = vrsn self._kind = kind self._size = size + def loads(self, raw, size=None, kind=Serials.json): """Utility static method to handle deserialization by kind @@ -975,7 +979,7 @@ def _loads(self, raw, size=None): pass - def _exhale(self, sad): + def _exhale(self, sad, cvrsn=None): """Serializes sad and sets the serialized size in its version string. As classmethod enables bootstrap of valid sad dict that has correct size @@ -992,6 +996,8 @@ def _exhale(self, sad): Parameters: clas (Serder): class reference sad (dict): serializable attribute dict of saidified data + cvrsn (Versionage | None): instance desired CESR code table version + If None then its extracted from raw or uses default .CVrsn """ @@ -1007,6 +1013,8 @@ def _exhale(self, sad): # generate new version string with correct size vs = versify(protocol=proto, version=vrsn, kind=kind, size=size) + cvrsn = cvrsn if cvrsn is not None else self.CVrsn + # find location of old version string inside raw match = Rever.search(raw) # Rever's regex takes bytes if not match or match.start() > 12: @@ -1021,6 +1029,7 @@ def _exhale(self, sad): self._raw = raw self._sad = sad + self._cvrsn = cvrsn self._proto = proto self._vrsn = vrsn self._kind = kind @@ -1259,6 +1268,15 @@ def sad(self): """ return dict(self._sad) # return copy + @property + def cvrsn(self): + """cvrsn (version) property getter + + Returns: + cvrsn (Versionage): instance of CESR code table version for this Serder + """ + return self._cvrsn + @property def kind(self): From fd8b3fbf341365e68e7fb98305684c43014be3e0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 24 Mar 2024 20:47:28 -0600 Subject: [PATCH 067/418] updated doc strings added couple of property aliases --- src/keri/core/serdering.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 4b0d66b9f..f3351cc23 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -201,8 +201,13 @@ class Serder: Properties: raw (bytes): of serialized event only sad (dict): self addressed data dict + cvrsn (Versionage): CESR code table version (Major, Minor) proto (str): Protocolage value as protocol identifier such as KERI, ACDC - version (Versionage): protocol version (Major, Minor) + alias of .protocol + protocol (str): Protocolage value as protocol identifier such as KERI, ACDC + alias of .proto + vrsn (Versionage): protocol version (Major, Minor) alias of .version + version (Versionage): protocol version (Major, Minor) alias of .vrsn kind (str): serialization kind coring.Serials such as JSON, CBOR, MGPK, CESR size (int): number of bytes in serialization said (str): qb64 said of .raw given by appropriate field @@ -211,10 +216,11 @@ class Serder: Hidden Attributes: - ._raw is bytes of serialized event only - ._sad is key event dict + ._raw (bytes): serialized message + ._sad (dict): sad dict (key event dict) + ._cvrsn (Versionage): CESR code table version ._proto (str): Protocolage value as protocol type identifier - ._version is Versionage instance of event version + ._vrsn is Versionage instance of event version ._kind is serialization kind string value (see namedtuple coring.Serials) supported kinds are 'json', 'cbor', 'msgpack', 'binary' ._size is int of number of bytes in serialed event only @@ -1288,7 +1294,7 @@ def kind(self): @property def proto(self): - """proto property getter + """proto property getter, alias of .protocol protocol identifier type value of Protocolage such as 'KERI' or 'ACDC' Returns: @@ -1296,10 +1302,20 @@ def proto(self): """ return self._proto + @property + def protocol(self): + """protocp; property getter, alias of .proto + protocol identifier type value of Protocolage such as 'KERI' or 'ACDC' + + Returns: + protocol (str): Protocolage value as protocol type + """ + return self._proto + @property def vrsn(self): - """vrsn (version) property getter + """vrsn (version) property getter, alias of .version Returns: vrsn (Versionage): instance of protocol version for this Serder @@ -1308,7 +1324,7 @@ def vrsn(self): @property def version(self): - """version property getter alias of .vrsn + """version property getter, alias of .vrsn Returns: version (Versionage): instance of protocol version for this Serder From ad7edbd8a114285d2bfa7a9d72f141aad9286797 Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Mon, 25 Mar 2024 18:06:54 -0400 Subject: [PATCH 068/418] Removed watcher commands from keripy (#719) --- src/keri/app/cli/commands/watcher/__init__.py | 0 src/keri/app/cli/commands/watcher/list.py | 37 --------- src/keri/app/cli/commands/watcher/start.py | 80 ------------------- src/keri/app/cli/commands/watcher/update.py | 73 ----------------- 4 files changed, 190 deletions(-) delete mode 100644 src/keri/app/cli/commands/watcher/__init__.py delete mode 100644 src/keri/app/cli/commands/watcher/list.py delete mode 100644 src/keri/app/cli/commands/watcher/start.py delete mode 100644 src/keri/app/cli/commands/watcher/update.py diff --git a/src/keri/app/cli/commands/watcher/__init__.py b/src/keri/app/cli/commands/watcher/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/keri/app/cli/commands/watcher/list.py b/src/keri/app/cli/commands/watcher/list.py deleted file mode 100644 index 43d50ac18..000000000 --- a/src/keri/app/cli/commands/watcher/list.py +++ /dev/null @@ -1,37 +0,0 @@ -import argparse - -from hio.base import doing - -from keri import kering -from keri.app.cli.common import existing - -parser = argparse.ArgumentParser(description='List set of watchers for this environment (Habitat)') -parser.set_defaults(handler=lambda args: handler(args)) -parser.add_argument('--name', '-n', help='Human readable reference', required=True) - - -def handler(args): - kwa = dict(args=args) - return [doing.doify(listWatchers, **kwa)] - - -def listWatchers(tymth, tock=0.0, **opts): - """ Command line status handler - - """ - _ = (yield tock) - args = opts["args"] - name = args.name - alias = args.alias - - try: - with existing.existingHab(name=name, alias=alias) as hab: - habr = hab.db.habs.get(name) - print("Watcher Set:") - for wat in habr.watchers: - print("\t{}".format(wat)) - - except kering.ConfigurationError: - print(f"identifier prefix for {name} does not exist, incept must be run first", ) - return -1 - diff --git a/src/keri/app/cli/commands/watcher/start.py b/src/keri/app/cli/commands/watcher/start.py deleted file mode 100644 index 5c9801daf..000000000 --- a/src/keri/app/cli/commands/watcher/start.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- encoding: utf-8 -*- -""" -keri.kli.commands.watcher module - -""" -import argparse -import logging - -import falcon -from hio.core import http -from hio.core.tcp import serving - -from keri import help -from keri.app import directing, indirecting, habbing, storing - -parser = argparse.ArgumentParser(description='Start watcher') -parser.set_defaults(handler=lambda args: startWatcher(args)) -parser.add_argument('-H', '--http', - action='store', - default=5651, - help="Local port number the HTTP server listens on. Default is 5651.") -parser.add_argument('-T', '--tcp', - action='store', - default=5652, - help="Local port number the HTTP server listens on. Default is 5652.") -parser.add_argument('-n', '--name', - action='store', - default="watcher", - help="Name of controller. Default is watcher.") -parser.add_argument('--controller', - action='store', - default="", - help="Identifier prefix of controller of this watcher") -parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', - required=False, default="") -parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', - dest="bran", default=None) # passcode => bran - - -def startWatcher(args): - name = args.name - tcpPort = int(args.tcp) - httpPort = int(args.http) - - help.ogler.level = logging.INFO - help.ogler.reopen(name="keri", temp=True, clear=True) - doers = setupWatcher(name, controller=args.controller, alias=args.alias, base=args.base, bran=args.bran, - tcpPort=tcpPort, httpPort=httpPort) - return doers - - -def setupWatcher(name="watcher", controller=None, alias="watcher", base="", bran=None, tcpPort=5651, httpPort=5652): - """ - """ - - hby = habbing.Habery(name=name, base=base, bran=bran) - hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer - doers = [hbyDoer] - hab = hby.makeHab(name=alias, transferable=False) - - app = falcon.App(cors_enable=True) - - mbx = storing.Mailboxer(name=name) - rep = storing.Respondant(hby=hby, mbx=mbx) - - httpEnd = indirecting.HttpEnd(hab=hab, app=app, rep=rep, mbx=mbx) - app.add_route("/", httpEnd) - - server = http.Server(port=httpPort, app=app) - httpServerDoer = http.ServerDoer(server=server) - - server = serving.Server(host="", port=tcpPort) - serverDoer = serving.ServerDoer(server=server) - - directant = directing.Directant(hab=hab, server=server) - - doers.extend([directant, serverDoer, httpServerDoer, httpEnd, rep]) - - return doers diff --git a/src/keri/app/cli/commands/watcher/update.py b/src/keri/app/cli/commands/watcher/update.py deleted file mode 100644 index 3c37e5044..000000000 --- a/src/keri/app/cli/commands/watcher/update.py +++ /dev/null @@ -1,73 +0,0 @@ -import argparse - -from hio.base import doing - -from keri import kering -from keri.app import habbing -from keri.core import coring - -parser = argparse.ArgumentParser(description='Update set of watchers for this environment (Habitat)') -parser.set_defaults(handler=lambda args: handler(args)) -parser.add_argument('--name', '-n', help='Human readable reference', required=True) -parser.add_argument('--watchers', '-w', help='New set of watchers, replaces all existing watchers. Can appear ' - 'multiple times', metavar="", default=[], - action="append", required=False) -parser.add_argument('--cut', '-c', help='Watchers to remove. Can appear multiple times', metavar="", - default=[], - action="append", required=False) -parser.add_argument('--add', '-a', help='Watchers to add. Can appear multiple times', metavar="", - default=[], - action="append", required=False) - - -def handler(args): - kwa = dict(args=args) - return [doing.doify(updateWatchers, **kwa)] - - -def updateWatchers(tymth, tock=0.0, **opts): - """ Command line status handler - - """ - _ = (yield tock) - args = opts["args"] - name = args.name - - try: - with habbing.existingHabitat(name=name) as hab: - - habr = hab.db.habs.get(name) - - cuts = set(args.cut) - adds = set(args.add) - ewats = set(habr.watchers) - if args.watchers: - if args.add or args.cut: - raise kering.ConfigurationError("you can only specify witnesses or cuts and add") - - ewats = set(args.watchers) - else: - ewats |= adds - ewats -= cuts - - for wat in ewats: - pre = coring.Prefixer(qb64=wat) - if pre.transferable: - raise ValueError("{} is an illegal watcher identifier, must be non-transferable" - "".format(wat)) - - habr.watchers = list(ewats) - - hab.db.habs.pin(name, habr) - - print("New Watcher Set:") - for wat in habr.watchers: - print("\t{}".format(wat)) - - except kering.ConfigurationError: - print(f"identifier prefix for {name} does not exist, incept must be run first", ) - return -1 - except ValueError as e: - print(e.args[0]) - return -1 - From 4999b707b4156c8095075b068d3db1d72e0511cd Mon Sep 17 00:00:00 2001 From: Jason Colburne Date: Mon, 25 Mar 2024 19:07:28 -0300 Subject: [PATCH 069/418] ensure creder.attrib is not a string before treating it like a dict (#720) --- src/keri/vdr/verifying.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/vdr/verifying.py b/src/keri/vdr/verifying.py index d7fcb245b..7f1c1a237 100644 --- a/src/keri/vdr/verifying.py +++ b/src/keri/vdr/verifying.py @@ -300,7 +300,7 @@ def saveCredential(self, creder, prefixer, seqner, saider): self.reger.issus.add(keys=issuer, val=saider) self.reger.schms.add(keys=schema, val=saider) - if 'i' in creder.attrib: + if not isinstance(creder.attrib, str) and 'i' in creder.attrib: subject = creder.attrib["i"].encode("utf-8") self.reger.subjs.add(keys=subject, val=saider) From 11e051cdfcbaa2f83db04d2161933e9dd97ba573 Mon Sep 17 00:00:00 2001 From: Rodolfo Date: Mon, 25 Mar 2024 20:35:09 -0500 Subject: [PATCH 070/418] Catching socket exception when creating witness http client (#723) --- src/keri/app/agenting.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 526479aca..7f90b41c5 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -12,6 +12,8 @@ from hio.core.tcp import clienting from hio.help import decking, Hict +from socket import gaierror + from . import httping, forwarding from .. import help from .. import kering @@ -74,10 +76,13 @@ def receipt(self, pre, sn=None): clients = dict() doers = [] for wit in wits: - client, clientDoer = httpClient(hab, wit) - clients[wit] = client - doers.append(clientDoer) - self.extend([clientDoer]) + try: + client, clientDoer = httpClient(hab, wit) + clients[wit] = client + doers.append(clientDoer) + self.extend([clientDoer]) + except (kering.MissingEntryError, gaierror) as e: + logger.error(f"unable to create http client for witness {wit}: {e}") rcts = dict() for wit, client in clients.items(): From bcb6f1790e70205698c490752f7b3deba3c71f5a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 25 Mar 2024 20:48:46 -0600 Subject: [PATCH 071/418] more prelim work to support CESR native message --- src/keri/core/coring.py | 23 --- src/keri/core/counting.py | 56 ++++-- src/keri/core/serdering.py | 391 +++++++++++++++++++++++------------- src/keri/kering.py | 101 ++++++++-- tests/core/test_coring.py | 26 +-- tests/core/test_counting.py | 42 +++- tests/test_kering.py | 241 +++++++++++++++++++--- 7 files changed, 629 insertions(+), 251 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 7b55182e8..9d1a09094 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -4696,27 +4696,6 @@ def __iter__(self): CtrDex = CounterCodex() -@dataclass(frozen=True) -class GenusCodex: - """GenusCodex is codex of protocol genera for code table. - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - KERI_ACDC_SPAC: str = '--AAA' # KERI, ACDC, and SPAC Protocol Stacks share the same tables - KERI: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables - ACDC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables - SPAC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables - - - def __iter__(self): - return iter(astuple(self)) # enables inclusion test with "in" - # duplicate values above just result in multiple entries in tuple so - # in inclusion still works - -GenDex = GenusCodex() # Make instance - - class Counter: """ Counter is fully qualified cryptographic material primitive base class for @@ -5186,8 +5165,6 @@ def _inhale(self, raw): if version != Version: raise VersionError("Unsupported version = {}.{}, expected {}." "".format(version.major, version.minor, Version)) - #if len(raw) < size: - #raise ShortageError("Need more bytes.") ked = loads(raw=raw, size=size, kind=kind) diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index c000f1693..8e162e2f9 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -19,6 +19,29 @@ from ..core.coring import Sizage + +@dataclass(frozen=True) +class GenusCodex: + """GenusCodex is codex of protocol genera for code table. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + KERI_ACDC_SPAC: str = '--AAA' # KERI, ACDC, and SPAC Protocol Stacks share the same tables + KERI: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables + ACDC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables + SPAC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables + + + def __iter__(self): + return iter(astuple(self)) # enables inclusion test with "in" + # duplicate values above just result in multiple entries in tuple so + # in inclusion still works + +GenDex = GenusCodex() # Make instance + + + @dataclass class MapDom: """Base class for dataclasses that support map syntax @@ -173,25 +196,6 @@ def __iter__(self): CtrDex_2_0 = CounterCodex_2_0() - -@dataclass(frozen=True) -class GenusCodex(MapCodex): - """GenusCodex is codex of protocol genera for code table. - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - KERI: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables - ACDC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables - - - def __iter__(self): - return iter(astuple(self)) # enables value not key inclusion test with "in" - # duplicate values above just result in multiple entries in tuple so - # in inclusion still works - -GenDex = GenusCodex() # Make instance - # keys and values as strings of keys Codict1 = asdict(CtrDex_1_0) Tagage_1_0 = namedtuple("Tagage_1_0", list(Codict1), defaults=list(Codict1)) @@ -581,10 +585,24 @@ def count(self): """ Returns ._count Makes ._count read only + + number of quadlets of b64 chars or triplets of b2 bytes of material + framed by counter """ return self._count + @property + def fullSize(self): + """ + Returns full size of counter in bytes + + """ + _, _, fs, _ = self.sizes[self.code] # get from sizes table + + return fs + + @property def qb64b(self): """ diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index f3351cc23..a6fb9b8cb 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -33,7 +33,7 @@ from .coring import Matter, Saider, Verfer, Diger, Number, Tholder from ..core import counting -from ..core.counting import AllTags, Counter +from ..core.counting import GenDex, AllTags, Counter from .. import help from ..help import helping @@ -112,12 +112,80 @@ class FieldDom: def __iter__(self): return iter(asdict(self)) +"""Design Notes: + +Problems is that sniff only determines if counter not which type of +counter. Then smell does regex lookahead to find out which serialization +when not count code but extractor does not look ahead but strips from +stream. So when possibility that CESR message is next either need to +not strip from stream when extracting or if counter is message +then grab rest of frame and reattach so raw in Serder includes the +message counter. Latter is better since always keep counter around +until later. So need to check counter type and if message then +extract rest of counter frame (message) and reattach counter raw. +Then can call Serder with raw and smellage that indicates CESR kind + +But this does not solve the problem of using the Serder subclass +for the given protocol. Merely knowing is a CESR message is not +enough also have to know the protocol which comes in the version +field (not version string). + +One solution is to modify smell so that it also can lookahead and +see the version field. Or lookahead and see the version field with +count codes in front. Problem is that the Regexes don't separate +cleanly. + +Another solution is to use distinct function for cesr native called +snuff like smell but regex only for CESR native. Reap can be told which +because sniff tells which it is. +So question for snuff is should it be searching over the counter or should it +start at version field. This changes regex so forced start at front of raw. +so if reattach counter but use skip then can snuff at start of string. +Begine regex with b'^' or b'\A' to match at start of string. + +So change Smellage to return extra field that has gvrsn when used by snuff +so can use Smellage for both smell and snuff in both reap and inhale +where smellage is used. Change egacy uses of smell to ignore extra value. + + + +Lets try that as it works the best. + +while True: # extract, deserialize, and strip message from ims + try: + serder = serdery.reap(ims=ims) # can set version here + except kering.ShortageError as ex: # need more bytes + yield + else: # extracted and stripped successfully + break # break out of while loop + +ctr = yield from self._extractor(ims=ims, klas=Counter, cold=cold) +if ctr.code == CtrDex.AttachmentGroup: # pipeline ctr? + pipelined = True + +@staticmethod +def _extractor(ims, klas, cold=Colds.txt, abort=False): + while True: + try: + if cold == Colds.txt: + return klas(qb64b=ims, strip=True) + elif cold == Colds.bny: + return klas(qb2=ims, strip=True) + else: + raise kering.ColdStartError("Invalid stream state cold={}.".format(cold)) + except kering.ShortageError as ex: + if abort: # pipelined pre-collects full frame before extracting + raise # bad pipelined frame so abort by raising error + yield + + + +""" + class Serdery: """Serder factory class for generating serder instances by protocol type from an incoming message stream. - - """ def __init__(self, *pa, **kwa): @@ -129,7 +197,7 @@ def __init__(self, *pa, **kwa): pass - def reap(self, ims): + def reap(self, ims, skip=0, native=False): """Extract and return Serder subclass based on protocol type reaped from version string inside serialized raw of Serder. @@ -140,12 +208,16 @@ def reap(self, ims): Parameters: ims (bytearray) of serialized incoming message stream. Assumes start of stream is raw Serder. - version (Versionage | None): instance supported protocol version - None means do not enforce a supported version - """ - #version = version if version is not None else self.version + skip (int): bytes to skip at front of ims. Useful when CESR native + serialization where skip is size of the message counter so smell + does need to see counter + native (bool): True means sniff determined may be CESR native message + so snuff instead of smell. + False means sniff determined not CESR native i.e + JSON, CBOR, MGPK field map. so use smell. Default False - smellage = smell(ims) + """ + smellage = smell(memoryview(ims)[skip:]) # does not copy to skip if smellage.protocol == Protocols.keri: return SerderKERI(raw=ims, strip=True, smellage=smellage) @@ -201,7 +273,8 @@ class Serder: Properties: raw (bytes): of serialized event only sad (dict): self addressed data dict - cvrsn (Versionage): CESR code table version (Major, Minor) + genus (str): CESR genus code + gvrsn (Versionage): instance CESR genus code table version (Major, Minor) proto (str): Protocolage value as protocol identifier such as KERI, ACDC alias of .protocol protocol (str): Protocolage value as protocol identifier such as KERI, ACDC @@ -252,8 +325,6 @@ class Serder: saids (dict): strict (bool): - - """ Dummy = "#" # dummy spaceholder char for SAID. Must not be a valid Base64 char @@ -447,7 +518,7 @@ class Serder: def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, - cvrsn=None, verify=True, makify=False, + genus=GenDex.KERI, gvrsn=Vrsn_2_0, verify=True, makify=False, proto=None, vrsn=None, kind=None, ilk=None, saids=None): """Deserialize raw if provided. Update properties from deserialized raw. Verifies said(s) embedded in sad as given by labels. @@ -459,7 +530,7 @@ def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, given by label(s) according to proto and ilk and code. Parameters: - raw (bytes): serialized event + raw (bytes | bytearray): serialized event sad (dict): serializable saidified field map of message. Ignored if raw provided strip (bool): True means strip (delete) raw from input stream @@ -469,8 +540,11 @@ def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, version string elements. If none or empty ignore otherwise assume that raw already had its version string extracted (reaped) into the elements of smellage. - cvrsn (Versionage | None): instance desired CESR code table version - If None then its extracted from raw or uses default .CVrsn + genus (str): CESR genus str. Either provided by parser from stream + or generated by Serder to stream + gvrsn (Versionage): instance CESR genus code table version + Either provided by parser from stream genus version or desired when + generating Serder instance to stream verify (bool): True means verify said(s) of given raw or sad. Raises ValidationError if verification fails Ignore when raw not provided or when raw and saidify is True @@ -495,11 +569,12 @@ def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, """ - cvrsn = cvrsn if cvrsn is not None else self.CVrsn + self._gvrsn = gvrsn + self._genus = genus if raw: # deserialize raw using property setter - self._inhale(raw=raw, smellage=smellage, cvrsn=cvrsn) - # ._inhale updates ._raw, ._sad, ._cvrsn, ._proto, ._vrsn, ._kind, ._size + self._inhale(raw=raw, smellage=smellage) + # ._inhale updates ._raw, ._sad, ._proto, ._vrsn, ._kind, ._size # primary said field label try: @@ -516,7 +591,7 @@ def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, except TypeError: pass # ignore if bytes - if verify: # verify the said(s) provided in raw + if verify: # verify fields including the said(s) provided in raw try: self._verify() # raises exception when not verify except Exception as ex: @@ -528,13 +603,13 @@ def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, elif sad or makify: # serialize sad into raw or make sad if makify: # recompute properties and said(s) and reset sad # makify resets sad, raw, proto, vrsn, kind, ilk, and size - self.makify(sad=sad, cvrsn=cvrsn, proto=proto, vrsn=vrsn, kind=kind, + self.makify(sad=sad, proto=proto, vrsn=vrsn, kind=kind, ilk=ilk, saids=saids) - # .makify updates ._raw, ._sad, ._cvrsn, ._proto, ._vrsn, ._kind, ._size + # .makify updates ._raw, ._sad, ._proto, ._vrsn, ._kind, ._size else: - self._exhale(sad=sad, cvrsn=cvrsn) - # .exhale updates ._raw, ._sad, ._cvrsn, ._proto, ._vrsn, ._kind, ._size + self._exhale(sad=sad) + # .exhale updates ._raw, ._sad, ._proto, ._vrsn, ._kind, ._size # primary said field label try: @@ -545,7 +620,7 @@ def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, except Exception: self._said = None # no saidive field - if verify: # verify the said(s) provided in sad + if verify: # verify fields including the said(s) provided in sad try: self._verify() # raises exception when not verify except Exception as ex: @@ -588,17 +663,34 @@ def _verify(self): Raises a ValidationError (or subclass) if any verification fails """ - if self.Protocol and self.proto != self.Protocol: - raise ValidationError(f"Expected protocol = {self.Protocol}, got " + if self.Protocol and self.proto != self.Protocol: # class required + raise ValidationError(f"Required protocol = {self.Protocol}, got " f"{self.proto} instead.") if self.proto not in self.Fields: raise ValidationError(f"Invalid protocol type = {self.proto}.") + if self.genus not in GenDex: # ensures self.genus != None + raise SerializeError(f"Invalid genus={self.genus}.") + + if getattr(GenDex, self.proto, None) != self.genus: + raise SerializeError(f"Incompatible protocol={self.proto} with " + f"genus={self.genus}.") + + if self.vrsn.major > self.gvrsn.major: + raise SerializeError(f"Incompatible major protocol version={self.vrsn}" + f" with major genus version={self.gvrsn}.") + + if self.vrsn not in self.Fields[self.proto]: + raise SerializeError(f"Invalid version={self.vrsn} for " + f"protocol={self.proto}.") + if self.ilk not in self.Fields[self.proto][self.vrsn]: raise ValidationError(f"Invalid packet type (ilk) = {self.ilk} for" f"protocol = {self.proto}.") + + fields = self.Fields[self.proto][self.vrsn][self.ilk] # get labelage alls = fields.alls # faster local reference @@ -649,9 +741,8 @@ def _verify(self): if saids[label] in DigDex: # if digestive then replace with dummy sad[label] = self.Dummy * len(sad[label]) - + # compute saidive digestive field values using raw from sized dummied sad raw = self.dumps(sad, kind=self.kind) # serialize dummied sad copy - for label, code in saids.items(): if code in DigDex: # subclass override if non digestive allowed klas, size, length = self.Digests[code] # digest algo size & length @@ -667,14 +758,37 @@ def _verify(self): f" = {self._sad}, should be {dig}.") sad[label] = dig - raw = self.dumps(sad, kind=self.kind) + raw = self.dumps(sad, kind=self.kind) # compute final raw + if raw != self.raw: raise ValidationError(f"Invalid round trip of {sad} != \n" f"{self.sad}.") + + if "v" not in sad: + raise ValidationError(f"Missing version string field in {sad}.") + + # extract version string elements to verify consistency with attributes + proto, vrsn, kind, size = deversify(sad["v"]) + if self.proto != proto: + raise ValidationError(f"Inconsistent protocol={self.proto} in {sad}.") + + if self.vrsn != vrsn: + raise ValidationError(f"Inconsistent version={self.vrsn} in {sad}.") + + if self.kind != kind: + raise ValidationError(f"Inconsistent kind={self.kind} in {sad}.") + + if self.kind in (Serials.json, Serials.cbor, Serials.mgpk): + if size != self.size != len(raw): + raise ValidationError(f"Inconsistent size={self.size} in {sad}.") + else: # size is not set in version string when kind is CESR + if self.size != len(raw): + raise ValidationError(f"Inconsistent size={self.size} in {sad}.") + # verified successfully since no exception - def makify(self, sad, *, cvrsn=None, proto=None, vrsn=None, kind=None, + def makify(self, sad, *, proto=None, vrsn=None, kind=None, ilk=None, saids=None): """Makify given sad dict makes the versions string and computes the said field values and sets associated properties: @@ -693,8 +807,6 @@ def makify(self, sad, *, cvrsn=None, proto=None, vrsn=None, kind=None, Parameters: sad (dict): serializable saidified field map of message. Ignored if raw provided - cvrsn (Versionage | None): instance desired CESR code table version - If None then its extracted from raw or uses default .CVrsn proto (str | None): desired protocol type str value of Protocols If None then its extracted from sad or uses default .Proto vrsn (Versionage | None): instance desired protocol version @@ -721,19 +833,32 @@ def makify(self, sad, *, cvrsn=None, proto=None, vrsn=None, kind=None, else: silk = sad.get('t') # if 't' not in sad .get returns None which may be valid - cvrsn = cvrsn if cvrsn is not None else self.CVrsn - if proto is None: proto = sproto if sproto is not None else self.Proto if proto not in self.Fields: - raise SerializeError(f"Invalid protocol type = {proto}.") + raise SerializeError(f"Invalid protocol={proto}.") + + if self.Protocol and proto != self.Protocol: # required by class + raise SerializeError(f"Required protocol={self.Protocol}, got " + f"protocol={proto} instead.") + + if self.genus not in GenDex: # ensures self.genus != None + raise SerializeError(f"Invalid genus={self.genus}.") + + if getattr(GenDex, proto, None) != self.genus: + raise SerializeError(f"Incompatible protocol={proto} with " + f"genus={self.genus}.") if vrsn is None: vrsn = svrsn if svrsn is not None else self.Vrsn if vrsn not in self.Fields[proto]: - raise SerializeError(f"Invalid version = {vrsn} for protocol = {proto}.") + raise SerializeError(f"Invalid version={vrsn} for protocol={proto}.") + + if vrsn.major > self.gvrsn.major: + raise SerializeError(f"Incompatible major protocol version={vrsn} " + f"with major genus version={self.gvrsn}.") if kind is None: kind = skind if skind is not None else self.Kind @@ -742,23 +867,15 @@ def makify(self, sad, *, cvrsn=None, proto=None, vrsn=None, kind=None, ilk = (silk if silk is not None else list(self.Fields[proto][vrsn])[0]) # list(dict) gives list of keys - if self.Protocol and proto != self.Protocol: - raise SerializeError(f"Expected protocol = {self.Protocol}, got " - f"{proto} instead.") - - if kind not in Serials: raise SerializeError(f"Invalid serialization kind = {kind}") - if ilk not in self.Fields[proto][vrsn]: raise SerializeError(f"Invalid packet type (ilk) = {ilk} for" f"protocol = {proto}.") fields = self.Fields[proto][vrsn][ilk] # get FieldDom of fields - - alls = fields.alls # faster local reference oalls = oset(alls) # ordereset of field labels oopts = oset(fields.opts) # ordereset of field labels @@ -838,21 +955,22 @@ def makify(self, sad, *, cvrsn=None, proto=None, vrsn=None, kind=None, raise SerializeError(f"Missing requires version string field 'v'" f" in sad = {sad}.") - # this size of sad needs to be computed based on actual version string span - # since not same for all versions - sad['v'] = self.Dummy * self.Spans[vrsn] # ensure span of vs is dummied MAXVERFULLSPAN + if kind in (Serials.json, Serials.cbor, Serials.mgpk): + # this size of sad needs to be computed based on actual version string span + # since not same for all versions + sad['v'] = self.Dummy * self.Spans[vrsn] # ensure span of vs is dummied MAXVERFULLSPAN - raw = self.dumps(sad, kind) # get size of sad with fully dummied vs and saids - size = len(raw) + raw = self.dumps(sad, kind) # get size of sad with fully dummied vs and saids + size = len(raw) - # generate new version string with correct size - vs = versify(protocol=proto, version=vrsn, kind=kind, size=size) - sad["v"] = vs # update version string in sad + # generate new version string with correct size + vs = versify(protocol=proto, version=vrsn, kind=kind, size=size) + sad["v"] = vs # update version string in sad + # now have correctly sized version string in sad - # now have correctly sized version string in sad - # now compute saidive digestive field values using sized dummied sad - raw = self.dumps(sad, kind=kind) # serialize sized dummied sad + # compute saidive digestive field values using raw from sized dummied sad + raw = self.dumps(sad, kind=kind) # serialize sized dummied sad for label, code in _saids.items(): if code in DigDex: # subclass override if non digestive allowed klas, dsize, dlen = self.Digests[code] # digest algo size & length @@ -866,17 +984,18 @@ def makify(self, sad, *, cvrsn=None, proto=None, vrsn=None, kind=None, sad[label] = dig raw = self.dumps(sad, kind=kind) # compute final raw + if kind == Serials.cesr:# cesr kind version string does not set size + size = len(raw) # size of whole message self._raw = raw self._sad = sad - self._cvrsn = cvrsn self._proto = proto self._vrsn = vrsn self._kind = kind self._size = size - def _inhale(self, raw, *, smellage=None, cvrsn=None): + def _inhale(self, raw, *, smellage=None): """Deserializes raw. Parses serilized event ser of serialization kind and assigns to instance attributes and returns tuple of associated elements. @@ -899,12 +1018,8 @@ def _inhale(self, raw, *, smellage=None, cvrsn=None): elements. If none or empty ignore otherwise assume that raw already had its version string extracted (reaped) into the elements of smellage. - cvrsn (Versionage | None): instance desired CESR code table version - If None then its extracted from raw or uses default .CVrsn - Note: - loads and jumps of json use str whereas cbor and msgpack use bytes - Assumes only supports Version + """ if smellage: # passed in so don't need to smell raw again @@ -913,15 +1028,14 @@ def _inhale(self, raw, *, smellage=None, cvrsn=None): proto, vrsn, kind, size = smell(raw) sad = self.loads(raw=raw, size=size, kind=kind) - - cvrsn = cvrsn if cvrsn is not None else self.CVrsn + # ._gvrsn may be set in loads when CESR native deserialization provides _gvrsn if "v" not in sad: # Regex does not check for version string label itself raise FieldError(f"Missing version string field in {sad}.") - self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray + # cypto opts want bytes not bytearray + self._raw = bytes(raw[:size]) # make copy so strip not affect self._sad = sad - self._cvrsn = cvrsn self._proto = proto self._vrsn = vrsn self._kind = kind @@ -930,18 +1044,23 @@ def _inhale(self, raw, *, smellage=None, cvrsn=None): def loads(self, raw, size=None, kind=Serials.json): - """Utility static method to handle deserialization by kind + """method to handle deserialization by kind + assumes already sniffed and smelled to determine + serialization size and kind Returns: sad (dict | list): deserialized dict or list. Assumes attribute dict of saidified data. Parameters: - raw (bytes |bytearray): raw serialization to deserialze as dict + raw (bytes | bytearray): raw serialization to deserialze as dict size (int): number of bytes to consume for the deserialization. If None then consume all bytes in raw kind (str): value of Serials (Serialage) serialization kind "JSON", "MGPK", "CBOR" + + Notes: + loads of json uses str whereas loads of cbor and msgpack use bytes """ if kind == Serials.json: try: @@ -982,30 +1101,17 @@ def _loads(self, raw, size=None): size (int): number of bytes to consume for the deserialization. If None then consume all bytes in raw """ + # ._gvrsn may be set in loads when CESR native deserialization provides _gvrsn pass - def _exhale(self, sad, cvrsn=None): - """Serializes sad and sets the serialized size in its version string. - - As classmethod enables bootstrap of valid sad dict that has correct size - in version string. This obviates sizeify. This can be called on self as - well because it only ever accesses clas attributes not instance attributes. - - Returns tuple of (raw, proto, kind, sad, vrsn) where: - raw (str): serialized event as bytes of kind - proto (str): protocol type as value of Protocolage - kind (str): serialzation kind as value of Serialage - sad (dict): modified serializable attribute dict of saidified data - vrsn (Versionage): tuple value (major, minor) + def _exhale(self, sad): + """Serializes sad and assigns attributes. + Asssumes all field values in sad are valid. + Call .verify to otherwise Parameters: - clas (Serder): class reference sad (dict): serializable attribute dict of saidified data - cvrsn (Versionage | None): instance desired CESR code table version - If None then its extracted from raw or uses default .CVrsn - - """ if "v" not in sad: raise SerializeError(f"Missing version string field in {sad}.") @@ -1014,28 +1120,13 @@ def _exhale(self, sad, cvrsn=None): proto, vrsn, kind, size = deversify(sad["v"]) raw = self.dumps(sad, kind) - size = len(raw) - - # generate new version string with correct size - vs = versify(protocol=proto, version=vrsn, kind=kind, size=size) - - cvrsn = cvrsn if cvrsn is not None else self.CVrsn - # find location of old version string inside raw - match = Rever.search(raw) # Rever's regex takes bytes - if not match or match.start() > 12: - raise SerializeError(f"Invalid version string in raw = {raw}.") - fore, back = match.span() # start and end positions of version string + if kind in (Serials.cesr): # cesr kind version string does not set size + size = len(raw) # size of whole message - # replace old version string in raw with new one - raw = b'%b%b%b' % (raw[:fore], vs.encode("utf-8"), raw[back:]) - if size != len(raw): # substitution messed up - raise SerializeError(f"Malformed size of raw in version string == {vs}") - sad["v"] = vs # update sad - - self._raw = raw + # must call .verify to ensure these are compatible + self._raw = raw # crypto opts want bytes not bytearray self._sad = sad - self._cvrsn = cvrsn self._proto = proto self._vrsn = vrsn self._kind = kind @@ -1043,7 +1134,8 @@ def _exhale(self, sad, cvrsn=None): def dumps(self, sad, kind=Serials.json): - """Utility static method to handle serialization by kind + """Method to handle serialization by kind + Assumes sad fields are properly filled out for serialization kind. Returns: raw (bytes): serialization of sad dict using serialization kind @@ -1051,7 +1143,11 @@ def dumps(self, sad, kind=Serials.json): Parameters: sad (dict | list)): serializable dict or list to serialize kind (str): value of Serials (Serialage) serialization kind - "JSON", "MGPK", "CBOR" + "JSON", "MGPK", "CBOR", "CSER" + + Notes: + dumps of json uses str whereas dumps of cbor and msgpack use bytes + crypto opts want bytes not bytearray """ if kind == Serials.json: raw = json.dumps(sad, separators=(",", ":"), @@ -1062,54 +1158,61 @@ def dumps(self, sad, kind=Serials.json): elif kind == Serials.cbor: raw = cbor.dumps(sad) + + elif kind == Serials.cser: + raw = self._dumps(sad) + else: raise SerializeError(f"Invalid serialization kind = {kind}") return raw - def _dumps(self, sad, protocol, version, cversion=Vrsn_2_0): + def _dumps(self, sad): """CESR native serialization of sad Returns: raw (bytes): CESR native serialization of sad dict Parameters: - clas (Serder): class reference sad (dict | list)): serializable dict or list to serialize - protocol (str): message protocol - version (Versionage): message protocol version - cversion (Versionage): CESR code table genus version Versioning: - CESR native serialization includes message protocol, protocol version - and CESR (genus) version. Assumes genus is compatible with message - protocol so genus is not needed. Protects from malleability attack + CESR native serialization includes in its fixed version field + a version primitive that includes message protocol+protocol version + +genus version: 0NPPPPMmmMmm (12 B64 characters) + + This assumes that genus is compatible with message + protocol so genus is not needed. This protects from malleability attack and ensure compatible cesr codes especially count (group) codes. Primitive codes are less problematic since so far all primitive codes tables are backwards compatible across major versions. - 0NPPPPMmmMmm (12 B64 characters) + """ + if (self.gvrsn.major < Vrsn_2_0.major or + self.vrsn.major < Vrsn_2_0.major): + raise SerializeError(f"Invalid major genus version={self.gvrsn}" + f"or Invalid major protocol version={self.vrsn}" + f" for native CESR serialization.") + if self.genus not in GenDex: # ensures self.genus != None + raise SerializeError(f"Invalid genus={self.genus}.") + + if getattr(GenDex, self.proto, None) != self.genus: + raise SerializeError(f"Incompatible protocol={self.proto} with " + f"genus={self.genus}.") - """ fixed = True # True = use fixed field, False= use field map raw = bytearray() - - if protocol not in Protocols: - raise SerializeError(f"Invalid {protocol=}.") - - if version not in self.Fields[protocol]: - raise SerializeError(f"Invalid {version=} for {protocol=}.") - - ilks = self.Fields[protocol][version] # get fields keyed by ilk + ilks = self.Fields[self.proto][self.vrsn] # get fields keyed by ilk ilk = sad.get('t') # returns None if missing message type (ilk) if ilk not in ilks: # raise SerializeError(f"Missing message type field " - f"'t' for {protocol=} {version=} with {sad=}.") + f"'t' for protocol={self.proto} " + f"version={self.vrsn} with {sad=}.") fields = ilks[ilk] # FieldDom for given protocol and ilk @@ -1121,7 +1224,7 @@ def _dumps(self, sad, protocol, version, cversion=Vrsn_2_0): # so can serialize in order to compute saidive fields # need to fix ._verify and .makify to account for CESR native serialization - if protocol == Protocols.keri: + if self.proto == Protocols.keri: for l, v in sad.items(): # assumes valid field order & presence if not fixed: # prepend label pass @@ -1129,8 +1232,9 @@ def _dumps(self, sad, protocol, version, cversion=Vrsn_2_0): # should dispatch or use match instead of big if else match l: # label case "v": # protocol+version - val = (MtrDex.Tag7 + protocol + - Counter.verToB64(version)).encode("utf-8") + val = (MtrDex.Tag10 + self.proto + + Counter.verToB64(self.vrsn) + + Counter.verToB64(self.gvrsn)).encode("utf-8") case "t": # message type val = (MtrDex.Tag3 + ilk).encode("utf-8") # add code @@ -1151,7 +1255,7 @@ def _dumps(self, sad, protocol, version, cversion=Vrsn_2_0): val = bytearray(Counter(tag=AllTags.GenericListGroup, count=len(frame) % 4, - version=cversion).qb64b) + version=self.gvrsn).qb64b) val.extend(frame) case "c": # list of config traits strings @@ -1162,7 +1266,7 @@ def _dumps(self, sad, protocol, version, cversion=Vrsn_2_0): val = bytearray(Counter(tag=AllTags.GenericListGroup, count=len(frame) % 4, - version=cversion).qb64b) + version=self.gvrsn).qb64b) val.extend(frame) case "a": # list of seals or field map of attributes @@ -1192,7 +1296,7 @@ def _dumps(self, sad, protocol, version, cversion=Vrsn_2_0): raw.extend(val) - elif protocol == Protocols.acdc: + elif self.protocol == Protocols.acdc: for l, val in sad.items(): # assumes valid field order & presence if not fixed: pass # prepend label @@ -1200,7 +1304,7 @@ def _dumps(self, sad, protocol, version, cversion=Vrsn_2_0): else: - raise SerializeError(f"Unsupported {protocol=}.") + raise SerializeError(f"Unsupported protocol={self.protocol}.") # prepend count code for message @@ -1274,14 +1378,25 @@ def sad(self): """ return dict(self._sad) # return copy + + @property + def genus(self): + """genus (CESR genu code) property getter + + Returns: + genus (stre): CESR genus code for this Serder + """ + return self._genus + + @property - def cvrsn(self): - """cvrsn (version) property getter + def gvrsn(self): + """gvrsn (CESR genus version) property getter Returns: - cvrsn (Versionage): instance of CESR code table version for this Serder + gvrsn (Versionage): instance, CESR genus code table version for this Serder """ - return self._cvrsn + return self._gvrsn @property diff --git a/src/keri/kering.py b/src/keri/kering.py index 7ed23ef36..338f87a39 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -36,32 +36,41 @@ #VEREX0 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' #Rever = re.compile(VEREX0) # compile is faster +# version string in JSON, CBOR, or MGPK field map serialization version 1 VER1FULLSPAN = 17 # number of characters in full version string VER1TERM = b'_' VEREX1 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' +# version string in JSON, CBOR, or MGPK field map serialization version 2 VER2FULLSPAN = 16 # number of characters in full version string VER2TERM = b'.' -VEREX2 = b'(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[A-Z]{4})(?P[0-9A-Za-z_-]{4}).' +VEREX2 = b'(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[A-Z]{4})(?P[0-9A-Za-z_-]{4})\.' VEREX = VEREX2 + b'|' + VEREX1 -MAXVERFULLSPAN = max(VER1FULLSPAN, VER2FULLSPAN) # max number of characters in full version string +# max number of characters in full version string +MAXVERFULLSPAN = max(VER2FULLSPAN, VER1FULLSPAN) Rever = re.compile(VEREX) # compile is faster - MAXVSOFFSET = 12 SMELLSIZE = MAXVSOFFSET + MAXVERFULLSPAN # min buffer size to inhale +# version field in CESR native serialization +VFFULLSPAN = 12 # number of characters in full version string +VFREX = b'0N(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})' + +Revfer = re.compile(VFREX) # compile is faster + """ Smellage (results of smelling a version string such as in a Serder) protocol (str): protocol type value of Protocols examples 'KERI', 'ACDC' - version (Versionage): named tuple (major, minor) ints of major minor version + version (Versionage): protocol version namedtuple (major, minor) of ints kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' - size (str): int size of raw serialization + size (int | Versionage): int size of raw serialization or + genus version namedtuple (major, minor) of ints """ Smellage = namedtuple("Smellage", "protocol version kind size") @@ -75,7 +84,9 @@ def rematch(match): Parameters: match (re.Match): instance of Match class - + Notes: + regular expressions work with memoryview objects not just bytes or + bytearrays """ full = match.group() # full matched version string if len(full) == VER2FULLSPAN and full[-1] == ord(VER2TERM): @@ -90,11 +101,6 @@ def rematch(match): vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) if vrsn.major < 2: # version2 vs but major < 2 raise VersionError(f"Incompatible {vrsn=} with version string.") - #if version is not None: # compatible version with vrsn - #if (vrsn.major > version.major or - #(vrsn.major == version.major and vrsn.minor > version.minor)): - #raise VersionError(f"Incompatible {version=}, with " - #f"{vrsn=}.") kind = kind.decode("utf-8") if kind not in Serials: @@ -113,19 +119,16 @@ def rematch(match): vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) if vrsn.major > 1: # version1 vs but major > 1 raise VersionError(f"Incompatible {vrsn=} with version string.") - #if version is not None: # compatible version with vrsn - #if (vrsn.major > version.major or - #(vrsn.major == version.major and vrsn.minor > version.minor)): - #raise VersionError(f"Incompatible {version=}, with " - #f"{vrsn=}.") kind = kind.decode("utf-8") if kind not in Serials: raise KindError(f"Invalid serialization kind = {kind}.") size = int(size, 16) - return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) + else: + raise VersionError(f"Bad rematch.") + return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) def versify(protocol=Protocols.keri, version=Version, kind=Serials.json, size=0): @@ -151,7 +154,6 @@ def versify(protocol=Protocols.keri, version=Version, kind=Serials.json, size=0) f"{intToB64(version.minor, l=2)}{kind}{intToB64(size, l=4)}.") - def deversify(vs): """ Returns: tuple(proto, kind, version, size) Where: @@ -182,8 +184,6 @@ def deversify(vs): return rematch(match) - - def smell(raw): """Extract and return instance of Smellage from version string inside raw serialization. @@ -196,6 +196,7 @@ def smell(raw): of stream is JSON, CBOR, or MGPK field map with first field is labeled 'v' and value is version string. + """ if len(raw) < SMELLSIZE: raise ShortageError(f"Need more raw bytes to smell full version string.") @@ -207,7 +208,67 @@ def smell(raw): return rematch(match) +def snatch(match, size=0): + """ Returns: + smellage (Smellage): named tuple extracted from version string regex match + (protocol, version, kind, size) + + Parameters: + match (re.Match): instance of Match class + size (int): provided size to substitute when missing + + Notes: + regular expressions work with memoryview objects not just bytes or + bytearrays + """ + if len(full) == VER0FULLSPAN: + proto, major, minor, gmajor, gminor = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + protocol = proto.decode("utf-8") + if protocol not in Protocols: + raise ProtocolError(f"Invalid protocol type = {protocol}.") + vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) + if vrsn.major < 2: # version2 vs but major < 2 + raise VersionError(f"Incompatible {vrsn=} with version string.") + + gvrsn = Versionage(major=b64ToInt(gmajor), minor=b64ToInt(gminor)) + if gvrsn.major < 2: # version2 vs but major < 2 + raise VersionError(f"Incompatible {gvrsn=} with CESR native version" + f"field.") + kind = Serials.cesr + size = size + else: + raise VersionError(f"Bad snatch.") + + return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) + + +def snuff(raw, size=0): + """Extract and return instance of Smellage from version string inside + raw serialization. + + Returns: + smellage (Smellage): named Tuple of (protocol, version, kind, size) + + Parameters: + raw (bytearray) of serialized incoming message stream. Assumes start + of stream is JSON, CBOR, or MGPK field map with first field + is labeled 'v' and value is version string. + size (int): provided size to substitute when missing + + """ + if len(raw) < SMELLSIZE: + raise ShortageError(f"Need more raw bytes to smell full version string.") + + match = Rever.search(raw) # Rever regex takes bytes/bytearray not str + if not match or match.start() > MAXVSOFFSET: + raise VersionError(f"Invalid version string from smelled raw = " + f"{raw[: SMELLSIZE]}.") + return snatch(match, size=size) @dataclass(frozen=True) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 4687c0dd7..98970e84d 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -31,7 +31,7 @@ from keri.core.coring import Serialage, Serials, Tiers from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, Indexer, - CtrDex, Counter, GenDex) + CtrDex, Counter) from keri.core.coring import (Verfer, Cigar, Signer, Salter, Saider, DigDex, Diger, Prefixer, Cipher, Encrypter, Decrypter) from keri.core.coring import versify, deversify, Rever, MAXVERFULLSPAN @@ -46,30 +46,6 @@ from keri.kering import Version, Versionage, VersionError -def test_genus_codex(): - """ - Test protocol genera in GenDex as instance of GenusCodex - - """ - - assert dataclasses.asdict(GenDex) == \ - { - 'KERI_ACDC_SPAC': '--AAA', - 'KERI': '--AAA', - 'ACDC': '--AAA', - 'SPAC': '--AAA' - } - - assert '--AAA' in GenDex - assert GenDex.KERI == "--AAA" - assert GenDex.ACDC == "--AAA" - assert GenDex.SPAC == "--AAA" - assert GenDex.KERI_ACDC_SPAC == "--AAA" - assert GenDex.KERI == GenDex.ACDC - - """End Test""" - - def test_matter(): """ diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 521d13b65..d50a3fe4e 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -19,10 +19,41 @@ nabSextets) from keri.core import counting -from keri.core.counting import Sizage, MapDom, MapCodex, Counter +from keri.core.counting import GenDex, Sizage, MapDom, MapCodex, Counter from keri.core.counting import Versionage, Vrsn_1_0, Vrsn_2_0, AllTags + +def test_genus_codex(): + """ + Test protocol genera in GenDex as instance of GenusCodex + + """ + + assert asdict(GenDex) == \ + { + 'KERI_ACDC_SPAC': '--AAA', + 'KERI': '--AAA', + 'ACDC': '--AAA', + 'SPAC': '--AAA' + } + + assert '--AAA' in GenDex + assert GenDex.KERI == "--AAA" + assert GenDex.ACDC == "--AAA" + assert GenDex.SPAC == "--AAA" + assert GenDex.KERI_ACDC_SPAC == "--AAA" + assert GenDex.KERI == GenDex.ACDC + + assert hasattr(GenDex, "KERI") + assert hasattr(GenDex, "ACDC") + assert hasattr(GenDex, "SPAC") + assert hasattr(GenDex, "KERI_ACDC_SPAC") + + """End Test""" + + + def test_mapdom(): """Test MapDom base dataclass""" @@ -166,6 +197,7 @@ def __iter__(self): # so value in dataclass not key in dataclass """End Test""" + def test_codexes_tags(): """ Test supporting module attributes @@ -619,6 +651,7 @@ def test_counter_v1(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_1_0 + assert counter.fullSize == 4 counter = Counter(tag=AllTags.ControllerIdxSigs, count=count, version=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs @@ -865,6 +898,7 @@ def test_counter_v1(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_1_0 + assert counter.fullSize == 8 assert not ims ims = bytearray(qscb2) @@ -875,6 +909,7 @@ def test_counter_v1(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_1_0 + assert counter.fullSize == 8 assert not ims # test protocol genus with CESR protocol genus version @@ -940,6 +975,7 @@ def test_counter_v2(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_2_0 + assert counter.fullSize == 4 # default count = 1 counter = Counter(code=CtrDex.ControllerIdxSigs, version=Vrsn_2_0) @@ -1223,6 +1259,7 @@ def test_counter_v2(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_2_0 + assert counter.fullSize == 8 counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup @@ -1263,6 +1300,7 @@ def test_counter_v2(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_2_0 + assert counter.fullSize == 8 assert not ims ims = bytearray(qscb2) @@ -1319,6 +1357,7 @@ def test_counter_v2(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_2_0 + assert counter.fullSize == 8 counter = Counter(code=CtrDex.KERIACDCGenusVersion, countB64=genver, @@ -1336,6 +1375,7 @@ def test_counter_v2(): if __name__ == "__main__": + test_genus_codex() test_mapdom() test_mapcodex() test_codexes_tags() diff --git a/tests/test_kering.py b/tests/test_kering.py index 80d0a070b..039e75d3e 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -16,19 +16,17 @@ from keri.kering import Serialage, Serials from keri.kering import Ilkage, Ilks from keri.kering import (Versionage, Version, MAXVERFULLSPAN, - versify, deversify, Rever) -from keri.kering import (VER1FULLSPAN, VER1TERM, VEREX1, - VER2FULLSPAN, VER2TERM, VEREX2, VEREX) + versify, deversify, Rever, Smellage, smell, + VER1FULLSPAN, VER1TERM, VEREX1, + VER2FULLSPAN, VER2TERM, VEREX2, + VEREX) +from keri.kering import VFFULLSPAN, VFREX, Revfer from keri.kering import VersionError, ProtocolError, KindError from keri.help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, codeB64ToB2, codeB2ToB64, Reb64, nabSextets) - - - - def test_protos(): """ Test protocols namedtuple instance Protocols @@ -49,18 +47,20 @@ def test_protos(): def test_version_regex(): """ Test version string regexing - """ - #VER1FULLSPAN = 17 # number of characters in full version string - #VER1TERM = b'_' - #VEREX1 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' + VER1FULLSPAN = 17 # number of characters in full version string + VER1TERM = b'_' + VEREX1 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' + + VER2FULLSPAN = 16 # number of characters in full version string + VER2TERM = b'.' + VEREX2 = b'(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[A-Z]{4})(?P[0-9A-Za-z_-]{4}).' - #VER2FULLSPAN = 16 # number of characters in full version string - #VER2TERM = b'.' - #VEREX2 = b'(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[A-Z]{4})(?P[0-9A-Za-z_-]{4}).' + VEREX = VEREX2 + b'|' + VEREX1 + """ - #VEREX = VEREX2 + b'|' + VEREX1 + # Test VEREX2 by itself pattern = re.compile(VEREX2) # compile is faster vs = b'KERICAAJSONAAAB.' @@ -83,7 +83,7 @@ def test_version_regex(): assert groups == (b'KERI', b'C', b'AA', b'JSON', b'AAAB') - + # Test VEREX with combined VEREXes pattern = re.compile(VEREX) # compile is faster vs = b'KERICAAJSONAAAB.' @@ -146,7 +146,6 @@ def test_version_regex(): assert groups == (b'KERI', b'C', b'AA', b'JSON', b'AAAB') - raw = b'{"vs":"KERI10JSON000002_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"}' match = pattern.search(raw) @@ -169,6 +168,203 @@ def test_version_regex(): """End Test""" +def test_smell(): + """ + Test smell function to parse into Serializations + """ + + raw = b'{"vs":"KERICAAJSONAAAB.","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' + assert smell(raw) == Smellage(protocol='KERI', + version=Versionage(major=2, minor=0), + kind='JSON', + size=1) + + raw = b'{"vs":"KERI10JSON000002_","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' + assert smell(raw) == Smellage(protocol='KERI', + version=Versionage(major=1, minor=0), + kind='JSON', + size=2) + + raw = b'{"vs":"KERICAAJSONAAABX.","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' + with pytest.raises(ProtocolError): + smell(raw) + + raw = b'{"vs":"KERI1XJSON000002_","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' + with pytest.raises(VersionError): + smell(raw) + + """End Test""" + + +def test_snuff(): + """ + Test snuff for looking ahead at CESR native messages from stream + + VER0FULLSPAN = 12 # number of characters in full version string + VEREX0 = b'0N(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})' + + + """ + #pattern = re.compile(VFREX) # compile is faster + pattern = Revfer + + vv = b'0NKERICAACAB' + + match = pattern.match(vv) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + raw = b'-FAM' + vv + assert raw == b'-FAM0NKERICAACAB' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + raw = b'-0FAAAAM' + vv + assert raw == b'-0FAAAAM0NKERICAACAB' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + + vv = b'0NKERICAACAB' + + match = pattern.match(vv) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + raw = b'-FAM' + vv + assert raw == b'-FAM0NKERICAACAB' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + raw = b'-0FAAAAM' + vv + assert raw == b'-0FAAAAM0NKERICAACAB' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + vv = b'0NKERI______' + raw = b'-0FAAAAM' + vv + assert raw == b'-0FAAAAM0NKERI______' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'_', b'__', b'_', b'__') + + + + #raw = b'-FAM0NKERICAACABXicpEPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf' + #assert smell(raw) == Smellage(protocol='KERI', + #version=Versionage(major=2, minor=0), + #kind='CESR', + #size=0) + + #raw =b'-0FAAAAM0NKERICAACABXrotEPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf' + #assert smell(raw, size=4096) == Smellage(protocol='KERI', + #version=Versionage(major=2, minor=0), + #kind='CESR', + #size=4096) + + #raw =b'-0FAAAAM0MKERICAACABXrotEPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf' + #with pytest.raises(VersionError): + #smell(raw) + + #raw =b'-0FAAAAMNKERICAACABXrotEPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf' + #with pytest.raises(VersionError): + #smell(raw) + + + def test_serials(): """ Test Serializations namedtuple instance Serials @@ -573,20 +769,15 @@ def test_ilks(): """End Test """ -def test_smell(): - """ - Test smell function to parse into Serializations - """ - pass - - """End Test""" if __name__ == "__main__": test_protos() test_version_regex() + test_smell() + test_snuff() test_serials() test_versify_v1() test_versify_v2() test_ilks() - test_smell() + From 82d34910051efb636d7b8f2bbdf5c072367fa22d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 26 Mar 2024 10:58:16 -0600 Subject: [PATCH 072/418] some more refactor --- src/keri/core/coring.py | 12 ++++---- src/keri/core/serdering.py | 39 ++++++++++++++--------- src/keri/kering.py | 35 +++++++++++---------- tests/core/test_serdering.py | 55 ++------------------------------- tests/test_kering.py | 60 +++++++++++++++++++----------------- 5 files changed, 82 insertions(+), 119 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 9d1a09094..8683ad7b0 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -83,7 +83,7 @@ def sizeify(ked, kind=None, version=Version): raise ValueError("Missing or empty version string in key event " "dict = {}".format(ked)) - proto, vrsn, knd, size = deversify(ked["v"]) # extract kind and version + proto, vrsn, knd, size, _ = deversify(ked["v"]) # extract kind and version if vrsn != version: raise ValueError("Unsupported version = {}.{}".format(vrsn.major, vrsn.minor)) @@ -3754,7 +3754,7 @@ def _serialize(clas, sad: dict, kind: str = None): """ knd = Serials.json if 'v' in sad: # versioned sad - _, _, knd, _ = deversify(sad['v']) + _, _, knd, _, _ = deversify(sad['v']) if not kind: # match logic of Serder for kind kind = knd @@ -5161,14 +5161,14 @@ def _inhale(self, raw): loads and jumps of json use str whereas cbor and msgpack use bytes """ - proto, version, kind, size = smell(raw) - if version != Version: + proto, vrsn, kind, size, _ = smell(raw) + if vrsn != Version: raise VersionError("Unsupported version = {}.{}, expected {}." - "".format(version.major, version.minor, Version)) + "".format(vrsn.major, vrsn.minor, Version)) ked = loads(raw=raw, size=size, kind=kind) - return ked, proto, kind, version, size + return ked, proto, kind, vrsn, size def _exhale(self, ked, kind=None): diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index a6fb9b8cb..f7f046480 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -197,7 +197,7 @@ def __init__(self, *pa, **kwa): pass - def reap(self, ims, skip=0, native=False): + def reap(self, ims, genus=GenDex.KERI, gvrsn=Vrsn_2_0, native=False, skip=0): """Extract and return Serder subclass based on protocol type reaped from version string inside serialized raw of Serder. @@ -208,21 +208,32 @@ def reap(self, ims, skip=0, native=False): Parameters: ims (bytearray) of serialized incoming message stream. Assumes start of stream is raw Serder. - skip (int): bytes to skip at front of ims. Useful when CESR native - serialization where skip is size of the message counter so smell - does need to see counter + genus (str): CESR genus code from stream parser. + Provides genus of enclosing stream top-level or nested group + gvrsn (Versionage): instance CESR genus code table version (Major, Minor) + Provides genus of enclosing stream top-level or nested group native (bool): True means sniff determined may be CESR native message so snuff instead of smell. False means sniff determined not CESR native i.e JSON, CBOR, MGPK field map. so use smell. Default False + skip (int): bytes to skip at front of ims. Useful when CESR native + serialization where skip is size of the message counter so smell + does need to see counter """ - smellage = smell(memoryview(ims)[skip:]) # does not copy to skip + if native: + pass + #smellage = smell(memoryview(ims)[skip:]) # does not copy to skip - if smellage.protocol == Protocols.keri: - return SerderKERI(raw=ims, strip=True, smellage=smellage) - elif smellage.protocol == Protocols.acdc: - return SerderACDC(raw=ims, strip=True, smellage=smellage) + else: + smellage = smell(ims) + + if smellage.proto == Protocols.keri: + return SerderKERI(raw=ims, strip=True, smellage=smellage, + genus=genus, gvrsn=gvrsn) + elif smellage.proto == Protocols.acdc: + return SerderACDC(raw=ims, strip=True, smellage=smellage, + genus=genus, gvrsn=gvrsn) else: raise ProtocolError(f"Unsupported protocol type = {smellage.proto}.") @@ -768,7 +779,7 @@ def _verify(self): raise ValidationError(f"Missing version string field in {sad}.") # extract version string elements to verify consistency with attributes - proto, vrsn, kind, size = deversify(sad["v"]) + proto, vrsn, kind, size, opt = deversify(sad["v"]) if self.proto != proto: raise ValidationError(f"Inconsistent protocol={self.proto} in {sad}.") @@ -827,7 +838,7 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, sproto = svrsn = skind = silk = None if sad and 'v' in sad: # attempt to get from vs in sad try: # extract version string elements as defaults if provided - sproto, svrsn, skind, _ = deversify(sad["v"]) + sproto, svrsn, skind, _, _ = deversify(sad["v"]) except ValueError as ex: pass else: @@ -1023,9 +1034,9 @@ def _inhale(self, raw, *, smellage=None): """ if smellage: # passed in so don't need to smell raw again - proto, vrsn, kind, size = smellage # tuple unpack + proto, vrsn, kind, size, gvrsn = smellage # tuple unpack else: # not passed in so smell raw - proto, vrsn, kind, size = smell(raw) + proto, vrsn, kind, size, gvrsn = smell(raw) sad = self.loads(raw=raw, size=size, kind=kind) # ._gvrsn may be set in loads when CESR native deserialization provides _gvrsn @@ -1117,7 +1128,7 @@ def _exhale(self, sad): raise SerializeError(f"Missing version string field in {sad}.") # extract elements so can replace size element but keep others - proto, vrsn, kind, size = deversify(sad["v"]) + proto, vrsn, kind, size, opt = deversify(sad["v"]) raw = self.dumps(sad, kind) diff --git a/src/keri/kering.py b/src/keri/kering.py index 338f87a39..9d653a86b 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -66,14 +66,15 @@ """ Smellage (results of smelling a version string such as in a Serder) - protocol (str): protocol type value of Protocols examples 'KERI', 'ACDC' - version (Versionage): protocol version namedtuple (major, minor) of ints + proto (str): protocol type value of Protocols examples 'KERI', 'ACDC' + vrsn (Versionage): protocol version namedtuple (major, minor) of ints kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' - size (int | Versionage): int size of raw serialization or - genus version namedtuple (major, minor) of ints + size (int): int size of raw serialization or + gvrsn (None | Versionage): optional default is None + For CESR native genus version namedtuple (major, minor) of ints """ -Smellage = namedtuple("Smellage", "protocol version kind size") +Smellage = namedtuple("Smellage", "proto vrsn kind size gvrsn", defaults=(None, )) def rematch(match): """ @@ -95,9 +96,9 @@ def rematch(match): "minor2", "kind2", "size2") - protocol = proto.decode("utf-8") - if protocol not in Protocols: - raise ProtocolError(f"Invalid protocol type = {protocol}.") + proto = proto.decode("utf-8") + if proto not in Protocols: + raise ProtocolError(f"Invalid protocol={proto}.") vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) if vrsn.major < 2: # version2 vs but major < 2 raise VersionError(f"Incompatible {vrsn=} with version string.") @@ -113,9 +114,9 @@ def rematch(match): "minor1", "kind1", "size1") - protocol = proto.decode("utf-8") - if protocol not in Protocols: - raise ProtocolError(f"Invalid protocol type = {protocol}.") + proto = proto.decode("utf-8") + if proto not in Protocols: + raise ProtocolError(f"Invalid protocol={proto}.") vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) if vrsn.major > 1: # version1 vs but major > 1 raise VersionError(f"Incompatible {vrsn=} with version string.") @@ -128,7 +129,7 @@ def rematch(match): else: raise VersionError(f"Bad rematch.") - return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) + return Smellage(proto=proto, vrsn=vrsn, kind=kind, size=size) def versify(protocol=Protocols.keri, version=Version, kind=Serials.json, size=0): @@ -221,15 +222,15 @@ def snatch(match, size=0): regular expressions work with memoryview objects not just bytes or bytearrays """ - if len(full) == VER0FULLSPAN: + if len(full) == VFFULLSPAN: proto, major, minor, gmajor, gminor = match.group("proto0", "major0", "minor0", "gmajor0", "gminor0") - protocol = proto.decode("utf-8") - if protocol not in Protocols: - raise ProtocolError(f"Invalid protocol type = {protocol}.") + proto = proto.decode("utf-8") + if proto not in Protocols: + raise ProtocolError(f"Invalid protocol type = {proto}.") vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) if vrsn.major < 2: # version2 vs but major < 2 raise VersionError(f"Incompatible {vrsn=} with version string.") @@ -243,7 +244,7 @@ def snatch(match, size=0): else: raise VersionError(f"Bad snatch.") - return Smellage(protocol=protocol, version=vrsn, kind=kind, size=size) + return Smellage(proto=proto, vrsn=vrsn, kind=kind, size=size, gvrsn=gvrsn) def snuff(raw, size=0): diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index b1a395a0c..421284148 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -2616,7 +2616,7 @@ def test_serdery(): b'kj_S6LJQDRNOiGohW327FMA6D2","s":""}Not a Serder here or there or' b' anywhere.') - serdery = Serdery(version=Version) + serdery = Serdery() serder = serdery.reap(ims) assert isinstance(serder, SerderKERI) @@ -2636,58 +2636,7 @@ def test_serdery(): """End Test""" -def test_serdery_noversion(): - """Test Serdery unsupported version""" - #Create incoming message stream for Serdery to reap - serder = SerderKERI(makify=True, ilk=kering.Ilks.ixn, verify=False) # make with defaults - sad = serder.sad - pre = "EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf87Bc70J" - sad['i'] = pre - sad['s'] = 2 - sad['a'] = [] - serderKeri = SerderKERI(sad=sad, makify=True) - assert serderKeri.verify() - - ims = bytearray(serderKeri.raw) - - serder = SerderACDC(makify=True, proto=Protocols.acdc, verify=False) # make defaults for ACDC - sad = serder.sad - isr = 'EO8CE5RH1X8QJwHHhPkj_S6LJQDRNOiGohW327FMA6D2' - sad['i'] = isr - serderAcdc = SerderACDC(sad=sad, makify=True) - assert serderAcdc.verify() - - ims.extend(serderAcdc.raw) - - ims.extend(b"Not a Serder here or there or anywhere.") - - assert ims == bytearray(b'{"v":"KERI10JSON00009d_","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYML' - b'Od2eYjmclndQN4bArjSf","i":"EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf' - b'87Bc70J","s":2,"p":"","a":[]}{"v":"ACDC10JSON000086_","d":"EJxJ1' - b'GB8oGD4JAH7YpiMCSWKDV3ulpt37zg9vq1QnOh_","i":"EO8CE5RH1X8QJwHHhP' - b'kj_S6LJQDRNOiGohW327FMA6D2","s":""}Not a Serder here or there or' - b' anywhere.') - - serdery = Serdery() # effective version is None - - serder = serdery.reap(ims) - assert isinstance(serder, SerderKERI) - assert serder.raw == serderKeri.raw - - serder = serdery.reap(ims) - assert isinstance(serder, SerderACDC) - assert serder.raw == serderAcdc.raw - - assert ims == bytearray(b'Not a Serder here or there or anywhere.') - - with pytest.raises(kering.VersionError): - serder = serdery.reap(ims) - - assert ims == bytearray(b'Not a Serder here or there or anywhere.') - - - """End Test""" if __name__ == "__main__": @@ -2710,4 +2659,4 @@ def test_serdery_noversion(): test_serderacdc() test_serder_v2() test_serdery() - test_serdery_noversion() + diff --git a/tests/test_kering.py b/tests/test_kering.py index 039e75d3e..4041bced0 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -174,16 +174,18 @@ def test_smell(): """ raw = b'{"vs":"KERICAAJSONAAAB.","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' - assert smell(raw) == Smellage(protocol='KERI', - version=Versionage(major=2, minor=0), + assert smell(raw) == Smellage(proto='KERI', + vrsn=Versionage(major=2, minor=0), kind='JSON', - size=1) + size=1, + gvrsn=None) raw = b'{"vs":"KERI10JSON000002_","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' - assert smell(raw) == Smellage(protocol='KERI', - version=Versionage(major=1, minor=0), + assert smell(raw) == Smellage(proto='KERI', + vrsn=Versionage(major=1, minor=0), kind='JSON', - size=2) + size=2, + gvrsn=None) raw = b'{"vs":"KERICAAJSONAAABX.","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' with pytest.raises(ProtocolError): @@ -503,70 +505,70 @@ def test_versify_v1(): vs = versify() # defaults assert vs == "KERI10JSON000000_" assert len(vs) == VER1FULLSPAN - proto, version, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.json - assert version == Version + assert vrsn == Version assert size == 0 vs = versify(kind=Serials.json, size=65) assert vs == "KERI10JSON000041_" assert len(vs) == VER1FULLSPAN - proto, version, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.json - assert version == Version + assert vrsn == Version assert size == 65 vs = versify(protocol=Protocols.acdc, kind=Serials.json, size=86) assert vs == "ACDC10JSON000056_" assert len(vs) == VER1FULLSPAN - proto, version, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.acdc assert kind == Serials.json - assert version == Version + assert vrsn == Version assert size == 86 vs = versify(kind=Serials.mgpk, size=0) assert vs == "KERI10MGPK000000_" assert len(vs) == VER1FULLSPAN - proto, version, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.mgpk - assert version == Version + assert vrsn == Version assert size == 0 vs = versify(kind=Serials.mgpk, size=65) assert vs == "KERI10MGPK000041_" assert len(vs) == VER1FULLSPAN - proto, version, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.mgpk - assert version == Version + assert vrsn == Version assert size == 65 vs = versify(kind=Serials.cbor, size=0) assert vs == "KERI10CBOR000000_" assert len(vs) == VER1FULLSPAN - proto, version, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.cbor - assert version == Version + assert vrsn == Version assert size == 0 vs = versify(kind=Serials.cbor, size=65) assert vs == "KERI10CBOR000041_" assert len(vs) == VER1FULLSPAN - proto, version, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.cbor - assert version == Version + assert vrsn == Version assert size == 65 vs = versify(version=Versionage(major=1, minor=1)) # defaults assert vs == "KERI11JSON000000_" assert len(vs) == VER1FULLSPAN - proto, vrsn, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.json assert vrsn == (1, 1) @@ -599,7 +601,7 @@ def test_versify_v2(): vs = versify(version=version) # defaults assert vs == "KERICAAJSONAAAA." assert len(vs) == VER2FULLSPAN - proto, vrsn, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.json assert vrsn == version @@ -608,7 +610,7 @@ def test_versify_v2(): vs = versify(version=version, kind=Serials.json, size=65) assert vs == "KERICAAJSONAABB." assert len(vs) == VER2FULLSPAN - proto, vrsn, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.json assert vrsn == version @@ -617,7 +619,7 @@ def test_versify_v2(): vs = versify(protocol=Protocols.acdc, version=version, kind=Serials.json, size=86) assert vs == "ACDCCAAJSONAABW." assert len(vs) == VER2FULLSPAN - proto, vrsn, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.acdc assert kind == Serials.json assert version == version @@ -626,7 +628,7 @@ def test_versify_v2(): vs = versify(version=version, kind=Serials.mgpk, size=0) assert vs == 'KERICAAMGPKAAAA.' assert len(vs) == VER2FULLSPAN - proto, vrsn, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.mgpk assert vrsn == version @@ -635,7 +637,7 @@ def test_versify_v2(): vs = versify(version=version, kind=Serials.mgpk, size=65) assert vs == 'KERICAAMGPKAABB.' assert len(vs) == VER2FULLSPAN - proto, vrsn, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.mgpk assert vrsn == version @@ -644,7 +646,7 @@ def test_versify_v2(): vs = versify(version=version, kind=Serials.cbor, size=0) assert vs == 'KERICAACBORAAAA.' assert len(vs) == VER2FULLSPAN - proto, vrsn, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.cbor assert vrsn == version @@ -653,7 +655,7 @@ def test_versify_v2(): vs = versify(version=version, kind=Serials.cbor, size=65) assert vs == 'KERICAACBORAABB.' assert len(vs) == VER2FULLSPAN - proto, vrsn, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.cbor assert vrsn == version @@ -662,7 +664,7 @@ def test_versify_v2(): vs = versify(version=Versionage(major=2, minor=1)) # defaults assert vs == "KERICABJSONAAAA." assert len(vs) == VER2FULLSPAN - proto, vrsn, kind, size = deversify(vs) + proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri assert kind == Serials.json assert vrsn == (2, 1) From 1502445f06c6cf358650ae683f9cc2c83440f3ef Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 26 Mar 2024 19:55:31 -0600 Subject: [PATCH 073/418] fixed new table entries for TagX got prepad calcs wrong exposed in testing so realigned Added Verser class to handle version field for CESR Native message --- src/keri/core/coring.py | 247 +++++++++++++++++++++++++++++++++---- src/keri/core/serdering.py | 9 +- tests/core/test_coring.py | 64 +++++++--- 3 files changed, 272 insertions(+), 48 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 8683ad7b0..859615647 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -35,11 +35,11 @@ EmptyListError, ShortageError, UnexpectedCodeError, DeserializeError, UnexpectedCountCodeError, UnexpectedOpCodeError) -from ..kering import (Versionage, Version, VERRAWSIZE, VERFMT, MAXVERFULLSPAN, +from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, + VERRAWSIZE, VERFMT, MAXVERFULLSPAN, versify, deversify, Rever, smell) from ..kering import Serials, Serialage, Protocols, Protocolage, Ilkage, Ilks - from ..help import helping from ..help.helping import sceil, nonStringIterable, nonStringSequence from ..help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, @@ -280,9 +280,10 @@ class MatterCodex: Vast: str = 'U' # Vast 17 byte b2 number Label1: str = 'V' # Label1 as one char (bytes) field map label lead size 1 Label2: str = 'W' # Label2 as two char (bytes) field map label lead size 0 - Tag3: str = 'X' # Tag3 3 B64 encoded chars for field tag or packet type, semver, trait like 'DND' - Tag7: str = 'Y' # Tag7 7 B64 encoded chars for field tag or packet kind and version KERIVVV - Blind: str = 'Z' # Blinding factor 256 bits, Cryptographic strength deterministically generated from random salt + Tag2: str = 'X' # Tag2 1 prepad + 2 B64 encoded chars for field tag + Tag6: str = 'Y' # Tag6 1 prepad + 6 B64 encoded chars for field tag + Tag10: str = 'Z' # Tag10 1 prepad + 10 B64 encoded chars for field tag + Blind: str = 'a' # Blinding factor 256 bits, Cryptographic strength deterministically generated from random salt Salt_128: str = '0A' # random salt/seed/nonce/private key or number of length 128 bits (Huge) Ed25519_Sig: str = '0B' # Ed25519 signature. ECDSA_256k1_Sig: str = '0C' # ECDSA secp256k1 signature. @@ -292,24 +293,25 @@ class MatterCodex: SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. Long: str = '0H' # Long 4 byte b2 number ECDSA_256r1_Sig: str = '0I' # ECDSA secp256r1 signature. - Tag1: str = '0J' # Tag1 1 B64 encoded char with pre pad for field tag - Tag2: str = '0K' # Tag2 2 B64 encoded chars for field tag or version VV or trait like 'EO' - Tag5: str = '0L' # Tag5 5 B64 encoded chars with pre pad for field tag - Tag6: str = '0M' # Tag6 6 B64 encoded chars for field tag - Tag10: str = '0N' # Tag10 10 B64 encoded chars for field tag or version PPPPMmmMmm + Tag1: str = '0J' # Tag1 1 prepad + 1 B64 encoded char for field tag + Tag5: str = '0K' # Tag5 1 prepad + 5 B64 encoded chars for field tag + Tag9: str = '0L' # Tag9 1 prepad + 9 B64 encoded chars for field tag ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. Ed448: str = '1AAD' # Ed448 public signing verification key. Basic derivation. Ed448_Sig: str = '1AAE' # Ed448 signature. Self-signing derivation. - Tag4: str = '1AAF' # Tag4 4 B64 encoded chars for field tag or message kind + Tag4: str = '1AAF' # Tag4 4 B64 encoded chars for field tag DateTime: str = '1AAG' # Base64 custom encoded 32 char ISO-8601 DateTime X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt ECDSA_256r1N: str = '1AAI' # ECDSA secp256r1 verification key non-transferable, basic derivation. ECDSA_256r1: str = '1AAJ' # ECDSA secp256r1 verification or encryption key, basic derivation - Null: str = '1AAK' # Null None or empty value - Yes: str = '1AAL' # Yes Truthy Boolean value - No: str = '1AAM' # No Falsey Boolean value + Tag3: str = '1AAK' # Tag1 1 prepad + 3 B64 encoded chars for field tag + Tag7: str = '1AAL' # Tag7 1 prepad + 7 B64 encoded chars for field tag + Tag8: str = '1AAM' # Tag8 8 B64 encoded chars for field tag + Null: str = '1AAN' # Null None or empty value + No: str = '1AAO' # No Falsey Boolean value + Yes: str = '1AAP' # Yes Truthy Boolean value TBD1: str = '2AAA' # Testing purposes only fixed with lead size 1 TBD2: str = '3AAA' # Testing purposes only of fixed with lead size 2 StrB64_L0: str = '4A' # String Base64 only lead size 0 @@ -710,7 +712,8 @@ class Matter: 'W': Sizage(hs=1, ss=0, fs=4, ls=0), 'X': Sizage(hs=1, ss=0, fs=4, ls=0), 'Y': Sizage(hs=1, ss=0, fs=8, ls=0), - 'Z': Sizage(hs=1, ss=0, fs=44, ls=0), + 'Z': Sizage(hs=1, ss=0, fs=12, ls=0), + 'a': Sizage(hs=1, ss=0, fs=44, ls=0), '0A': Sizage(hs=2, ss=0, fs=24, ls=0), '0B': Sizage(hs=2, ss=0, fs=88, ls=0), '0C': Sizage(hs=2, ss=0, fs=88, ls=0), @@ -721,10 +724,8 @@ class Matter: '0H': Sizage(hs=2, ss=0, fs=8, ls=0), '0I': Sizage(hs=2, ss=0, fs=88, ls=0), '0J': Sizage(hs=2, ss=0, fs=4, ls=0), - '0K': Sizage(hs=2, ss=0, fs=4, ls=0), - '0L': Sizage(hs=2, ss=0, fs=8, ls=0), - '0M': Sizage(hs=2, ss=0, fs=8, ls=0), - '0N': Sizage(hs=2, ss=0, fs=12, ls=0), + '0K': Sizage(hs=2, ss=0, fs=8, ls=0), + '0L': Sizage(hs=2, ss=0, fs=12, ls=0), '1AAA': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAB': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), @@ -735,9 +736,12 @@ class Matter: '1AAH': Sizage(hs=4, ss=0, fs=100, ls=0), '1AAI': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAJ': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAK': Sizage(hs=4, ss=0, fs=8, ls=0), + '1AAL': Sizage(hs=4, ss=0, fs=12, ls=0), + '1AAM': Sizage(hs=4, ss=0, fs=12, ls=0), + '1AAN': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAO': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAP': Sizage(hs=4, ss=0, fs=4, ls=0), '2AAA': Sizage(hs=4, ss=0, fs=8, ls=1), '3AAA': Sizage(hs=4, ss=0, fs=8, ls=2), '4A': Sizage(hs=2, ss=2, fs=None, ls=0), @@ -1703,8 +1707,7 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, if raw is None and qb64b is None and qb64 is None and qb2 is None: if dts is None: # defaults to now dts = helping.nowIso8601() - # if len(dts) != 32: - # raise ValueError("Invalid length of date time string") + if hasattr(dts, "decode"): dts = dts.decode("utf-8") qb64 = MtrDex.DateTime + dts.translate(self.ToB64) @@ -1739,6 +1742,202 @@ def datetime(self): """ return helping.fromIso8601(self.dts) +# Versage namedtuple +# proto (str): protocol element of Protocols +# vrsn (Versionage): instance protocol version namedtuple (major, minor) ints +# vrsn (Versionage | None): instance genus version namedtuple (major, minor) ints +Versage = namedtuple("Versage", "proto vrsn gvrsn", defaults=(None, )) + + +class Verser(Matter): + """ + Verser is subclass of Matter, cryptographic material, for formatted + version primitives in Base64. Supports different primitives based on code. + + Verser provides a more compact representation than would be obtained by + converting the raw ASCII representation to Base64. + + Attributes: + + Inherited Properties: (See Matter) + code (str): hard part of derivation code to indicate cypher suite + both (int): hard and soft parts of full text code + size (int): Number of triplets of bytes including lead bytes + (quadlets of chars) of variable sized material. Value of soft size, + ss, part of full text code. + Otherwise None. + rize (int): number of bytes of raw material not including + lead bytes + raw (bytes): crypto material only without code + qb64 (str): Base64 fully qualified with derivation code + crypto mat + qb64b (bytes): Base64 fully qualified with derivation code + crypto mat + qb2 (bytes): binary with derivation code + crypto material + transferable (bool): True means transferable derivation code False otherwise + digestive (bool): True means digest derivation code False otherwise + prefixive (bool): True means identifier prefix derivation code False otherwise + + Properties: + proto (str): protocol from Protocols + vrsn (Versionage): instance protocol version. + namedtuple (major, minor) of ints + gvrsn (Versionage | None): instance genus version. + namedtuple (major, minor) of ints + + Hidden Inherited: + _code (str): value for .code property + _raw (bytes): value for .raw property + _rsize (bytes): value for .rsize property. Raw size in bytes when + variable sized material else None. + _size (int): value for .size property. Number of triplets of bytes + including lead bytes (quadlets of chars) of variable sized material + else None. + _infil (types.MethodType): creates qb64b from .raw and .code + (fully qualified Base64) + _exfil (types.MethodType): extracts .code and .raw from qb64b + (fully qualified Base64) + + Hidden: + _proto (str): value for .proto property + _vrsn (Versionage): value for .vrsn property + _gvrsn (Versionage | None): value for .gvrsn property + + Methods: + + """ + + + def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, + code=MtrDex.Tag10, versage=None, proto=Protocols.keri, + vrsn=Vrsn_2_0, gvrsn=Vrsn_2_0, **kwa): + """ + Inherited Parameters: (see Matter) + raw (bytes): unqualified crypto material usable for crypto operations + code (str): stable (hard) part of derivation code + rize (int): raw size in bytes when variable sized material else None + qb64b (bytes): fully qualified crypto material Base64 + qb64 (str, bytes): fully qualified crypto material Base64 + qb2 (bytes): fully qualified crypto material Base2 + strip (bool): True means strip (delete) matter from input stream + bytearray after parsing qb64b or qb2. False means do not strip + + Parameters: + versage (Versage | None): namedtuple of (proto, vrsn, gvrsn) + proto (str | None): protocol from Protocols + vrsn (Versionage | None): instance protocol version. + namedtuple (major, minor) of ints + gvrsn (Versionage | None): instance genus version. + namedtuple (major, minor) of ints + + Notes: + prepad = 'A' + """ + if raw is None and qb64b is None and qb64 is None and qb2 is None: + if versage: + proto, vrsn, gvrsn = versage + if code == MtrDex.Tag10: + if not gvrsn: + raise InvalidValueError(f"Missing genus version.") + + qb64 = (code + 'A' + proto + + self.verToB64(vrsn) + + self.verToB64(gvrsn)).encode("utf-8") + + elif code == MtrDex.Tag7: + qb64 = (code + 'A' + proto + self.verToB64(vrsn)) + + else: + raise InvalidCodeError(f"Invalid {code=} for Verser.") + + + super(Verser, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, + code=code, **kwa) + + if self.code not in (MtrDex.Tag10, MtrDex.Tag7): + raise InvalidCodeError(f"Invalid code={self.code} for Verser.") + + @property + def versage(self): + """Property versage (Versage): named tuple of (proto, vrsn, gvrsn) + + """ + gvrsn = None + clp = len(self.code) + 1 # code plus prepad + proto = self.qb64[clp:clp+4] + vrsn = self.b64ToVer(self.qb64[clp+4:clp+7]) + if self.fullSize == clp + 10: + gvrsn = self.b64ToVer(self.qb64[clp+7:clp+10]) + + return Versage(proto=proto, vrsn=vrsn, gvrsn=gvrsn) + + + @staticmethod + def verToB64(version=None, *, text="", major=0, minor=0): + """ Converts version to Base64 representation + + Returns: + verB64 (str): + + Example: + Verser.verToB64(verstr = "1.0")) + + Parameters: + version (Versionage): instange of namedtuple + Versionage(major=major,minor=minor) + text (str): text format of version as dotted decimal "major.minor" + major (int): When version is None and verstr is empty then use major minor + range [0, 63] for one Base64 character + minor (int): When version is None and verstr is empty then use major minor + range [0, 4095] for two Base64 characters + + """ + if version: + major = version.major + minor = version.minor + + elif text: + splits = text.split(".", maxsplit=2) + splits = [(int(s) if s else 0) for s in splits] + parts = [major, minor] + for i in range(2-len(splits),0, -1): # append missing minor and/or major + splits.append(parts[-i]) + major = splits[0] + minor = splits[1] + + if major < 0 or major > 63 or minor < 0 or minor > 4095: + raise ValueError(f"Out of bounds version = {major}.{minor}.") + + return (f"{intToB64(major)}{intToB64(minor, l=2)}") + + + @staticmethod + def b64ToVer(b64, *, texted=False): + """ Converts Base64 representation of version to Versionage or + text dotted decimal format + + default is Versionage + + Returns: + version (Versionage | str): + + Example: + .b64ToVer("BAA")) + + Parameters: + b64 (str): base64 string of three characters Mmm for Major minor + texted (bool): return text format dotted decimal string + + + """ + if not Reb64.match(b64.encode("utf-8")): + raise ValueError("Invalid Base64.") + + if texted: + return ".".join([f"{b64ToInt(b64[0])}", f"{b64ToInt(b64[1:3])}"]) + + return Versionage(major=b64ToInt(b64[0]), minor=b64ToInt(b64[1:3])) + + + class Streamer: """ diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index f7f046480..8141bfa43 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -30,7 +30,7 @@ from ..kering import Protocols, Serials, Rever, versify, deversify, Ilks from ..core import coring from .coring import MtrDex, DigDex, PreDex, Saids, Digestage -from .coring import Matter, Saider, Verfer, Diger, Number, Tholder +from .coring import Matter, Saider, Verfer, Diger, Number, Tholder, Verser from ..core import counting from ..core.counting import GenDex, AllTags, Counter @@ -1243,9 +1243,10 @@ def _dumps(self, sad): # should dispatch or use match instead of big if else match l: # label case "v": # protocol+version - val = (MtrDex.Tag10 + self.proto + - Counter.verToB64(self.vrsn) + - Counter.verToB64(self.gvrsn)).encode("utf-8") + val = Verser(code=MtrDex.Tag10, + proto=self.proto, + vrsn=self.vrsn, + gvrsn=self.gvrsn).qb64b case "t": # message type val = (MtrDex.Tag3 + ilk).encode("utf-8") # add code diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 98970e84d..dbc5ac81e 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -27,7 +27,8 @@ from keri.core import eventing from keri.core.coring import (Ilkage, Ilks, Saids, Protocols, Protocolage, Sadder, Tholder, Seqner, - NumDex, Number, Siger, Dater, Bexter, Texter) + NumDex, Number, Siger, Dater, Bexter, Texter, + Verser, Versage) from keri.core.coring import Serialage, Serials, Tiers from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, Indexer, @@ -76,9 +77,10 @@ def test_matter(): 'Vast': 'U', 'Label1': 'V', 'Label2': 'W', - 'Tag3': 'X', - 'Tag7': 'Y', - 'Blind': 'Z', + 'Tag2': 'X', + 'Tag6': 'Y', + 'Tag10': 'Z', + 'Blind': 'a', 'Salt_128': '0A', 'Ed25519_Sig': '0B', 'ECDSA_256k1_Sig': '0C', @@ -89,10 +91,8 @@ def test_matter(): 'Long': '0H', 'ECDSA_256r1_Sig': '0I', 'Tag1': '0J', - 'Tag2': '0K', - 'Tag5': '0L', - 'Tag6': '0M', - 'Tag10': '0N', + 'Tag5': '0K', + 'Tag9': '0L', 'ECDSA_256k1N': '1AAA', 'ECDSA_256k1': '1AAB', 'Ed448N': '1AAC', @@ -103,9 +103,12 @@ def test_matter(): 'X25519_Cipher_Salt': '1AAH', 'ECDSA_256r1N': '1AAI', 'ECDSA_256r1': '1AAJ', - 'Null': '1AAK', - 'Yes': '1AAL', - 'No': '1AAM', + 'Tag3': '1AAK', + 'Tag7': '1AAL', + 'Tag8': '1AAM', + 'Null': '1AAN', + 'No': '1AAO', + 'Yes': '1AAP', 'TBD1': '2AAA', 'TBD2': '3AAA', 'StrB64_L0': '4A', @@ -156,7 +159,8 @@ def test_matter(): } # Codes table with sizes of code (hard) and full primitive material - assert Matter.Sizes == { + assert Matter.Sizes == \ + { 'A': Sizage(hs=1, ss=0, fs=44, ls=0), 'B': Sizage(hs=1, ss=0, fs=44, ls=0), 'C': Sizage(hs=1, ss=0, fs=44, ls=0), @@ -182,7 +186,8 @@ def test_matter(): 'W': Sizage(hs=1, ss=0, fs=4, ls=0), 'X': Sizage(hs=1, ss=0, fs=4, ls=0), 'Y': Sizage(hs=1, ss=0, fs=8, ls=0), - 'Z': Sizage(hs=1, ss=0, fs=44, ls=0), + 'Z': Sizage(hs=1, ss=0, fs=12, ls=0), + 'a': Sizage(hs=1, ss=0, fs=44, ls=0), '0A': Sizage(hs=2, ss=0, fs=24, ls=0), '0B': Sizage(hs=2, ss=0, fs=88, ls=0), '0C': Sizage(hs=2, ss=0, fs=88, ls=0), @@ -193,10 +198,8 @@ def test_matter(): '0H': Sizage(hs=2, ss=0, fs=8, ls=0), '0I': Sizage(hs=2, ss=0, fs=88, ls=0), '0J': Sizage(hs=2, ss=0, fs=4, ls=0), - '0K': Sizage(hs=2, ss=0, fs=4, ls=0), - '0L': Sizage(hs=2, ss=0, fs=8, ls=0), - '0M': Sizage(hs=2, ss=0, fs=8, ls=0), - '0N': Sizage(hs=2, ss=0, fs=12, ls=0), + '0K': Sizage(hs=2, ss=0, fs=8, ls=0), + '0L': Sizage(hs=2, ss=0, fs=12, ls=0), '1AAA': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAB': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), @@ -207,9 +210,12 @@ def test_matter(): '1AAH': Sizage(hs=4, ss=0, fs=100, ls=0), '1AAI': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAJ': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAK': Sizage(hs=4, ss=0, fs=8, ls=0), + '1AAL': Sizage(hs=4, ss=0, fs=12, ls=0), + '1AAM': Sizage(hs=4, ss=0, fs=12, ls=0), + '1AAN': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAO': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAP': Sizage(hs=4, ss=0, fs=4, ls=0), '2AAA': Sizage(hs=4, ss=0, fs=8, ls=1), '3AAA': Sizage(hs=4, ss=0, fs=8, ls=2), '4A': Sizage(hs=2, ss=2, fs=None, ls=0), @@ -3695,6 +3701,23 @@ def test_dater(): """ Done Test """ +def test_verser(): + """ + Test Verser version primitive subclass of Matter + """ + verser = Verser() # defaults + assert verser.code == MtrDex.Tag10 + assert verser.raw == b'\x02\x84D\x80\x80\x00 \x00' + assert verser.qb64 == 'ZAKERICAACAA' + assert verser.qb2 == b'd\x02\x84D\x80\x80\x00 \x00' + assert verser.versage == Versage(proto='KERI', + vrsn=Versionage(major=2, minor=0), + gvrsn=Versionage(major=2, minor=0)) + + + + """ Done Test """ + def test_texter(): """ @@ -6450,6 +6473,7 @@ def test_tholder(): if __name__ == "__main__": test_matter() + test_verser() test_texter() #test_counter() #test_prodex() From 582ed80ca3dbcbe7727d9c14baa84dcd9b480be4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 27 Mar 2024 14:51:58 -0600 Subject: [PATCH 074/418] some refactor of Counter and Matter added hard soft properties started adding support for special soft values of Matter codes with empty raw for tags --- src/keri/core/coring.py | 143 +++++++++++++++++++++++++++--------- src/keri/core/counting.py | 47 ++++++++++-- src/keri/core/serdering.py | 6 +- src/keri/help/helping.py | 3 +- tests/core/test_coring.py | 41 +++++++---- tests/core/test_counting.py | 22 ++++-- 6 files changed, 196 insertions(+), 66 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 859615647..c2042972d 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -639,18 +639,27 @@ class Matter: Includes the following attributes and properties: + Class Attributes: + Codex (MatterCodex): MtrDex + Hards (dict): hard sizes keyed by qb64 selector + Bards (dict): hard size keyed by qb2 selector + Sizes (dict): sizes tables for codes + Attributes: Properties: code (str): hard part of derivation code to indicate cypher suite - both (int): hard and soft parts of full text code - size (int): Number of triplets of bytes including lead bytes - (quadlets of chars) of variable sized material. Value of soft size, - ss, part of full text code. - Otherwise None. - rize (int): number of bytes of raw material not including - lead bytes - raw (bytes): crypto material only without code + hard (str): hard part of derivation code. alias for code + soft (str): soft part of derivation code fs any. + Empty when ss = 0. + both (str): hard + soft parts of full text code + size (int | None): Number of quadlets/triplets of chars/bytes including + lead bytes of variable sized material (fs = None). + Converted value of the soft part (of len ss) of full + derivation code. + Otherwise None when not variably sized (fs != None) + rize (int): number of bytes of raw material not including lead bytes + raw (bytes): crypto material only. Not derivation code or lead bytes. qb64 (str): Base64 fully qualified with derivation code + crypto mat qb64b (bytes): Base64 fully qualified with derivation code + crypto mat qb2 (bytes): binary with derivation code + crypto material @@ -671,6 +680,16 @@ class Matter: _exfil (types.MethodType): extracts .code and .raw from qb64b (fully qualified Base64) + When fs is None then ss must not be 0 as size must be defined + When fs is not None then ss may be 0 and must have fs >= hs + ss + When fs > hs + ss then must have fs = hs + ss + size where size = + converted soft value + ls + When fs = hs + ss then raw must be empty and soft value is special + this also includes the case where soft value = 0 because raw + must still be empty to satisfy fs = hs + ss + 0 + so no need to compute soft value when fs = hs + ss only ensure + raw is empty. This allows treating soft value as special not a size. + """ Codex = MtrDex # Hards table maps from bytes Base64 first code char to int of hard size, hs, @@ -681,6 +700,12 @@ class Matter: Hards.update({chr(c): 1 for c in range(97, 97 + 26)}) Hards.update([('0', 2), ('1', 4), ('2', 4), ('3', 4), ('4', 2), ('5', 2), ('6', 2), ('7', 4), ('8', 4), ('9', 4)]) + + + # Bards table maps first code char. converted to binary sextext of hard size, + # hs. Used for ._bexfil. + Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) + # Sizes table maps from value of hs chars of code to Sizage namedtuple of # (hs, ss, fs, ls) where hs is hard size, ss is soft size, and fs is full size # and ls is lead size @@ -777,36 +802,37 @@ class Matter: } - # Bards table maps first code char. converted to binary sextext of hard size, - # hs. Used for ._bexfil. - Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, qb64b=None, qb64=None, qb2=None, strip=False): """ Validate as fully qualified Parameters: - raw (bytes): unqualified crypto material usable for crypto operations + raw (bytes | bytearray | None): unqualified crypto material usable + for crypto operations. code (str): stable (hard) part of derivation code - rize (int): raw size in bytes when variable sized material else None - qb64b (bytes): fully qualified crypto material Base64 - qb64 (str, bytes): fully qualified crypto material Base64 - qb2 (bytes): fully qualified crypto material Base2 + rize (int | None): raw size in bytes when variable sized material not + including lead bytes if any + Otherwise None + qb64b (bytes | None): fully qualified crypto material Base64 + qb64 (str | bytes | None): fully qualified crypto material Base64 + qb2 (bytes | None): fully qualified crypto material Base2 strip (bool): True means strip (delete) matter from input stream bytearray after parsing qb64b or qb2. False means do not strip - Needs either (raw and code and optionally size and rsize) + Needs either (raw and code and optionally rsize) or qb64b or qb64 or qb2 Otherwise raises EmptyMaterialError - When raw and code and optional size and rsize provided - then validate that code is correct for length of raw, size, rsize - and assign .raw + When raw and code and optional rsize provided + then validate that code is correct for length of raw, rsize, + computed size from Sizes and assign .raw Else when qb64b or qb64 or qb2 provided extract and assign .raw and .code and .size and .rsize """ size = None # variable raw binary size including leader in quadlets + soft = '' # soft portion of code, if raw is not None: # raw provided if not code: raise EmptyMaterialError(f"Improper initialization need either " @@ -819,7 +845,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, raise InvalidCodeError("Unsupported code={}.".format(code)) if code[0] in SmallVrzDex or code[0] in LargeVrzDex: # dynamic size - if rize: # use rsize to determin length of raw to extract + if rize: # use rsize to determine length of raw to extract if rize < 0: raise InvalidVarRawSizeError(f"Missing var raw size for " f"code={code}.") @@ -865,8 +891,9 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, raise RawMaterialError(f"Not enougth raw bytes for code={code}" f"expected {rize} got {len(raw)}.") - self._code = code # hard value part of code - self._size = size # soft value part of code in int + self._code = code # str hard part of code + self._soft = soft # srtsoft part of code, empty when ss=0 + self._size = size # int of soft part value, None when fs != None self._raw = bytes(raw) # crypto ops require bytes not bytearray elif qb64b is not None: @@ -886,6 +913,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, raise EmptyMaterialError(f"Improper initialization need either " f"(raw and code) or qb64b or qb64 or qb2.") + @classmethod def _rawSize(cls, code): """ @@ -899,6 +927,7 @@ def _rawSize(cls, code): raise InvalidCodeSizeError(f"Non-fixed raw size code {code}.") return (((fs - cs) * 3 // 4) - ls) + @classmethod def _leadSize(cls, code): """ @@ -909,36 +938,76 @@ def _leadSize(cls, code): _, _, _, ls = cls.Sizes[code] # get lead size from .Sizes table return ls + @property def code(self): """ - Returns ._code which is the hard part only of full text code. - Some codes only have a hard part. Soft part is for variable sized matter. - Makes .code read only + Returns: + code (str): hard part only of full text code. + + Getter for ._code. Makes ._code read only + + Some codes only have a hard part. Soft part may be for variable sized + matter or for special codes that are code only (raw is empty) """ return self._code + @property - def both(self): + def hard(self): + """ + Returns: + hard (str): hard part only of full text code. Alias for .code. + + """ + return self.code + + + @property + def soft(self): """ - Returns both hard and soft parts of full text code + Returns: + soft (str): soft part only of full text code. + + Getter for ._soft. Make ._soft read only """ - _, ss, _, _ = self.Sizes[self.code] - return (f"{self.code}{intToB64(self.size, l=ss)}") + return self._soft @property def size(self): """ - Returns ._size int or None if not variable sized matter - Makes .size read only + Returns: + size(int | None): Number of variably sized b64 quadlets/b2 triplets + in primitive when varibly sized + None when not variably sized (fs==None) + + Getter for ._size. Makes ._size read only + + Number of quadlets/triplets of chars/bytes of variable sized material or + None when not variably sized. + + Converted qb64 value to int of ss portion of full text code when variably + sized primitive material (fs = None). - Number of triplets of bytes including lead bytes (quadlets of chars) - of variable sized material. Value of soft size, ss, part of full text code. """ return self._size + @property + def both(self): + """ + Returns: + both (str): hard + soft parts of full text code + """ + _, ss, _, fs = self.Sizes[self.code] + + if self.size is not None: + return (f"{self.code}{intToB64(self.size, l=ss)}") + else: + return (f"{self.code}{self.soft}") + + @property def fullSize(self): """ @@ -952,6 +1021,7 @@ def fullSize(self): fs = hs + ss + (self.size * 4) return fs + @property def raw(self): """ @@ -960,6 +1030,7 @@ def raw(self): """ return self._raw + @property def qb64b(self): """ @@ -969,6 +1040,7 @@ def qb64b(self): """ return self._infil() + @property def qb64(self): """ @@ -978,6 +1050,7 @@ def qb64(self): """ return self.qb64b.decode("utf-8") + @property def qb2(self): """ @@ -986,6 +1059,7 @@ def qb2(self): """ return self._binfil() + @property def transferable(self): """ @@ -995,6 +1069,7 @@ def transferable(self): """ return (self.code not in NonTransDex) + @property def digestive(self): """ diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 8e162e2f9..21792db6c 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -574,24 +574,57 @@ def sizes(self): @property def code(self): """ - Returns ._code - Makes .code read only + Returns: + code (str): hard part only of full text code. + Getter for ._code. Makes .code read only + + Soft part is count """ return self._code @property - def count(self): + def hard(self): + """ + Returns: + hard (str): hard part only of full text code. Alias for .code. + """ - Returns ._count - Makes ._count read only + return self.code - number of quadlets of b64 chars or triplets of b2 bytes of material - framed by counter + + @property + def count(self): + """ + Returns: + count (int): count value in quadlets/triples chars/bytes of material + framed by counter. + Getter for ._count. Makes ._count read only """ return self._count + @property + def soft(self): + """ + Returns: + soft (str): Base64 soft part of full counter code. Count value in + quadlets/triples chars/bytes of material framed by counter. + Converts .count to b64 + """ + _, ss, _, _ = self.sizes[self.code] + return intToB64(self._count, l=ss) + + + @property + def both(self): + """ + Returns: + both (str): hard + soft parts of full text code + """ + return f"{self.hard}{self.soft}" + + @property def fullSize(self): """ diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 8141bfa43..0a83a36cf 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1421,7 +1421,7 @@ def kind(self): @property def proto(self): - """proto property getter, alias of .protocol + """proto property getter, protocol identifier type value of Protocolage such as 'KERI' or 'ACDC' Returns: @@ -1437,12 +1437,12 @@ def protocol(self): Returns: protocol (str): Protocolage value as protocol type """ - return self._proto + return self.proto @property def vrsn(self): - """vrsn (version) property getter, alias of .version + """vrsn (version) property getter Returns: vrsn (Versionage): instance of protocol version for this Serder diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index 038d97203..75a709ee5 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -331,7 +331,8 @@ def codeB2ToB64(b, l): One char for each of l sextets from front (left) of b. This is useful for encoding as code characters, sextets from the front of a Base2 bytes (byte string). Must provide l because of ambiguity between l=3 - and l=4. Both require 3 bytes in b. + and l=4. Both require 3 bytes in b. Trailing pad bits are removed so + returned sextets as characters are right aligned . """ if hasattr(b, 'encode'): b = b.encode("utf-8") # convert to bytes diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index dbc5ac81e..02c9c1ef1 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -48,10 +48,11 @@ -def test_matter(): +def test_matter_class(): """ - Test Matter class + Test Matter class attributes """ + assert dataclasses.asdict(MtrDex) == \ { 'Ed25519_Seed': 'A', @@ -268,7 +269,7 @@ def test_matter(): assert not (val.hs + val.ss) % 4 - # Bizes maps bytes of sextet of decoded first character of code with hard size of code + # Test .Bards # verify equivalents of items for Sizes and Bizes for skey, sval in Matter.Hards.items(): ckey = codeB64ToB2(skey) @@ -277,6 +278,14 @@ def test_matter(): assert Matter._rawSize(MtrDex.Ed25519) == 32 assert Matter._leadSize(MtrDex.Ed25519) == 0 + + +def test_matter(): + """ + Test Matter instances + """ + + # verkey, sigkey = pysodium.crypto_sign_keypair() verkey = b'iN\x89Gi\xe6\xc3&~\x8bG|%\x90(L\xd6G\xddB\xef`\x07\xd2T\xfc\xe1\xcd.\x9b\xe4#' prefix = 'BGlOiUdp5sMmfotHfCWQKEzWR91C72AH0lT84c0um-Qj' # str @@ -297,7 +306,8 @@ def test_matter(): # test from raw matter = Matter(raw=verkey) # default code is MtrDex.Ed25519N assert matter.raw == verkey - assert matter.code == MtrDex.Ed25519N + assert matter.code == MtrDex.Ed25519N == matter.hard + assert matter.soft == "" assert matter.both == MtrDex.Ed25519N assert matter.size == None assert matter.fullSize == 44 @@ -313,6 +323,7 @@ def test_matter(): assert matter.digestive == False assert matter.prefixive == True + # test round trip assert matter.qb64 == encodeB64(matter.qb2).decode("utf-8") assert matter.qb2 == decodeB64(matter.qb64.encode("utf-8")) @@ -678,7 +689,8 @@ def test_matter(): qb2 = b'\xe4\x10\x02\x00abcde' matter = Matter(raw=raw, code=code) assert matter.raw == raw - assert matter.code == code + assert matter.code == code == matter.hard + assert matter.soft == "" assert matter.both == both assert matter.size == 2 # quadlets assert matter.fullSize == 12 # chars @@ -3705,14 +3717,14 @@ def test_verser(): """ Test Verser version primitive subclass of Matter """ - verser = Verser() # defaults - assert verser.code == MtrDex.Tag10 - assert verser.raw == b'\x02\x84D\x80\x80\x00 \x00' - assert verser.qb64 == 'ZAKERICAACAA' - assert verser.qb2 == b'd\x02\x84D\x80\x80\x00 \x00' - assert verser.versage == Versage(proto='KERI', - vrsn=Versionage(major=2, minor=0), - gvrsn=Versionage(major=2, minor=0)) + #verser = Verser() # defaults + #assert verser.code == MtrDex.Tag10 + #assert verser.raw == b'' + #assert verser.qb64 == 'ZAKERICAACAA' + #assert verser.qb2 == b'd\x02\x84D\x80\x80\x00 \x00' + #assert verser.versage == Versage(proto='KERI', + #vrsn=Versionage(major=2, minor=0), + #gvrsn=Versionage(major=2, minor=0)) @@ -6472,9 +6484,10 @@ def test_tholder(): if __name__ == "__main__": + test_matter_class() test_matter() test_verser() - test_texter() + #test_texter() #test_counter() #test_prodex() #test_indexer() diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index d50a3fe4e..578c5db08 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -412,9 +412,9 @@ def test_codexes_tags(): """End Test""" -def test_counter(): +def test_counter_class(): """ - Test Counter class + Test Counter class variables """ # test class attributes @@ -645,13 +645,15 @@ def test_counter_v1(): qscb2 = decodeB64(qscb) counter = Counter(tag="ControllerIdxSigs", count=count, version=Vrsn_1_0) - assert counter.code == CtrDex.ControllerIdxSigs + assert counter.code == CtrDex.ControllerIdxSigs == counter.hard assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_1_0 assert counter.fullSize == 4 + assert counter.soft =='AB' + assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 counter = Counter(tag=AllTags.ControllerIdxSigs, count=count, version=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs @@ -969,13 +971,15 @@ def test_counter_v2(): # default version and default count = 1 counter = Counter(code=CtrDex.ControllerIdxSigs) - assert counter.code == CtrDex.ControllerIdxSigs + assert counter.code == CtrDex.ControllerIdxSigs == counter.hard assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_2_0 assert counter.fullSize == 4 + assert counter.soft =='AB' + assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 # default count = 1 counter = Counter(code=CtrDex.ControllerIdxSigs, version=Vrsn_2_0) @@ -1253,13 +1257,15 @@ def test_counter_v2(): qscb2 = decodeB64(qscb) counter = Counter(code=CtrDex.BigGenericGroup, count=count, version=Vrsn_2_0) - assert counter.code == CtrDex.BigGenericGroup + assert counter.code == CtrDex.BigGenericGroup == counter.hard assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_2_0 assert counter.fullSize == 8 + assert counter.soft == 'AACAB' + assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 counter = Counter(qb64b=qscb, version=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup @@ -1321,12 +1327,14 @@ def test_counter_v2(): qscb2 = decodeB64(qscb) counter = Counter(code=CtrDex.GenericGroup, count=count, version=Vrsn_2_0) - assert counter.code == CtrDex.BigGenericGroup + assert counter.code == CtrDex.BigGenericGroup == counter.hard assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_2_0 + assert counter.soft =='AACAB' + assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 counter = Counter(tag=AllTags.GenericGroup, count=count, version=Vrsn_2_0) assert counter.code == CtrDex.BigGenericGroup @@ -1379,7 +1387,7 @@ def test_counter_v2(): test_mapdom() test_mapcodex() test_codexes_tags() - test_counter() + test_counter_class() test_counter_v1() test_counter_v2() From 333d1a7d3d8988170439e2bf2e18be9c6e4ef65b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 27 Mar 2024 21:24:05 -0600 Subject: [PATCH 075/418] Refactor Matter to support special Tag codes with special soft parts of code Change logic for interpeting size table --- src/keri/core/coring.py | 222 +++++++++++++++++++++++++++----------- src/keri/kering.py | 7 ++ tests/core/test_coring.py | 66 +++++++++--- 3 files changed, 220 insertions(+), 75 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index c2042972d..6dce3491d 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -27,7 +27,9 @@ from ..kering import MaxON -from ..kering import (EmptyMaterialError, RawMaterialError, InvalidCodeError, +from ..kering import (EmptyMaterialError, RawMaterialError, + InvalidCodeError, InvalidSoftError, + InvalidSizeError, InvalidCodeSizeError, InvalidVarIndexError, InvalidVarSizeError, InvalidVarRawSizeError, ConversionError, InvalidValueError, InvalidTypeError, @@ -680,15 +682,29 @@ class Matter: _exfil (types.MethodType): extracts .code and .raw from qb64b (fully qualified Base64) - When fs is None then ss must not be 0 as size must be defined - When fs is not None then ss may be 0 and must have fs >= hs + ss - When fs > hs + ss then must have fs = hs + ss + size where size = - converted soft value + ls - When fs = hs + ss then raw must be empty and soft value is special - this also includes the case where soft value = 0 because raw - must still be empty to satisfy fs = hs + ss + 0 - so no need to compute soft value when fs = hs + ss only ensure - raw is empty. This allows treating soft value as special not a size. + When fs in table is None then ss must not be 0 as size must be defined + + When fs computed is not None then ss may be 0 + When fs > hs + ss then must have ss != 0, fs = hs + ss + size where size = + converted soft value from ss chars + ls, size is not None + + When ss == 0 raw must be empty and size must be None, soft must be empty + + + When fs = hs + ss then raw must be empty, size must be None, + and soft value may be special: + + When fs = hs + ss and ss == 0, raw is empty, soft is empty and soft not special + but fs == hs == hs + 0 is still valid code + When fs = hs + ss and ss > 0, raw is empty soft not empty and soft is spectial + + These includes the case where ss > 0 but if we compute size == 0 + from soft not empty. because when size == 0, raw must still be empty to + satisfy fs = hs + ss + 0 = hs + ss. + So no need to compute soft value when fs = hs + ss only ensure + raw is empty. + This allows treating soft value as special not a size. + and force size = None. """ Codex = MtrDex @@ -737,7 +753,7 @@ class Matter: 'W': Sizage(hs=1, ss=0, fs=4, ls=0), 'X': Sizage(hs=1, ss=0, fs=4, ls=0), 'Y': Sizage(hs=1, ss=0, fs=8, ls=0), - 'Z': Sizage(hs=1, ss=0, fs=12, ls=0), + 'Z': Sizage(hs=1, ss=11, fs=12, ls=0), 'a': Sizage(hs=1, ss=0, fs=44, ls=0), '0A': Sizage(hs=2, ss=0, fs=24, ls=0), '0B': Sizage(hs=2, ss=0, fs=88, ls=0), @@ -804,7 +820,7 @@ class Matter: def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, - qb64b=None, qb64=None, qb2=None, strip=False): + qb64b=None, qb64=None, qb2=None, soft='', strip=False): """ Validate as fully qualified Parameters: @@ -817,6 +833,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, qb64b (bytes | None): fully qualified crypto material Base64 qb64 (str | bytes | None): fully qualified crypto material Base64 qb2 (bytes | None): fully qualified crypto material Base2 + soft (str | None): soft part for special codes strip (bool): True means strip (delete) matter from input stream bytearray after parsing qb64b or qb2. False means do not strip @@ -832,17 +849,20 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, """ size = None # variable raw binary size including leader in quadlets - soft = '' # soft portion of code, - if raw is not None: # raw provided + soft = soft # soft portion of code, + if raw is not None: # raw provided but may be empty if not code: raise EmptyMaterialError(f"Improper initialization need either " - f"(raw and code) or qb64b or qb64 or qb2.") + f"(raw not None and code) or " + f"(raw empty and code and soft) or " + f"(code and soft) or " + f"qb64b or qb64 or qb2.") if not isinstance(raw, (bytes, bytearray)): - raise TypeError(f"Not a bytes or bytearray, raw={raw}.") + raise TypeError(f"Not a bytes or bytearray {raw=}.") if code not in self.Sizes: - raise InvalidCodeError("Unsupported code={}.".format(code)) + raise InvalidCodeError(f"Unsupported {code=}.") if code[0] in SmallVrzDex or code[0] in LargeVrzDex: # dynamic size if rize: # use rsize to determine length of raw to extract @@ -865,37 +885,69 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, s = astuple(LargeVrzDex)[ls] code = f"{s}{'A' * (hs - 2)}{code[1]}" else: - raise InvalidVarRawSizeError(r"Unsupported raw size for " - f"code={code}.") + raise InvalidVarRawSizeError(f"Unsupported raw size for " + f"{code=}.") elif code[0] in LargeVrzDex: # compute code with sizes if size <= (64 ** 4 - 1): hs = 4 s = astuple(LargeVrzDex)[ls] code = f"{s}{code[1:hs]}" else: - raise InvalidVarRawSizeError(r"Unsupported raw size for " - f"code={code}.") + raise InvalidVarRawSizeError(f"Unsupported raw size for " + f"{code=}.") else: - raise InvalidVarRawSizeError(r"Unsupported variable raw size " - f"code={code}.") + raise InvalidVarRawSizeError(f"Unsupported variable raw size " + f"{code=}.") else: hs, ss, fs, ls = self.Sizes[code] # get sizes assumes ls consistent if not fs: # invalid - raise InvalidVarSizeError(r"Unsupported variable size " - f"code={code}.") + raise InvalidVarSizeError(f"Unsupported variable size {code=}.") rize = Matter._rawSize(code) + if ss == 0 and soft: + raise InvalidSoftError(f"Non-empty {soft=} part when not special.") + + if fs == hs + ss and ss > 0: # special soft size + if not soft or len(soft) != ss or ls != 0: # invalid code + raise InvalidSoftError(f"Invalid {soft=} or {code=}" + f" when special.") + if rize: # raw must be empty + raise InvalidSizeError(f"Nonzero raw size {rize=} when " + f" special {code=}.") + raw = raw[:rize] # copy only exact size from raw stream if len(raw) != rize: # forbids shorter raise RawMaterialError(f"Not enougth raw bytes for code={code}" - f"expected {rize} got {len(raw)}.") + f"expected {rize=} got {len(raw)}.") self._code = code # str hard part of code - self._soft = soft # srtsoft part of code, empty when ss=0 + self._soft = soft # str soft part of code, empty when ss=0 self._size = size # int of soft part value, None when fs != None self._raw = bytes(raw) # crypto ops require bytes not bytearray + elif soft and code: # special when raw None + hs, ss, fs, ls = self.Sizes[code] # get sizes assumes ls consistent + if not fs: # invalid can be variable size + raise InvalidVarSizeError(f"Missing raw for variable size {code=}.") + + if ss == 0: + raise InvalidCodeError("Nonempty {soft=} part for zero soft " + f"size {ss=} for {code=}.") + + if fs != hs + ss or len(soft) != ss or ls != 0: # not special soft code + raise InvalidSoftError("Invalid {soft=} or {code=} when special.") + + rize = Matter._rawSize(code) + if rize: + raise InvalidSizeError(f"Nonzero raw size {rize=} when special" + f" {code=}.") + + self._code = code # str hard part of code + self._soft = soft # str soft part of code, empty when ss=0 + self._size = size # int of soft part value, None when fs != None + self._raw = b'' # empty raw when special + elif qb64b is not None: self._exfil(qb64b) if strip: # assumes bytearray @@ -911,7 +963,10 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, else: raise EmptyMaterialError(f"Improper initialization need either " - f"(raw and code) or qb64b or qb64 or qb2.") + f"(raw not None and code) or " + f"(raw empty and code and soft) or " + f"(code and soft) or " + f"qb64b or qb64 or qb2.") @classmethod @@ -1000,7 +1055,7 @@ def both(self): Returns: both (str): hard + soft parts of full text code """ - _, ss, _, fs = self.Sizes[self.code] + _, ss, _, _ = self.Sizes[self.code] if self.size is not None: return (f"{self.code}{intToB64(self.size, l=ss)}") @@ -1008,6 +1063,21 @@ def both(self): return (f"{self.code}{self.soft}") + @property + def special(self): + """ + Returns: + special (bool): True when .soft is special i.e. when + .soft is not empty (ss > 0) and + .size is None (fs is not None) and + fs = hs + ss (raw is empty) + False otherwise + + """ + hs, ss, fs, _ = self.Sizes[self.code] + return (self.soft and self.size is None and fs == hs + ss) + + @property def fullSize(self): """ @@ -1096,10 +1166,14 @@ def _infil(self): self.code + converted self.raw to Base64 with pad chars stripped cs = hs + ss - fs = (size * 4) + cs + if fs is None in table: + fs = (size * 4) + cs + else: # fs set by table where + fs = hs + cs + ls + rawsize """ - code = self.code # hard size codex value + code = self.code # hard part of full code == codex value + soft = self.soft # soft part of full code may be empty size = self.size # size if variable length, None otherwise raw = self.raw # bytes or bytearray @@ -1107,6 +1181,10 @@ def _infil(self): hs, ss, fs, ls = self.Sizes[code] if not fs: # variable sized, compute code ss value from .size cs = hs + ss # both hard + soft size + + if ss < 1: # ss < 1 so not variable sized + raise InvalidCodeSizeError(f"Soft size {ss=} must be positive for " + f" variable length material.") if cs % 4: raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " f"variable length material. cs={cs}.") @@ -1124,14 +1202,14 @@ def _infil(self): return (both.encode("utf-8") + encodeB64(bytes([0] * ls) + raw)) else: # fixed size so prepad but lead ls may not be zero - both = code + both = code + soft cs = len(both) if (cs % 4) != ps - ls: # adjusted pad given lead bytes raise InvalidCodeSizeError(f"Invalid code={both} for converted" f" raw pad size={ps}.") # prepad, convert, and replace upfront # when fixed and ls != 0 then cs % 4 is zero and ps==ls - # otherwise fixed and ls == 0 then cs % 4 == ps + # otherwise when fixed and ls == 0 then cs % 4 == ps return (both.encode("utf-8") + encodeB64(bytes([0] * ps) + raw)[cs % 4:]) @@ -1142,6 +1220,7 @@ def _binfil(self): equivalent of Base64 decode of .qb64 into .qb2 """ code = self.code # codex value + soft = self.soft size = self.size # optional size if variable length raw = self.raw # bytes or bytearray @@ -1149,6 +1228,9 @@ def _binfil(self): cs = hs + ss if not fs: # compute both and fs from size + if ss < 1: # ss < 1 so not variable sized + raise InvalidCodeSizeError(f"Soft size {ss=} must be positive for " + f" variable length material.") if cs % 4: raise InvalidCodeSizeError("Whole code size not multiple of 4 for " "variable length material. cs={}.".format(cs)) @@ -1160,7 +1242,7 @@ def _binfil(self): both = f"{code}{intToB64(size, l=ss)}" fs = hs + ss + (size * 4) else: - both = code + both = code + soft if len(both) != cs: raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." @@ -1213,21 +1295,25 @@ def _exfil(self, qb64b): hs, ss, fs, ls = self.Sizes[hard] # assumes hs in both tables match cs = hs + ss # both hs and ss + # assumes that unit tests on Matter .Sizes .Hards and .Bards ensure that + # these are well formed. + # when fs is None then ss > 0 otherwise fs > hs + ss when ss > 0 + + soft = qb64b[hs:hs + ss] # extract soft chars, empty when ss==0 + if hasattr(soft, "decode"): + soft = soft.decode("utf-8") + size = None if not fs: # compute fs from size chars in ss part of code + if ss < 1: # ss < 1 so not variable sized + raise ValidationError(f"Soft size {ss=} must be positive for " + f" variable length material.") if cs % 4: - raise ValidationError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - size = qb64b[hs:hs + ss] # extract size chars - if hasattr(size, "decode"): - size = size.decode("utf-8") - size = b64ToInt(size) # compute int size - fs = (size * 4) + cs + raise ValidationError(f"Whole code size, {cs=}, not multiple" + f" of 4 for variable length material.") - # assumes that unit tests on Matter and MatterCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and ss == 0 and not fs % 4 and hs > 0 and fs >= hs + ss - # unless fs is None + size = b64ToInt(soft) # compute variable size int may have value 0 + fs = (size * 4) + cs if len(qb64b) < fs: # need more bytes raise ShortageError(f"Need {fs - len(qb64b)} more chars.") @@ -1246,7 +1332,7 @@ def _exfil(self, qb64b): if pi & (2 ** pbs - 1 ): # masked pad bits non-zero raise ValueError(f"Non zeroed prepad bits = " f"{pi & (2 ** pbs - 1 ):<06b} in {qb64b[cs:cs+1]}.") - raw = paw[ps:] # strip off ps prepad paw bytes + raw = paw[ps:] # strip off ps prepad paw bytes, paw is bytes so raw is bytes else: # not ps. IF not ps THEN may or may not be ls (lead) base = qb64b[cs:] # strip off code leaving lead chars if any and value @@ -1265,8 +1351,9 @@ def _exfil(self, qb64b): raise ConversionError(f"Improperly qualified material = {qb64b}") self._code = hard # hard only - self._size = size - self._raw = raw # ensure bytes so immutable and for crypto ops + self._soft = soft # soft only + self._size = size # variable size or None + self._raw = raw # ensure bytes for crypto ops, may be empty def _bexfil(self, qb2): @@ -1301,9 +1388,23 @@ def _bexfil(self, qb2): hs, ss, fs, ls = self.Sizes[hard] cs = hs + ss # both hs and ss + # assumes that unit tests on Matter .Sizes .Hards and .Bards ensure that + # these are well formed. + # when fs is None then ss > 0 otherwise fs > hs + ss when ss > 0 + bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets + if len(qb2) < bcs: # need more bytes + raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) + + both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code + soft = both[hs:hs + ss] # get soft may be empty + size = None if not fs: # compute fs from size chars in ss part of code + if ss < 1: # ss < 1 so not variable sized + raise ValidationError(f"Soft size {ss=} must be positive for " + f" variable length material.") + if cs % 4: raise ValidationError("Whole code size not multiple of 4 for " "variable length material. cs={}.".format(cs)) @@ -1311,14 +1412,8 @@ def _bexfil(self, qb2): if len(qb2) < bcs: # need more bytes raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) - both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code - size = b64ToInt(both[hs:hs + ss]) # get size - fs = (size * 4) + cs - - # assumes that unit tests on Matter and MatterCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and ss == 0 and not fs % 4 and hs > 0 and - # (fs >= hs + ss if fs is not None else True) + size = b64ToInt(soft) # get size from soft + fs = (size * 4) + cs # compute fs bfs = sceil(fs * 3 / 4) # bfs is min bytes to hold fs sextets if len(qb2) < bfs: # need more bytes @@ -1342,14 +1437,16 @@ def _bexfil(self, qb2): else: raise ValueError(f"Non zeroed lead bytes = 0x{li:02x}.") - raw = qb2[(bcs + ls):] # strip code and leader bytes from qb2 to get raw + # strip code and leader bytes from qb2 to get raw + raw = qb2[(bcs + ls):] # may be empty if len(raw) != (len(qb2) - bcs - ls): # exact lengths raise ConversionError(r"Improperly qualified material = {qb2}") - self._code = hard - self._size = size - self._raw = bytes(raw) # ensure bytes so immutable and crypto operations + self._code = hard # hard only + self._soft = soft # soft only may be empty + self._size = size # variable size or None + self._raw = bytes(raw) # ensure bytes for crypto ops may be empty class Seqner(Matter): @@ -1930,6 +2027,9 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, if self.code not in (MtrDex.Tag10, MtrDex.Tag7): raise InvalidCodeError(f"Invalid code={self.code} for Verser.") + if not self.special: + raise MaterialError(f"Invalid code={self.code} for Verser.") + @property def versage(self): """Property versage (Versage): named tuple of (proto, vrsn, gvrsn) @@ -1939,7 +2039,7 @@ def versage(self): clp = len(self.code) + 1 # code plus prepad proto = self.qb64[clp:clp+4] vrsn = self.b64ToVer(self.qb64[clp+4:clp+7]) - if self.fullSize == clp + 10: + if self.fullSize == clp + 10: # assumes special gvrsn = self.b64ToVer(self.qb64[clp+7:clp+10]) return Versage(proto=proto, vrsn=vrsn, gvrsn=gvrsn) diff --git a/src/keri/kering.py b/src/keri/kering.py index 9d653a86b..556b1cd4e 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -532,6 +532,13 @@ class InvalidCodeError(MaterialError): raise InvalidCodeError("error message") """ +class InvalidSoftError(MaterialError): + """ + Invalid, Unknown, or unrecognized soft part encountered during crypto material init + Usage: + raise InvalidSoftError("error message") + """ + class InvalidTypeError(MaterialError): """ Invalid material value type encountered during crypto material init diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 02c9c1ef1..8e3102e6a 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -187,7 +187,7 @@ def test_matter_class(): 'W': Sizage(hs=1, ss=0, fs=4, ls=0), 'X': Sizage(hs=1, ss=0, fs=4, ls=0), 'Y': Sizage(hs=1, ss=0, fs=8, ls=0), - 'Z': Sizage(hs=1, ss=0, fs=12, ls=0), + 'Z': Sizage(hs=1, ss=11, fs=12, ls=0), 'a': Sizage(hs=1, ss=0, fs=44, ls=0), '0A': Sizage(hs=2, ss=0, fs=24, ls=0), '0B': Sizage(hs=2, ss=0, fs=88, ls=0), @@ -260,13 +260,17 @@ def test_matter_class(): for ckey in Matter.Sizes.keys(): assert Matter.Hards[ckey[0]] == Matter.Sizes[ckey].hs - # verify all Codes have ss == 0 and not fs % 4 and hs > 0 and fs > hs - # if fs is not None else not (hs + ss) % 4 + # verify all Codes + # if fs None else not (hs + ss) % 4 for val in Matter.Sizes.values(): if val.fs is not None: - assert val.ss == 0 and not val.fs % 4 and val.hs > 0 and val.fs >= (val.hs + val.ss) + assert not val.fs % 4 and val.hs > 0 and val.fs >= (val.hs + val.ss) + if val.ss > 0: # special soft value + assert val.fs == val.hs + val.ss # raw must be empty + assert val.ls == 0 # no lead + else: - assert not (val.hs + val.ss) % 4 + assert val.ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 # Test .Bards @@ -3717,15 +3721,49 @@ def test_verser(): """ Test Verser version primitive subclass of Matter """ - #verser = Verser() # defaults - #assert verser.code == MtrDex.Tag10 - #assert verser.raw == b'' - #assert verser.qb64 == 'ZAKERICAACAA' - #assert verser.qb2 == b'd\x02\x84D\x80\x80\x00 \x00' - #assert verser.versage == Versage(proto='KERI', - #vrsn=Versionage(major=2, minor=0), - #gvrsn=Versionage(major=2, minor=0)) - + code = MtrDex.Tag10 + soft = 'AKERICAACAA' + qb64 = 'ZAKERICAACAA' + qb2 = b'd\x02\x84D\x80\x80\x00 \x00' + raw = b'' + + verser = Verser() # defaults + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == Versage(proto='KERI', + vrsn=Versionage(major=2, minor=0), + gvrsn=Versionage(major=2, minor=0)) + + code = verser.code + soft = verser.soft + qb2 = verser.qb2 + qb64 = verser.qb64 + + verser = Verser(qb2=qb2) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == Versage(proto='KERI', + vrsn=Versionage(major=2, minor=0), + gvrsn=Versionage(major=2, minor=0)) + + verser = Verser(qb64=qb64) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == Versage(proto='KERI', + vrsn=Versionage(major=2, minor=0), + gvrsn=Versionage(major=2, minor=0)) """ Done Test """ From fda9bbe5a5e452aea2122409a339712b50ded7af Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Mar 2024 10:03:58 -0600 Subject: [PATCH 076/418] updated tests of Sizes --- src/keri/core/coring.py | 28 +++++----------------------- tests/core/test_coring.py | 20 +++++++++++++------- 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 6dce3491d..aabb7d54a 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -682,29 +682,11 @@ class Matter: _exfil (types.MethodType): extracts .code and .raw from qb64b (fully qualified Base64) - When fs in table is None then ss must not be 0 as size must be defined - - When fs computed is not None then ss may be 0 - When fs > hs + ss then must have ss != 0, fs = hs + ss + size where size = - converted soft value from ss chars + ls, size is not None - - When ss == 0 raw must be empty and size must be None, soft must be empty - - - When fs = hs + ss then raw must be empty, size must be None, - and soft value may be special: - - When fs = hs + ss and ss == 0, raw is empty, soft is empty and soft not special - but fs == hs == hs + 0 is still valid code - When fs = hs + ss and ss > 0, raw is empty soft not empty and soft is spectial - - These includes the case where ss > 0 but if we compute size == 0 - from soft not empty. because when size == 0, raw must still be empty to - satisfy fs = hs + ss + 0 = hs + ss. - So no need to compute soft value when fs = hs + ss only ensure - raw is empty. - This allows treating soft value as special not a size. - and force size = None. + Size table rules for special soft values: + if fn in table is None then must have ss > 0 + else (fn is not None) then + if ss > 0 and fn = hs + ss then special soft value + else must have ss == 0 """ Codex = MtrDex diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 8e3102e6a..c602463bb 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -256,22 +256,28 @@ def test_matter_class(): assert Matter.Sizes['A'].fs == 44 # full size assert Matter.Sizes['A'].ls == 0 # lead size - # verify first hs Sizes matches hs in Codes for same first char - for ckey in Matter.Sizes.keys(): - assert Matter.Hards[ckey[0]] == Matter.Sizes[ckey].hs # verify all Codes # if fs None else not (hs + ss) % 4 for val in Matter.Sizes.values(): - if val.fs is not None: - assert not val.fs % 4 and val.hs > 0 and val.fs >= (val.hs + val.ss) + assert (isinstance(val.hs, int) and isinstance(val.ss, int) and + isinstance(val.ls, int)) + assert val.hs > 0 and val.ss >= 0 and val.ls >= 0 + if val.fs is not None: # fixed sized + assert isinstance(val.fs, int) and val.fs > 0 and not val.fs % 4 + assert val.fs >= (val.hs + val.ss) if val.ss > 0: # special soft value assert val.fs == val.hs + val.ss # raw must be empty assert val.ls == 0 # no lead - - else: + else: + assert val.ss == 0 + else: # variable sized assert val.ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 + # Test .Hards + # verify first hs Sizes matches hs in Codes for same first char + for ckey in Matter.Sizes.keys(): + assert Matter.Hards[ckey[0]] == Matter.Sizes[ckey].hs # Test .Bards # verify equivalents of items for Sizes and Bizes From d77239c99c75e740a72bfc690fe225725014b6ef Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Mar 2024 10:49:47 -0600 Subject: [PATCH 077/418] remove unneccessry checks --- src/keri/core/coring.py | 126 ++++++++++++++++++++++++-------------- tests/core/test_coring.py | 5 +- 2 files changed, 83 insertions(+), 48 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index aabb7d54a..679e1611e 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -647,6 +647,9 @@ class Matter: Bards (dict): hard size keyed by qb2 selector Sizes (dict): sizes tables for codes + Calss Methods: + + Attributes: Properties: @@ -660,7 +663,7 @@ class Matter: Converted value of the soft part (of len ss) of full derivation code. Otherwise None when not variably sized (fs != None) - rize (int): number of bytes of raw material not including lead bytes + fullSize (int): full size of primitive raw (bytes): crypto material only. Not derivation code or lead bytes. qb64 (str): Base64 fully qualified with derivation code + crypto mat qb64b (bytes): Base64 fully qualified with derivation code + crypto mat @@ -668,25 +671,32 @@ class Matter: transferable (bool): True means transferable derivation code False otherwise digestive (bool): True means digest derivation code False otherwise prefixive (bool): True means identifier prefix derivation code False otherwise + special (bool): True when soft is special raw is empty and fixed size Hidden: _code (str): value for .code property - _raw (bytes): value for .raw property - _rsize (bytes): value for .rsize property. Raw size in bytes when - variable sized material else None. + _soft (str): soft value of full code _size (int): value for .size property. Number of triplets of bytes including lead bytes (quadlets of chars) of variable sized material else None. - _infil (types.MethodType): creates qb64b from .raw and .code - (fully qualified Base64) - _exfil (types.MethodType): extracts .code and .raw from qb64b - (fully qualified Base64) + _raw (bytes): value for .raw property + _rawSize(): + _leadSize(): + _special(): + _infil(): creates qb64b from .raw and .code (fully qualified Base64) + _binfil(): creates qb2 from .raw and .code (fully qualified Base2) + _exfil(): extracts .code and .raw from qb64b (fully qualified Base64) + _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) + + Size table rules for special soft values: - if fn in table is None then must have ss > 0 + if fn in table is None then must have ss > 0 and (hs + ss) % 4 else (fn is not None) then - if ss > 0 and fn = hs + ss then special soft value - else must have ss == 0 + if ss > 0 and fn = hs + ss and ls == 0 then + special soft value + else + not special must have ss == 0 """ Codex = MtrDex @@ -857,15 +867,19 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, ls = (3 - (rize % 3)) % 3 # calc actual lead (pad) size # raw binary size including leader in bytes size = (rize + ls) // 3 # calculate value of size in triplets + if code[0] in SmallVrzDex: # compute code with sizes if size <= (64 ** 2 - 1): hs = 2 s = astuple(SmallVrzDex)[ls] code = f"{s}{code[1:hs]}" + ss = 2 elif size <= (64 ** 4 - 1): # make big version of code hs = 4 s = astuple(LargeVrzDex)[ls] code = f"{s}{'A' * (hs - 2)}{code[1]}" + soft = intToB64(size, 4) + ss = 4 else: raise InvalidVarRawSizeError(f"Unsupported raw size for " f"{code=}.") @@ -874,12 +888,14 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, hs = 4 s = astuple(LargeVrzDex)[ls] code = f"{s}{code[1:hs]}" + ss = 4 else: raise InvalidVarRawSizeError(f"Unsupported raw size for " f"{code=}.") else: raise InvalidVarRawSizeError(f"Unsupported variable raw size " f"{code=}.") + soft = intToB64(size, ss) else: hs, ss, fs, ls = self.Sizes[code] # get sizes assumes ls consistent @@ -888,15 +904,22 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, rize = Matter._rawSize(code) if ss == 0 and soft: - raise InvalidSoftError(f"Non-empty {soft=} part when not special.") + raise InvalidSoftError(f"Non-empty {soft=} part when not" + f" special.") if fs == hs + ss and ss > 0: # special soft size - if not soft or len(soft) != ss or ls != 0: # invalid code - raise InvalidSoftError(f"Invalid {soft=} or {code=}" - f" when special.") + if not soft or len(soft) < ss: + raise ShortageError(f"Not enough chars in {code=} " + f"{soft=} with {ss=}.") + + if ls != 0: # lead must be zero + raise InvalidSoftError(f"Nonzero lead(ls)) for {code=}" + f" {soft=} when special.") + if rize: # raw must be empty - raise InvalidSizeError(f"Nonzero raw size {rize=} when " - f" special {code=}.") + raise RawMaterialError(f"Nonzero raw size {rize=} when " + f" special {code=} {soft=}.") + soft = soft[:ss] raw = raw[:rize] # copy only exact size from raw stream if len(raw) != rize: # forbids shorter @@ -976,6 +999,21 @@ def _leadSize(cls, code): return ls + @classmethod + def _special(cls, code): + """ + Returns: + special (bool): True when code has special soft i.e. when + fs is not None and ss > 0 and fs = hs + ss and ls = 0 + i.e. (fs fixed and soft not empty and raw is empty and no lead) + False otherwise + + """ + hs, ss, fs, ls = cls.Sizes[code] + + return (fs is not None and ss > 0 and fs == hs + ss and ls == 0) + + @property def code(self): """ @@ -1045,21 +1083,6 @@ def both(self): return (f"{self.code}{self.soft}") - @property - def special(self): - """ - Returns: - special (bool): True when .soft is special i.e. when - .soft is not empty (ss > 0) and - .size is None (fs is not None) and - fs = hs + ss (raw is empty) - False otherwise - - """ - hs, ss, fs, _ = self.Sizes[self.code] - return (self.soft and self.size is None and fs == hs + ss) - - @property def fullSize(self): """ @@ -1142,6 +1165,17 @@ def prefixive(self): return (self.code in PreDex) + @property + def special(self): + """ + special (bool): True when self.code has special self.soft i.e. when + fs is not None and ss > 0 and fs = hs + ss and ls = 0 + i.e. (fs fixed and soft not empty and raw is empty and no lead) + False otherwise + """ + return self._special(self.code) + + def _infil(self): """ Returns bytes of fully qualified base64 characters @@ -1287,12 +1321,12 @@ def _exfil(self, qb64b): size = None if not fs: # compute fs from size chars in ss part of code - if ss < 1: # ss < 1 so not variable sized - raise ValidationError(f"Soft size {ss=} must be positive for " - f" variable length material.") - if cs % 4: - raise ValidationError(f"Whole code size, {cs=}, not multiple" - f" of 4 for variable length material.") + #if ss < 1: # ss < 1 so not variable sized + #raise ValidationError(f"Soft size {ss=} must be positive for " + #f" variable length material.") + #if cs % 4: + #raise ValidationError(f"Whole code size, {cs=}, not multiple" + #f" of 4 for variable length material.") size = b64ToInt(soft) # compute variable size int may have value 0 fs = (size * 4) + cs @@ -1383,13 +1417,13 @@ def _bexfil(self, qb2): size = None if not fs: # compute fs from size chars in ss part of code - if ss < 1: # ss < 1 so not variable sized - raise ValidationError(f"Soft size {ss=} must be positive for " - f" variable length material.") + #if ss < 1: # ss < 1 so not variable sized + #raise ValidationError(f"Soft size {ss=} must be positive for " + #f" variable length material.") - if cs % 4: - raise ValidationError("Whole code size not multiple of 4 for " - "variable length material. cs={}.".format(cs)) + #if cs % 4: + #raise ValidationError("Whole code size not multiple of 4 for " + #"variable length material. cs={}.".format(cs)) if len(qb2) < bcs: # need more bytes raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) @@ -2009,8 +2043,8 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, if self.code not in (MtrDex.Tag10, MtrDex.Tag7): raise InvalidCodeError(f"Invalid code={self.code} for Verser.") - if not self.special: - raise MaterialError(f"Invalid code={self.code} for Verser.") + if not self._special(self.code): + raise InvalidCodeError(f"Not special code={self.code} for Verser.") @property def versage(self): diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index c602463bb..dbc43b046 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -695,14 +695,15 @@ def test_matter(): assert Matter._leadSize(code) == 1 raw = b'abcde' # 5 bytes two triplets with lead 1 both = '5BAC' # full code both hard and soft parts two quadlets/triplets + soft = 'AC' qb64 = '5BACAGFiY2Rl' qb2 = b'\xe4\x10\x02\x00abcde' matter = Matter(raw=raw, code=code) assert matter.raw == raw assert matter.code == code == matter.hard - assert matter.soft == "" - assert matter.both == both assert matter.size == 2 # quadlets + assert matter.soft == soft + assert matter.both == both assert matter.fullSize == 12 # chars assert matter.qb64 == qb64 assert matter.qb2 == qb2 From 72767742a2061032fe01b7f0b18d6b6d20fed21e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Mar 2024 16:28:35 -0600 Subject: [PATCH 078/418] refactor Matter to support efficient encoding of B64 tags for various small Base64 values like field tags or types or Traits etc. Need more unit tests. Update Codexes --- src/keri/core/coring.py | 79 ++++++++++++--------------- tests/core/test_coring.py | 109 ++++++++++++++++++++------------------ 2 files changed, 90 insertions(+), 98 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 679e1611e..820058c92 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -280,12 +280,11 @@ class MatterCodex: Large: str = 'S' # Large 11 byte b2 number Great: str = 'T' # Great 14 byte b2 number Vast: str = 'U' # Vast 17 byte b2 number - Label1: str = 'V' # Label1 as one char (bytes) field map label lead size 1 - Label2: str = 'W' # Label2 as two char (bytes) field map label lead size 0 - Tag2: str = 'X' # Tag2 1 prepad + 2 B64 encoded chars for field tag - Tag6: str = 'Y' # Tag6 1 prepad + 6 B64 encoded chars for field tag - Tag10: str = 'Z' # Tag10 1 prepad + 10 B64 encoded chars for field tag - Blind: str = 'a' # Blinding factor 256 bits, Cryptographic strength deterministically generated from random salt + Label1: str = 'V' # Label1 as 1 bytes for label lead size 1 + Label2: str = 'W' # Label2 as 2 bytes for label lead size 0 + Tag3: str = 'X' # Tag3 3 B64 encoded chars for field tag + Tag7: str = 'Y' # Tag7 7 B64 encoded chars for field tag + Blind: str = 'Z' # Blinding factor 256 bits, Cryptographic strength deterministically generated from random salt Salt_128: str = '0A' # random salt/seed/nonce/private key or number of length 128 bits (Huge) Ed25519_Sig: str = '0B' # Ed25519 signature. ECDSA_256k1_Sig: str = '0C' # ECDSA secp256k1 signature. @@ -296,24 +295,26 @@ class MatterCodex: Long: str = '0H' # Long 4 byte b2 number ECDSA_256r1_Sig: str = '0I' # ECDSA secp256r1 signature. Tag1: str = '0J' # Tag1 1 prepad + 1 B64 encoded char for field tag - Tag5: str = '0K' # Tag5 1 prepad + 5 B64 encoded chars for field tag - Tag9: str = '0L' # Tag9 1 prepad + 9 B64 encoded chars for field tag + Tag2: str = '0K' # Tag2 2 B64 encoded chars for field tag + Tag5: str = '0L' # Tag5 1 prepad + 5 B64 encoded chars for field tag + Tag6: str = '0M' # Tag6 6 B64 encoded chars for field tag + Tag9: str = '0N' # Tag9 1 prepad + 9 B64 encoded chars for field tag + Tag10: str = '0O' # Tag10 10 B64 encoded chars for field tag ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. Ed448: str = '1AAD' # Ed448 public signing verification key. Basic derivation. Ed448_Sig: str = '1AAE' # Ed448 signature. Self-signing derivation. - Tag4: str = '1AAF' # Tag4 4 B64 encoded chars for field tag + Label3: str = '1AAF' # Label3 as 3 bytes for label lead size 0 DateTime: str = '1AAG' # Base64 custom encoded 32 char ISO-8601 DateTime X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt ECDSA_256r1N: str = '1AAI' # ECDSA secp256r1 verification key non-transferable, basic derivation. ECDSA_256r1: str = '1AAJ' # ECDSA secp256r1 verification or encryption key, basic derivation - Tag3: str = '1AAK' # Tag1 1 prepad + 3 B64 encoded chars for field tag - Tag7: str = '1AAL' # Tag7 1 prepad + 7 B64 encoded chars for field tag - Tag8: str = '1AAM' # Tag8 8 B64 encoded chars for field tag - Null: str = '1AAN' # Null None or empty value - No: str = '1AAO' # No Falsey Boolean value - Yes: str = '1AAP' # Yes Truthy Boolean value + Null: str = '1AAK' # Null None or empty value + No: str = '1AAL' # No Falsey Boolean value + Yes: str = '1AAM' # Yes Truthy Boolean value + Tag4: str = '1AAN' # Tag4 4 B64 encoded chars for field tag + Tag8: str = '1AAO' # Tag8 8 B64 encoded chars for field tag TBD1: str = '2AAA' # Testing purposes only fixed with lead size 1 TBD2: str = '3AAA' # Testing purposes only of fixed with lead size 2 StrB64_L0: str = '4A' # String Base64 only lead size 0 @@ -743,10 +744,9 @@ class Matter: 'U': Sizage(hs=1, ss=0, fs=24, ls=0), 'V': Sizage(hs=1, ss=0, fs=4, ls=1), 'W': Sizage(hs=1, ss=0, fs=4, ls=0), - 'X': Sizage(hs=1, ss=0, fs=4, ls=0), - 'Y': Sizage(hs=1, ss=0, fs=8, ls=0), - 'Z': Sizage(hs=1, ss=11, fs=12, ls=0), - 'a': Sizage(hs=1, ss=0, fs=44, ls=0), + 'X': Sizage(hs=1, ss=3, fs=4, ls=0), + 'Y': Sizage(hs=1, ss=7, fs=8, ls=0), + 'Z': Sizage(hs=1, ss=0, fs=44, ls=0), '0A': Sizage(hs=2, ss=0, fs=24, ls=0), '0B': Sizage(hs=2, ss=0, fs=88, ls=0), '0C': Sizage(hs=2, ss=0, fs=88, ls=0), @@ -756,9 +756,12 @@ class Matter: '0G': Sizage(hs=2, ss=0, fs=88, ls=0), '0H': Sizage(hs=2, ss=0, fs=8, ls=0), '0I': Sizage(hs=2, ss=0, fs=88, ls=0), - '0J': Sizage(hs=2, ss=0, fs=4, ls=0), - '0K': Sizage(hs=2, ss=0, fs=8, ls=0), - '0L': Sizage(hs=2, ss=0, fs=12, ls=0), + '0J': Sizage(hs=2, ss=2, fs=4, ls=0), + '0K': Sizage(hs=2, ss=2, fs=4, ls=0), + '0L': Sizage(hs=2, ss=6, fs=8, ls=0), + '0M': Sizage(hs=2, ss=6, fs=8, ls=0), + '0N': Sizage(hs=2, ss=10, fs=12, ls=0), + '0O': Sizage(hs=2, ss=10, fs=12, ls=0), '1AAA': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAB': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), @@ -769,12 +772,11 @@ class Matter: '1AAH': Sizage(hs=4, ss=0, fs=100, ls=0), '1AAI': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAJ': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAK': Sizage(hs=4, ss=0, fs=8, ls=0), - '1AAL': Sizage(hs=4, ss=0, fs=12, ls=0), - '1AAM': Sizage(hs=4, ss=0, fs=12, ls=0), - '1AAN': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAO': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAP': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAN': Sizage(hs=4, ss=4, fs=8, ls=0), + '1AAO': Sizage(hs=4, ss=8, fs=12, ls=0), '2AAA': Sizage(hs=4, ss=0, fs=8, ls=1), '3AAA': Sizage(hs=4, ss=0, fs=8, ls=2), '4A': Sizage(hs=2, ss=2, fs=None, ls=0), @@ -1321,13 +1323,6 @@ def _exfil(self, qb64b): size = None if not fs: # compute fs from size chars in ss part of code - #if ss < 1: # ss < 1 so not variable sized - #raise ValidationError(f"Soft size {ss=} must be positive for " - #f" variable length material.") - #if cs % 4: - #raise ValidationError(f"Whole code size, {cs=}, not multiple" - #f" of 4 for variable length material.") - size = b64ToInt(soft) # compute variable size int may have value 0 fs = (size * 4) + cs @@ -1417,13 +1412,6 @@ def _bexfil(self, qb2): size = None if not fs: # compute fs from size chars in ss part of code - #if ss < 1: # ss < 1 so not variable sized - #raise ValidationError(f"Soft size {ss=} must be positive for " - #f" variable length material.") - - #if cs % 4: - #raise ValidationError("Whole code size not multiple of 4 for " - #"variable length material. cs={}.".format(cs)) if len(qb2) < bcs: # need more bytes raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) @@ -2026,12 +2014,13 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, if not gvrsn: raise InvalidValueError(f"Missing genus version.") - qb64 = (code + 'A' + proto + + qb64 = (code + + proto + self.verToB64(vrsn) + self.verToB64(gvrsn)).encode("utf-8") elif code == MtrDex.Tag7: - qb64 = (code + 'A' + proto + self.verToB64(vrsn)) + qb64 = (code + proto + self.verToB64(vrsn)).encode("utf-8") else: raise InvalidCodeError(f"Invalid {code=} for Verser.") @@ -2052,7 +2041,7 @@ def versage(self): """ gvrsn = None - clp = len(self.code) + 1 # code plus prepad + clp = len(self.code) proto = self.qb64[clp:clp+4] vrsn = self.b64ToVer(self.qb64[clp+4:clp+7]) if self.fullSize == clp + 10: # assumes special diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index dbc43b046..96e84ad0c 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -3,7 +3,7 @@ tests.core.test_coring module """ -import dataclasses +from dataclasses import asdict import hashlib import json from base64 import urlsafe_b64decode as decodeB64 @@ -53,7 +53,7 @@ def test_matter_class(): Test Matter class attributes """ - assert dataclasses.asdict(MtrDex) == \ + assert asdict(MtrDex) == \ { 'Ed25519_Seed': 'A', 'Ed25519N': 'B', @@ -78,10 +78,9 @@ def test_matter_class(): 'Vast': 'U', 'Label1': 'V', 'Label2': 'W', - 'Tag2': 'X', - 'Tag6': 'Y', - 'Tag10': 'Z', - 'Blind': 'a', + 'Tag3': 'X', + 'Tag7': 'Y', + 'Blind': 'Z', 'Salt_128': '0A', 'Ed25519_Sig': '0B', 'ECDSA_256k1_Sig': '0C', @@ -92,24 +91,26 @@ def test_matter_class(): 'Long': '0H', 'ECDSA_256r1_Sig': '0I', 'Tag1': '0J', - 'Tag5': '0K', - 'Tag9': '0L', + 'Tag2': '0K', + 'Tag5': '0L', + 'Tag6': '0M', + 'Tag9': '0N', + 'Tag10': '0O', 'ECDSA_256k1N': '1AAA', 'ECDSA_256k1': '1AAB', 'Ed448N': '1AAC', 'Ed448': '1AAD', 'Ed448_Sig': '1AAE', - 'Tag4': '1AAF', + 'Label3': '1AAF', 'DateTime': '1AAG', 'X25519_Cipher_Salt': '1AAH', 'ECDSA_256r1N': '1AAI', 'ECDSA_256r1': '1AAJ', - 'Tag3': '1AAK', - 'Tag7': '1AAL', - 'Tag8': '1AAM', - 'Null': '1AAN', - 'No': '1AAO', - 'Yes': '1AAP', + 'Null': '1AAK', + 'No': '1AAL', + 'Yes': '1AAM', + 'Tag4': '1AAN', + 'Tag8': '1AAO', 'TBD1': '2AAA', 'TBD2': '3AAA', 'StrB64_L0': '4A', @@ -185,10 +186,9 @@ def test_matter_class(): 'U': Sizage(hs=1, ss=0, fs=24, ls=0), 'V': Sizage(hs=1, ss=0, fs=4, ls=1), 'W': Sizage(hs=1, ss=0, fs=4, ls=0), - 'X': Sizage(hs=1, ss=0, fs=4, ls=0), - 'Y': Sizage(hs=1, ss=0, fs=8, ls=0), - 'Z': Sizage(hs=1, ss=11, fs=12, ls=0), - 'a': Sizage(hs=1, ss=0, fs=44, ls=0), + 'X': Sizage(hs=1, ss=3, fs=4, ls=0), + 'Y': Sizage(hs=1, ss=7, fs=8, ls=0), + 'Z': Sizage(hs=1, ss=0, fs=44, ls=0), '0A': Sizage(hs=2, ss=0, fs=24, ls=0), '0B': Sizage(hs=2, ss=0, fs=88, ls=0), '0C': Sizage(hs=2, ss=0, fs=88, ls=0), @@ -198,9 +198,12 @@ def test_matter_class(): '0G': Sizage(hs=2, ss=0, fs=88, ls=0), '0H': Sizage(hs=2, ss=0, fs=8, ls=0), '0I': Sizage(hs=2, ss=0, fs=88, ls=0), - '0J': Sizage(hs=2, ss=0, fs=4, ls=0), - '0K': Sizage(hs=2, ss=0, fs=8, ls=0), - '0L': Sizage(hs=2, ss=0, fs=12, ls=0), + '0J': Sizage(hs=2, ss=2, fs=4, ls=0), + '0K': Sizage(hs=2, ss=2, fs=4, ls=0), + '0L': Sizage(hs=2, ss=6, fs=8, ls=0), + '0M': Sizage(hs=2, ss=6, fs=8, ls=0), + '0N': Sizage(hs=2, ss=10, fs=12, ls=0), + '0O': Sizage(hs=2, ss=10, fs=12, ls=0), '1AAA': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAB': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), @@ -211,12 +214,11 @@ def test_matter_class(): '1AAH': Sizage(hs=4, ss=0, fs=100, ls=0), '1AAI': Sizage(hs=4, ss=0, fs=48, ls=0), '1AAJ': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAK': Sizage(hs=4, ss=0, fs=8, ls=0), - '1AAL': Sizage(hs=4, ss=0, fs=12, ls=0), - '1AAM': Sizage(hs=4, ss=0, fs=12, ls=0), - '1AAN': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAO': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAP': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), + '1AAN': Sizage(hs=4, ss=4, fs=8, ls=0), + '1AAO': Sizage(hs=4, ss=8, fs=12, ls=0), '2AAA': Sizage(hs=4, ss=0, fs=8, ls=1), '3AAA': Sizage(hs=4, ss=0, fs=8, ls=2), '4A': Sizage(hs=2, ss=2, fs=None, ls=0), @@ -251,6 +253,7 @@ def test_matter_class(): '9AAE': Sizage(hs=4, ss=4, fs=None, ls=2) } + assert Matter.Sizes['A'].hs == 1 # hard size assert Matter.Sizes['A'].ss == 0 # soft size assert Matter.Sizes['A'].fs == 44 # full size @@ -1332,20 +1335,20 @@ def test_matter(): assert matter.digestive == False assert matter.prefixive == False - # test Tag4 + # test Label3 #val = int("F89CFF", 16) #assert val == 16293119 #raw = val.to_bytes(3, 'big') #assert raw == b'\xf8\x9c\xff' raw = b'hio' - cs = len(MtrDex.Tag4) + cs = len(MtrDex.Label3) assert cs == 4 ps = cs % 4 assert ps == 0 txt = encodeB64(bytes([0]*ps) + raw) #assert txt == b'-Jz_' assert txt == b'aGlv' - qb64b = MtrDex.Tag4.encode("utf-8") + txt[ps:] + qb64b = MtrDex.Label3.encode("utf-8") + txt[ps:] #assert qb64b == b'1AAF-Jz_' assert qb64b == b'1AAFaGlv' qb64 = qb64b.decode("utf-8") @@ -1356,9 +1359,9 @@ def test_matter(): assert qb2[bs:] == raw # stable value in qb2 assert encodeB64(qb2) == qb64b - matter = Matter(raw=raw, code=MtrDex.Tag4) + matter = Matter(raw=raw, code=MtrDex.Label3) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == MtrDex.Label3 assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 @@ -1370,7 +1373,7 @@ def test_matter(): matter = Matter(qb64b=qb64b) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == MtrDex.Label3 assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 @@ -1382,7 +1385,7 @@ def test_matter(): matter = Matter(qb64=qb64) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == MtrDex.Label3 assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 @@ -1394,7 +1397,7 @@ def test_matter(): matter = Matter(qb2=qb2) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == MtrDex.Label3 assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 @@ -1404,18 +1407,18 @@ def test_matter(): assert matter.digestive == False assert matter.prefixive == False - # test Tag4 as chars + # test Label3 as chars txt = b'icp_' raw = decodeB64(txt) assert raw == b'\x89\xca\x7f' val = int.from_bytes(raw, 'big') assert val == 9030271 - cs = len(MtrDex.Tag4) + cs = len(MtrDex.Label3) assert cs == 4 ps = cs % 4 assert ps == 0 txt = encodeB64(bytes([0]*ps) + raw) - qb64b = MtrDex.Tag4.encode("utf-8") + txt + qb64b = MtrDex.Label3.encode("utf-8") + txt assert qb64b == b'1AAFicp_' qb64 = qb64b.decode("utf-8") qb2 = decodeB64(qb64b) @@ -1424,9 +1427,9 @@ def test_matter(): assert qb2[bs:] == raw # stable value in qb2 assert encodeB64(qb2) == qb64b - matter = Matter(raw=raw, code=MtrDex.Tag4) + matter = Matter(raw=raw, code=MtrDex.Label3) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == MtrDex.Label3 assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 @@ -1436,7 +1439,7 @@ def test_matter(): matter = Matter(qb64b=qb64b) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == MtrDex.Label3 assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 @@ -1448,7 +1451,7 @@ def test_matter(): matter = Matter(qb64=qb64) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == MtrDex.Label3 assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 @@ -1458,7 +1461,7 @@ def test_matter(): matter = Matter(qb2=qb2) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == MtrDex.Label3 assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 @@ -1477,7 +1480,7 @@ def test_indexer(): """ assert Indexer.Codex == IdrDex - assert dataclasses.asdict(IdrDex) == { + assert asdict(IdrDex) == { 'Ed25519_Sig': 'A', 'Ed25519_Crt_Sig': 'B', 'ECDSA_256k1_Sig': 'C', @@ -1519,7 +1522,7 @@ def test_indexer(): assert IdrDex.TBD1 == '1z' assert IdrDex.TBD4 == '4z' - assert dataclasses.asdict(IdxSigDex) == { + assert asdict(IdxSigDex) == { 'Ed25519_Sig': 'A', 'Ed25519_Crt_Sig': 'B', 'ECDSA_256k1_Sig': 'C', @@ -1556,7 +1559,7 @@ def test_indexer(): assert IdxSigDex.Ed448_Big_Crt_Sig == '3B' - assert dataclasses.asdict(IdxCrtSigDex) == { + assert asdict(IdxCrtSigDex) == { 'Ed25519_Crt_Sig': 'B', 'ECDSA_256k1_Crt_Sig': 'D', 'ECDSA_256r1_Crt_Sig': 'F', @@ -1577,7 +1580,7 @@ def test_indexer(): assert IdxCrtSigDex.Ed448_Big_Crt_Sig == '3B' - assert dataclasses.asdict(IdxBthSigDex) == { + assert asdict(IdxBthSigDex) == { 'Ed25519_Sig': 'A', 'ECDSA_256k1_Sig': 'C', 'ECDSA_256r1_Sig': 'E', @@ -2162,7 +2165,7 @@ def test_counter(): """ Test Counter class """ - assert dataclasses.asdict(CtrDex) == { + assert asdict(CtrDex) == { 'ControllerIdxSigs': '-A', 'WitnessIdxSigs': '-B', 'NonTransReceiptCouples': '-C', @@ -2697,7 +2700,7 @@ def test_number(): Test Number subclass of Matter """ - assert dataclasses.asdict(NumDex) == { + assert asdict(NumDex) == { 'Short': 'M', 'Long': '0H', 'Tall': 'R', @@ -3729,9 +3732,9 @@ def test_verser(): Test Verser version primitive subclass of Matter """ code = MtrDex.Tag10 - soft = 'AKERICAACAA' - qb64 = 'ZAKERICAACAA' - qb2 = b'd\x02\x84D\x80\x80\x00 \x00' + soft = 'KERICAACAA' + qb64 = '0OKERICAACAA' + qb2 = b'\xd0\xe2\x84D\x80\x80\x00 \x00' raw = b'' verser = Verser() # defaults From 9575a7e249289c3f89172012429938091a7003a8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Mar 2024 17:00:50 -0600 Subject: [PATCH 079/418] fix lint errors. fix checks for soft as Base64 etc --- src/keri/core/coring.py | 44 +++++++++++++++++++++++++------------- src/keri/core/serdering.py | 9 ++++---- src/keri/kering.py | 17 ++++++++++++++- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 820058c92..44f962c1d 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -27,7 +27,7 @@ from ..kering import MaxON -from ..kering import (EmptyMaterialError, RawMaterialError, +from ..kering import (EmptyMaterialError, RawMaterialError, SoftMaterialError, InvalidCodeError, InvalidSoftError, InvalidSizeError, InvalidCodeSizeError, InvalidVarIndexError, @@ -901,8 +901,10 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, else: hs, ss, fs, ls = self.Sizes[code] # get sizes assumes ls consistent - if not fs: # invalid - raise InvalidVarSizeError(f"Unsupported variable size {code=}.") + if not fs: # invalid must not be variable size + raise InvalidVarSizeError(f"Unsupported {code=} for " + f"variable size {fs=}.") + rize = Matter._rawSize(code) if ss == 0 and soft: @@ -910,10 +912,6 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, f" special.") if fs == hs + ss and ss > 0: # special soft size - if not soft or len(soft) < ss: - raise ShortageError(f"Not enough chars in {code=} " - f"{soft=} with {ss=}.") - if ls != 0: # lead must be zero raise InvalidSoftError(f"Nonzero lead(ls)) for {code=}" f" {soft=} when special.") @@ -921,8 +919,17 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, if rize: # raw must be empty raise RawMaterialError(f"Nonzero raw size {rize=} when " f" special {code=} {soft=}.") + + if not soft or len(soft) < ss: + raise SoftMaterialError(f"Not enough chars in {code=} " + f"{soft=} with {ss=}.") + soft = soft[:ss] + if not Reb64.match(soft): + raise InvalidSoftError(f"Non Base64 chars in {soft=}.") + + raw = raw[:rize] # copy only exact size from raw stream if len(raw) != rize: # forbids shorter raise RawMaterialError(f"Not enougth raw bytes for code={code}" @@ -935,21 +942,28 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, elif soft and code: # special when raw None hs, ss, fs, ls = self.Sizes[code] # get sizes assumes ls consistent - if not fs: # invalid can be variable size - raise InvalidVarSizeError(f"Missing raw for variable size {code=}.") - - if ss == 0: - raise InvalidCodeError("Nonempty {soft=} part for zero soft " - f"size {ss=} for {code=}.") + if not fs: # + raise InvalidSoftError(f"Unsupported {code=} {fs=} for special" + f" soft.") - if fs != hs + ss or len(soft) != ss or ls != 0: # not special soft code - raise InvalidSoftError("Invalid {soft=} or {code=} when special.") + if fs != hs + ss or ss == 0 or ls != 0: # not special soft code + raise InvalidSoftError("Invalid {code=} {fs=} or lead={ls} " + f" when special soft.") rize = Matter._rawSize(code) if rize: raise InvalidSizeError(f"Nonzero raw size {rize=} when special" f" {code=}.") + if not soft or len(soft) < ss: + raise SoftMaterialError(f"Not enough chars in {code=} " + f"{soft=} with {ss=}.") + + soft = soft[:ss] + + if not Reb64.match(soft): + raise InvalidSoftError(f"Non Base64 chars in {soft=}.") + self._code = code # str hard part of code self._soft = soft # str soft part of code, empty when ss=0 self._size = size # int of soft part value, None when fs != None diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 0a83a36cf..50a7bfc2f 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1288,7 +1288,7 @@ def _dumps(self, sad): pass val = bytearray(Counter(tag=AllTags.GenericMapGroup, count=len(frame) % 4, - version=cversion).qb64b) + version=self.gvrsn).qb64b) else: for e in v: # list pass @@ -1296,13 +1296,14 @@ def _dumps(self, sad): val = bytearray(Counter(tag=AllTags.GenericListGroup, count=len(frame) % 4, - version=cversion).qb64b) + version=self.gvrsn).qb64b) val.extend(frame) case _: # if extra fields this is where logic would be raise SerializeError(f"Unsupported protocol field label" - f"='{l}' for {protocol=} {version=}.") + f"='{l}' for protocol={self.proto}" + f" version={self.vrsn}.") raw.extend(val) @@ -1316,7 +1317,7 @@ def _dumps(self, sad): else: - raise SerializeError(f"Unsupported protocol={self.protocol}.") + raise SerializeError(f"Unsupported protocol={self.proto}.") # prepend count code for message diff --git a/src/keri/kering.py b/src/keri/kering.py index 556b1cd4e..0ab43c76a 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -222,6 +222,7 @@ def snatch(match, size=0): regular expressions work with memoryview objects not just bytes or bytearrays """ + full = match.group() # full matched version string if len(full) == VFFULLSPAN: proto, major, minor, gmajor, gminor = match.group("proto0", "major0", @@ -506,7 +507,15 @@ class RawMaterialError(MaterialError): """ Not Enough bytes in buffer bytearray for raw material Usage: - raise ShortageError("error message") + raise RawMaterialError("error message") + """ + + +class SoftMaterialError(MaterialError): + """ + Not Enough chars in soft for soft material + Usage: + raise SoftMaterialError("error message") """ @@ -517,6 +526,7 @@ class EmptyMaterialError(MaterialError): raise EmptyMaterialError("error message") """ + class InvalidVersionError(MaterialError): """ Invalid, Unknown, or unrecognized CESR code table version encountered during @@ -525,6 +535,7 @@ class InvalidVersionError(MaterialError): raise InvalidVersionError("error message") """ + class InvalidCodeError(MaterialError): """ Invalid, Unknown, or unrecognized code encountered during crypto material init @@ -532,6 +543,7 @@ class InvalidCodeError(MaterialError): raise InvalidCodeError("error message") """ + class InvalidSoftError(MaterialError): """ Invalid, Unknown, or unrecognized soft part encountered during crypto material init @@ -539,6 +551,7 @@ class InvalidSoftError(MaterialError): raise InvalidSoftError("error message") """ + class InvalidTypeError(MaterialError): """ Invalid material value type encountered during crypto material init @@ -546,6 +559,7 @@ class InvalidTypeError(MaterialError): raise InvalidTypeError("error message") """ + class InvalidValueError(MaterialError): """ Invalid material value encountered during crypto material init @@ -553,6 +567,7 @@ class InvalidValueError(MaterialError): raise InvalidValueError("error message") """ + class InvalidSizeError(MaterialError): """ Invalid size encountered during crypto material init From 8ae368317f8daf1b5c0b6dbf1bee564c4649573b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Mar 2024 18:04:34 -0600 Subject: [PATCH 080/418] fixed comments in Codex --- src/keri/core/coring.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 44f962c1d..7b6c8f0f3 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -280,10 +280,10 @@ class MatterCodex: Large: str = 'S' # Large 11 byte b2 number Great: str = 'T' # Great 14 byte b2 number Vast: str = 'U' # Vast 17 byte b2 number - Label1: str = 'V' # Label1 as 1 bytes for label lead size 1 - Label2: str = 'W' # Label2 as 2 bytes for label lead size 0 - Tag3: str = 'X' # Tag3 3 B64 encoded chars for field tag - Tag7: str = 'Y' # Tag7 7 B64 encoded chars for field tag + Label1: str = 'V' # Label1 1 bytes for label lead size 1 + Label2: str = 'W' # Label2 2 bytes for label lead size 0 + Tag3: str = 'X' # Tag3 3 B64 encoded chars for special values + Tag7: str = 'Y' # Tag7 7 B64 encoded chars for special values Blind: str = 'Z' # Blinding factor 256 bits, Cryptographic strength deterministically generated from random salt Salt_128: str = '0A' # random salt/seed/nonce/private key or number of length 128 bits (Huge) Ed25519_Sig: str = '0B' # Ed25519 signature. @@ -294,12 +294,12 @@ class MatterCodex: SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. Long: str = '0H' # Long 4 byte b2 number ECDSA_256r1_Sig: str = '0I' # ECDSA secp256r1 signature. - Tag1: str = '0J' # Tag1 1 prepad + 1 B64 encoded char for field tag - Tag2: str = '0K' # Tag2 2 B64 encoded chars for field tag - Tag5: str = '0L' # Tag5 1 prepad + 5 B64 encoded chars for field tag - Tag6: str = '0M' # Tag6 6 B64 encoded chars for field tag - Tag9: str = '0N' # Tag9 1 prepad + 9 B64 encoded chars for field tag - Tag10: str = '0O' # Tag10 10 B64 encoded chars for field tag + Tag1: str = '0J' # Tag1 1 B64 encoded char + 1 prepad for special values + Tag2: str = '0K' # Tag2 2 B64 encoded chars for for special values + Tag5: str = '0L' # Tag5 5 B64 encoded chars + 1 prepad for special values + Tag6: str = '0M' # Tag6 6 B64 encoded chars for special values + Tag9: str = '0N' # Tag9 9 B64 encoded chars + 1 prepad for special values + Tag10: str = '0O' # Tag10 10 B64 encoded chars for special values ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. @@ -313,8 +313,8 @@ class MatterCodex: Null: str = '1AAK' # Null None or empty value No: str = '1AAL' # No Falsey Boolean value Yes: str = '1AAM' # Yes Truthy Boolean value - Tag4: str = '1AAN' # Tag4 4 B64 encoded chars for field tag - Tag8: str = '1AAO' # Tag8 8 B64 encoded chars for field tag + Tag4: str = '1AAN' # Tag4 4 B64 encoded chars for special values + Tag8: str = '1AAO' # Tag8 8 B64 encoded chars for special values TBD1: str = '2AAA' # Testing purposes only fixed with lead size 1 TBD2: str = '3AAA' # Testing purposes only of fixed with lead size 2 StrB64_L0: str = '4A' # String Base64 only lead size 0 From 0808304af1fb5771fe47fdf399e3062d39d7a98b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Mar 2024 18:49:41 -0600 Subject: [PATCH 081/418] started unit test dev for Matter with special soft --- src/keri/core/coring.py | 4 +-- tests/core/test_coring.py | 57 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 7b6c8f0f3..797b25163 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -926,7 +926,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, soft = soft[:ss] - if not Reb64.match(soft): + if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") @@ -961,7 +961,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, soft = soft[:ss] - if not Reb64.match(soft): + if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") self._code = code # str hard part of code diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 96e84ad0c..d8b52195e 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -1471,6 +1471,62 @@ def test_matter(): assert matter.digestive == False assert matter.prefixive == False + """ Done Test """ + +def test_matter_special(): + """ + Test Matter instances using code with special soft values + """ + # test Tag3 + + code = MtrDex.Tag3 + soft = 'icp' + qb64 = 'Xicp' + qb2 = b"^')" + raw = b'' + + matter = Matter(code=code, soft=soft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + + code = matter.code + soft = matter.soft + qb2 = matter.qb2 + qb64 = matter.qb64 + + matter = Matter(qb2=qb2) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + + matter = Matter(qb64=qb64) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + + # Test corner conditions + # Empty raw + matter = Matter(raw=b'', code=code, soft=soft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + + + + """ Done Test """ @@ -6534,6 +6590,7 @@ def test_tholder(): if __name__ == "__main__": test_matter_class() test_matter() + test_matter_special() test_verser() #test_texter() #test_counter() From 6473d7efe811254763986555dc1d9873b9d8524d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 29 Mar 2024 08:32:09 -0600 Subject: [PATCH 082/418] Refactor so Matter.soft is determinative and .size is derived --- src/keri/core/coring.py | 35 +++++++++++++---------------------- tests/core/test_coring.py | 11 ++++++++--- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 797b25163..91ea41e1d 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -871,12 +871,12 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, size = (rize + ls) // 3 # calculate value of size in triplets if code[0] in SmallVrzDex: # compute code with sizes - if size <= (64 ** 2 - 1): + if size <= (64 ** 2 - 1): # ss = 2 hs = 2 s = astuple(SmallVrzDex)[ls] code = f"{s}{code[1:hs]}" ss = 2 - elif size <= (64 ** 4 - 1): # make big version of code + elif size <= (64 ** 4 - 1): # ss = 4 make big version of code hs = 4 s = astuple(LargeVrzDex)[ls] code = f"{s}{'A' * (hs - 2)}{code[1]}" @@ -886,7 +886,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, raise InvalidVarRawSizeError(f"Unsupported raw size for " f"{code=}.") elif code[0] in LargeVrzDex: # compute code with sizes - if size <= (64 ** 4 - 1): + if size <= (64 ** 4 - 1): # ss = 4 hs = 4 s = astuple(LargeVrzDex)[ls] code = f"{s}{code[1:hs]}" @@ -1082,7 +1082,8 @@ def size(self): sized primitive material (fs = None). """ - return self._size + #return self._size + return (b64ToInt(self.soft) if self.soft else None) @property @@ -1093,10 +1094,12 @@ def both(self): """ _, ss, _, _ = self.Sizes[self.code] - if self.size is not None: - return (f"{self.code}{intToB64(self.size, l=ss)}") - else: - return (f"{self.code}{self.soft}") + #if self.size is not None: + #return (f"{self.code}{intToB64(self.size, l=ss)}") + #else: + #return (f"{self.code}{self.soft}") + + return (f"{self.code}{self.soft}") @property @@ -1211,16 +1214,10 @@ def _infil(self): ps = ((3 - (len(raw) % 3)) % 3) # pad size chars or lead size bytes hs, ss, fs, ls = self.Sizes[code] + # assumes unit tests on Matter.Sizes ensure valid size entries if not fs: # variable sized, compute code ss value from .size cs = hs + ss # both hard + soft size - if ss < 1: # ss < 1 so not variable sized - raise InvalidCodeSizeError(f"Soft size {ss=} must be positive for " - f" variable length material.") - if cs % 4: - raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if size < 0 or size > (64 ** ss - 1): raise InvalidVarSizeError("Invalid size={} for code={}." "".format(size, code)) @@ -1258,14 +1255,8 @@ def _binfil(self): hs, ss, fs, ls = self.Sizes[code] cs = hs + ss - + # assumes unit tests on Matter.Sizes ensure valid size entries if not fs: # compute both and fs from size - if ss < 1: # ss < 1 so not variable sized - raise InvalidCodeSizeError(f"Soft size {ss=} must be positive for " - f" variable length material.") - if cs % 4: - raise InvalidCodeSizeError("Whole code size not multiple of 4 for " - "variable length material. cs={}.".format(cs)) if size < 0 or size > (64 ** ss - 1): raise InvalidVarSizeError("Invalid size={} for code={}." diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index d8b52195e..7011892b4 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -3,7 +3,7 @@ tests.core.test_coring module """ -from dataclasses import asdict +from dataclasses import asdict, astuple import hashlib import json from base64 import urlsafe_b64decode as decodeB64 @@ -261,8 +261,7 @@ def test_matter_class(): # verify all Codes - # if fs None else not (hs + ss) % 4 - for val in Matter.Sizes.values(): + for code, val in Matter.Sizes.items(): # hard code assert (isinstance(val.hs, int) and isinstance(val.ss, int) and isinstance(val.ls, int)) assert val.hs > 0 and val.ss >= 0 and val.ls >= 0 @@ -276,6 +275,12 @@ def test_matter_class(): assert val.ss == 0 else: # variable sized assert val.ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 + if code[0] in coring.SmallVrzDex: # small variable sized code + assert val.hs == 2 and val.ss == 2 and val.fs == None + assert code[0] == astuple(coring.SmallVrzDex)[val.ls] + elif code[0] in coring.LargeVrzDex: # large veriable sized code + assert val.hs == 4 and val.ss == 4 and val.fs == None + assert code[0] == astuple(coring.LargeVrzDex)[val.ls] # Test .Hards # verify first hs Sizes matches hs in Codes for same first char From 338507db9a15cb6bb99d939d75b7855db54239c5 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 29 Mar 2024 15:44:41 -0600 Subject: [PATCH 083/418] added support for special soft when fixed size and non-empty raw in code table. Need to update Matter init still. Added more thorough unit tests of valid Matter.Sizes table entries. --- src/keri/core/coring.py | 141 +++++++++++++++++--------------------- src/keri/help/helping.py | 2 +- tests/core/test_coring.py | 109 +++++++++++++++++++---------- 3 files changed, 136 insertions(+), 116 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 91ea41e1d..244333f89 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -315,8 +315,12 @@ class MatterCodex: Yes: str = '1AAM' # Yes Truthy Boolean value Tag4: str = '1AAN' # Tag4 4 B64 encoded chars for special values Tag8: str = '1AAO' # Tag8 8 B64 encoded chars for special values - TBD1: str = '2AAA' # Testing purposes only fixed with lead size 1 - TBD2: str = '3AAA' # Testing purposes only of fixed with lead size 2 + TBD0S: str = '1__-' # Testing purposes only, fixed special values with non-empty raw lead size 0 + TBD0: str = '1___' # Testing purposes only, fixed with lead size 0 + TBD1S: str = '2__-' # Testing purposes only, fixed special values with non-empty raw lead size 1 + TBD1: str = '2___' # Testing purposes only, fixed with lead size 1 + TBD2S: str = '3__-' # Testing purposes only, fixed special values with non-empty raw lead size 1 + TBD2: str = '3___' # Testing purposes only, fixed with lead size 2 StrB64_L0: str = '4A' # String Base64 only lead size 0 StrB64_L1: str = '5A' # String Base64 only lead size 1 StrB64_L2: str = '6A' # String Base64 only lead size 2 @@ -677,9 +681,6 @@ class Matter: Hidden: _code (str): value for .code property _soft (str): soft value of full code - _size (int): value for .size property. Number of triplets of bytes - including lead bytes (quadlets of chars) of variable sized material - else None. _raw (bytes): value for .raw property _rawSize(): _leadSize(): @@ -690,7 +691,6 @@ class Matter: _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) - Size table rules for special soft values: if fn in table is None then must have ss > 0 and (hs + ss) % 4 else (fn is not None) then @@ -776,9 +776,13 @@ class Matter: '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAN': Sizage(hs=4, ss=4, fs=8, ls=0), - '1AAO': Sizage(hs=4, ss=8, fs=12, ls=0), - '2AAA': Sizage(hs=4, ss=0, fs=8, ls=1), - '3AAA': Sizage(hs=4, ss=0, fs=8, ls=2), + '1AAO': Sizage(hs=4, ss=2, fs=12, ls=0), + '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), + '1___': Sizage(hs=4, ss=0, fs=8, ls=0), + '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), + '2___': Sizage(hs=4, ss=0, fs=8, ls=1), + '3__-': Sizage(hs=4, ss=2, fs=12, ls=2), + '3___': Sizage(hs=4, ss=0, fs=8, ls=2), '4A': Sizage(hs=2, ss=2, fs=None, ls=0), '5A': Sizage(hs=2, ss=2, fs=None, ls=1), '6A': Sizage(hs=2, ss=2, fs=None, ls=2), @@ -937,7 +941,6 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, self._code = code # str hard part of code self._soft = soft # str soft part of code, empty when ss=0 - self._size = size # int of soft part value, None when fs != None self._raw = bytes(raw) # crypto ops require bytes not bytearray elif soft and code: # special when raw None @@ -966,7 +969,6 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, self._code = code # str hard part of code self._soft = soft # str soft part of code, empty when ss=0 - self._size = size # int of soft part value, None when fs != None self._raw = b'' # empty raw when special elif qb64b is not None: @@ -999,9 +1001,10 @@ def _rawSize(cls, code): """ hs, ss, fs, ls = cls.Sizes[code] # get sizes cs = hs + ss # both hard + soft code size + # assumes .Sizes only has valid entries if fs is None: raise InvalidCodeSizeError(f"Non-fixed raw size code {code}.") - return (((fs - cs) * 3 // 4) - ls) + return (((fs - cs) * 3 // 4) - ls) # assumes cs % 4 != 3 and fs % 4 == 0 @classmethod @@ -1071,18 +1074,14 @@ def size(self): Returns: size(int | None): Number of variably sized b64 quadlets/b2 triplets in primitive when varibly sized - None when not variably sized (fs==None) - - Getter for ._size. Makes ._size read only + None when not variably sized when (fs!=None) Number of quadlets/triplets of chars/bytes of variable sized material or None when not variably sized. - Converted qb64 value to int of ss portion of full text code when variably - sized primitive material (fs = None). - + Converted qb64 value to int of soft ss portion of full text code + when variably sized primitive material (fs == None). """ - #return self._size return (b64ToInt(self.soft) if self.soft else None) @@ -1092,7 +1091,7 @@ def both(self): Returns: both (str): hard + soft parts of full text code """ - _, ss, _, _ = self.Sizes[self.code] + #_, ss, _, _ = self.Sizes[self.code] #if self.size is not None: #return (f"{self.code}{intToB64(self.size, l=ss)}") @@ -1208,37 +1207,35 @@ def _infil(self): """ code = self.code # hard part of full code == codex value - soft = self.soft # soft part of full code may be empty - size = self.size # size if variable length, None otherwise - raw = self.raw # bytes or bytearray + both = self.both # code + soft, soft may be empty + raw = self.raw # bytes or bytearray, raw may be empty - ps = ((3 - (len(raw) % 3)) % 3) # pad size chars or lead size bytes hs, ss, fs, ls = self.Sizes[code] + cs = hs + ss # assumes unit tests on Matter.Sizes ensure valid size entries - if not fs: # variable sized, compute code ss value from .size - cs = hs + ss # both hard + soft size - - if size < 0 or size > (64 ** ss - 1): - raise InvalidVarSizeError("Invalid size={} for code={}." - "".format(size, code)) - # both is hard code + size converted to ss B64 chars - both = f"{code}{intToB64(size, l=ss)}" - - if len(both) % 4 != ps - ls: # adjusted pad given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted" - f" raw pad size={ps}.") - # prepad, convert, and prepend + + if cs != len(both): + InvalidCodeSizeError(f"Invalid full code={both} for sizes {hs=} and" + f" {ss=}.") + + ps = ((3 - (len(raw) % 3)) % 3) # pad size chars or lead size bytes + if (cs % 4) != ps - ls: # adjusted pad provided by full code given lead bytes + raise InvalidCodeSizeError(f"Invalid code={both} for converted " + f"raw pad size={ps} and lead size={ls}.") + + if not fs: # variable sized + # prepad, convert, and prepend. When ls and ps then ls accounts for + # which ensures encodeB64 does not have trailing B64 pad chars return (both.encode("utf-8") + encodeB64(bytes([0] * ls) + raw)) else: # fixed size so prepad but lead ls may not be zero - both = code + soft - cs = len(both) - if (cs % 4) != ps - ls: # adjusted pad given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted" - f" raw pad size={ps}.") # prepad, convert, and replace upfront - # when fixed and ls != 0 then cs % 4 is zero and ps==ls - # otherwise when fixed and ls == 0 then cs % 4 == ps + # When fixed fs, raw may have ps != 0 when ls = 0 which hs+ss must + # provide so encodeB64 may have trailing B64 pad chars so need to + # strip these off. When fixed and ls != 0 then ps == ls + # When fixed and ls != 0 then cs % 4 is zero and ps==ls so using + # ps instead of ls for prepad works. Otherwise when fixed and + # ls == 0 then cs % 4 == ps so strip compensates for prepad. return (both.encode("utf-8") + encodeB64(bytes([0] * ps) + raw)[cs % 4:]) @@ -1248,47 +1245,36 @@ def _binfil(self): self.code converted to Base2 + self.raw left shifted with pad bits equivalent of Base64 decode of .qb64 into .qb2 """ - code = self.code # codex value - soft = self.soft - size = self.size # optional size if variable length - raw = self.raw # bytes or bytearray + code = self.code # hard part of full code == codex value + both = self.both # code + soft, soft may be empty + raw = self.raw # bytes or bytearray may be empty hs, ss, fs, ls = self.Sizes[code] cs = hs + ss # assumes unit tests on Matter.Sizes ensure valid size entries - if not fs: # compute both and fs from size - - if size < 0 or size > (64 ** ss - 1): - raise InvalidVarSizeError("Invalid size={} for code={}." - "".format(size, code)) - # both is hard code + converted index - both = f"{code}{intToB64(size, l=ss)}" - fs = hs + ss + (size * 4) - else: - both = code + soft - - if len(both) != cs: - raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." - .format(cs, len(code))) - n = sceil(cs * 3 / 4) # number of b2 bytes to hold b64 code # convert code both to right align b2 int then left shift in pad bits # then convert to bytes bcode = (b64ToInt(both) << (2 * (cs % 4))).to_bytes(n, 'big') - full = bcode + bytes([0] * ls) + raw + full = bcode + bytes([0] * ls) + raw # includes lead bytes + bfs = len(full) + if not fs: # compute fs + fs = hs + ss + (len(raw) + ls) * 4 // 3 # hs + ss + (size * 4) if bfs % 3 or (bfs * 4 // 3) != fs: # invalid size - raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") - + raise InvalidCodeSizeError(f"Invalid full code={both} for raw size" + f"={len(raw)}.") return full def _exfil(self, qb64b): """ - Extracts self.code and self.raw from qualified base64 bytes qb64b + Extracts self.code and self.raw from qualified base64 str or bytes qb64b + Detects is str and converts to bytes + + Parameters: + qb64b (str | bytes | bytearray): fully qualified base64 from stream - cs = hs + ss - fs = (size * 4) + cs """ if not qb64b: # empty need more bytes raise ShortageError("Empty material.") @@ -1326,10 +1312,9 @@ def _exfil(self, qb64b): if hasattr(soft, "decode"): soft = soft.decode("utf-8") - size = None - if not fs: # compute fs from size chars in ss part of code - size = b64ToInt(soft) # compute variable size int may have value 0 - fs = (size * 4) + cs + if not fs: # compute fs from soft from ss part which provides size B64 + # compute variable size as int may have value 0 + fs = (b64ToInt(soft) * 4) + cs if len(qb64b) < fs: # need more bytes raise ShortageError(f"Need {fs - len(qb64b)} more chars.") @@ -1339,7 +1324,7 @@ def _exfil(self, qb64b): qb64b = qb64b.encode("utf-8") # check for non-zeroed pad bits or lead bytes - ps = cs % 4 # code pad size ps = cs mod 4 + ps = cs % 4 # code pad size ps = cs mod 4 assumes cs mod 4 != 3 pbs = 2 * (ps if ps else ls) # pad bit size in bits if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls base = ps * b'A' + qb64b[cs:] # replace pre code with prepad chars of zero @@ -1368,7 +1353,6 @@ def _exfil(self, qb64b): self._code = hard # hard only self._soft = soft # soft only - self._size = size # variable size or None self._raw = raw # ensure bytes for crypto ops, may be empty @@ -1415,14 +1399,12 @@ def _bexfil(self, qb2): both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code soft = both[hs:hs + ss] # get soft may be empty - size = None if not fs: # compute fs from size chars in ss part of code - if len(qb2) < bcs: # need more bytes raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) - size = b64ToInt(soft) # get size from soft - fs = (size * 4) + cs # compute fs + # compute size as int from soft part given by ss B64 chars + fs = (b64ToInt(soft) * 4) + cs # compute fs bfs = sceil(fs * 3 / 4) # bfs is min bytes to hold fs sextets if len(qb2) < bfs: # need more bytes @@ -1454,7 +1436,6 @@ def _bexfil(self, qb2): self._code = hard # hard only self._soft = soft # soft only may be empty - self._size = size # variable size or None self._raw = bytes(raw) # ensure bytes for crypto ops may be empty diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index 75a709ee5..401d23db2 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -36,7 +36,7 @@ def sceil(r): """ Symmetric ceiling function Returns: - sc (int): value that is symmetric ceiling of r away from zero + sceil (int): value that is symmetric ceiling of r away from zero Because int() provides a symmetric floor towards zero, just inc int(r) by: 1 when r - int(r) > 0 (r positive) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 7011892b4..1c8d9275c 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -23,6 +23,12 @@ from cryptography.hazmat.primitives import hashes from cryptography import exceptions +from keri.kering import (EmptyMaterialError, RawMaterialError, DerivationError, + ShortageError, InvalidCodeSizeError, InvalidVarIndexError, + InvalidValueError, DeserializeError, ValidationError, + InvalidVarRawSizeError) +from keri.kering import Version, Versionage, VersionError + from keri.core import coring from keri.core import eventing from keri.core.coring import (Ilkage, Ilks, Saids, Protocols, Protocolage, @@ -37,14 +43,10 @@ Diger, Prefixer, Cipher, Encrypter, Decrypter) from keri.core.coring import versify, deversify, Rever, MAXVERFULLSPAN from keri.core.coring import generateSigners, generatePrivates -from keri.help.helping import (intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, - B64_CHARS, Reb64, nabSextets) + from keri.help import helping -from keri.kering import (EmptyMaterialError, RawMaterialError, DerivationError, - ShortageError, InvalidCodeSizeError, InvalidVarIndexError, - InvalidValueError, DeserializeError, ValidationError, - InvalidVarRawSizeError) -from keri.kering import Version, Versionage, VersionError +from keri.help.helping import (sceil, intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, + B64_CHARS, Reb64, nabSextets) @@ -111,8 +113,12 @@ def test_matter_class(): 'Yes': '1AAM', 'Tag4': '1AAN', 'Tag8': '1AAO', - 'TBD1': '2AAA', - 'TBD2': '3AAA', + 'TBD0S': '1__-', + 'TBD0': '1___', + 'TBD1S': '2__-', + 'TBD1': '2___', + 'TBD2S': '3__-', + 'TBD2': '3___', 'StrB64_L0': '4A', 'StrB64_L1': '5A', 'StrB64_L2': '6A', @@ -218,9 +224,13 @@ def test_matter_class(): '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAN': Sizage(hs=4, ss=4, fs=8, ls=0), - '1AAO': Sizage(hs=4, ss=8, fs=12, ls=0), - '2AAA': Sizage(hs=4, ss=0, fs=8, ls=1), - '3AAA': Sizage(hs=4, ss=0, fs=8, ls=2), + '1AAO': Sizage(hs=4, ss=2, fs=12, ls=0), + '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), + '1___': Sizage(hs=4, ss=0, fs=8, ls=0), + '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), + '2___': Sizage(hs=4, ss=0, fs=8, ls=1), + '3__-': Sizage(hs=4, ss=2, fs=12, ls=2), + '3___': Sizage(hs=4, ss=0, fs=8, ls=2), '4A': Sizage(hs=2, ss=2, fs=None, ls=0), '5A': Sizage(hs=2, ss=2, fs=None, ls=1), '6A': Sizage(hs=2, ss=2, fs=None, ls=2), @@ -262,24 +272,51 @@ def test_matter_class(): # verify all Codes for code, val in Matter.Sizes.items(): # hard code - assert (isinstance(val.hs, int) and isinstance(val.ss, int) and - isinstance(val.ls, int)) - assert val.hs > 0 and val.ss >= 0 and val.ls >= 0 - if val.fs is not None: # fixed sized - assert isinstance(val.fs, int) and val.fs > 0 and not val.fs % 4 - assert val.fs >= (val.hs + val.ss) - if val.ss > 0: # special soft value - assert val.fs == val.hs + val.ss # raw must be empty - assert val.ls == 0 # no lead - else: - assert val.ss == 0 - else: # variable sized - assert val.ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 + hs = val.hs + ss = val.ss + fs = val.fs + ls = val.ls + cs = hs + ss + + assert (isinstance(hs, int) and isinstance(ss, int) and + isinstance(ls, int)) + assert hs > 0 and ss >= 0 and ls in (0, 1, 2) + assert len(code) == hs + + if fs is None: # variable sized + assert ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 + + else: # fixed size + assert isinstance(fs, int) and fs > 0 and not fs % 4 + assert fs >= cs + assert cs % 4 != 3 # prevent ambiguous conversion + if ss > 0 and fs == cs: # special soft value with raw empty + assert ls == 0 # no lead + rs = (fs - cs) * 3 // 4 - ls # raw size bytes sans lead + ps = (3 - (rs + ls) % 3) % 3 # pad size given raw + lead + assert ps == (cs % 4) # cs % 4 is pad size given cs % 4 != 3 + assert sceil((rs + ls) * 4 / 3) + cs == fs # sextets add up + + + if code[0] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz': + assert len(code) == 1 + elif code[0] in '0': + assert len(code) == 2 + elif code[0] in '1': + assert len(code) == 4 and ls == 0 + elif code[0] in '2': + assert len(code) == 4 and ls == 1 + elif code[0] in '3': + assert len(code) == 4 and ls == 2 + else: + assert code[0] not in '-_' # count or op code + if code[0] in coring.SmallVrzDex: # small variable sized code - assert val.hs == 2 and val.ss == 2 and val.fs == None + assert val.hs == 2 and val.ss == 2 and val.fs is None assert code[0] == astuple(coring.SmallVrzDex)[val.ls] + elif code[0] in coring.LargeVrzDex: # large veriable sized code - assert val.hs == 4 and val.ss == 4 and val.fs == None + assert val.hs == 4 and val.ss == 4 and val.fs is None assert code[0] == astuple(coring.LargeVrzDex)[val.ls] # Test .Hards @@ -295,6 +332,8 @@ def test_matter_class(): assert Matter._rawSize(MtrDex.Ed25519) == 32 assert Matter._leadSize(MtrDex.Ed25519) == 0 + assert not Matter._special(MtrDex.Ed25519) + assert Matter._special(MtrDex.Tag3) @@ -546,14 +585,14 @@ def test_matter(): assert ims == extra # stripped not include extra # test fix sized with leader 1 - # TBD1 = '2AAA' # Testing purposes only fixed with lead size 1 + # TBD1 = '2___' # Testing purposes only fixed with lead size 1 code = MtrDex.TBD1 # '2AAA' assert Matter._rawSize(code) == 2 assert Matter._leadSize(code) == 1 raw = b'ab' - qb64 = '2AAAAGFi' # '2AAA' + encodeB64(b'\x00ab').decode("utf-8") - qb2 = decodeB64(qb64) # b'\xd8\x00\x00\x00ab' + qb64 = '2___AGFi' # '2AAA' + encodeB64(b'\x00ab').decode("utf-8") + qb2 = decodeB64(qb64) matter = Matter(raw=raw, code=code) assert matter.raw == raw assert matter.code == code @@ -616,7 +655,7 @@ def test_matter(): assert matter.prefixive == False # test with bad pad or lead - badqb64 = '2AAA_2Fi' # '2AAA' + encodeB64(b'\xffab').decode("utf-8") + badqb64 = '2____2Fi' # '2___' + encodeB64(b'\xffab').decode("utf-8") badqb2 = decodeB64(badqb64) # b'\xd8\x00\x00\xffab' with pytest.raises(ValueError) as ex: @@ -629,13 +668,13 @@ def test_matter(): # test fix sized with leader 2 - # TBD2 = '3AAA' # Testing purposes only of fixed with lead size 2 + # TBD2 = '3___' # Testing purposes only of fixed with lead size 2 code = MtrDex.TBD2 # '3AAA' assert Matter._rawSize(code) == 1 assert Matter._leadSize(code) == 2 raw = b'z' - qb64 = '3AAAAAB6' - qb2 = b'\xdc\x00\x00\x00\x00z' + qb64 = '3___AAB6' + qb2 = decodeB64(qb64) matter = Matter(raw=raw, code=code) assert matter.raw == raw assert matter.code == code @@ -685,7 +724,7 @@ def test_matter(): assert matter.prefixive == False # test with bad pad or lead - badqb64 = '3AAA__96' # '3AAA' + encodeB64(b'\xff\xffz').decode("utf-8") + badqb64 = '3_____96' # '3AAA' + encodeB64(b'\xff\xffz').decode("utf-8") badqb2 = decodeB64(badqb64) #b'\xdc\x00\x00\xff\xffz' with pytest.raises(ValueError) as ex: From 4158da4b7b0f189dd614443dfea078cfb7561ea7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 29 Mar 2024 16:46:48 -0600 Subject: [PATCH 084/418] clean up Matter code given new functionality, refinement and optimization get rid of unneccessary checks assuming unit text validates Sizes table thouroughly run time --- src/keri/core/coring.py | 82 +++++++++++++++++++-------------------- tests/core/test_coring.py | 19 +++++---- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 244333f89..2e006b26c 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -846,13 +846,10 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, .raw and .code and .size and .rsize """ - size = None # variable raw binary size including leader in quadlets - soft = soft # soft portion of code, if raw is not None: # raw provided but may be empty if not code: raise EmptyMaterialError(f"Improper initialization need either " f"(raw not None and code) or " - f"(raw empty and code and soft) or " f"(code and soft) or " f"qb64b or qb64 or qb2.") @@ -862,7 +859,12 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, if code not in self.Sizes: raise InvalidCodeError(f"Unsupported {code=}.") - if code[0] in SmallVrzDex or code[0] in LargeVrzDex: # dynamic size + hs, ss, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes + + if fs is None: # variable sized assumes code[0] in SmallVrzDex or LargeVrzDex + #if not (code[0] in SmallVrzDex or code[0] in LargeVrzDex): + #raise InvalidCodeError(f"Non-variable size {code=}.") + if rize: # use rsize to determine length of raw to extract if rize < 0: raise InvalidVarRawSizeError(f"Missing var raw size for " @@ -903,32 +905,27 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, f"{code=}.") soft = intToB64(size, ss) - else: - hs, ss, fs, ls = self.Sizes[code] # get sizes assumes ls consistent - if not fs: # invalid must not be variable size - raise InvalidVarSizeError(f"Unsupported {code=} for " - f"variable size {fs=}.") - + else: # fixed size but raw may be empty and/or special soft rize = Matter._rawSize(code) - if ss == 0 and soft: - raise InvalidSoftError(f"Non-empty {soft=} part when not" - f" special.") - - if fs == hs + ss and ss > 0: # special soft size - if ls != 0: # lead must be zero - raise InvalidSoftError(f"Nonzero lead(ls)) for {code=}" - f" {soft=} when special.") + #if ss == 0 and soft: + #raise InvalidSoftError(f"Non-empty {soft=} part when not" + #f" special.") - if rize: # raw must be empty - raise RawMaterialError(f"Nonzero raw size {rize=} when " - f" special {code=} {soft=}.") + if ss > 0: # special soft size, so soft must be provided + #if fs == hs + ss: + #if ls != 0: # lead must be zero + #raise InvalidSoftError(f"Nonzero lead(ls)) for {code=}" + #f" {soft=} when special.") - if not soft or len(soft) < ss: - raise SoftMaterialError(f"Not enough chars in {code=} " - f"{soft=} with {ss=}.") + #if rize: # raw must be empty + #raise RawMaterialError(f"Nonzero raw size {rize=} when " + #f" special {code=} {soft=}.") soft = soft[:ss] + if len(soft) != ss: + raise SoftMaterialError(f"Not enough chars in {soft=} " + f"with {ss=} for {code=}.") if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") @@ -943,33 +940,37 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, self._soft = soft # str soft part of code, empty when ss=0 self._raw = bytes(raw) # crypto ops require bytes not bytearray - elif soft and code: # special when raw None - hs, ss, fs, ls = self.Sizes[code] # get sizes assumes ls consistent - if not fs: # - raise InvalidSoftError(f"Unsupported {code=} {fs=} for special" - f" soft.") + elif soft and code: # fixed size and special when raw None + hs, ss, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes + if not fs: # variable sized code so can't be soft + raise InvalidSoftError(f"Unsupported variable sized {code=} " + f" with {fs=} for special {soft=}.") - if fs != hs + ss or ss == 0 or ls != 0: # not special soft code - raise InvalidSoftError("Invalid {code=} {fs=} or lead={ls} " - f" when special soft.") + if not ss > 0 or not ls == 0 or not fs == hs + ss: # not special soft + raise InvalidSoftError("Invalid soft size={ss} or lead={ls} " + f" or {code=} {fs=} when special soft.") - rize = Matter._rawSize(code) - if rize: - raise InvalidSizeError(f"Nonzero raw size {rize=} when special" - f" {code=}.") + #rize = Matter._rawSize(code) + #if rize: + #raise InvalidSizeError(f"Nonzero raw size {rize=} when special" + #f" soft {code=}.") - if not soft or len(soft) < ss: - raise SoftMaterialError(f"Not enough chars in {code=} " - f"{soft=} with {ss=}.") + #if not soft or len(soft) < ss: + #raise SoftMaterialError(f"Not enough chars in {soft=} with " + #f"{ss=} for {code=}.") soft = soft[:ss] + if len(soft) != ss: + raise SoftMaterialError(f"Not enough chars in {soft=} " + f"with {ss=} for {code=}.") + if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") self._code = code # str hard part of code self._soft = soft # str soft part of code, empty when ss=0 - self._raw = b'' # empty raw when special + self._raw = b'' # make raw empty when None and when special soft elif qb64b is not None: self._exfil(qb64b) @@ -987,7 +988,6 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, else: raise EmptyMaterialError(f"Improper initialization need either " f"(raw not None and code) or " - f"(raw empty and code and soft) or " f"(code and soft) or " f"qb64b or qb64 or qb2.") diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 1c8d9275c..e94bf0e71 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -285,13 +285,25 @@ def test_matter_class(): if fs is None: # variable sized assert ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 + assert code[0] in coring.SmallVrzDex or code[0] in coring.LargeVrzDex + + if code[0] in coring.SmallVrzDex: # small variable sized code + assert val.hs == 2 and val.ss == 2 and val.fs is None + assert code[0] == astuple(coring.SmallVrzDex)[val.ls] + + elif code[0] in coring.LargeVrzDex: # large veriable sized code + assert val.hs == 4 and val.ss == 4 and val.fs is None + assert code[0] == astuple(coring.LargeVrzDex)[val.ls] else: # fixed size + assert not (code[0] in coring.SmallVrzDex or code[0] in coring.LargeVrzDex) assert isinstance(fs, int) and fs > 0 and not fs % 4 assert fs >= cs assert cs % 4 != 3 # prevent ambiguous conversion if ss > 0 and fs == cs: # special soft value with raw empty assert ls == 0 # no lead + assert Matter._rawSize(code) == 0 + rs = (fs - cs) * 3 // 4 - ls # raw size bytes sans lead ps = (3 - (rs + ls) % 3) % 3 # pad size given raw + lead assert ps == (cs % 4) # cs % 4 is pad size given cs % 4 != 3 @@ -311,13 +323,6 @@ def test_matter_class(): else: assert code[0] not in '-_' # count or op code - if code[0] in coring.SmallVrzDex: # small variable sized code - assert val.hs == 2 and val.ss == 2 and val.fs is None - assert code[0] == astuple(coring.SmallVrzDex)[val.ls] - - elif code[0] in coring.LargeVrzDex: # large veriable sized code - assert val.hs == 4 and val.ss == 4 and val.fs is None - assert code[0] == astuple(coring.LargeVrzDex)[val.ls] # Test .Hards # verify first hs Sizes matches hs in Codes for same first char From 5f6120c58a729bd83f2b0ebd92da0dc679ea2c1a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 29 Mar 2024 16:52:52 -0600 Subject: [PATCH 085/418] remove commented out code --- src/keri/core/coring.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 2e006b26c..bd65d3a6d 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -862,9 +862,6 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, hs, ss, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes if fs is None: # variable sized assumes code[0] in SmallVrzDex or LargeVrzDex - #if not (code[0] in SmallVrzDex or code[0] in LargeVrzDex): - #raise InvalidCodeError(f"Non-variable size {code=}.") - if rize: # use rsize to determine length of raw to extract if rize < 0: raise InvalidVarRawSizeError(f"Missing var raw size for " @@ -906,22 +903,9 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, soft = intToB64(size, ss) else: # fixed size but raw may be empty and/or special soft - rize = Matter._rawSize(code) - - #if ss == 0 and soft: - #raise InvalidSoftError(f"Non-empty {soft=} part when not" - #f" special.") + rize = Matter._rawSize(code) # get raw size from Sizes for code if ss > 0: # special soft size, so soft must be provided - #if fs == hs + ss: - #if ls != 0: # lead must be zero - #raise InvalidSoftError(f"Nonzero lead(ls)) for {code=}" - #f" {soft=} when special.") - - #if rize: # raw must be empty - #raise RawMaterialError(f"Nonzero raw size {rize=} when " - #f" special {code=} {soft=}.") - soft = soft[:ss] if len(soft) != ss: raise SoftMaterialError(f"Not enough chars in {soft=} " @@ -950,21 +934,11 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, raise InvalidSoftError("Invalid soft size={ss} or lead={ls} " f" or {code=} {fs=} when special soft.") - #rize = Matter._rawSize(code) - #if rize: - #raise InvalidSizeError(f"Nonzero raw size {rize=} when special" - #f" soft {code=}.") - - #if not soft or len(soft) < ss: - #raise SoftMaterialError(f"Not enough chars in {soft=} with " - #f"{ss=} for {code=}.") - soft = soft[:ss] if len(soft) != ss: raise SoftMaterialError(f"Not enough chars in {soft=} " f"with {ss=} for {code=}.") - if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") From 28085dabdc299e182c0577ebaa58e3a6f6f2c723 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 30 Mar 2024 11:24:04 -0600 Subject: [PATCH 086/418] Fixed refined logic to fix bugs in corner cases. Updated comments added tests. --- src/keri/core/coring.py | 76 ++++++++++++++++++++++++--------------- tests/core/test_coring.py | 63 +++++++++++++++++++++----------- 2 files changed, 89 insertions(+), 50 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index bd65d3a6d..b35acf32f 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -975,10 +975,10 @@ def _rawSize(cls, code): """ hs, ss, fs, ls = cls.Sizes[code] # get sizes cs = hs + ss # both hard + soft code size - # assumes .Sizes only has valid entries if fs is None: raise InvalidCodeSizeError(f"Non-fixed raw size code {code}.") - return (((fs - cs) * 3 // 4) - ls) # assumes cs % 4 != 3 and fs % 4 == 0 + # assumes .Sizes only has valid entries, cs % 4 != 3, and fs % 4 == 0 + return (((fs - cs) * 3 // 4) - ls) @classmethod @@ -1170,20 +1170,13 @@ def special(self): def _infil(self): """ - Returns bytes of fully qualified base64 characters - self.code + converted self.raw to Base64 with pad chars stripped - - cs = hs + ss - if fs is None in table: - fs = (size * 4) + cs - else: # fs set by table where - fs = hs + cs + ls + rawsize - + Returns: + primitive (bytes): fully qualified base64 characters. """ code = self.code # hard part of full code == codex value both = self.both # code + soft, soft may be empty raw = self.raw # bytes or bytearray, raw may be empty - + rs = len(raw) # raw size hs, ss, fs, ls = self.Sizes[code] cs = hs + ss # assumes unit tests on Matter.Sizes ensure valid size entries @@ -1192,25 +1185,50 @@ def _infil(self): InvalidCodeSizeError(f"Invalid full code={both} for sizes {hs=} and" f" {ss=}.") - ps = ((3 - (len(raw) % 3)) % 3) # pad size chars or lead size bytes - if (cs % 4) != ps - ls: # adjusted pad provided by full code given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted " - f"raw pad size={ps} and lead size={ls}.") + if not fs: # variable sized - # prepad, convert, and prepend. When ls and ps then ls accounts for - # which ensures encodeB64 does not have trailing B64 pad chars - return (both.encode("utf-8") + encodeB64(bytes([0] * ls) + raw)) - - else: # fixed size so prepad but lead ls may not be zero - # prepad, convert, and replace upfront - # When fixed fs, raw may have ps != 0 when ls = 0 which hs+ss must - # provide so encodeB64 may have trailing B64 pad chars so need to - # strip these off. When fixed and ls != 0 then ps == ls - # When fixed and ls != 0 then cs % 4 is zero and ps==ls so using - # ps instead of ls for prepad works. Otherwise when fixed and - # ls == 0 then cs % 4 == ps so strip compensates for prepad. - return (both.encode("utf-8") + encodeB64(bytes([0] * ps) + raw)[cs % 4:]) + # Tests on .Sizes table must ensure ls in (0,1,2) and cs % 4 == 0 but + # can't know the variable size. So instance methods must ensure that + # (ls + rs) % 3 == 0 i.e. both full code (B64) and lead+raw (B2) + # are both 24 bit aligned. + # If so then should not need following check. + if (ls + rs) % 3 or cs % 4: + raise InvalidCodeSizeError(f"Invalid full code{both=} with " + f"variable raw size={rs} given " + f" {cs=}, {hs=}, {ss=}, {fs=}, and " + f"{ls=}.") + + # When ls+rs is 24 bit aligned then encodeB64 has no trailing + # pad chars that need to be stripped. So simply prepad raw with + # ls zero bytes and convert (encodeB64). + full = (both.encode("utf-8") + encodeB64(bytes([0] * ls) + raw)) + + else: # fixed size + ps = (3 - ((rs + ls) % 3)) % 3 # net pad size given raw with lead + # net pad size must equal both code size remainder so that primitive + # both + converted padded raw is fs long. Assumes ls in (0,1,2) and + # cs % 4 != 3, fs % 4 == 0. Sizes table test must ensure these properties. + # If so then should not need following check. + if ps != (cs % 4): # given cs % 4 != 3 then cs % 4 is pad size + raise InvalidCodeSizeError(f"Invalid full code{both=} with " + f"fixed raw size={rs} given " + f" {cs=}, {hs=}, {ss=}, {fs=}, and " + f"{ls=}.") + + # Predpad raw so we midpad the full primitive. Prepad with ps+ls + # zero bytes ensures encodeB64 of prepad+lead+raw has no trailing + # pad characters. Finally skip first ps == cs % 4 of the converted + # characters to ensure that when full code is prepended, the full + # primitive size is fs but midpad bits are zeros. + full = (both.encode("utf-8") + encodeB64(bytes([0] * (ps + ls)) + raw)[ps:]) + + if (len(full) % 4) or (fs and len(full) != fs): + raise InvalidCodeSizeError(f"Invalid full size given code{both=} " + f" with raw size={rs}, {cs=}, {hs=}, " + f"{ss=}, {fs=}, and {ls=}.") + + return full def _binfil(self): diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index e94bf0e71..91d869ab5 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -284,16 +284,37 @@ def test_matter_class(): assert len(code) == hs if fs is None: # variable sized - assert ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 + assert ss > 0 and not (cs % 4) # full code is 24 bit aligned + # assumes that Matter methods also ensure (ls + rs) % 3 == 0 i.e. + # variable raw with lead is 24 bit aligned, where rs is raw size. assert code[0] in coring.SmallVrzDex or code[0] in coring.LargeVrzDex if code[0] in coring.SmallVrzDex: # small variable sized code - assert val.hs == 2 and val.ss == 2 and val.fs is None - assert code[0] == astuple(coring.SmallVrzDex)[val.ls] + assert hs == 2 and ss == 2 and fs is None + assert code[0] == astuple(coring.SmallVrzDex)[ls] + if code[0] in '4': + assert ls == 0 + elif code[0] in '5': + assert ls == 1 + elif code[0] in '6': + assert ls == 2 + else: + assert False elif code[0] in coring.LargeVrzDex: # large veriable sized code assert val.hs == 4 and val.ss == 4 and val.fs is None - assert code[0] == astuple(coring.LargeVrzDex)[val.ls] + assert code[0] == astuple(coring.LargeVrzDex)[ls] + if code[0] in '7': + assert ls == 0 + elif code[0] in '8': + assert ls == 1 + elif code[0] in '9': + assert ls == 2 + else: + assert False + + else: + assert False else: # fixed size assert not (code[0] in coring.SmallVrzDex or code[0] in coring.LargeVrzDex) @@ -304,24 +325,24 @@ def test_matter_class(): assert ls == 0 # no lead assert Matter._rawSize(code) == 0 - rs = (fs - cs) * 3 // 4 - ls # raw size bytes sans lead - ps = (3 - (rs + ls) % 3) % 3 # pad size given raw + lead - assert ps == (cs % 4) # cs % 4 is pad size given cs % 4 != 3 + # verify correct sizes given raw size. Assumes properties above + rs = ((fs - cs) * 3 // 4) - ls # raw size bytes sans lead assert sceil((rs + ls) * 4 / 3) + cs == fs # sextets add up - - - if code[0] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz': - assert len(code) == 1 - elif code[0] in '0': - assert len(code) == 2 - elif code[0] in '1': - assert len(code) == 4 and ls == 0 - elif code[0] in '2': - assert len(code) == 4 and ls == 1 - elif code[0] in '3': - assert len(code) == 4 and ls == 2 - else: - assert code[0] not in '-_' # count or op code + ps = (3 - ((rs + ls) % 3)) % 3 # net pad size given raw with lead + assert ps == (cs % 4) # ensure correct midpad zero bits for cs + + if code[0] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz': + assert len(code) == 1 + elif code[0] in '0': + assert len(code) == 2 + elif code[0] in '1': + assert len(code) == 4 and ls == 0 + elif code[0] in '2': + assert len(code) == 4 and ls == 1 + elif code[0] in '3': + assert len(code) == 4 and ls == 2 + else: + assert code[0] not in '456789-_' # count or op code # Test .Hards From fa3875761fe3af8b9e85281e33ad07a4dc5c763b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 30 Mar 2024 17:00:19 -0600 Subject: [PATCH 087/418] refactored Matter._exfil fixed bug and with better error reporting. --- src/keri/core/coring.py | 95 ++++++++++------ tests/app/test_keeping.py | 3 +- tests/core/test_coring.py | 227 ++++++++++++++++++++++++++++++++++---- 3 files changed, 271 insertions(+), 54 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index b35acf32f..29339b304 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -677,6 +677,7 @@ class Matter: digestive (bool): True means digest derivation code False otherwise prefixive (bool): True means identifier prefix derivation code False otherwise special (bool): True when soft is special raw is empty and fixed size + composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip Hidden: _code (str): value for .code property @@ -831,7 +832,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, qb64b (bytes | None): fully qualified crypto material Base64 qb64 (str | bytes | None): fully qualified crypto material Base64 qb2 (bytes | None): fully qualified crypto material Base2 - soft (str | None): soft part for special codes + soft (str | bytes): soft part for special codes strip (bool): True means strip (delete) matter from input stream bytearray after parsing qb64b or qb2. False means do not strip @@ -846,6 +847,9 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, .raw and .code and .size and .rsize """ + if hasattr(soft, "decode"): # make soft str + soft = soft.decode("utf-8") + if raw is not None: # raw provided but may be empty if not code: raise EmptyMaterialError(f"Improper initialization need either " @@ -913,6 +917,8 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") + else: + soft = '' # must be empty when ss == 0 raw = raw[:rize] # copy only exact size from raw stream @@ -997,14 +1003,13 @@ def _special(cls, code): """ Returns: special (bool): True when code has special soft i.e. when - fs is not None and ss > 0 and fs = hs + ss and ls = 0 - i.e. (fs fixed and soft not empty and raw is empty and no lead) + fs is not None and ss > 0 False otherwise """ hs, ss, fs, ls = cls.Sizes[code] - return (fs is not None and ss > 0 and fs == hs + ss and ls == 0) + return (fs is not None and ss > 0) @property @@ -1167,6 +1172,18 @@ def special(self): """ return self._special(self.code) + @property + def composable(self): + """ + composable (bool): True when both .qb64b and .qb2 are 24 bit aligned and + round trip using encodeB64 and decodeB64. + False otherwise + """ + qb64b = self.qb64b + qb2 = self.qb2 + return (len(qb64b) % 4 == 0 and len(qb2) % 3 == 0 and + encodeB64(qb2) == qb64b and decodeB64(qb64b) == qb2) + def _infil(self): """ @@ -1185,8 +1202,6 @@ def _infil(self): InvalidCodeSizeError(f"Invalid full code={both} for sizes {hs=} and" f" {ss=}.") - - if not fs: # variable sized # Tests on .Sizes table must ensure ls in (0,1,2) and cs % 4 == 0 but # can't know the variable size. So instance methods must ensure that @@ -1315,30 +1330,44 @@ def _exfil(self, qb64b): if hasattr(qb64b, "encode"): # only convert extracted chars from stream qb64b = qb64b.encode("utf-8") - # check for non-zeroed pad bits or lead bytes - ps = cs % 4 # code pad size ps = cs mod 4 assumes cs mod 4 != 3 - pbs = 2 * (ps if ps else ls) # pad bit size in bits - if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls - base = ps * b'A' + qb64b[cs:] # replace pre code with prepad chars of zero - paw = decodeB64(base) # decode base to leave prepadded raw - pi = (int.from_bytes(paw[:ps], "big")) # prepad as int - if pi & (2 ** pbs - 1 ): # masked pad bits non-zero - raise ValueError(f"Non zeroed prepad bits = " - f"{pi & (2 ** pbs - 1 ):<06b} in {qb64b[cs:cs+1]}.") - raw = paw[ps:] # strip off ps prepad paw bytes, paw is bytes so raw is bytes - - else: # not ps. IF not ps THEN may or may not be ls (lead) - base = qb64b[cs:] # strip off code leaving lead chars if any and value - # decode lead chars + val leaving lead bytes + raw bytes - # then strip off ls lead bytes leaving raw - paw = decodeB64(base) # decode base to leave prepadded paw bytes - li = int.from_bytes(paw[:ls], "big") # lead as int - if li: # pre pad lead bytes must be zero - if ls == 1: - raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") - else: - raise ValueError(f"Non zeroed lead bytes = 0x{li:04x}.") - raw = paw[ls:] # paw is bytes so raw is bytes + # check for non-zeroed pad bits and/or lead bytes + # net prepad ps == cs % 4 (remainer). Assumes ps != 3 i.e ps in (0,1,2) + # To ensure number of prepad bytes and prepad chars are same. + # need net prepad chars ps to invert using decodeB64 of lead + raw + + ps = cs % 4 # net prepad bytes to ensure 24 bit align when encodeB64 + base = ps * b'A' + qb64b[cs:] # prepad ps 'A's to B64 of (lead + raw) + paw = decodeB64(base) # now should have ps + ls leading sextexts of zeros + raw = paw[ps+ls:] # remove prepad midpat bytes to invert back to raw + # ensure midpad bytes are zero + pi = int.from_bytes(paw[:ps+ls], "big") + if pi != 0: + raise ConversionError(f"Non zeroed midpad bytes=0x{pi:0{(ps + ls) * 2}x}.") + + + #ps = cs % 4 # net prepad bytes to ensure 24 bit align when encodeB64 + #pbs = 2 * (ps if ps else ls) # pad bit size in bits + #if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls + #base = ps * b'A' + qb64b[cs:] # replace pre code with prepad chars of zero + #paw = decodeB64(base) # decode base to leave prepadded raw + #pi = (int.from_bytes(paw[:ps], "big")) # prepad as int + #if pi & (2 ** pbs - 1 ): # masked pad bits non-zero + #raise ValueError(f"Non zeroed prepad bits = " + #f"{pi & (2 ** pbs - 1 ):<06b} in {qb64b[cs:cs+1]}.") + #raw = paw[ps:] # strip off ps prepad paw bytes, paw is bytes so raw is bytes + + #else: # not ps. IF not ps THEN may or may not be ls (lead) + #base = qb64b[cs:] # strip off code leaving lead chars if any and value + ## decode lead chars + val leaving lead bytes + raw bytes + ## then strip off ls lead bytes leaving raw + #paw = decodeB64(base) # decode base to leave prepadded paw bytes + #li = int.from_bytes(paw[:ls], "big") # lead as int + #if li: # pre pad lead bytes must be zero + #if ls == 1: + #raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") + #else: + #raise ValueError(f"Non zeroed lead bytes = 0x{li:04x}.") + #raw = paw[ls:] # paw is bytes so raw is bytes if len(raw) != ((len(qb64b) - cs) * 3 // 4) - ls: # exact lengths raise ConversionError(f"Improperly qualified material = {qb64b}") @@ -1410,15 +1439,15 @@ def _bexfil(self, qb2): # convert last byte of code bytes in which are pad bits to int pi = (int.from_bytes(qb2[bcs-1:bcs], "big")) if pi & (2 ** pbs - 1 ): # masked pad bits non-zero - raise ValueError(f"Non zeroed pad bits = " + raise ConversionError(f"Non zeroed pad bits = " f"{pi & (2 ** pbs - 1 ):>08b} in 0x{pi:02x}.") else: # not ps. IF not ps THEN may or may not be ls (lead) li = int.from_bytes(qb2[bcs:bcs+ls], "big") # lead as int if li: # pre pad lead bytes must be zero if ls == 1: - raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") + raise ConversionError(f"Non zeroed lead byte = 0x{li:02x}.") else: - raise ValueError(f"Non zeroed lead bytes = 0x{li:02x}.") + raise ConversionError(f"Non zeroed lead bytes = 0x{li:02x}.") # strip code and leader bytes from qb2 to get raw raw = qb2[(bcs + ls):] # may be empty diff --git a/tests/app/test_keeping.py b/tests/app/test_keeping.py index 0912413c3..4f289a0d9 100644 --- a/tests/app/test_keeping.py +++ b/tests/app/test_keeping.py @@ -16,6 +16,7 @@ from hio.base import doing +from keri import kering from keri.help import helping from keri.core import coring from keri.core.coring import IdrDex @@ -708,7 +709,7 @@ def test_manager(): with keeping.openKS() as keeper: - with pytest.raises(ValueError): + with pytest.raises(kering.ConversionError): #test invalid qb64 of Salt manager = keeping.Manager(ks=keeper, salt='0AzwMTIzNDU2Nzg5YWJjZGVm') diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 91d869ab5..9dc5d6e4b 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -23,10 +23,12 @@ from cryptography.hazmat.primitives import hashes from cryptography import exceptions +from keri import kering from keri.kering import (EmptyMaterialError, RawMaterialError, DerivationError, ShortageError, InvalidCodeSizeError, InvalidVarIndexError, InvalidValueError, DeserializeError, ValidationError, - InvalidVarRawSizeError) + InvalidVarRawSizeError, ConversionError, + SoftMaterialError, InvalidSoftError) from keri.kering import Version, Versionage, VersionError from keri.core import coring @@ -405,11 +407,14 @@ def test_matter(): assert matter.transferable == False assert matter.digestive == False assert matter.prefixive == True + assert not matter.special + assert matter.composable # test round trip assert matter.qb64 == encodeB64(matter.qb2).decode("utf-8") assert matter.qb2 == decodeB64(matter.qb64.encode("utf-8")) + assert matter.composable # Test from qb64b matter = Matter(qb64b=prefixb) @@ -433,15 +438,17 @@ def test_matter(): # test non-zero pad bits in qb64 init ps == 1 badprefix1 = 'B_AAY2RlZmdoaWprbG1ub3BxcnN0dXYwMTIzNDU2Nzg5' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badprefix1) - assert str(ex.value) == "Non zeroed prepad bits = 110000 in b'_'." + #assert str(ex.value) == "Non zeroed prepad bits = 110000 in b'_'." + assert str(ex.value) == 'Non zeroed midpad bytes=0x03.' # test non-zero pad bits in qb64 init ps == 2 badprefix2 = '0A_wMTIzNDU2Nzg5YWJjZGVm' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badprefix2) - assert str(ex.value) == "Non zeroed prepad bits = 111100 in b'_'." + #assert str(ex.value) == "Non zeroed prepad bits = 111100 in b'_'." + assert str(ex.value) == 'Non zeroed midpad bytes=0x000f.' # test truncates extra bytes from qb64 parameter longprefix = prefix + "ABCD" # extra bytes in size @@ -466,13 +473,13 @@ def test_matter(): # test non-zero pad bits in qb2 init ps ==1 badprebin1 = decodeB64(badprefix1) # b'\x07\xf0\x00cdefghijklmnopqrstuv0123456789' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badprebin1) assert str(ex.value) == 'Non zeroed pad bits = 00000011 in 0x07.' # test non-zero pad bits in qb2 init ps ==2 badprebin2 = decodeB64(badprefix2) # b'\xd0\x0f\xf0123456789abcdef' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badprebin2) assert str(ex.value) == 'Non zeroed pad bits = 00001111 in 0x0f.' @@ -630,6 +637,8 @@ def test_matter(): assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable assert matter.qb64 == encodeB64(matter.qb2).decode("utf-8") assert matter.qb2 == decodeB64(matter.qb64.encode("utf-8")) @@ -684,13 +693,15 @@ def test_matter(): badqb64 = '2____2Fi' # '2___' + encodeB64(b'\xffab').decode("utf-8") badqb2 = decodeB64(badqb64) # b'\xd8\x00\x00\xffab' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) - assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + assert str(ex.value) == 'Non zeroed midpad bytes=0xff.' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' # test fix sized with leader 2 @@ -712,6 +723,8 @@ def test_matter(): assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable assert matter.qb64 == encodeB64(matter.qb2).decode("utf-8") assert matter.qb2 == decodeB64(matter.qb64.encode("utf-8")) @@ -753,11 +766,12 @@ def test_matter(): badqb64 = '3_____96' # '3AAA' + encodeB64(b'\xff\xffz').decode("utf-8") badqb2 = decodeB64(badqb64) #b'\xdc\x00\x00\xff\xffz' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) - assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + #assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + assert str(ex.value) == 'Non zeroed midpad bytes=0xffff.' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' @@ -841,11 +855,12 @@ def test_matter(): badqb64 = '5BAC_2FiY2Rl' # '5BAC' + encodeB64(b'\xffabcde').decode("utf-8") badqb2 = decodeB64(badqb64) # b'\xe4\x10\x02\xffabcde' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) - assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + assert str(ex.value) == 'Non zeroed midpad bytes=0xff.' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) assert str(ex.value) == 'Non zeroed lead byte = 0xff.' @@ -974,11 +989,12 @@ def test_matter(): badqb64 = '6BAC__9hYmNk' # '5BAC' + encodeB64(b'\xff\xffabcd').decode("utf-8") badqb2 = decodeB64(badqb64) # b'\xe8\x10\x02\xff\xffabcd' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) - assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + #assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + assert str(ex.value) == 'Non zeroed midpad bytes=0xffff.' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' @@ -1548,7 +1564,6 @@ def test_matter_special(): Test Matter instances using code with special soft values """ # test Tag3 - code = MtrDex.Tag3 soft = 'icp' qb64 = 'Xicp' @@ -1562,6 +1577,7 @@ def test_matter_special(): assert matter.qb64 == qb64 assert matter.qb2 == qb2 assert matter.special + assert matter.composable code = matter.code soft = matter.soft @@ -1575,6 +1591,7 @@ def test_matter_special(): assert matter.qb64 == qb64 assert matter.qb2 == qb2 assert matter.special + assert matter.composable matter = Matter(qb64=qb64) assert matter.code == matter.hard == code @@ -1583,6 +1600,7 @@ def test_matter_special(): assert matter.qb64 == qb64 assert matter.qb2 == qb2 assert matter.special + assert matter.composable # Test corner conditions # Empty raw @@ -1593,8 +1611,177 @@ def test_matter_special(): assert matter.qb64 == qb64 assert matter.qb2 == qb2 assert matter.special + assert matter.composable + + #non empty raw ignored since code special, forces empty raw + badraw = b'abcdefg' + matter = Matter(raw=badraw, code=code, soft=soft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + #raw None + matter = Matter(code=code, soft=soft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # soft extra chars ignored + bigsoft = 'icprot' + matter = Matter(code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # soft bytes not str + bigsoft = b'icprot' + matter = Matter(code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # soft too small + weesoft = 'ic' + with pytest.raises(SoftMaterialError): + matter = Matter(code=code, soft=weesoft) + + # soft not B64 chars + badsoft = b'#@$%^&*!' + with pytest.raises(InvalidSoftError): + matter = Matter(code=code, soft=badsoft) + + #non empty raw and badsoft + badraw = b'abcdefg' + badsoft = b'#@$%^&*!' + with pytest.raises(InvalidSoftError): + matter = Matter(raw=badraw, code=code, soft=badsoft) + + # soft but not special code + numraw = b'\xf7\x7f' + matter = Matter(raw=numraw, code=MtrDex.Short, soft=soft) + assert matter.code == matter.hard == MtrDex.Short + assert matter.soft == '' + assert matter.raw == numraw + assert not matter.special + assert matter.composable + + # Test TBD0S '1__-' + # soft special but valid non-empty raw as part of primitive + code = MtrDex.TBD0S # sizes '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), + rs = Matter._rawSize(code) # raw size + soft = 'TG' + qb64 = '1__-TGB1dnd4' + qb2 = b'\xd7\xff\xfeL`uvwx' + raw = b'uvwx' + + assert rs == 4 + + bigsoft = 'TGIF' + extraw = b'uvwxyz' + + matter = Matter(raw=extraw, code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + matter = Matter(qb2=qb2) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + matter = Matter(qb64=qb64) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # Same as above but raw all zeros + + qb64 = '1__-TGAAAAAA' + qb2 = b'\xd7\xff\xfeL`\x00\x00\x00\x00' + raw = b'\x00\x00\x00\x00' + + assert rs == 4 + + bigsoft = 'TGIF' + extraw = bytearray([0] * 7) + + matter = Matter(raw=extraw, code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # Test TBD1S '2__-' + # soft special but valid non-empty raw as part of primitive + code = MtrDex.TBD1S # sizes '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), + rs = Matter._rawSize(code) # raw size + soft = 'TG' + qb64 = '2__-TGAAdXZ3' # see lead byte + qb2 = b'\xdb\xff\xfeL`\x00uvw' + raw = b'uvw' + + assert rs == 3 + + bigsoft = 'TGIF' + extraw = b'uvwxyz' + + matter = Matter(raw=extraw, code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + matter = Matter(qb2=qb2) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + matter = Matter(qb64=qb64) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable """ Done Test """ From 0b2a0d3c65cb41a93372bacb476c68f302c49792 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 30 Mar 2024 18:56:00 -0600 Subject: [PATCH 088/418] refactor Matter.binfil to be cleaner and account for special soft codes --- src/keri/core/coring.py | 58 +++++++++++---------------------------- tests/core/test_coring.py | 31 ++++++++++++--------- 2 files changed, 34 insertions(+), 55 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 29339b304..1cad89b12 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -1342,32 +1342,7 @@ def _exfil(self, qb64b): # ensure midpad bytes are zero pi = int.from_bytes(paw[:ps+ls], "big") if pi != 0: - raise ConversionError(f"Non zeroed midpad bytes=0x{pi:0{(ps + ls) * 2}x}.") - - - #ps = cs % 4 # net prepad bytes to ensure 24 bit align when encodeB64 - #pbs = 2 * (ps if ps else ls) # pad bit size in bits - #if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls - #base = ps * b'A' + qb64b[cs:] # replace pre code with prepad chars of zero - #paw = decodeB64(base) # decode base to leave prepadded raw - #pi = (int.from_bytes(paw[:ps], "big")) # prepad as int - #if pi & (2 ** pbs - 1 ): # masked pad bits non-zero - #raise ValueError(f"Non zeroed prepad bits = " - #f"{pi & (2 ** pbs - 1 ):<06b} in {qb64b[cs:cs+1]}.") - #raw = paw[ps:] # strip off ps prepad paw bytes, paw is bytes so raw is bytes - - #else: # not ps. IF not ps THEN may or may not be ls (lead) - #base = qb64b[cs:] # strip off code leaving lead chars if any and value - ## decode lead chars + val leaving lead bytes + raw bytes - ## then strip off ls lead bytes leaving raw - #paw = decodeB64(base) # decode base to leave prepadded paw bytes - #li = int.from_bytes(paw[:ls], "big") # lead as int - #if li: # pre pad lead bytes must be zero - #if ls == 1: - #raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") - #else: - #raise ValueError(f"Non zeroed lead bytes = 0x{li:04x}.") - #raw = paw[ls:] # paw is bytes so raw is bytes + raise ConversionError(f"Nonzero midpad bytes=0x{pi:0{(ps + ls) * 2}x}.") if len(raw) != ((len(qb64b) - cs) * 3 // 4) - ls: # exact lengths raise ConversionError(f"Improperly qualified material = {qb64b}") @@ -1432,22 +1407,21 @@ def _bexfil(self, qb2): raise ShortageError("Need {} more bytes.".format(bfs - len(qb2))) qb2 = qb2[:bfs] # extract qb2 fully qualified primitive code plus material - # check for non-zeroed prepad bits or lead bytes - ps = cs % 4 # code pad size ps = cs mod 4 - pbs = 2 * (ps if ps else ls) # pad bit size in bits - if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls - # convert last byte of code bytes in which are pad bits to int - pi = (int.from_bytes(qb2[bcs-1:bcs], "big")) - if pi & (2 ** pbs - 1 ): # masked pad bits non-zero - raise ConversionError(f"Non zeroed pad bits = " - f"{pi & (2 ** pbs - 1 ):>08b} in 0x{pi:02x}.") - else: # not ps. IF not ps THEN may or may not be ls (lead) - li = int.from_bytes(qb2[bcs:bcs+ls], "big") # lead as int - if li: # pre pad lead bytes must be zero - if ls == 1: - raise ConversionError(f"Non zeroed lead byte = 0x{li:02x}.") - else: - raise ConversionError(f"Non zeroed lead bytes = 0x{li:02x}.") + + + # check for nonzero trailing full code mid pad bits + ps = cs % 4 # full code (both) net pad size for 24 bit alignment + pbs = 2 * ps # mid pad bits = 2 per net pad + # get pad bits in last byte of full code + pi = (int.from_bytes(qb2[bcs-1:bcs], "big")) # convert byte to int + pi = pi & (2 ** pbs - 1 ) # mask with 1's in pad bit locations + if pi: # not zero so raise error + raise ConversionError(f"Nonzero code mid pad bits=0b{pi:0{pbs}b}.") + + # check nonzero leading mid pad lead bytes in lead + raw + li = int.from_bytes(qb2[bcs:bcs+ls], "big") # lead as int + if li: # midpad lead bytes must be zero + raise ConversionError(f"Nonzero lead midpad bytes=0x{li:0{ls*2}x}.") # strip code and leader bytes from qb2 to get raw raw = qb2[(bcs + ls):] # may be empty diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 9dc5d6e4b..33db26eac 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -441,14 +441,14 @@ def test_matter(): with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badprefix1) #assert str(ex.value) == "Non zeroed prepad bits = 110000 in b'_'." - assert str(ex.value) == 'Non zeroed midpad bytes=0x03.' + assert str(ex.value) == 'Nonzero midpad bytes=0x03.' # test non-zero pad bits in qb64 init ps == 2 badprefix2 = '0A_wMTIzNDU2Nzg5YWJjZGVm' with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badprefix2) #assert str(ex.value) == "Non zeroed prepad bits = 111100 in b'_'." - assert str(ex.value) == 'Non zeroed midpad bytes=0x000f.' + assert str(ex.value) == 'Nonzero midpad bytes=0x000f.' # test truncates extra bytes from qb64 parameter longprefix = prefix + "ABCD" # extra bytes in size @@ -475,13 +475,15 @@ def test_matter(): badprebin1 = decodeB64(badprefix1) # b'\x07\xf0\x00cdefghijklmnopqrstuv0123456789' with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badprebin1) - assert str(ex.value) == 'Non zeroed pad bits = 00000011 in 0x07.' + #assert str(ex.value) == 'Non zeroed pad bits = 00000011 in 0x07.' + assert str(ex.value) == 'Nonzero code mid pad bits=0b11.' # test non-zero pad bits in qb2 init ps ==2 badprebin2 = decodeB64(badprefix2) # b'\xd0\x0f\xf0123456789abcdef' with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badprebin2) - assert str(ex.value) == 'Non zeroed pad bits = 00001111 in 0x0f.' + #assert str(ex.value) == 'Non zeroed pad bits = 00001111 in 0x0f.' + assert str(ex.value) == 'Nonzero code mid pad bits=0b1111.' # test raises ShortageError if not enough bytes in qb2 parameter @@ -696,12 +698,12 @@ def test_matter(): with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' - assert str(ex.value) == 'Non zeroed midpad bytes=0xff.' + assert str(ex.value) == 'Nonzero midpad bytes=0xff.' with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) - assert str(ex.value) == 'Non zeroed lead byte = 0xff.' - #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + assert str(ex.value) == 'Nonzero lead midpad bytes=0xff.' # test fix sized with leader 2 @@ -769,11 +771,12 @@ def test_matter(): with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) #assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' - assert str(ex.value) == 'Non zeroed midpad bytes=0xffff.' + assert str(ex.value) == 'Nonzero midpad bytes=0xffff.' with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) - assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + #assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + assert str(ex.value) == 'Nonzero lead midpad bytes=0xffff.' # test variable sized with leader 1 code = MtrDex.Bytes_L1 @@ -858,11 +861,12 @@ def test_matter(): with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' - assert str(ex.value) == 'Non zeroed midpad bytes=0xff.' + assert str(ex.value) == 'Nonzero midpad bytes=0xff.' with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) - assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + assert str(ex.value) == 'Nonzero lead midpad bytes=0xff.' # test variable sized with leader 1 with code replacement code0 = MtrDex.Bytes_L0 # use leader 0 code but with lead size 1 raw @@ -992,11 +996,12 @@ def test_matter(): with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) #assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' - assert str(ex.value) == 'Non zeroed midpad bytes=0xffff.' + assert str(ex.value) == 'Nonzero midpad bytes=0xffff.' with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) - assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + #assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + assert str(ex.value) == 'Nonzero lead midpad bytes=0xffff.' # test variable sized with leader 2 with code replacement code0 = MtrDex.Bytes_L0 # use leader 0 code but with lead size 2 raw From e870f259f4d4050ff9a777ecd51dd8c4bf7d163b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 30 Mar 2024 19:18:25 -0600 Subject: [PATCH 089/418] added unit test for TBD0 --- tests/core/test_coring.py | 160 +++++++++++++++++++++++++++++++++++++- 1 file changed, 158 insertions(+), 2 deletions(-) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 33db26eac..48623fcf0 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -619,14 +619,89 @@ def test_matter(): assert matter.prefixive == True assert ims == extra # stripped not include extra + # test fixed size with leader 0 + # TBD0 = '1___' # Testing purposes only fixed with lead size 0 + + code = MtrDex.TBD0 # '1___' + assert Matter._rawSize(code) == 3 + assert Matter._leadSize(code) == 0 + raw = b'abc' + qb64 = '1___YWJj' # + qb2 = decodeB64(qb64) + matter = Matter(raw=raw, code=code) + assert matter.raw == raw + assert matter.code == code + assert matter.both == code + assert matter.size == None + assert matter.fullSize == 8 + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.transferable == True + assert matter.digestive == False + assert matter.prefixive == False + assert not matter.special + assert matter.composable + + assert matter.qb64 == encodeB64(matter.qb2).decode("utf-8") + assert matter.qb2 == decodeB64(matter.qb64.encode("utf-8")) + + matter._exfil(qb64.encode("utf-8")) + assert matter.code == code + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + + matter = Matter(qb64b=qb64.encode("utf-8")) + assert matter.code == code + assert matter.raw == raw + + matter = Matter(qb64=qb64) + assert matter.code == code + assert matter.raw == raw + + matter = Matter(qb64=qb64.encode("utf-8")) # works for either + assert matter.code == code + assert matter.raw == raw + + # Test ._bexfil + matter._bexfil(qb2) + assert matter.raw == raw + assert matter.code == code + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + + matter = Matter(qb2=qb2) + assert matter.code == code + assert matter.raw == raw + assert matter.qb64b == qb64.encode("utf-8") + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.transferable == True + assert matter.digestive == False + + matter = Matter(raw=raw, code=code) + assert matter.raw == raw + assert matter.code == code + assert matter.both == code + assert matter.size == None + assert matter.fullSize == 8 + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.transferable == True + assert matter.digestive == False + assert matter.prefixive == False + + # Can't have bad pad because cs % 4 == 0 + # Can't habe bad lead because ls ==0 + # test fix sized with leader 1 # TBD1 = '2___' # Testing purposes only fixed with lead size 1 - code = MtrDex.TBD1 # '2AAA' + code = MtrDex.TBD1 # '2___' assert Matter._rawSize(code) == 2 assert Matter._leadSize(code) == 1 raw = b'ab' - qb64 = '2___AGFi' # '2AAA' + encodeB64(b'\x00ab').decode("utf-8") + qb64 = '2___AGFi' # '2___' + encodeB64(b'\x00ab').decode("utf-8") qb2 = decodeB64(qb64) matter = Matter(raw=raw, code=code) assert matter.raw == raw @@ -1788,6 +1863,87 @@ def test_matter_special(): assert matter.special assert matter.composable + # Same as above but raw all zeros + + qb64 = '2__-TGAAAAAA' + qb2 = b'\xdb\xff\xfeL`\x00\x00\x00\x00' + raw = b'\x00\x00\x00' + + assert rs == 3 + + bigsoft = 'TGIF' + extraw = bytearray([0] * 7) + + matter = Matter(raw=extraw, code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # Test TBD2S '3__-' + # soft special but valid non-empty raw as part of primitive + code = MtrDex.TBD2S # sizes '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), + rs = Matter._rawSize(code) # raw size + soft = 'TG' + qb64 = '3__-TGAAAHV2' # see lead byte + qb2 = b'\xdf\xff\xfeL`\x00\x00uv' + raw = b'uv' + + assert rs == 2 + + bigsoft = 'TGIF' + extraw = b'uvwxyz' + + matter = Matter(raw=extraw, code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + matter = Matter(qb2=qb2) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + matter = Matter(qb64=qb64) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # Same as above but raw all zeros + + qb64 = '3__-TGAAAAAA' + qb2 = b'\xdf\xff\xfeL`\x00\x00\x00\x00' + raw = b'\x00\x00' + + assert rs == 2 + + bigsoft = 'TGIF' + extraw = bytearray([0] * 7) + + matter = Matter(raw=extraw, code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + """ Done Test """ From 62ddc3be1f655f417b6d136a7f6730ba73c63fbc Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 31 Mar 2024 07:49:25 -0600 Subject: [PATCH 090/418] revert Label3 to Tag4 fix unit tests for Labe1 and Label2 --- src/keri/core/coring.py | 12 ++--- tests/core/test_coring.py | 111 ++++++++++++++------------------------ 2 files changed, 46 insertions(+), 77 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 1cad89b12..f3e7e80eb 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -305,7 +305,7 @@ class MatterCodex: Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. Ed448: str = '1AAD' # Ed448 public signing verification key. Basic derivation. Ed448_Sig: str = '1AAE' # Ed448 signature. Self-signing derivation. - Label3: str = '1AAF' # Label3 as 3 bytes for label lead size 0 + Tag4: str = '1AAF' # Tag4 4 B64 encoded chars for special values DateTime: str = '1AAG' # Base64 custom encoded 32 char ISO-8601 DateTime X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt ECDSA_256r1N: str = '1AAI' # ECDSA secp256r1 verification key non-transferable, basic derivation. @@ -313,13 +313,12 @@ class MatterCodex: Null: str = '1AAK' # Null None or empty value No: str = '1AAL' # No Falsey Boolean value Yes: str = '1AAM' # Yes Truthy Boolean value - Tag4: str = '1AAN' # Tag4 4 B64 encoded chars for special values - Tag8: str = '1AAO' # Tag8 8 B64 encoded chars for special values + Tag8: str = '1AAN' # Tag8 8 B64 encoded chars for special values TBD0S: str = '1__-' # Testing purposes only, fixed special values with non-empty raw lead size 0 TBD0: str = '1___' # Testing purposes only, fixed with lead size 0 TBD1S: str = '2__-' # Testing purposes only, fixed special values with non-empty raw lead size 1 TBD1: str = '2___' # Testing purposes only, fixed with lead size 1 - TBD2S: str = '3__-' # Testing purposes only, fixed special values with non-empty raw lead size 1 + TBD2S: str = '3__-' # Testing purposes only, fixed special values with non-empty raw lead size 2 TBD2: str = '3___' # Testing purposes only, fixed with lead size 2 StrB64_L0: str = '4A' # String Base64 only lead size 0 StrB64_L1: str = '5A' # String Base64 only lead size 1 @@ -768,7 +767,7 @@ class Matter: '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), '1AAD': Sizage(hs=4, ss=0, fs=80, ls=0), '1AAE': Sizage(hs=4, ss=0, fs=56, ls=0), - '1AAF': Sizage(hs=4, ss=0, fs=8, ls=0), + '1AAF': Sizage(hs=4, ss=4, fs=8, ls=0), '1AAG': Sizage(hs=4, ss=0, fs=36, ls=0), '1AAH': Sizage(hs=4, ss=0, fs=100, ls=0), '1AAI': Sizage(hs=4, ss=0, fs=48, ls=0), @@ -776,8 +775,7 @@ class Matter: '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAN': Sizage(hs=4, ss=4, fs=8, ls=0), - '1AAO': Sizage(hs=4, ss=2, fs=12, ls=0), + '1AAN': Sizage(hs=4, ss=2, fs=12, ls=0), '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), '1___': Sizage(hs=4, ss=0, fs=8, ls=0), '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 48623fcf0..04be04d2f 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -105,7 +105,7 @@ def test_matter_class(): 'Ed448N': '1AAC', 'Ed448': '1AAD', 'Ed448_Sig': '1AAE', - 'Label3': '1AAF', + 'Tag4': '1AAF', 'DateTime': '1AAG', 'X25519_Cipher_Salt': '1AAH', 'ECDSA_256r1N': '1AAI', @@ -113,8 +113,7 @@ def test_matter_class(): 'Null': '1AAK', 'No': '1AAL', 'Yes': '1AAM', - 'Tag4': '1AAN', - 'Tag8': '1AAO', + 'Tag8': '1AAN', 'TBD0S': '1__-', 'TBD0': '1___', 'TBD1S': '2__-', @@ -217,7 +216,7 @@ def test_matter_class(): '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), '1AAD': Sizage(hs=4, ss=0, fs=80, ls=0), '1AAE': Sizage(hs=4, ss=0, fs=56, ls=0), - '1AAF': Sizage(hs=4, ss=0, fs=8, ls=0), + '1AAF': Sizage(hs=4, ss=4, fs=8, ls=0), '1AAG': Sizage(hs=4, ss=0, fs=36, ls=0), '1AAH': Sizage(hs=4, ss=0, fs=100, ls=0), '1AAI': Sizage(hs=4, ss=0, fs=48, ls=0), @@ -225,8 +224,7 @@ def test_matter_class(): '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAN': Sizage(hs=4, ss=4, fs=8, ls=0), - '1AAO': Sizage(hs=4, ss=2, fs=12, ls=0), + '1AAN': Sizage(hs=4, ss=2, fs=12, ls=0), '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), '1___': Sizage(hs=4, ss=0, fs=8, ls=0), '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), @@ -1501,141 +1499,114 @@ def test_matter(): assert matter.digestive == False assert matter.prefixive == False - # test Label3 - #val = int("F89CFF", 16) - #assert val == 16293119 - #raw = val.to_bytes(3, 'big') - #assert raw == b'\xf8\x9c\xff' - raw = b'hio' - cs = len(MtrDex.Label3) - assert cs == 4 - ps = cs % 4 - assert ps == 0 - txt = encodeB64(bytes([0]*ps) + raw) - #assert txt == b'-Jz_' - assert txt == b'aGlv' - qb64b = MtrDex.Label3.encode("utf-8") + txt[ps:] - #assert qb64b == b'1AAF-Jz_' - assert qb64b == b'1AAFaGlv' - qb64 = qb64b.decode("utf-8") + # test Label1 + code = MtrDex.Label1 + raw = b'*' + qb64 = 'VAAq' + qb64b = qb64.encode("utf-8") qb2 = decodeB64(qb64b) - assert qb2 == b'\xd4\x00\x05hio' - #assert qb2 == b'\xd4\x00\x05\xf8\x9c\xff' - bs = ceil((cs * 3) / 4) - assert qb2[bs:] == raw # stable value in qb2 - assert encodeB64(qb2) == qb64b - matter = Matter(raw=raw, code=MtrDex.Label3) + matter = Matter(raw=raw, code=code) assert matter.raw == raw - assert matter.code == MtrDex.Label3 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb64b=qb64b) assert matter.raw == raw - assert matter.code == MtrDex.Label3 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb64=qb64) assert matter.raw == raw - assert matter.code == MtrDex.Label3 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b - assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb2=qb2) assert matter.raw == raw - assert matter.code == MtrDex.Label3 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable - # test Label3 as chars - txt = b'icp_' - raw = decodeB64(txt) - assert raw == b'\x89\xca\x7f' - val = int.from_bytes(raw, 'big') - assert val == 9030271 - cs = len(MtrDex.Label3) - assert cs == 4 - ps = cs % 4 - assert ps == 0 - txt = encodeB64(bytes([0]*ps) + raw) - qb64b = MtrDex.Label3.encode("utf-8") + txt - assert qb64b == b'1AAFicp_' - qb64 = qb64b.decode("utf-8") + # test Label2 + code = MtrDex.Label2 + raw = b'@&' + qb64 = 'WEAm' + qb64b = qb64.encode("utf-8") qb2 = decodeB64(qb64b) - assert qb2 == b'\xd4\x00\x05\x89\xca\x7f' - bs = ceil((cs * 3) / 4) - assert qb2[bs:] == raw # stable value in qb2 - assert encodeB64(qb2) == qb64b - matter = Matter(raw=raw, code=MtrDex.Label3) + matter = Matter(raw=raw, code=code) assert matter.raw == raw - assert matter.code == MtrDex.Label3 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb64b=qb64b) assert matter.raw == raw - assert matter.code == MtrDex.Label3 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb64=qb64) assert matter.raw == raw - assert matter.code == MtrDex.Label3 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b - assert matter.qb2 == qb2 assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb2=qb2) assert matter.raw == raw - assert matter.code == MtrDex.Label3 + assert matter.code ==code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable + """ Done Test """ From ebb0830904aca0746661c498ff163ac773bef731 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 31 Mar 2024 15:55:42 -0600 Subject: [PATCH 091/418] created Tagger class with unit tests --- src/keri/core/coring.py | 189 ++++++++++++++++++++++++++++++++++---- tests/core/test_coring.py | 98 ++++++++++++++++++-- 2 files changed, 265 insertions(+), 22 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index f3e7e80eb..485b282fb 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -602,7 +602,6 @@ def __iter__(self): DigDex = DigCodex() # Make instance - @dataclass(frozen=True) class NumCodex: """ @@ -628,6 +627,54 @@ def __iter__(self): NumDex = NumCodex() # Make instance +@dataclass(frozen=True) +class TagCodex: + """ + TagCodex is codex of Base64 derivation codes for compactly representing + various small Base64 tag values as special code soft part values. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Tag1: str = '0J' # 1 B64 char tag with 1 pre pad + Tag2: str = '0K' # 1 B64 char tag + Tag3: str = 'X' # 1 B64 char tag + Tag4: str = '1AAF' # 1 B64 char tag + Tag5: str = '0L' # 1 B64 char tag with 1 pre pad + Tag6: str = '0M' # 1 B64 char tag + Tag7: str = 'Y' # 1 B64 char tag + Tag8: str = '1AAN' # 1 B64 char tag + Tag9: str = '0N' # 1 B64 char tag with 1 pre pad + Tag10: str = '0O' # 1 B64 char tag + + def __iter__(self): + return iter(astuple(self)) + + +TagDex = TagCodex() # Make instance + + +@dataclass(frozen=True) +class PadTagCodex: + """ + TagCodex is codex of Base64 derivation codes for compactly representing + various small Base64 tag values as prepadded special code soft part values. + Prepad is 1 B64 char. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Tag1: str = '0J' # 1 B64 char tag with 1 pre pad + Tag5: str = '0L' # 1 B64 char tag with 1 pre pad + Tag9: str = '0N' # 1 B64 char tag with 1 pre pad + + def __iter__(self): + return iter(astuple(self)) + + +PadTagDex = PadTagCodex() # Make instance + + # namedtuple for size entries in Matter and Counter derivation code tables # hs is the hard size int number of chars in hard (stable) part of code # ss is the soft size int number of chars in soft (unstable) part of code @@ -659,7 +706,7 @@ class Matter: Properties: code (str): hard part of derivation code to indicate cypher suite hard (str): hard part of derivation code. alias for code - soft (str): soft part of derivation code fs any. + soft (str | bytes): soft part of derivation code fs any. Empty when ss = 0. both (str): hard + soft parts of full text code size (int | None): Number of quadlets/triplets of chars/bytes including @@ -691,16 +738,9 @@ class Matter: _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) - Size table rules for special soft values: - if fn in table is None then must have ss > 0 and (hs + ss) % 4 - else (fn is not None) then - if ss > 0 and fn = hs + ss and ls == 0 then - special soft value - else - not special must have ss == 0 + Special soft values are indicated when fn in table is None and ss > 0. """ - Codex = MtrDex # Hards table maps from bytes Base64 first code char to int of hard size, hs, # (stable) of code. The soft size, ss, (unstable) is always 0 for Matter # unless fs is None which allows for variable size multiple of 4, i.e. @@ -775,7 +815,7 @@ class Matter: '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAN': Sizage(hs=4, ss=2, fs=12, ls=0), + '1AAN': Sizage(hs=4, ss=8, fs=12, ls=0), '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), '1___': Sizage(hs=4, ss=0, fs=8, ls=0), '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), @@ -816,21 +856,21 @@ class Matter: - def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, - qb64b=None, qb64=None, qb2=None, soft='', strip=False): + def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, + qb64b=None, qb64=None, qb2=None, strip=False): """ Validate as fully qualified Parameters: raw (bytes | bytearray | None): unqualified crypto material usable for crypto operations. code (str): stable (hard) part of derivation code + soft (str | bytes): soft part for special codes rize (int | None): raw size in bytes when variable sized material not including lead bytes if any Otherwise None qb64b (bytes | None): fully qualified crypto material Base64 qb64 (str | bytes | None): fully qualified crypto material Base64 qb2 (bytes | None): fully qualified crypto material Base2 - soft (str | bytes): soft part for special codes strip (bool): True means strip (delete) matter from input stream bytearray after parsing qb64b or qb2. False means do not strip @@ -934,7 +974,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, raise InvalidSoftError(f"Unsupported variable sized {code=} " f" with {fs=} for special {soft=}.") - if not ss > 0 or not ls == 0 or not fs == hs + ss: # not special soft + if not ss > 0 or (fs == hs + ss and not ls == 0): # not special soft raise InvalidSoftError("Invalid soft size={ss} or lead={ls} " f" or {code=} {fs=} when special soft.") @@ -1593,7 +1633,6 @@ class Number(Matter): Methods: """ - Codex = NumDex def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, code=NumDex.Short, num=None, numh=None, **kwa): @@ -1897,6 +1936,124 @@ def datetime(self): """ return helping.fromIso8601(self.dts) + +class Tagger(Matter): + """ + Tagger is subclass of Matter, cryptographic material, for compact special + fixed size primitive with non-empty soft part and empty raw part. + + Tagger provides a more compact representation of small Base64 values in + as soft part of code rather than would be obtained by by using a small raw + part whose ASCII representation is converted to Base64. + + Attributes: + + Inherited Properties: (See Matter) + code (str): hard part of derivation code to indicate cypher suite + hard (str): hard part of derivation code. alias for code + soft (str): soft part of derivation code fs any. + Empty when ss = 0. + both (str): hard + soft parts of full text code + size (int | None): Number of quadlets/triplets of chars/bytes including + lead bytes of variable sized material (fs = None). + Converted value of the soft part (of len ss) of full + derivation code. + Otherwise None when not variably sized (fs != None) + fullSize (int): full size of primitive + raw (bytes): crypto material only. Not derivation code or lead bytes. + qb64 (str): Base64 fully qualified with derivation code + crypto mat + qb64b (bytes): Base64 fully qualified with derivation code + crypto mat + qb2 (bytes): binary with derivation code + crypto material + transferable (bool): True means transferable derivation code False otherwise + digestive (bool): True means digest derivation code False otherwise + prefixive (bool): True means identifier prefix derivation code False otherwise + special (bool): True when soft is special raw is empty and fixed size + composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip + + Properties: + + + Hidden Inherited: + _code (str): value for .code property + _soft (str): soft value of full code + _raw (bytes): value for .raw property + _rawSize(): + _leadSize(): + _special(): + _infil(): creates qb64b from .raw and .code (fully qualified Base64) + _binfil(): creates qb2 from .raw and .code (fully qualified Base2) + _exfil(): extracts .code and .raw from qb64b (fully qualified Base64) + _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) + + + Hidden: + + + Methods: + + def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, + qb64b=None, qb64=None, qb2=None, strip=False): + + """ + Pad = '_' # B64 pad char for tag codes with pre-padded soft values + + def __init__(self, tag='', soft='', code=None, **kwa): + """ + Inherited Parameters: (see Matter) + raw (bytes | bytearray | None): unqualified crypto material usable + for crypto operations. + code (str): stable (hard) part of derivation code + soft (str | bytes): soft part for special codes + rize (int | None): raw size in bytes when variable sized material not + including lead bytes if any + Otherwise None + qb64b (bytes | None): fully qualified crypto material Base64 + qb64 (str | bytes | None): fully qualified crypto material Base64 + qb2 (bytes | None): fully qualified crypto material Base2 + strip (bool): True means strip (delete) matter from input stream + bytearray after parsing qb64b or qb2. False means do not strip + + Parameters: + tag (str | bytes): Base64 plain. Prepad is added as needed. + + """ + if tag: + if hasattr(tag, "decode"): # make tag str + tag = tag.decode("utf-8") + if not Reb64.match(tag.encode("utf-8")): + raise InvalidSoftError(f"Non Base64 chars in {tag=}.") + codes = astuple(TagDex) + l = len(tag) # soft not empty so l > 0 + if l > len(codes): + raise InvalidSoftError("Oversized tag={soft}.") + code = codes[l-1] # get code for size of soft + if code in PadTagDex: + soft = self.Pad + tag # pre pad for those that need it + else: + soft = tag + + + super(Tagger, self).__init__(soft=soft, code=code, **kwa) + + if (not self._special(self.code)) or self.code not in TagDex: + raise InvalidCodeError(f"Invalid code={self.code} for Tagger.") + + @property + def tag(self): + """Property tag (str): plain without prepad (strips prepad from soft) + + """ + tag = self.soft + if self.code in PadTagDex: + pad = self.soft[0] + tag = self.soft[1:] + if pad != self.Pad: + raise InvalidSoftError("Invaid pre {pad=} for {tag=}.") + + return tag + + + # Versage namedtuple # proto (str): protocol element of Protocols # vrsn (Versionage): instance protocol version namedtuple (major, minor) ints diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 04be04d2f..2da05af8f 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -36,7 +36,7 @@ from keri.core.coring import (Ilkage, Ilks, Saids, Protocols, Protocolage, Sadder, Tholder, Seqner, NumDex, Number, Siger, Dater, Bexter, Texter, - Verser, Versage) + Verser, Versage, TagDex, PadTagDex, Tagger) from keri.core.coring import Serialage, Serials, Tiers from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, Indexer, @@ -153,8 +153,6 @@ def test_matter_class(): } - assert Matter.Codex == MtrDex - # first character of code with hard size of code assert Matter.Hards == { 'A': 1, 'B': 1, 'C': 1, 'D': 1, 'E': 1, 'F': 1, 'G': 1, 'H': 1, 'I': 1, @@ -224,7 +222,7 @@ def test_matter_class(): '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAN': Sizage(hs=4, ss=2, fs=12, ls=0), + '1AAN': Sizage(hs=4, ss=8, fs=12, ls=0), '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), '1___': Sizage(hs=4, ss=0, fs=8, ls=0), '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), @@ -3156,8 +3154,6 @@ def test_number(): 'Vast': 'U' } - assert Number.Codex == NumDex - with pytest.raises(RawMaterialError): number = Number(raw=b'') @@ -4172,6 +4168,95 @@ def test_dater(): """ Done Test """ +def test_tagger(): + """ + Test Tagger version primitive subclass of Matter + """ + # Test TagCodex PadTagCodex and associated Sizes to be valid specials + + + + + with pytest.raises(EmptyMaterialError): + tagger = Tagger() # defaults + + # Tag1 + tag = 'v' + code = MtrDex.Tag1 + soft = '_v' + qb64 = '0J_v' + qb64b = qb64.encode("utf-8") + qb2 = decodeB64(qb64b) + raw = b'' + + tagger = Tagger(tag=tag) # defaults + assert tagger.code == tagger.hard == code + assert tagger.soft == soft + assert tagger.raw == raw + assert tagger.qb64 == qb64 + assert tagger.qb2 == qb2 + assert tagger.special + assert tagger.composable + assert tagger.tag == tag + + tagger = Tagger(qb2=qb2) + assert tagger.code == tagger.hard == code + assert tagger.soft == soft + assert tagger.raw == raw + assert tagger.qb64 == qb64 + assert tagger.qb2 == qb2 + assert tagger.special + assert tagger.composable + assert tagger.tag == tag + + tagger = Tagger(qb64=qb64) + assert tagger.code == tagger.hard == code + assert tagger.soft == soft + assert tagger.raw == raw + assert tagger.qb64 == qb64 + assert tagger.qb2 == qb2 + assert tagger.special + assert tagger.composable + assert tagger.tag == tag + + + tagger = Tagger(qb64b=qb64b) + assert tagger.code == tagger.hard == code + assert tagger.soft == soft + assert tagger.raw == raw + assert tagger.qb64 == qb64 + assert tagger.qb2 == qb2 + assert tagger.special + assert tagger.composable + assert tagger.tag == tag + + + tags = 'abcdefghij' + alltags = dict() + for l in range(1, len(astuple(TagDex)) + 1): + tag = tags[:l] + tagger = Tagger(tag=tag) + assert tagger.tag == tag + assert len(tagger.tag) == l + assert tagger.code == astuple(TagDex)[l - 1] + alltags[l] = (tagger.tag, tagger.code) + + assert alltags == \ + { + 1: ('a', '0J'), + 2: ('ab', '0K'), + 3: ('abc', 'X'), + 4: ('abcd', '1AAF'), + 5: ('abcde', '0L'), + 6: ('abcdef', '0M'), + 7: ('abcdefg', 'Y'), + 8: ('abcdefgh', '1AAN'), + 9: ('abcdefghi', '0N'), + 10: ('abcdefghij', '0O') + } + """ Done Test """ + + def test_verser(): """ Test Verser version primitive subclass of Matter @@ -6980,6 +7065,7 @@ def test_tholder(): test_matter_class() test_matter() test_matter_special() + test_tagger() test_verser() #test_texter() #test_counter() From d8af121b3b9e22fe4b92862fd4617c8ef4ff7740 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 31 Mar 2024 18:19:58 -0600 Subject: [PATCH 092/418] refactor Verser to be subclass of Tagger --- src/keri/core/coring.py | 128 ++++++++++++++++++------------------- src/keri/core/serdering.py | 5 +- tests/core/test_coring.py | 125 ++++++++++++++++++++++++++++++------ 3 files changed, 169 insertions(+), 89 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 485b282fb..f02ad5c60 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -1971,9 +1971,10 @@ class Tagger(Matter): composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip Properties: + tag (str): B64 primitive without prepad (strips prepad from soft) - Hidden Inherited: + Inherited Hidden: (See Matter) _code (str): value for .code property _soft (str): soft value of full code _raw (bytes): value for .raw property @@ -2040,7 +2041,8 @@ def __init__(self, tag='', soft='', code=None, **kwa): @property def tag(self): - """Property tag (str): plain without prepad (strips prepad from soft) + """Returns: + tag (str): B64 primitive without prepad (strips prepad from soft) """ tag = self.soft @@ -2061,76 +2063,81 @@ def tag(self): Versage = namedtuple("Versage", "proto vrsn gvrsn", defaults=(None, )) -class Verser(Matter): +class Verser(Tagger): """ - Verser is subclass of Matter, cryptographic material, for formatted - version primitives in Base64. Supports different primitives based on code. + Verser is subclass of Tagger, cryptographic material, for formatted + version primitives in Base64. Leverages Tagger support compact special + fixed size primitives with non-empty soft part and empty raw part. Verser provides a more compact representation than would be obtained by converting the raw ASCII representation to Base64. Attributes: - Inherited Properties: (See Matter) + Inherited Properties: (See Tagger) code (str): hard part of derivation code to indicate cypher suite - both (int): hard and soft parts of full text code - size (int): Number of triplets of bytes including lead bytes - (quadlets of chars) of variable sized material. Value of soft size, - ss, part of full text code. - Otherwise None. - rize (int): number of bytes of raw material not including - lead bytes - raw (bytes): crypto material only without code + hard (str): hard part of derivation code. alias for code + soft (str): soft part of derivation code fs any. + Empty when ss = 0. + both (str): hard + soft parts of full text code + size (int | None): Number of quadlets/triplets of chars/bytes including + lead bytes of variable sized material (fs = None). + Converted value of the soft part (of len ss) of full + derivation code. + Otherwise None when not variably sized (fs != None) + fullSize (int): full size of primitive + raw (bytes): crypto material only. Not derivation code or lead bytes. qb64 (str): Base64 fully qualified with derivation code + crypto mat qb64b (bytes): Base64 fully qualified with derivation code + crypto mat qb2 (bytes): binary with derivation code + crypto material transferable (bool): True means transferable derivation code False otherwise digestive (bool): True means digest derivation code False otherwise prefixive (bool): True means identifier prefix derivation code False otherwise + special (bool): True when soft is special raw is empty and fixed size + composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip + tag (str): B64 primitive without prepad (strips prepad from soft) + Properties: - proto (str): protocol from Protocols - vrsn (Versionage): instance protocol version. - namedtuple (major, minor) of ints - gvrsn (Versionage | None): instance genus version. - namedtuple (major, minor) of ints + versage (Versage): named tuple of (proto, vrsn, gvrsn) - Hidden Inherited: + Inherited Hidden: (See Tagger) _code (str): value for .code property + _soft (str): soft value of full code _raw (bytes): value for .raw property - _rsize (bytes): value for .rsize property. Raw size in bytes when - variable sized material else None. - _size (int): value for .size property. Number of triplets of bytes - including lead bytes (quadlets of chars) of variable sized material - else None. - _infil (types.MethodType): creates qb64b from .raw and .code - (fully qualified Base64) - _exfil (types.MethodType): extracts .code and .raw from qb64b - (fully qualified Base64) + _rawSize(): + _leadSize(): + _special(): + _infil(): creates qb64b from .raw and .code (fully qualified Base64) + _binfil(): creates qb2 from .raw and .code (fully qualified Base2) + _exfil(): extracts .code and .raw from qb64b (fully qualified Base64) + _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) Hidden: - _proto (str): value for .proto property - _vrsn (Versionage): value for .vrsn property - _gvrsn (Versionage | None): value for .gvrsn property + Methods: """ - def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, - code=MtrDex.Tag10, versage=None, proto=Protocols.keri, - vrsn=Vrsn_2_0, gvrsn=Vrsn_2_0, **kwa): + def __init__(self, qb64b=None, qb64=None, qb2=None, versage=None, + proto=Protocols.keri, vrsn=Vrsn_2_0, gvrsn=None, tag='', **kwa): """ - Inherited Parameters: (see Matter) - raw (bytes): unqualified crypto material usable for crypto operations + Inherited Parameters: (see Tagger) + raw (bytes | bytearray | None): unqualified crypto material usable + for crypto operations. code (str): stable (hard) part of derivation code - rize (int): raw size in bytes when variable sized material else None - qb64b (bytes): fully qualified crypto material Base64 - qb64 (str, bytes): fully qualified crypto material Base64 - qb2 (bytes): fully qualified crypto material Base2 + soft (str | bytes): soft part for special codes + rize (int | None): raw size in bytes when variable sized material not + including lead bytes if any + Otherwise None + qb64b (bytes | None): fully qualified crypto material Base64 + qb64 (str | bytes | None): fully qualified crypto material Base64 + qb2 (bytes | None): fully qualified crypto material Base2 strip (bool): True means strip (delete) matter from input stream bytearray after parsing qb64b or qb2. False means do not strip + tag (str | bytes): Base64 plain. Prepad is added as needed. Parameters: versage (Versage | None): namedtuple of (proto, vrsn, gvrsn) @@ -2143,45 +2150,32 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, Notes: prepad = 'A' """ - if raw is None and qb64b is None and qb64 is None and qb2 is None: + if not (qb64b or qb64 or qb2): if versage: proto, vrsn, gvrsn = versage - if code == MtrDex.Tag10: - if not gvrsn: - raise InvalidValueError(f"Missing genus version.") - qb64 = (code + - proto + - self.verToB64(vrsn) + - self.verToB64(gvrsn)).encode("utf-8") + tag = proto + self.verToB64(vrsn) - elif code == MtrDex.Tag7: - qb64 = (code + proto + self.verToB64(vrsn)).encode("utf-8") + if gvrsn: + tag += self.verToB64(gvrsn) - else: - raise InvalidCodeError(f"Invalid {code=} for Verser.") - - - super(Verser, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, - code=code, **kwa) + super(Verser, self).__init__(qb64b=qb64b, qb64=qb64, qb2=qb2, tag=tag, **kwa) - if self.code not in (MtrDex.Tag10, MtrDex.Tag7): - raise InvalidCodeError(f"Invalid code={self.code} for Verser.") + if self.code not in (MtrDex.Tag7, MtrDex.Tag10, ): + raise InvalidCodeError(f"Invalid code={self.code} for Verser " + f"{self.tag=}.") - if not self._special(self.code): - raise InvalidCodeError(f"Not special code={self.code} for Verser.") @property def versage(self): - """Property versage (Versage): named tuple of (proto, vrsn, gvrsn) + """Returns: + versage (Versage): named tuple of (proto, vrsn, gvrsn) """ gvrsn = None - clp = len(self.code) - proto = self.qb64[clp:clp+4] - vrsn = self.b64ToVer(self.qb64[clp+4:clp+7]) - if self.fullSize == clp + 10: # assumes special - gvrsn = self.b64ToVer(self.qb64[clp+7:clp+10]) + proto = self.tag[:4] + vrsn = self.b64ToVer(self.tag[4:7]) + gvrsn = self.b64ToVer(self.tag[7:10]) if len(self.tag) == 10 else None return Versage(proto=proto, vrsn=vrsn, gvrsn=gvrsn) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 50a7bfc2f..89bf53807 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1243,10 +1243,7 @@ def _dumps(self, sad): # should dispatch or use match instead of big if else match l: # label case "v": # protocol+version - val = Verser(code=MtrDex.Tag10, - proto=self.proto, - vrsn=self.vrsn, - gvrsn=self.gvrsn).qb64b + val = Verser(proto=self.proto, vrsn=self.vrsn).qb64b case "t": # message type val = (MtrDex.Tag3 + ilk).encode("utf-8") # add code diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 2da05af8f..32963378f 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -29,7 +29,7 @@ InvalidValueError, DeserializeError, ValidationError, InvalidVarRawSizeError, ConversionError, SoftMaterialError, InvalidSoftError) -from keri.kering import Version, Versionage, VersionError +from keri.kering import Version, Versionage, VersionError, Vrsn_1_0, Vrsn_2_0 from keri.core import coring from keri.core import eventing @@ -4261,50 +4261,139 @@ def test_verser(): """ Test Verser version primitive subclass of Matter """ - code = MtrDex.Tag10 - soft = 'KERICAACAA' - qb64 = '0OKERICAACAA' - qb2 = b'\xd0\xe2\x84D\x80\x80\x00 \x00' + # Test defaults + code = MtrDex.Tag7 + soft = 'KERICAA' + tag = 'KERICAA' + qb64 = 'YKERICAA' + qb64b = qb64.encode() + qb2 = decodeB64(qb64b) raw = b'' + versage = Versage(proto=Protocols.keri, vrsn=Vrsn_2_0, gvrsn=None) verser = Verser() # defaults assert verser.code == verser.hard == code assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.composable + assert verser.versage == versage + + # test with default equivalent values + verser = Verser(versage=versage) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.composable + assert verser.versage == versage + + verser = Verser(proto=Protocols.keri, vrsn=Vrsn_2_0) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + verser = Verser(qb2=qb2) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + verser = Verser(qb64=qb64) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + verser = Verser(qb64b=qb64b) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag assert verser.raw == raw assert verser.qb64 == qb64 assert verser.qb2 == qb2 assert verser.special - assert verser.versage == Versage(proto='KERI', - vrsn=Versionage(major=2, minor=0), - gvrsn=Versionage(major=2, minor=0)) + assert verser.versage == versage - code = verser.code - soft = verser.soft - qb2 = verser.qb2 - qb64 = verser.qb64 + # Test with gvrsn + code = MtrDex.Tag10 + soft = 'ACDCCAACAA' + tag = 'ACDCCAACAA' + qb64 = '0OACDCCAACAA' + qb64b = qb64.encode() + qb2 = decodeB64(qb64b) + raw = b'' + versage = Versage(proto=Protocols.acdc, vrsn=Vrsn_2_0, gvrsn=Vrsn_2_0) + + verser = Verser(versage=versage) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.composable + assert verser.versage == versage + + verser = Verser(proto=Protocols.acdc, vrsn=Vrsn_2_0, gvrsn=Vrsn_2_0) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage verser = Verser(qb2=qb2) assert verser.code == verser.hard == code assert verser.soft == soft + assert verser.tag == tag assert verser.raw == raw assert verser.qb64 == qb64 assert verser.qb2 == qb2 assert verser.special - assert verser.versage == Versage(proto='KERI', - vrsn=Versionage(major=2, minor=0), - gvrsn=Versionage(major=2, minor=0)) + assert verser.versage == versage verser = Verser(qb64=qb64) assert verser.code == verser.hard == code assert verser.soft == soft + assert verser.tag == tag assert verser.raw == raw assert verser.qb64 == qb64 assert verser.qb2 == qb2 assert verser.special - assert verser.versage == Versage(proto='KERI', - vrsn=Versionage(major=2, minor=0), - gvrsn=Versionage(major=2, minor=0)) + assert verser.versage == versage + verser = Verser(qb64b=qb64b) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage """ Done Test """ From 2b02fa6c6fceddb5c256e7784358092de3c5ea3c Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 29 Mar 2024 13:14:35 -0700 Subject: [PATCH 093/418] Resolving rebase conflict Signed-off-by: pfeairheller --- src/keri/app/cli/commands/rename.py | 17 ++- src/keri/app/habbing.py | 163 ++++++++-------------------- src/keri/db/basing.py | 44 +------- src/keri/db/versioning.py | 24 ++++ tests/app/test_habbing.py | 34 ++---- tests/app/test_signify.py | 4 +- tests/core/test_keystate.py | 4 +- tests/db/test_basing.py | 4 +- 8 files changed, 102 insertions(+), 192 deletions(-) create mode 100644 src/keri/db/versioning.py diff --git a/src/keri/app/cli/commands/rename.py b/src/keri/app/cli/commands/rename.py index 60fff68ef..a12420650 100644 --- a/src/keri/app/cli/commands/rename.py +++ b/src/keri/app/cli/commands/rename.py @@ -45,12 +45,19 @@ def rename(tymth, tock=0.0, **opts): try: with existing.existingHab(name=name, alias=alias, base=base, bran=bran) as (hby, hab): - habord = hab.db.habs.get(keys=alias) - hab.db.habs.put(keys=newAlias, - val=habord) - hab.db.habs.rem(keys=alias) + if hby.habByName(newAlias) is not None: + print(f"{newAlias} is already in use") - print(f"Hab {alias} renamed to {newAlias}") + if (prefixer := hab.db.names.get(keys=("", name))) is not None: + + habord = hab.db.habs.get(keys=prefixer.qb64) + habord.name = name + hab.db.habs.pin(keys=habord.hid, + val=habord) + hab.db.names.pin(keys=("", name), val=prefixer) + hab.db.names.rem(keys=("", alias)) + + print(f"Hab {alias} renamed to {newAlias}") except ConfigurationError as e: print(f"identifier prefix for {name} does not exist, incept must be run first", ) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 4fec262fa..795a8000a 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -226,7 +226,6 @@ def __init__(self, *, name='test', base="", temp=False, self.psr = parsing.Parser(framed=True, kvy=self.kvy, rvy=self.rvy, exc=self.exc, local=True) self.habs = {} # empty .habs - self.namespaces = {} # empty .namespaces self._signator = None self.inited = False @@ -327,86 +326,46 @@ def loadHabs(self): self.reconfigure() # pre hab load reconfiguration groups = [] - for name, habord in self.db.habs.getItemIter(): - name = ".".join(name) # detupleize the database key name + for prefix, habord in self.db.habs.getItemIter(): pre = habord.hid # create Hab instance and inject dependencies if habord.mid and not habord.sid: hab = GroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=pre, temp=self.temp, smids=habord.smids) + name=habord.name, pre=pre, temp=self.temp, smids=habord.smids) groups.append(habord) elif habord.sid and not habord.mid: hab = SignifyHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=habord.sid) + name=habord.name, pre=habord.sid) elif habord.sid and habord.mid: hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=pre) + name=habord.name, pre=pre) groups.append(habord) else: hab = Hab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=pre, temp=self.temp) + name=habord.name, pre=pre, temp=self.temp) # Rules for acceptance: # It is accepted into its own local KEL even if it has not been fully # witnessed and if delegated, its delegator has not yet sealed it if not hab.accepted and not habord.mid: raise kering.ConfigurationError(f"Problem loading Hab pre=" - f"{pre} name={name} from db.") + f"{pre} name={habord.name} from db.") # read in config file and process any oobis or endpoints for hab hab.inited = True self.habs[hab.pre] = hab - for keys, habord in self.db.nmsp.getItemIter(): - ns = keys[0] - name = ".".join(keys[1:]) # detupleize the database key name - pre = habord.hid - - # create Hab instance and inject dependencies - if habord.mid and not habord.sid: - hab = GroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, ns=ns, pre=pre, temp=self.temp, smids=habord.smids) - groups.append(habord) - elif habord.sid and not habord.mid: - hab = SignifyHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, ns=ns, pre=habord.sid) - elif habord.sid and habord.mid: - hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=habord.sid) - groups.append(habord) - else: - hab = Hab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, ns=ns, pre=pre, temp=self.temp) - - # Rules for acceptance: - # It is accepted into its own local KEL even if it has not been fully - # witnessed and if delegated, its delegator has not yet sealed it - if not hab.accepted and not habord.mid: - raise kering.ConfigurationError(f"Problem loading Hab pre=" - f"{pre} name={name} from db.") - - # read in config file and process any oobis or endpoints for hab - hab.inited = True - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - # Populate the participant hab after loading all habs for habord in groups: self.habs[habord.hid].mhab = self.habs[habord.mid] self.reconfigure() # post hab load reconfiguration - def makeHab(self, name, ns=None, cf=None, **kwa): """Make new Hab with name, pre is generated from **kwa @@ -437,13 +396,7 @@ def makeHab(self, name, ns=None, cf=None, **kwa): hab.make(**kwa) - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def makeGroupHab(self, group, mhab, smids, rmids=None, ns=None, **kwa): @@ -514,13 +467,7 @@ def makeGroupHab(self, group, mhab, smids, rmids=None, ns=None, **kwa): name=group, ns=ns, mhab=mhab, smids=smids, rmids=rmids, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def joinGroupHab(self, pre, group, mhab, smids, rmids=None, ns=None): @@ -568,6 +515,8 @@ def joinGroupHab(self, pre, group, mhab, smids, rmids=None, ns=None): hab.pre = pre habord = basing.HabitatRecord(hid=hab.pre, + name=self.name, + domain=ns, mid=mhab.pre, smids=smids, rmids=rmids) @@ -576,13 +525,7 @@ def joinGroupHab(self, pre, group, mhab, smids, rmids=None, ns=None): hab.prefixes.add(pre) hab.inited = True - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def makeSignifyHab(self, name, ns=None, **kwa): @@ -592,13 +535,7 @@ def makeSignifyHab(self, name, ns=None, **kwa): name=name, ns=ns, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def makeSignifyGroupHab(self, name, mhab, ns=None, **kwa): @@ -608,13 +545,8 @@ def makeSignifyGroupHab(self, name, mhab, ns=None, **kwa): name=name, mhab=mhab, ns=ns, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab + self.habs[hab.pre] = hab return hab def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): @@ -663,6 +595,8 @@ def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): hab.pre = pre habord = basing.HabitatRecord(hid=hab.pre, sid=mhab.pre, + name=name, + domain=ns, smids=smids, rmids=rmids) @@ -670,21 +604,19 @@ def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): hab.prefixes.add(pre) hab.inited = True - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab - def deleteHab(self, name): - hab = self.habByName(name) + def deleteHab(self, name, ns=None): + hab = self.habByName(name, ns=ns) if not hab: return False - if not self.db.habs.rem(keys=(name,)): + if not self.db.habs.rem(keys=(hab.pre,)): + return False + + ns = "" if ns is None else ns + if not self.db.names.rem(keys=(ns, name)): return False del self.habs[hab.pre] @@ -760,7 +692,7 @@ def prefixes(self): def habByPre(self, pre): """ - Returns the Hab instance from .habs or .namespace + Returns the Hab instance from .habs or None including the default namespace. Args: @@ -769,15 +701,10 @@ def habByPre(self, pre): Returns: Hab: Hab instance for the aid pre or None """ - hab = None if pre in self.habs: - hab = self.habs[pre] - else: - for nsp in self.namespaces.values(): - if pre in nsp: - hab = nsp[pre] + return self.habs[pre] - return hab + return None def habByName(self, name, ns=None): """ @@ -790,13 +717,11 @@ def habByName(self, name, ns=None): ns (str): optional namespace of hab """ - if ns is not None: - if (habord := self.db.nmsp.get(keys=(ns, name))) is not None: - habs = self.namespaces[ns] - return habs[habord.hid] if habord.hid in habs else None - - elif (habord := self.db.habs.get(name)) is not None: - return self.habs[habord.hid] if habord.hid in self.habs else None + ns = "" if ns is None else ns + if (prefixer := self.db.names.get(keys=(ns, name))) is not None: + pre = prefixer.qb64 + if pre in self.habs: + return self.habs[pre] return None @@ -1131,12 +1056,11 @@ def make(self, DnD, code, data, delpre, estOnly, isith, verfers, nsith, digers, return serder def save(self, habord): - if self.ns is None: - self.db.habs.pin(keys=self.name, - val=habord) - else: - self.db.nmsp.put(keys=(self.ns, self.name), - val=habord) + self.db.habs.pin(keys=self.pre, + val=habord) + ns = "" if self.ns is None else self.ns + self.db.names.put(keys=(ns, self.name), + val=coring.Prefixer(qb64=self.pre)) def reconfigure(self): """Apply configuration from config file managed by .cf. to this Hab. @@ -2349,7 +2273,7 @@ def make(self, *, secrecies=None, iridx=0, code=coring.MtrDex.Blake3_256, dcode= self.mgr.move(old=opre, new=self.pre) # move to incept event pre # may want db method that updates .habs. and .prefixes together - habord = basing.HabitatRecord(hid=self.pre) + habord = basing.HabitatRecord(hid=self.pre, name=self.name, domain=self.ns) if not hidden: self.save(habord) @@ -2437,7 +2361,7 @@ def make(self, *, serder, sigers, **kwargs): self.processEvent(serder, sigers) - habord = basing.HabitatRecord(hid=self.pre, sid=self.pre) + habord = basing.HabitatRecord(hid=self.pre, sid=self.pre, name=self.name, domain=self.ns) self.save(habord) self.inited = True @@ -2596,7 +2520,7 @@ def make(self, *, serder, sigers, **kwargs): self.processEvent(serder, sigers) - habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, sid=self.pre) + habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, sid=self.pre, name=self.name, domain=self.ns) self.save(habord) self.inited = True @@ -2622,6 +2546,10 @@ def processEvent(self, serder, sigers): raise kering.ValidationError(f"Improper Habitat event type={serder.ked['t']} for " f"pre={self.pre}.") + def rotate(self, *, smids=None, rmids=None, serder=None, sigers=None, **kwargs): + # TODO: save smids and rmids here + super(SignifyGroupHab, self).rotate(serder=serder, sigers=sigers, **kwargs) + class GroupHab(BaseHab): """ @@ -2785,6 +2713,8 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, + name=self.name, + domain=self.ns, smids=self.smids, rmids=self.rmids) @@ -2793,7 +2723,8 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, self.inited = True - def rotate(self, serder=None, **kwargs): + def rotate(self, smids=None, rmids=None, serder=None, **kwargs): + # TODO: save smids and rmids here if serder is None: return super(GroupHab, self).rotate(**kwargs) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index d8bf0039e..f3021f501 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -274,6 +274,8 @@ class HabitatRecord: # baser.habs """ hid: str # hab own identifier prefix qb64 + name: str | None = None + domain: str | None = None mid: str | None = None # group member identifier qb64 when hid is group smids: list | None = None # group signing member ids when hid is group rmids: list | None = None # group rotating member ids when hid is group @@ -953,11 +955,8 @@ def reopen(self, **kwa): self.habs = koming.Komer(db=self, subkey='habs.', schema=HabitatRecord, ) - - # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix - self.nmsp = koming.Komer(db=self, - subkey='nmsp.', - schema=HabitatRecord, ) + # habitat name database mapping (domain,name) as key to Prefixer + self.names = subing.CesrSuber(db=self, subkey='names.', klas=coring.Prefixer, sep="^") # SAD support datetime stamps and signatures indexed and not-indexed # all sad sdts (sad datetime serializations) maps said to date-time @@ -1173,7 +1172,6 @@ def reopen(self, **kwa): return self.env - def reload(self): """ Reload stored prefixes and Kevers from .habs @@ -1200,28 +1198,6 @@ def reload(self): for keys in removes: # remove bare .habs records self.habs.rem(keys=keys) - # Load namespaced Habs - removes = [] - for keys, data in self.nmsp.getItemIter(): - if (ksr := self.states.get(keys=data.hid)) is not None: - try: - kever = eventing.Kever(state=ksr, - db=self, - local=True) - except kering.MissingEntryError as ex: # no kel event for keystate - removes.append(keys) # remove from .habs - continue - self.kevers[kever.prefixer.qb64] = kever - self.prefixes.add(kever.prefixer.qb64) - if data.mid: # group hab - self.groups.add(data.hid) - elif data.mid is None: # in .habs but no corresponding key state and not a group so remove - removes.append(keys) # no key state or KEL event for .hab record - - for keys in removes: # remove bare .habs records - self.nmsp.rem(keys=keys) - - def clean(self): """ Clean database by creating re-verified cleaned cloned copy @@ -1259,17 +1235,12 @@ def clean(self): for keys, val in self.habs.getItemIter(): if val.hid in copy.kevers: # only copy habs that verified copy.habs.put(keys=keys, val=val) + ns = "" if val.domain is None else val.domain + copy.names.put(keys=(ns, val.name), val=coring.Prefixer(qb64=val.hid)) copy.prefixes.add(val.hid) if val.mid: # a group hab copy.groups.add(val.hid) - # ToDo XXXX - # is this obsolete? Should this be removed or should this be - # be the Signator name not the default name of the Habery? - if not copy.habs.get(keys=(self.name,)): - raise ValueError("Error cloning habs, missing orig name={}." - "".format(self.name)) - # clone .ends and .locs databases for keys, val in self.ends.getItemIter(): exists = False # only copy if entries in both .ends and .locs @@ -1315,7 +1286,6 @@ def clean(self): if os.path.exists(copy.path): shutil.rmtree(copy.path) - def clonePreIter(self, pre, fn=0): """ Returns iterator of first seen event messages with attachments for the @@ -1504,8 +1474,6 @@ def findAnchoringSeal(self, pre, seal, sn=0): return srdr return None - - def signingMembers(self, pre: str): """ Find signing members of a multisig group aid. diff --git a/src/keri/db/versioning.py b/src/keri/db/versioning.py new file mode 100644 index 000000000..d1e6b21c7 --- /dev/null +++ b/src/keri/db/versioning.py @@ -0,0 +1,24 @@ +# -*- encoding: utf-8 -*- +""" +keri.db.dbing module + + +""" +from keri.db import koming +from keri.db.basing import HabitatRecord + + +class Upgrader: + + def __init__(self, db): + self.db = db + + # habitat application state keyed by habitat name, includes prefix + self.habs = koming.Komer(db=self.db, + subkey='habs.', + schema=HabitatRecord, ) + + # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix + self.nmsp = koming.Komer(db=self.db, + subkey='nmsp.', + schema=HabitatRecord, ) diff --git a/tests/app/test_habbing.py b/tests/app/test_habbing.py index ed35dd295..9b23675d4 100644 --- a/tests/app/test_habbing.py +++ b/tests/app/test_habbing.py @@ -727,8 +727,7 @@ def test_namespaced_habs(): nshab = hby.makeHab(name="test2", ns="agent") assert nshab.pre == "EErXOolQNmKrTMKfXdQ1sj8YsgZZe4wMXZwsX-j1V6Dd" - assert len(hby.habs) == 1 - assert len(hby.namespaces) == 1 + assert len(hby.habs) == 2 assert len(hby.prefixes) == 2 found = hby.habByName(name="test2") @@ -742,11 +741,8 @@ def test_namespaced_habs(): nshab = hby.makeHab(name="test.3", ns="agent") assert nshab.pre == "EG5FUOzW_KKVB8JGlNGoZAADDC8cZ6Jt079nLEaFnYcg" - assert len(hby.habs) == 1 - assert len(hby.namespaces) == 1 + assert len(hby.habs) == 3 assert len(hby.prefixes) == 3 - ns = hby.namespaces['agent'] - assert len(ns) == 2 # '.' characters not allowed in namespace names with pytest.raises(kering.ConfigurationError): @@ -773,14 +769,9 @@ def test_namespaced_habs(): assert pre in hby.db.kevers # read through cache assert pre in hby.db.prefixes - assert len(hby.habs) == 2 + assert len(hby.habs) == 5 assert len(hby.db.prefixes) == 5 - agent = hby.namespaces["agent"] - assert len(agent) == 2 - ctrl = hby.namespaces["controller"] - assert len(ctrl) == 1 - found = hby.habByName(name=name) assert found.pre == opre found = hby.habByName(name="test.1") @@ -840,10 +831,10 @@ def test_hab_by_pre(): # Only habs in default namespace are in hby.habs assert hab1.pre in hby.habs assert hab2.pre in hby.habs - assert hab3.pre not in hby.habs - assert hab4.pre not in hby.habs - assert hab5.pre not in hby.habs - assert hab6.pre not in hby.habs + assert hab3.pre in hby.habs + assert hab4.pre in hby.habs + assert hab5.pre in hby.habs + assert hab6.pre in hby.habs assert hby.habByPre("EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3") is None @@ -854,17 +845,6 @@ def test_hab_by_pre(): assert hby.habByPre(pre=hab5.pre) == hab5 assert hby.habByPre(pre=hab6.pre) == hab6 - assert "one" in hby.namespaces - assert hab3.pre in hby.namespaces["one"] - assert hab4.pre in hby.namespaces["one"] - assert hab1.pre not in hby.namespaces["one"] - assert hab2.pre not in hby.namespaces["one"] - assert "two" in hby.namespaces - assert hab5.pre in hby.namespaces["two"] - assert hab6.pre in hby.namespaces["two"] - assert hab1.pre not in hby.namespaces["two"] - assert hab2.pre not in hby.namespaces["two"] - def test_postman_endsfor(): with habbing.openHby(name="test", temp=True, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby, \ diff --git a/tests/app/test_signify.py b/tests/app/test_signify.py index 4afaa4647..1d9a7d08f 100644 --- a/tests/app/test_signify.py +++ b/tests/app/test_signify.py @@ -71,7 +71,7 @@ def test_remote_salty_hab(): assert [verfer.qb64 for verfer in kever.verfers] == keys assert [diger.qb64 for diger in kever.ndigers] == nxt - habord = remote.db.habs.get(name) + habord = remote.db.habs.get(hab.pre) assert habord.hid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" assert habord.sid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" @@ -113,7 +113,7 @@ def test_remote_salty_hab(): assert [verfer.qb64 for verfer in kever.verfers] == keys1 assert [diger.qb64 for diger in kever.ndigers] == nxt1 - habord = remote.db.habs.get(name) + habord = remote.db.habs.get(hab.pre) assert habord.hid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" assert habord.sid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" diff --git a/tests/core/test_keystate.py b/tests/core/test_keystate.py index 40c2c40d3..a9160a10f 100644 --- a/tests/core/test_keystate.py +++ b/tests/core/test_keystate.py @@ -163,9 +163,9 @@ def test_keystate(mockHelpingNowUTC): bamHab = bamHby.makeHab(name="bam", isith='1', icount=1, transferable=True) # Set Wes has Bam's watcher - habr = bamHab.db.habs.get("bam") + habr = bamHab.db.habs.get(bamHab.pre) habr.watchers = [wesHab.pre] - bamHab.db.habs.pin("bam", habr) + bamHab.db.habs.pin(bamHab.pre, habr) bamRtr = routing.Router() bamRvy = routing.Revery(db=bamHby.db, rtr=bamRtr) diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 791e88095..575342f21 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -1753,7 +1753,7 @@ def test_clean_baser(): assert natHab.db.env.stat()['entries'] <= 96 #68 # verify name pre kom in db - data = natHab.db.habs.get(keys=natHab.name) + data = natHab.db.habs.get(keys=natHab.pre) assert data.hid == natHab.pre # add garbage event to corrupt database @@ -1812,7 +1812,7 @@ def test_clean_baser(): assert state.f == '6' # verify name pre kom in db - data = natHab.db.habs.get(keys=natHab.name) + data = natHab.db.habs.get(keys=natHab.pre) assert data.hid == natHab.pre From 694e3066add4f458a648dc39479229c47f17a9b6 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Thu, 29 Feb 2024 15:17:52 -0800 Subject: [PATCH 094/418] Change names database to Suber, add check for Hab with existing name. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/rename.py | 6 +++--- src/keri/app/habbing.py | 10 ++++++---- src/keri/db/basing.py | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/keri/app/cli/commands/rename.py b/src/keri/app/cli/commands/rename.py index a12420650..af0fb0cf6 100644 --- a/src/keri/app/cli/commands/rename.py +++ b/src/keri/app/cli/commands/rename.py @@ -48,13 +48,13 @@ def rename(tymth, tock=0.0, **opts): if hby.habByName(newAlias) is not None: print(f"{newAlias} is already in use") - if (prefixer := hab.db.names.get(keys=("", name))) is not None: + if (pre := hab.db.names.get(keys=("", name))) is not None: - habord = hab.db.habs.get(keys=prefixer.qb64) + habord = hab.db.habs.get(keys=pre) habord.name = name hab.db.habs.pin(keys=habord.hid, val=habord) - hab.db.names.pin(keys=("", name), val=prefixer) + hab.db.names.pin(keys=("", name), val=pre) hab.db.names.rem(keys=("", alias)) print(f"Hab {alias} renamed to {newAlias}") diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 795a8000a..f146aca88 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -718,8 +718,7 @@ def habByName(self, name, ns=None): """ ns = "" if ns is None else ns - if (prefixer := self.db.names.get(keys=(ns, name))) is not None: - pre = prefixer.qb64 + if (pre := self.db.names.get(keys=(ns, name))) is not None: if pre in self.habs: return self.habs[pre] @@ -1059,8 +1058,11 @@ def save(self, habord): self.db.habs.pin(keys=self.pre, val=habord) ns = "" if self.ns is None else self.ns - self.db.names.put(keys=(ns, self.name), - val=coring.Prefixer(qb64=self.pre)) + if self.db.names.get(keys=(ns, self.name)) is not None: + raise ValueError("AID already exists with that name") + + self.db.names.pin(keys=(ns, self.name), + val=self.pre) def reconfigure(self): """Apply configuration from config file managed by .cf. to this Hab. diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index f3021f501..14daa5ab1 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -956,7 +956,7 @@ def reopen(self, **kwa): subkey='habs.', schema=HabitatRecord, ) # habitat name database mapping (domain,name) as key to Prefixer - self.names = subing.CesrSuber(db=self, subkey='names.', klas=coring.Prefixer, sep="^") + self.names = subing.Suber(db=self, subkey='names.', sep="^") # SAD support datetime stamps and signatures indexed and not-indexed # all sad sdts (sad datetime serializations) maps said to date-time @@ -1236,7 +1236,7 @@ def clean(self): if val.hid in copy.kevers: # only copy habs that verified copy.habs.put(keys=keys, val=val) ns = "" if val.domain is None else val.domain - copy.names.put(keys=(ns, val.name), val=coring.Prefixer(qb64=val.hid)) + copy.names.put(keys=(ns, val.name), val=val.hid) copy.prefixes.add(val.hid) if val.mid: # a group hab copy.groups.add(val.hid) From 7c89fcc7f96e67bbffc86ec9541d22eb296f3bd9 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 1 Mar 2024 12:42:31 -0800 Subject: [PATCH 095/418] Moving the tracking of signing members and rotation members from tracking public keys to static tracking of smids and rmids in the HabitatRecord in the database for the group Habs. Signed-off-by: pfeairheller --- .../demo/basic/demo-witness-async-script.sh | 22 ++++++------ src/keri/app/habbing.py | 30 +++++++++++++--- src/keri/core/eventing.py | 5 --- src/keri/db/basing.py | 35 ++++--------------- tests/app/test_grouping.py | 8 ++--- 5 files changed, 46 insertions(+), 54 deletions(-) diff --git a/scripts/demo/basic/demo-witness-async-script.sh b/scripts/demo/basic/demo-witness-async-script.sh index 57885a7cb..26b6aac89 100755 --- a/scripts/demo/basic/demo-witness-async-script.sh +++ b/scripts/demo/basic/demo-witness-async-script.sh @@ -13,34 +13,34 @@ function isSuccess() { } # CREATE DATABASE AND KEYSTORE -kli init --name witness-test --base "${KERI_TEMP_DIR}" --nopasscode +kli init --name async-witness-test --base "${KERI_TEMP_DIR}" --nopasscode isSuccess # RESOLVE WITNESS OOBIs -kli oobi resolve --name witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wan --oobi http://127.0.0.1:5643/oobi/BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM/controller +kli oobi resolve --name async-witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wan --oobi http://127.0.0.1:5643/oobi/BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM/controller isSuccess -kli oobi resolve --name witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wil --oobi http://127.0.0.1:5642/oobi/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha/controller +kli oobi resolve --name async-witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wil --oobi http://127.0.0.1:5642/oobi/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha/controller isSuccess -kli oobi resolve --name witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wes --oobi http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller +kli oobi resolve --name async-witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wes --oobi http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller isSuccess ## INCEPT AND PROPOGATE EVENTS AND RECEIPTS TO WITNESSES -kli incept --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" +kli incept --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" isSuccess -kli incept --name witness-test --base "${KERI_TEMP_DIR}" --alias inquisitor --file "${KERI_DEMO_SCRIPT_DIR}/data/inquisitor-sample.json" +kli incept --name async-witness-test --base "${KERI_TEMP_DIR}" --alias inquisitor --file "${KERI_DEMO_SCRIPT_DIR}/data/inquisitor-sample.json" isSuccess -kli status --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits +kli status --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits -kli rotate --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-cut BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +kli rotate --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-cut BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX isSuccess -kli status --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits +kli status --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits -kli rotate --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-add BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +kli rotate --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-add BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX isSuccess -kli status --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits +kli status --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits echo 'Test Complete' \ No newline at end of file diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index f146aca88..2dd9c594f 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -2512,8 +2512,11 @@ def replyEndRole(self, cid, role=None, eids=None, scheme=""): class SignifyGroupHab(SignifyHab): - def __init__(self, mhab=None, **kwa): + def __init__(self, smids, mhab=None, rmids=None, **kwa): self.mhab = mhab + self.smids = smids # group signing member aids in this group hab + self.rmids = rmids or smids # group rotating member aids in this group hab + super(SignifyGroupHab, self).__init__(**kwa) def make(self, *, serder, sigers, **kwargs): @@ -2549,9 +2552,19 @@ def processEvent(self, serder, sigers): f"pre={self.pre}.") def rotate(self, *, smids=None, rmids=None, serder=None, sigers=None, **kwargs): - # TODO: save smids and rmids here + + if (habord := self.db.habs.get(keys=(self.pre,))) is None: + raise kering.ValidationError(f"Missing HabitatRecord for pre={self.pre}") + super(SignifyGroupHab, self).rotate(serder=serder, sigers=sigers, **kwargs) + self.smids = smids + self.rmids = rmids + habord.smids = smids + habord.rmids = rmids + + self.db.habs.pin(keys=(self.pre,), val=habord) + class GroupHab(BaseHab): """ @@ -2633,11 +2646,10 @@ def __init__(self, smids, mhab=None, rmids=None, **kwa): """ self.mhab = mhab # local participant Hab of this group hab self.smids = smids # group signing member aids in this group hab - self.rmids = rmids # group rotating member aids in this group hab + self.rmids = rmids or smids # group rotating member aids in this group hab super(GroupHab, self).__init__(**kwa) - def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, nsith=None, toad=None, wits=None, delpre=None, estOnly=False, DnD=False, merfers, migers=None, data=None): @@ -2726,11 +2738,13 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, self.inited = True def rotate(self, smids=None, rmids=None, serder=None, **kwargs): - # TODO: save smids and rmids here if serder is None: return super(GroupHab, self).rotate(**kwargs) + if (habord := self.db.habs.get(keys=(self.pre,))) is None: + raise kering.ValidationError(f"Missing HabitatRecord for pre={self.pre}") + # sign handles group hab with .mhab case sigers = self.sign(ser=serder.raw, verfers=serder.verfers, rotated=True) @@ -2745,6 +2759,12 @@ def rotate(self, smids=None, rmids=None, serder=None, **kwargs): raise kering.ValidationError("Improper Habitat rotation for " "pre={self.pre}.") from ex + self.smids = smids + self.rmids = rmids + habord.smids = smids + habord.rmids = rmids + self.db.habs.pin(keys=(self.pre,), val=habord) + return msg def sign(self, ser, verfers=None, indexed=True, rotated=False, indices=None, ondices=None): diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index d0c063aec..1d3e860f8 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2967,11 +2967,6 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, esr = basing.EventSourceRecord(local=local) self.db.esrs.put(keys=dgkeys, val=esr) - val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) - for verfer in (serder.verfers if serder.verfers is not None else []): - self.db.pubs.add(keys=(verfer.qb64,), val=val) - for diger in (serder.ndigers if serder.ndigers is not None else []): - self.db.digs.add(keys=(diger.qb64,), val=val) if first: # append event dig to first seen database in order if seqner and saider: # delegation for authorized delegated or issued event couple = seqner.qb64b + saider.qb64b diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 14daa5ab1..8fc1cc1b0 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1486,22 +1486,10 @@ def signingMembers(self, pre: str): list: qb64 identifier prefixes of signing members for provided aid """ - members = [] - if pre not in self.kevers: - return members - - kever = self.kevers[pre] - for verfer in kever.verfers: - if (couples := self.pubs.get(keys=(verfer.qb64,))) is None: - continue - - for couple in couples: - prefixer, seqner = couple - if prefixer.qb64 != pre: # Rule out aid being queried - members.append(prefixer.qb64) - - return members + if (habord := self.habs.get(keys=(pre,))) is None: + return None + return habord.smids def rotationMembers(self, pre: str): """ Find rotation members of a multisig group aid. @@ -1514,21 +1502,10 @@ def rotationMembers(self, pre: str): Returns: list: qb64 identifier prefixes of rotation members for provided aid """ - members = [] - if pre not in self.kevers: - return members - - kever = self.kevers[pre] - for diger in kever.ndigers: - if (couples := self.digs.get(keys=(diger.qb64,))) is None: - continue - - for couple in couples: - prefixer, seqner = couple - if prefixer.qb64 != pre: # Rule out aid being queried - members.append(prefixer.qb64) + if (habord := self.habs.get(keys=(pre,))) is None: + return None - return members + return habord.rmids def fullyWitnessed(self, serder): """ Verify the witness threshold on the event diff --git a/tests/app/test_grouping.py b/tests/app/test_grouping.py index 5504c5781..fca1bc91d 100644 --- a/tests/app/test_grouping.py +++ b/tests/app/test_grouping.py @@ -662,11 +662,11 @@ def test_multisig_rotate(mockHelpingNowUTC): exn, atc = grouping.multisigRotateExn(ghab=ghab1, smids=ghab1.smids, rmids=ghab1.rmids, rot=rot) assert exn.ked["r"] == '/multisig/rot' - assert exn.saidb == b'ENfCk9DUUck6Ixe6cYnbCbJfIsisA3H4kHPwm5Z-2Tf8' + assert exn.saidb == b'EAE5RLyDb_3W8fUOzDOHWwpMAvHePaz8Jpz1XWL0b0lQ' assert atc == (b'-FABEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba0AAAAAAAAAAAAAAA' - b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAADChiAf' - b'iExAQ2ETkzzf7MOubXV9mL-r6fPsOI4yn348yeE5dXqdI7ddn5-wnPwNVjqqKkDp' - b'xlOEFYRiBQEbwZQC') + b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAACzqlzb' + b'yRO_M5Kp6nyN1K5JZ9s9D4UAreliSs4OiyCe_y9KnDeP65tub3DW5hUKlVQWnPv4' + b'TrOBHk2vJNOEJtAF') data = exn.ked["a"] assert data["smids"] == ghab1.smids From f24fceda4be5f8700fcfb98282e1c4a43fb774d8 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 1 Mar 2024 14:29:57 -0800 Subject: [PATCH 096/418] Two fixes to ensure proper tracking of smids and rmids. Signed-off-by: pfeairheller --- src/keri/app/habbing.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 2dd9c594f..53059ffb5 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -538,11 +538,11 @@ def makeSignifyHab(self, name, ns=None, **kwa): self.habs[hab.pre] = hab return hab - def makeSignifyGroupHab(self, name, mhab, ns=None, **kwa): + def makeSignifyGroupHab(self, name, mhab, smids, rmids=None, ns=None, **kwa): # create group Hab in this Habery hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, mhab=mhab, ns=ns, temp=self.temp) + name=name, mhab=mhab, smids=smids, rmids=rmids, ns=ns, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs @@ -590,7 +590,7 @@ def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): # create group Hab in this Habery hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, mhab=mhab, ns=ns, temp=self.temp) + name=name, mhab=mhab, smids=smids, rmids=rmids, ns=ns, temp=self.temp) hab.pre = pre habord = basing.HabitatRecord(hid=hab.pre, @@ -2525,7 +2525,8 @@ def make(self, *, serder, sigers, **kwargs): self.processEvent(serder, sigers) - habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, sid=self.pre, name=self.name, domain=self.ns) + habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, smids=self.smids, rmids=self.rmids, + sid=self.pre, name=self.name, domain=self.ns) self.save(habord) self.inited = True From 1fe95f771311521d9c3278626c7497ca9a6fabd4 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Tue, 5 Mar 2024 14:52:06 -0800 Subject: [PATCH 097/418] First working draft of a migration framework Signed-off-by: pfeairheller --- scripts/demo/basic/clean.sh | 46 +++++++++ src/keri/app/cli/commands/clean.py | 66 +++++++++++++ src/keri/app/cli/commands/migrate/__init__.py | 0 src/keri/app/cli/commands/migrate/list.py | 65 +++++++++++++ src/keri/app/cli/commands/migrate/run.py | 58 ++++++++++++ src/keri/app/cli/commands/migrate/show.py | 62 +++++++++++++ src/keri/app/cli/common/existing.py | 5 +- src/keri/app/habbing.py | 1 - src/keri/db/basing.py | 93 ++++++++++++++----- src/keri/db/migrating.py | 45 +++++++++ src/keri/db/migrations/__init__.py | 0 src/keri/db/migrations/rekey_habs.py | 64 +++++++++++++ 12 files changed, 479 insertions(+), 26 deletions(-) create mode 100755 scripts/demo/basic/clean.sh create mode 100644 src/keri/app/cli/commands/clean.py create mode 100644 src/keri/app/cli/commands/migrate/__init__.py create mode 100644 src/keri/app/cli/commands/migrate/list.py create mode 100644 src/keri/app/cli/commands/migrate/run.py create mode 100644 src/keri/app/cli/commands/migrate/show.py create mode 100644 src/keri/db/migrating.py create mode 100644 src/keri/db/migrations/__init__.py create mode 100644 src/keri/db/migrations/rekey_habs.py diff --git a/scripts/demo/basic/clean.sh b/scripts/demo/basic/clean.sh new file mode 100755 index 000000000..2862075d9 --- /dev/null +++ b/scripts/demo/basic/clean.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +echo "Testing clean" + +function isSuccess() { + ret=$? + if [ $ret -ne 0 ]; then + echo "Error $ret" + exit $ret + fi +} + +# CREATE DATABASE AND KEYSTORE +kli init --name nat --base "${KERI_TEMP_DIR}" --nopasscode +isSuccess + +# RESOLVE WITNESS OOBIs +kli oobi resolve --name nat --base "${KERI_TEMP_DIR}" --oobi-alias wan --oobi http://127.0.0.1:5643/oobi/BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM/controller +isSuccess +kli oobi resolve --name nat --base "${KERI_TEMP_DIR}" --oobi-alias wil --oobi http://127.0.0.1:5642/oobi/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha/controller +isSuccess +kli oobi resolve --name nat --base "${KERI_TEMP_DIR}" --oobi-alias wes --oobi http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller +isSuccess + +## INCEPT AND PROPOGATE EVENTS AND RECEIPTS TO WITNESSES +kli incept --name nat --base "${KERI_TEMP_DIR}" --receipt-endpoint --alias nat --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli rotate --name nat --base "${KERI_TEMP_DIR}" --receipt-endpoint --alias nat --witness-cut BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +isSuccess +kli rotate --name nat --base "${KERI_TEMP_DIR}" --receipt-endpoint --alias nat --witness-add BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess + +kli clean --name nat --base "${KERI_TEMP_DIR}" +isSuccess + +kli status --name nat --alias nat diff --git a/src/keri/app/cli/commands/clean.py b/src/keri/app/cli/commands/clean.py new file mode 100644 index 000000000..a7b6e1226 --- /dev/null +++ b/src/keri/app/cli/commands/clean.py @@ -0,0 +1,66 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app.cli.common import existing +from keri.db import migrating + +logger = help.ogler.getLogger() + + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + clean = CleanDoer(args) + return [clean] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class CleanDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(CleanDoer, self).__init__() + + def recur(self, tyme): + + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + print("Migrating...") + migrator = migrating.Migrator(db=hby.db) + migrator.migrate() + print("Finished") + + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + + print("Database open, performing clean...") + hby.db.clean() + print("Finished.") + + return True diff --git a/src/keri/app/cli/commands/migrate/__init__.py b/src/keri/app/cli/commands/migrate/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py new file mode 100644 index 000000000..ebe9722ae --- /dev/null +++ b/src/keri/app/cli/commands/migrate/list.py @@ -0,0 +1,65 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing +from prettytable import PrettyTable + +from keri.app.cli.common import existing +from keri.db import migrating + +logger = help.ogler.getLogger() + + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + lister = ListDoer(args) + return [lister] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class ListDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(ListDoer, self).__init__() + + def recur(self, tyme): + tab = PrettyTable() + tab.field_names = ["Num", "Name", "Date Completed"] + tab.align["Name"] = "l" + + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + + migrator = migrating.Migrator(db=hby.db) + for idx, (name, dater) in enumerate(migrator.complete()): + date = dater.datetime.strftime("%Y-%m-%d %H:%M") if dater is not None else "Not Run" + tab.add_row((f"{idx+1}", f"{name}", date)) + + print(tab) + return True diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py new file mode 100644 index 000000000..f4ae6e33c --- /dev/null +++ b/src/keri/app/cli/commands/migrate/run.py @@ -0,0 +1,58 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app.cli.common import existing +from keri.db import migrating + +logger = help.ogler.getLogger() + + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + clean = CleanDoer(args) + return [clean] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class CleanDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(CleanDoer, self).__init__() + + def recur(self, tyme): + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + print("Migrating...") + migrator = migrating.Migrator(db=hby.db) + migrator.migrate() + print("Finished") + + return True diff --git a/src/keri/app/cli/commands/migrate/show.py b/src/keri/app/cli/commands/migrate/show.py new file mode 100644 index 000000000..3ce45c2ca --- /dev/null +++ b/src/keri/app/cli/commands/migrate/show.py @@ -0,0 +1,62 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app.cli.common import existing +from keri.db import migrating + +logger = help.ogler.getLogger() + + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + clean = CleanDoer(args) + return [clean] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) +parser.add_argument('--migration', '-m', help='migration name', required=True) + + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class CleanDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(CleanDoer, self).__init__() + + def recur(self, tyme): + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + + migrator = migrating.Migrator(db=hby.db) + [(name, dater)] = migrator.complete(name=self.args.migration) + date = dater.datetime.strftime("%Y-%m-%d %H:%M") if dater is not None else "Not Run" + + print(f"{self.args.migration} -> {date}") + + return True diff --git a/src/keri/app/cli/common/existing.py b/src/keri/app/cli/common/existing.py index 287d4e033..ec0f16e3c 100644 --- a/src/keri/app/cli/common/existing.py +++ b/src/keri/app/cli/common/existing.py @@ -12,7 +12,7 @@ from keri.app import habbing, keeping -def setupHby(name, base="", bran=None, cf=None): +def setupHby(name, base="", bran=None, cf=None, temp=False): """ Create Habery off of existing directory Parameters: @@ -20,6 +20,7 @@ def setupHby(name, base="", bran=None, cf=None): base(str): optional base directory prefix bran(str): optional passcode if the Habery was created encrypted cf (Configer): optional configuration for loading reference data + temp (bool): True means create database in /tmp Returns: Habery: the configured habery @@ -27,7 +28,7 @@ def setupHby(name, base="", bran=None, cf=None): """ ks = keeping.Keeper(name=name, base=base, - temp=False, + temp=temp, cf=cf, reopen=True) aeid = ks.gbls.get('aeid') diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 53059ffb5..eba307ea5 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -313,7 +313,6 @@ def setup(self, *, seed=None, aeid=None, bran=None, pidx=None, algo=None, self.loadHabs() self.inited = True - def loadHabs(self): """Load Habs instance from db diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 8fc1cc1b0..91b8c9f14 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -34,7 +34,7 @@ from hio.base import doing -from . import dbing, koming, subing +from . import dbing, koming, subing, migrating from .. import kering from ..core import coring, eventing, parsing, serdering @@ -931,6 +931,8 @@ def reopen(self, **kwa): self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) self.qnfs = self.env.open_db(key=b'qnfs.', dupsort=True) + self.migs = subing.CesrSuber(db=self, subkey="migs.", klas=coring.Dater) + # event source local (protected) or non-local (remote not protected) self.esrs = koming.Komer(db=self, schema=EventSourceRecord, @@ -945,6 +947,7 @@ def reopen(self, **kwa): # events as ordered by first seen ordinals self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=coring.Seqner) # Kever state made of KeyStateRecord key states + # TODO: clean self.states = koming.Komer(db=self, schema=KeyStateRecord, subkey='stts.') @@ -977,6 +980,7 @@ def reopen(self, **kwa): # all reply messages. Maps reply said to serialization. Replys are # versioned sads ( with version string) so use Serder to deserialize and # use .sdts, .ssgs, and .scgs for datetimes and signatures + # TODO: clean self.rpys = subing.SerderSuber(db=self, subkey='rpys.') # all reply escrows indices of partially signed reply messages. Maps @@ -987,6 +991,7 @@ def reopen(self, **kwa): # auth AuthN/AuthZ by controller at cid of endpoint provider at eid # maps key=cid.role.eid to val=said of end reply + # TODO: clean self.eans = subing.CesrSuber(db=self, subkey='eans.', klas=coring.Saider) # auth AuthN/AuthZ by endpoint provider at eid of location at scheme url @@ -1005,6 +1010,7 @@ def reopen(self, **kwa): schema=LocationRecord, ) # index of last retrieved message from witness mailbox + # TODO: clean self.tops = koming.Komer(db=self, subkey='witm.', schema=TopicsRecord, ) @@ -1022,6 +1028,7 @@ def reopen(self, **kwa): klas=(coring.Seqner, coring.Saider)) # completed group multisig + # TODO: clean self.cgms = subing.CesrSuber(db=self, subkey='cgms.', klas=coring.Saider) @@ -1029,50 +1036,58 @@ def reopen(self, **kwa): self.epse = subing.SerderSuber(db=self, subkey="epse.") # exchange messages + # TODO: clean self.exns = subing.SerderSuber(db=self, subkey="exns.") # Forward pointer to a provided reply message + # TODO: clean self.erpy = subing.CesrSuber(db=self, subkey="erpy.", klas=coring.Saider) - # exchange messages - self.sxns = subing.SerderSuber(db=self, subkey="sxns.") - # exchange message signatures + # TODO: clean self.esigs = subing.CesrIoSetSuber(db=self, subkey='esigs.', klas=coring.Siger) # exchange message signatures + # TODO: clean self.ecigs = subing.CatCesrIoSetSuber(db=self, subkey='ecigs.', klas=(coring.Verfer, coring.Cigar)) # exchange pathed attachments + # TODO: clean self.epath = subing.IoSetSuber(db=self, subkey=".epath") # accepted signed 12-word challenge response exn messages keys by prefix of signer + # TODO: clean self.chas = subing.CesrIoSetSuber(db=self, subkey='chas.', klas=coring.Saider) # successfull signed 12-word challenge response exn messages keys by prefix of signer + # TODO: clean self.reps = subing.CesrIoSetSuber(db=self, subkey='reps.', klas=coring.Saider) # authorzied well known OOBIs + # TODO: clean self.wkas = koming.IoSetKomer(db=self, subkey='wkas.', schema=WellKnownAuthN) # KSN support datetime stamps and signatures indexed and not-indexed # all ksn kdts (key state datetime serializations) maps said to date-time + # TODO: clean self.kdts = subing.CesrSuber(db=self, subkey='kdts.', klas=coring.Dater) # all key state messages. Maps key state said to serialization. ksns are # KeyStateRecords so use ._asdict or ._asjson as appropriate # use .kdts, .ksgs, and .kcgs for datetimes and signatures + # TODO: clean self.ksns = koming.Komer(db=self, schema=KeyStateRecord, subkey='ksns.') - #self.ksns = subing.SerderSuber(db=self, subkey='ksns.') # key state SAID database for successfully saved key state notices # maps key=(prefix, aid) to val=said of key state + # TODO: clean self.knas = subing.CesrSuber(db=self, subkey='knas.', klas=coring.Saider) # config loaded oobis to be processed asynchronously, keyed by oobi URL + # TODO: clean self.oobis = koming.Komer(db=self, subkey='oobis.', schema=OobiRecord, @@ -1091,52 +1106,62 @@ def reopen(self, **kwa): sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Resolved OOBIs (those that have been processed successfully for this database. + # TODO: clean self.roobi = koming.Komer(db=self, subkey='roobi.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Well known OOBIs that are to be used for mfa against a resolved OOBI. + # TODO: clean self.woobi = koming.Komer(db=self, subkey='woobi.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Well known OOBIs that are to be used for mfa against a resolved OOBI. + # TODO: clean self.moobi = koming.Komer(db=self, subkey='moobi.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Multifactor well known OOBI auth records to process. Keys by controller URL + # TODO: clean self.mfa = koming.Komer(db=self, subkey='mfa.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Resolved multifactor well known OOBI auth records. Keys by controller URL + # TODO: clean self.rmfa = koming.Komer(db=self, subkey='rmfa.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # JSON schema SADs keys by the SAID + # TODO: clean self.schema = subing.SchemerSuber(db=self, subkey='schema.') # Field values for contact information for remote identifiers. Keyed by prefix/field + # TODO: clean self.cfld = subing.Suber(db=self, subkey="cfld.") # Global settings for the Habery environment self.hbys = subing.Suber(db=self, subkey='hbys.') # Signed contact data, keys by prefix + # TODO: clean self.cons = subing.Suber(db=self, subkey="cons.") # Transferable signatures on contact data + # TODO: clean self.ccigs = subing.CesrSuber(db=self, subkey='ccigs.', klas=coring.Cigar) # Chunked image data for contact information for remote identifiers + # TODO: clean self.imgs = self.env.open_db(key=b'imgs.') # Delegation escrow dbs # @@ -1146,26 +1171,17 @@ def reopen(self, **kwa): # delegated unanchored escrow self.dune = subing.SerderSuber(db=self, subkey='dune.') - # completed group multisig + # completed group delegated AIDs + # TODO: clean self.cdel = subing.CesrSuber(db=self, subkey='cdel.', klas=coring.Saider) - # siging public keys mapped to the AID and event seq no they appeared in so - # can look up member event designating as signing keys - # updated by Kever.logEvent. - self.pubs = subing.CatCesrIoSetSuber(db=self, subkey="pubs.", - klas=(coring.Prefixer, coring.Seqner)) - - # next key digests mapped to the AID and event seq no they appeared in so - # can look up event designating as next signing key digest - # updated by Kever.logEvent. - self.digs = subing.CatCesrIoSetSuber(db=self, subkey="digs.", - klas=(coring.Prefixer, coring.Seqner)) - # multisig sig embed payload SAID mapped to containing exn messages across group multisig participants + # TODO: clean self.meids = subing.CesrIoSetSuber(db=self, subkey="meids.", klas=coring.Saider) # multisig sig embed payload SAID mapped to group multisig participants AIDs + # TODO: clean self.maids = subing.CesrIoSetSuber(db=self, subkey="maids.", klas=coring.Prefixer) self.reload() @@ -1177,6 +1193,10 @@ def reload(self): Reload stored prefixes and Kevers from .habs """ + # Check migrations to see if this database is up to date. Error otherwise + if not migrating.Migrator(db=self).current(): + raise kering.DatabaseError("Database migrations must be run.") + removes = [] for keys, data in self.habs.getItemIter(): if (ksr := self.states.get(keys=data.hid)) is not None: @@ -1230,6 +1250,33 @@ def clean(self): for msg in self.cloneAllPreIter(): # clone into copy psr.parseOne(ims=msg) + # This is the list of non-set based databases that are not created as part of event processing. + # for now we are just copying them from self to copy without worrying about being able to + # reprocess them. We need a more secure method in the future + unsecured = ["hbys", "schema", "states", "rpys", "eans", "tops", "cgms", "exns", "erpy", + "kdts", "ksns", "knas", "oobis", "roobi", "woobi", "moobi", "mfa", "rmfa", + "cfld", "cons", "ccigs", "cdel", "migs"] + + for name in unsecured: + srcdb = getattr(self, name) + cpydb = getattr(copy, name) + for keys, val in srcdb.getItemIter(): + cpydb.put(keys=keys, val=val) + + # This is the list of set based databases that are not created as part of event processing. + # for now we are just copying them from self to copy without worrying about being able to + # reprocess them. We need a more secure method in the future + sets = ["esigs", "ecigs", "epath", "chas", "reps", "wkas", "meids", "maids"] + for name in sets: + srcdb = getattr(self, name) + cpydb = getattr(copy, name) + for keys, val in srcdb.getItemIter(): + cpydb.add(keys=keys, val=val) + + # Insecure raw imgs database copy. + for (key, val) in self.getAllItemIter(self.imgs): + copy.imgs.setVal(key=key, val=val) + # clone .habs habitat name prefix Komer subdb # copy.habs = koming.Komer(db=copy, schema=HabitatRecord, subkey='habs.') # copy for keys, val in self.habs.getItemIter(): @@ -1242,15 +1289,15 @@ def clean(self): copy.groups.add(val.hid) # clone .ends and .locs databases - for keys, val in self.ends.getItemIter(): + for (cid, role, eid), val in self.ends.getItemIter(): exists = False # only copy if entries in both .ends and .locs for scheme in ("https", "http", "tcp"): # all supported schemes - lval = self.locs.get(keys=(val.eid, scheme)) - if lval and lval.cid == keys[0] and lval.role == keys[1]: + lval = self.locs.get(keys=(eid, scheme)) + if lval: exists = True # loc with matching cid and rol - copy.locs.put(keys=(val.eid, scheme), val=lval) + copy.locs.put(keys=(eid, scheme), val=lval) if exists: # only copy end if has at least one matching loc - copy.ends.put(keys=keys, vals=[val]) + copy.ends.put(keys=(cid, role, eid), val=val) # remove own db directory replace with clean clone copy if os.path.exists(self.path): diff --git a/src/keri/db/migrating.py b/src/keri/db/migrating.py new file mode 100644 index 000000000..b2c173523 --- /dev/null +++ b/src/keri/db/migrating.py @@ -0,0 +1,45 @@ +import importlib +import sys + +from keri.core import coring + +MIGRATIONS = ["rekey_habs"] + + +class Migrator: + + def __init__(self, db): + self.db = db + + def migrate(self): + for migration in MIGRATIONS: + if self.db.migs.get(keys=(migration,)) is not None: + continue + + modName = f"keri.db.migrations.{migration}" + mod = importlib.import_module(modName) + try: + sys.stdout.write(f"Running migration {modName}... ") + mod.migrate(self.db) + print("done.") + except Exception as e: + print(f"Abandoning migratoin {migration} with error: {e}") + return + + self.db.migs.pin(keys=(migration,), val=coring.Dater()) + + def current(self): + return self.db.migs.get(MIGRATIONS[-1]) is not None + + def complete(self, name=None): + migrations = [] + if not name: + for mig in MIGRATIONS: + dater = self.db.migs.get(keys=(mig,)) + migrations.append((mig, dater)) + else: + if name not in MIGRATIONS: + raise ValueError(f"No migration named {name}") + migrations.append((name, self.db.migs.get(keys=(name,)))) + + return migrations diff --git a/src/keri/db/migrations/__init__.py b/src/keri/db/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/keri/db/migrations/rekey_habs.py b/src/keri/db/migrations/rekey_habs.py new file mode 100644 index 000000000..9275a25e2 --- /dev/null +++ b/src/keri/db/migrations/rekey_habs.py @@ -0,0 +1,64 @@ +from dataclasses import dataclass, field, asdict + +from keri.db import koming, basing + + +@dataclass +class OldHabitatRecord: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + hid (str): identifier prefix of hab qb64 + mid (str | None): group member identifier qb64 when hid is group + smids (list | None): group signing member identifiers qb64 when hid is group + rmids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + + + """ + hid: str # hab own identifier prefix qb64 + mid: str | None = None # group member identifier qb64 when hid is group + smids: list | None = None # group signing member ids when hid is group + rmids: list | None = None # group rotating member ids when hid is group + sid: str | None = None # Signify identifier qb64 when hid is Signify + watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers + + +def migrate(db): + habs = koming.Komer(db=db, + subkey='habs.', + schema=OldHabitatRecord, ) + + # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix + nmsp = koming.Komer(db=db, + subkey='nmsp.', + schema=OldHabitatRecord, ) + + habords = dict() + for name, habord in habs.getItemIter(): + name = ".".join(name) # detupleize the database key name + nhabord = basing.HabitatRecord(**asdict(habord)) + nhabord.name = name + habords[habord.hid] = nhabord + + habs.trim() + + for keys, habord in nmsp.getItemIter(): + ns = keys[0] + name = ".".join(keys[1:]) # detupleize the database key name + nhabord = basing.HabitatRecord(**asdict(habord)) + nhabord.name = name + nhabord.domain = ns + habords[habord.hid] = nhabord + + nmsp.trim() + + for pre, habord in habords.items(): + print(pre) + print(habord) + db.habs.pin(keys=(pre,), val=habord) + ns = "" if habord.domain is None else habord.domain + print(ns) + print(habord.name) + db.names.pin(keys=(ns, habord.name), val=pre) From cf336d7bee2155ae916a35712905adb80c95838f Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Mon, 11 Mar 2024 07:29:58 -0700 Subject: [PATCH 098/418] Adding version table. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/multisig/join.py | 3 --- src/keri/db/basing.py | 1 + src/keri/db/migrating.py | 8 ++++++-- src/keri/db/versioning.py | 24 ---------------------- 4 files changed, 7 insertions(+), 29 deletions(-) delete mode 100644 src/keri/db/versioning.py diff --git a/src/keri/app/cli/commands/multisig/join.py b/src/keri/app/cli/commands/multisig/join.py index dfca28c15..e52dae7a1 100644 --- a/src/keri/app/cli/commands/multisig/join.py +++ b/src/keri/app/cli/commands/multisig/join.py @@ -146,9 +146,6 @@ def incept(self, attrs): """ Incept group multisig """ - if True: - return True - smids = attrs["smids"] # change body mids for group member ids rmids = attrs["rmids"] if "rmids" in attrs else None ked = attrs["ked"] diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 91b8c9f14..862288741 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -932,6 +932,7 @@ def reopen(self, **kwa): self.qnfs = self.env.open_db(key=b'qnfs.', dupsort=True) self.migs = subing.CesrSuber(db=self, subkey="migs.", klas=coring.Dater) + self.vers = subing.Suber(db=self, subkey="vers.") # event source local (protected) or non-local (remote not protected) self.esrs = koming.Komer(db=self, diff --git a/src/keri/db/migrating.py b/src/keri/db/migrating.py index b2c173523..687252a49 100644 --- a/src/keri/db/migrating.py +++ b/src/keri/db/migrating.py @@ -3,7 +3,11 @@ from keri.core import coring -MIGRATIONS = ["rekey_habs"] +MIGRATIONS = [ + ("1", ["rekey_habs"]) +] + +VERSION = "1" class Migrator: @@ -23,7 +27,7 @@ def migrate(self): mod.migrate(self.db) print("done.") except Exception as e: - print(f"Abandoning migratoin {migration} with error: {e}") + print(f"\nAbandoning migratoin {migration} with error: {e}") return self.db.migs.pin(keys=(migration,), val=coring.Dater()) diff --git a/src/keri/db/versioning.py b/src/keri/db/versioning.py deleted file mode 100644 index d1e6b21c7..000000000 --- a/src/keri/db/versioning.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- encoding: utf-8 -*- -""" -keri.db.dbing module - - -""" -from keri.db import koming -from keri.db.basing import HabitatRecord - - -class Upgrader: - - def __init__(self, db): - self.db = db - - # habitat application state keyed by habitat name, includes prefix - self.habs = koming.Komer(db=self.db, - subkey='habs.', - schema=HabitatRecord, ) - - # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix - self.nmsp = koming.Komer(db=self.db, - subkey='nmsp.', - schema=HabitatRecord, ) From c8fa849cfc8ee718bf672071ecb18f2df948754a Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 15 Mar 2024 06:00:23 -0700 Subject: [PATCH 099/418] Creating a VERSION file for databases and tracking current version of software. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/init.py | 1 + src/keri/db/basing.py | 2 +- src/keri/db/dbing.py | 12 ++++++++++++ src/keri/db/migrating.py | 10 +++++----- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/keri/app/cli/commands/init.py b/src/keri/app/cli/commands/init.py index aa8479ec6..f66f936fb 100644 --- a/src/keri/app/cli/commands/init.py +++ b/src/keri/app/cli/commands/init.py @@ -12,6 +12,7 @@ import keri.app.oobiing from keri.app import habbing, configing, oobiing, connecting from keri.app.keeping import Algos +from keri.db import migrating from keri.kering import ConfigurationError from keri.vdr import credentialing diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 862288741..5bc4b2ac1 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1195,7 +1195,7 @@ def reload(self): """ # Check migrations to see if this database is up to date. Error otherwise - if not migrating.Migrator(db=self).current(): + if not migrating.Migrator(db=self).current(self.version): raise kering.DatabaseError("Database migrations must be run.") removes = [] diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index f186a0b7d..1e9ae8fbb 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -53,12 +53,14 @@ from typing import Union import lmdb +from hio.help.helping import ocfn from ordered_set import OrderedSet as oset from hio.base import filing from hio.base import filing +import keri from ..kering import MaxON # maximum ordinal number for seqence or first seen from ..help import helping @@ -383,6 +385,16 @@ def reopen(self, readonly=False, **kwa): # creates files data.mdb and lock.mdb in .dbDirPath self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=104857600, mode=self.perm, readonly=self.readonly) + + fver = ocfn(path=f"{self.path}/VERSION", perm=self.perm) + + if ver := fver.read(): + self.version = ver + else: + self.version = keri.__version__ + fver.write(self.version) + fver.close() + self.opened = True if opened and self.env else False return self.opened diff --git a/src/keri/db/migrating.py b/src/keri/db/migrating.py index 687252a49..af9574f48 100644 --- a/src/keri/db/migrating.py +++ b/src/keri/db/migrating.py @@ -1,14 +1,13 @@ import importlib import sys +import keri from keri.core import coring MIGRATIONS = [ - ("1", ["rekey_habs"]) + ("1.1.0", ["rekey_habs"]) ] -VERSION = "1" - class Migrator: @@ -32,8 +31,9 @@ def migrate(self): self.db.migs.pin(keys=(migration,), val=coring.Dater()) - def current(self): - return self.db.migs.get(MIGRATIONS[-1]) is not None + def current(self, version): + return version == keri.__version__ + # return self.db.migs.get(MIGRATIONS[-1]) is not None def complete(self, name=None): migrations = [] From be6f0b3b74e4991906774e362a04acc0e9df3747 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Tue, 26 Mar 2024 12:37:54 -0700 Subject: [PATCH 100/418] getting lastest version from database. Signed-off-by: pfeairheller --- src/keri/db/dbing.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 1e9ae8fbb..9150d9570 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -348,6 +348,7 @@ def __init__(self, readonly=False, **kwa): """ self.env = None + self.version = None self.readonly = True if readonly else False super(LMDBer, self).__init__(**kwa) @@ -386,18 +387,29 @@ def reopen(self, readonly=False, **kwa): self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=104857600, mode=self.perm, readonly=self.readonly) - fver = ocfn(path=f"{self.path}/VERSION", perm=self.perm) - - if ver := fver.read(): - self.version = ver - else: + self.version = self.getVersion() + if self.version is None: self.version = keri.__version__ - fver.write(self.version) - fver.close() + self.setVersion(keri.__version__) self.opened = True if opened and self.env else False return self.opened + def getVersion(self): + with self.env.begin() as txn: + cursor = txn.cursor() + version = cursor.get(b'__version__') + return version.decode("utf-8") if version is not None else None + + def setVersion(self, val): + if hasattr(val, "encode"): + val = val.encode("utf-8") # convert str to bytes + + with self.env.begin(write=True) as txn: + cursor = txn.cursor() + version = cursor.replace(b'__version__', val) + return version + def close(self, clear=False): """ From 1553418706a1071a1db1b6bc407f1b510e27471e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 1 Apr 2024 14:32:42 -0600 Subject: [PATCH 101/418] added Ilker class for message type (ilk) for native CESR value --- src/keri/core/coring.py | 112 ++++++++++++++++++++++++++++++++++++-- src/keri/core/eventing.py | 2 +- tests/core/test_coring.py | 109 ++++++++++++++++++++++++++++++++++--- 3 files changed, 209 insertions(+), 14 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index f02ad5c60..ebc1a435c 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -1446,7 +1446,6 @@ def _bexfil(self, qb2): qb2 = qb2[:bfs] # extract qb2 fully qualified primitive code plus material - # check for nonzero trailing full code mid pad bits ps = cs % 4 # full code (both) net pad size for 24 bit alignment pbs = 2 * ps # mid pad bits = 2 per net pad @@ -2055,6 +2054,111 @@ def tag(self): return tag +class Ilker(Tagger): + """ + Ilker is subclass of Tagger, cryptographic material, for formatted + message types (ilks) in Base64. Leverages Tagger support compact special + fixed size primitives with non-empty soft part and empty raw part. + + Ilker provides a more compact representation than would be obtained by + converting the raw ASCII representation to Base64. + + Attributes: + + Inherited Properties: (See Tagger) + code (str): hard part of derivation code to indicate cypher suite + hard (str): hard part of derivation code. alias for code + soft (str): soft part of derivation code fs any. + Empty when ss = 0. + both (str): hard + soft parts of full text code + size (int | None): Number of quadlets/triplets of chars/bytes including + lead bytes of variable sized material (fs = None). + Converted value of the soft part (of len ss) of full + derivation code. + Otherwise None when not variably sized (fs != None) + fullSize (int): full size of primitive + raw (bytes): crypto material only. Not derivation code or lead bytes. + qb64 (str): Base64 fully qualified with derivation code + crypto mat + qb64b (bytes): Base64 fully qualified with derivation code + crypto mat + qb2 (bytes): binary with derivation code + crypto material + transferable (bool): True means transferable derivation code False otherwise + digestive (bool): True means digest derivation code False otherwise + prefixive (bool): True means identifier prefix derivation code False otherwise + special (bool): True when soft is special raw is empty and fixed size + composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip + tag (str): B64 primitive without prepad (strips prepad from soft) + + + Properties: + ilk (str): message type from Ilks of Ilkage + + Inherited Hidden: (See Tagger) + _code (str): value for .code property + _soft (str): soft value of full code + _raw (bytes): value for .raw property + _rawSize(): + _leadSize(): + _special(): + _infil(): creates qb64b from .raw and .code (fully qualified Base64) + _binfil(): creates qb2 from .raw and .code (fully qualified Base2) + _exfil(): extracts .code and .raw from qb64b (fully qualified Base64) + _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) + + Hidden: + + + Methods: + + """ + + + def __init__(self, qb64b=None, qb64=None, qb2=None, tag='', ilk='', **kwa): + """ + Inherited Parameters: (see Tagger) + raw (bytes | bytearray | None): unqualified crypto material usable + for crypto operations. + code (str): stable (hard) part of derivation code + soft (str | bytes): soft part for special codes + rize (int | None): raw size in bytes when variable sized material not + including lead bytes if any + Otherwise None + qb64b (bytes | None): fully qualified crypto material Base64 + qb64 (str | bytes | None): fully qualified crypto material Base64 + qb2 (bytes | None): fully qualified crypto material Base2 + strip (bool): True means strip (delete) matter from input stream + bytearray after parsing qb64b or qb2. False means do not strip + tag (str | bytes): Base64 plain. Prepad is added as needed. + + Parameters: + + + """ + if not (qb64b or qb64 or qb2): + if ilk: + tag = ilk + + + super(Ilker, self).__init__(qb64b=qb64b, qb64=qb64, qb2=qb2, tag=tag, **kwa) + + if self.code not in (MtrDex.Tag3, ): + raise InvalidCodeError(f"Invalid code={self.code} for Ilker " + f"{self.ilk=}.") + if self.ilk not in Ilks: + raise InvalidSoftError("Ivalid ilk={self.ilk} for Ilker.") + + + + @property + def ilk(self): + """Returns: + tag (str): B64 primitive without prepad (strips prepad from soft) + + Alias for self.tag + + """ + return self.tag + + # Versage namedtuple # proto (str): protocol element of Protocols @@ -2147,8 +2251,6 @@ def __init__(self, qb64b=None, qb64=None, qb2=None, versage=None, gvrsn (Versionage | None): instance genus version. namedtuple (major, minor) of ints - Notes: - prepad = 'A' """ if not (qb64b or qb64 or qb2): if versage: @@ -2162,8 +2264,8 @@ def __init__(self, qb64b=None, qb64=None, qb2=None, versage=None, super(Verser, self).__init__(qb64b=qb64b, qb64=qb64, qb2=qb2, tag=tag, **kwa) if self.code not in (MtrDex.Tag7, MtrDex.Tag10, ): - raise InvalidCodeError(f"Invalid code={self.code} for Verser " - f"{self.tag=}.") + raise InvalidCodeError(f"Invalid code={self.code} for " + "Verser={self.tag}.") @property diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index d0c063aec..e443f56df 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -53,7 +53,7 @@ class TraitCodex: """ EstOnly: str = 'EO' # Only allow establishment events DoNotDelegate: str = 'DND' # Dot not allow delegated identifiers - NoRegistrarBackers: str = 'NB' # This should be NRB in next versionDo not allow any registrar backers + NoRegistrarBackers: str = 'NB' # Do not allow any registrar backers. This should be NRB in next version. RegistrarBackers: str = 'RB' # Registrar backer provided in Registrar seal def __iter__(self): diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 32963378f..c4b08ad58 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -28,15 +28,16 @@ ShortageError, InvalidCodeSizeError, InvalidVarIndexError, InvalidValueError, DeserializeError, ValidationError, InvalidVarRawSizeError, ConversionError, - SoftMaterialError, InvalidSoftError) + SoftMaterialError, InvalidSoftError, InvalidCodeError) from keri.kering import Version, Versionage, VersionError, Vrsn_1_0, Vrsn_2_0 from keri.core import coring from keri.core import eventing from keri.core.coring import (Ilkage, Ilks, Saids, Protocols, Protocolage, - Sadder, Tholder, Seqner, - NumDex, Number, Siger, Dater, Bexter, Texter, - Verser, Versage, TagDex, PadTagDex, Tagger) + Sadder, Tholder, Seqner, NumDex, Number, Siger, + Dater, Bexter, Texter, + TagDex, PadTagDex, Tagger, Ilker, + Verser, Versage, ) from keri.core.coring import Serialage, Serials, Tiers from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, Indexer, @@ -4174,9 +4175,6 @@ def test_tagger(): """ # Test TagCodex PadTagCodex and associated Sizes to be valid specials - - - with pytest.raises(EmptyMaterialError): tagger = Tagger() # defaults @@ -4219,7 +4217,6 @@ def test_tagger(): assert tagger.composable assert tagger.tag == tag - tagger = Tagger(qb64b=qb64b) assert tagger.code == tagger.hard == code assert tagger.soft == soft @@ -4257,6 +4254,101 @@ def test_tagger(): """ Done Test """ +def test_ilker(): + """ + Test Ilker version primitive subclass of Tagger + """ + with pytest.raises(EmptyMaterialError): + ilker = Ilker() # defaults + + # Tag1 + ilk = Ilks.rot + tag = ilk + code = MtrDex.Tag3 + soft = 'rot' + qb64 = 'Xrot' + qb64b = qb64.encode("utf-8") + qb2 = decodeB64(qb64b) + raw = b'' + + ilker = Ilker(ilk=ilk) # defaults + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + ilker = Ilker(qb2=qb2) + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + ilker = Ilker(qb64=qb64) + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + ilker = Ilker(qb64b=qb64b) + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + ilker = Ilker(tag=tag) + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + # test error condition + with pytest.raises(InvalidSoftError): + ilker = Ilker(ilk='bad') + + # ignores code + ilker = Ilker(ilk=ilk, code=MtrDex.Tag4) + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + # test error using soft and code + with pytest.raises(InvalidCodeError): + ilker = Ilker(soft='bady', code=MtrDex.Tag4) + + """End Test""" + + def test_verser(): """ Test Verser version primitive subclass of Matter @@ -7155,6 +7247,7 @@ def test_tholder(): test_matter() test_matter_special() test_tagger() + test_ilker() test_verser() #test_texter() #test_counter() From a1c4d26895acb6117b96e36ae14aaedfc233c4f6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 1 Apr 2024 15:09:32 -0600 Subject: [PATCH 102/418] moved TraitCodex to kering so no circular import Added new traits for V2 --- src/keri/app/cli/commands/multisig/join.py | 8 +++--- src/keri/app/habbing.py | 22 +++++++-------- src/keri/core/coring.py | 3 +- src/keri/core/eventing.py | 25 ++--------------- src/keri/kering.py | 22 +++++++++++++++ src/keri/vdr/credentialing.py | 2 +- src/keri/vdr/eventing.py | 10 +++---- tests/test_kering.py | 32 ++++++++++++++++++++-- tests/vdr/test_eventing.py | 6 ++-- 9 files changed, 80 insertions(+), 50 deletions(-) diff --git a/src/keri/app/cli/commands/multisig/join.py b/src/keri/app/cli/commands/multisig/join.py index dfca28c15..95fba0e4d 100644 --- a/src/keri/app/cli/commands/multisig/join.py +++ b/src/keri/app/cli/commands/multisig/join.py @@ -170,8 +170,8 @@ def incept(self, attrs): inits["isith"] = ked["kt"] inits["nsith"] = ked["nt"] - inits["estOnly"] = eventing.TraitCodex.EstOnly in ked["c"] - inits["DnD"] = eventing.TraitCodex.DoNotDelegate in ked["c"] + inits["estOnly"] = eventing.TraitDex.EstOnly in ked["c"] + inits["DnD"] = eventing.TraitDex.DoNotDelegate in ked["c"] inits["toad"] = ked["bt"] inits["wits"] = ked["b"] @@ -276,8 +276,8 @@ def showEvent(self, hab, mids, ked): if not thold.weighted: tab.add_row(["Signature Threshold", thold.num]) - tab.add_row(["Establishment Only", eventing.TraitCodex.EstOnly in ked["c"]]) - tab.add_row(["Do Not Delegate", eventing.TraitCodex.DoNotDelegate in ked["c"]]) + tab.add_row(["Establishment Only", eventing.TraitDex.EstOnly in ked["c"]]) + tab.add_row(["Do Not Delegate", eventing.TraitDex.DoNotDelegate in ked["c"]]) tab.add_row(["Witness Threshold", ked["bt"]]) tab.add_row(["Witnesses", "\n".join(ked["b"])]) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 4fec262fa..e03af42df 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -423,7 +423,7 @@ def makeHab(self, name, ns=None, cf=None, **kwa): toad (Union[int,str]): int or str hex of witness threshold wits (list): of qb64 prefixes of witnesses delpre (str): qb64 of delegator identifier prefix - estOnly (str): eventing.TraitCodex.EstOnly means only establishment + estOnly (str): eventing.TraitDex.EstOnly means only establishment events allowed in KEL for this Hab data (list | None): seal dicts """ @@ -475,9 +475,9 @@ def makeGroupHab(self, group, mhab, smids, rmids=None, ns=None, **kwa): toad (Union[int,str]): int or str hex of witness threshold wits (list): of qb64 prefixes of witnesses delpre (str): qb64 of delegator identifier prefix - estOnly (str): eventing.TraitCodex.EstOnly means only establishment + estOnly (str): eventing.TraitDex.EstOnly means only establishment events allowed in KEL for this Hab - DnD (bool): eventing.TraitCodex.DnD means do allow delegated identifiers from this identifier + DnD (bool): eventing.TraitDex.DnD means do allow delegated identifiers from this identifier ToDo: NRR add midxs tuples for each group member or all in group multisig. @@ -1083,10 +1083,10 @@ def make(self, DnD, code, data, delpre, estOnly, isith, verfers, nsith, digers, specified else compute default based on number of wits (backers) wits (list | None): qb64 prefixes of witnesses if any delpre (str | None): qb64 of delegator identifier prefix if any - estOnly (bool | None): True means add trait eventing.TraitCodex.EstOnly + estOnly (bool | None): True means add trait eventing.TraitDex.EstOnly which means only establishment events allowed in KEL for this Hab False (default) means allows non-est events and no trait is added. - DnD (bool): True means add trait of eventing.TraitCodex.DnD which + DnD (bool): True means add trait of eventing.TraitDex.DnD which means do not allow delegated identifiers from this identifier False (default) means do allow and no trait is added. @@ -1103,9 +1103,9 @@ def make(self, DnD, code, data, delpre, estOnly, isith, verfers, nsith, digers, nst = coring.Tholder(sith=nsith).sith # next signing threshold cnfg = [] if estOnly: - cnfg.append(eventing.TraitCodex.EstOnly) + cnfg.append(eventing.TraitDex.EstOnly) if DnD: - cnfg.append(eventing.TraitCodex.DoNotDelegate) + cnfg.append(eventing.TraitDex.DoNotDelegate) self.delpre = delpre keys = [verfer.qb64 for verfer in verfers] if self.delpre: @@ -2282,10 +2282,10 @@ def make(self, *, secrecies=None, iridx=0, code=coring.MtrDex.Blake3_256, dcode= specified else compute default based on number of wits (backers) wits (list | None): qb64 prefixes of witnesses if any delpre (str | None): qb64 of delegator identifier prefix if any - estOnly (bool | None): True means add trait eventing.TraitCodex.EstOnly + estOnly (bool | None): True means add trait eventing.TraitDex.EstOnly which means only establishment events allowed in KEL for this Hab False (default) means allows non-est events and no trait is added. - DnD (bool): True means add trait of eventing.TraitCodex.DnD which + DnD (bool): True means add trait of eventing.TraitDex.DnD which means do not allow delegated identifiers from this identifier False (default) means do allow and no trait is added. @@ -2729,10 +2729,10 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, specified else compute default based on number of wits (backers) wits (list | None): qb64 prefixes of witnesses if any delpre (str | None): qb64 of delegator identifier prefix if any - estOnly (bool | None): True means add trait eventing.TraitCodex.EstOnly + estOnly (bool | None): True means add trait eventing.TraitDex.EstOnly which means only establishment events allowed in KEL for this Hab False (default) means allows non-est events and no trait is added. - DnD (bool): True means add trait of eventing.TraitCodex.DnD which + DnD (bool): True means add trait of eventing.TraitDex.DnD which means do not allow delegated identifiers from this identifier False (default) means do allow and no trait is added. merfers (list[Verfer] | None): group member Verfer instances of diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index ebc1a435c..30bfa061e 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -6,10 +6,9 @@ import re import json from typing import Union +from collections import namedtuple, deque from collections.abc import Sequence, Mapping - from dataclasses import dataclass, astuple -from collections import namedtuple, deque from base64 import urlsafe_b64encode as encodeB64 from base64 import urlsafe_b64decode as decodeB64 from fractions import Fraction diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index e443f56df..9968694df 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -33,7 +33,7 @@ UnverifiedReceiptError, UnverifiedTransferableReceiptError, QueryNotFoundError, MisfitEventSourceError, MissingDelegableApprovalError) -from ..kering import Version, Versionage +from ..kering import Version, Versionage, TraitCodex, TraitDex from ..help import helping @@ -43,25 +43,6 @@ MaxIntThold = 2 ** 32 - 1 -@dataclass(frozen=True) -class TraitCodex: - """ - TraitCodex is codex of inception configuration trait code strings - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - - """ - EstOnly: str = 'EO' # Only allow establishment events - DoNotDelegate: str = 'DND' # Dot not allow delegated identifiers - NoRegistrarBackers: str = 'NB' # Do not allow any registrar backers. This should be NRB in next version. - RegistrarBackers: str = 'RB' # Registrar backer provided in Registrar seal - - def __iter__(self): - return iter(astuple(self)) - - -TraitDex = TraitCodex() # Make instance - # Location of last establishment key event: sn is int, dig is qb64 digest LastEstLoc = namedtuple("LastEstLoc", 's d') @@ -1852,8 +1833,8 @@ def reload(self, state): self.cuts = state.ee.br self.adds = state.ee.ba self.estOnly = False - self.doNotDelegate = True if TraitCodex.DoNotDelegate in state.c else False - self.estOnly = True if TraitCodex.EstOnly in state.c else False + self.doNotDelegate = True if TraitDex.DoNotDelegate in state.c else False + self.estOnly = True if TraitDex.EstOnly in state.c else False self.lastEst = LastEstLoc(s=int(state.ee.s, 16), d=state.ee.d) self.delpre = state.di if state.di else None diff --git a/src/keri/kering.py b/src/keri/kering.py index 0ab43c76a..089fc53ab 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -413,6 +413,28 @@ def sniff(ims): watcher='watcher', judge='judge', juror='juror', peer='peer', mailbox="mailbox", agent="agent") +@dataclass(frozen=True) +class TraitCodex: + """ + TraitCodex is codex of inception configuration trait code strings + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + """ + EstOnly: str = 'EO' # Only allow establishment events. Inception only. + DoNotDelegate: str = 'DND' # Dot not allow delegated identifiers. Inception only. + RegistrarBackers: str = 'RB' # Registrar backer provided in Registrar seal in this event + NoBackers: str = 'NB' # Do not allow any (registrar backers). + # Inception and Rotation in v2. This should be NRB in next version. + NoRegistrarBackers: str = 'NRB' # Do not allow any registrar backers. Inception and Rotation. + DelegateIsDelegator: str = 'DID' # Treat delegate AIDs same as their delegator. Inception only + + def __iter__(self): + return iter(astuple(self)) + + +TraitDex = TraitCodex() # Make instance + # Exception Subclasses class KeriError(Exception): diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index f26a01b9d..8a85c8e54 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -279,7 +279,7 @@ def make(self, *, nonce=None, noBackers=True, baks=None, toad=None, estOnly=Fals if vcp is None: baks = baks if baks is not None else [] - self.cnfg = [TraitDex.NoRegistrarBackers] if noBackers else [] + self.cnfg = [TraitDex.NoBackers] if noBackers else [] if estOnly: self.cnfg.append(TraitDex.EstOnly) diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 47a1b54be..baa6eed41 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -69,7 +69,7 @@ def incept( cnfg = cnfg if cnfg is not None else [] baks = baks if baks is not None else [] - if TraitDex.NoRegistrarBackers in cnfg and len(baks) > 0: + if TraitDex.NoBackers in cnfg and len(baks) > 0: raise ValueError("{} backers specified for NB vcp, 0 allowed".format(len(baks))) if len(oset(baks)) != len(baks): @@ -763,7 +763,7 @@ def reload(self, rsr): self.toad = int(ked["bt"], 16) self.baks = ked["b"] - self.noBackers = True if TraitDex.NoRegistrarBackers in ked["c"] else False + self.noBackers = True if TraitDex.NoBackers in ked["c"] else False self.estOnly = True if TraitDex.EstOnly in ked["c"] else False if (raw := self.reger.getTvt(key=dgKey(pre=self.prefixer.qb64, @@ -784,7 +784,7 @@ def state(self): #state(self, kind=Serials.json) cnfg = [] if self.noBackers: - cnfg.append(TraitDex.NoRegistrarBackers) + cnfg.append(TraitDex.NoBackers) dgkey = dbing.dgKey(self.regk, self.serder.said) couple = self.reger.getAnc(dgkey) @@ -871,7 +871,7 @@ def config(self, serder, noBackers=None, estOnly=None): else False) # ensure default estOnly is boolean cnfg = serder.ked["c"] # process cnfg for traits - if TraitDex.NoRegistrarBackers in cnfg: + if TraitDex.NoBackers in cnfg: self.noBackers = True if TraitDex.EstOnly in cnfg: self.estOnly = True @@ -1758,7 +1758,7 @@ def processReplyRegistryTxnState(self, *, serder, saider, route, cigars=None, ts # Load backers from either tsn or Kever of issuer cnfg = rsr.c - if TraitDex.NoRegistrarBackers in cnfg: + if TraitDex.NoBackers in cnfg: kevers = self.kevers[pre] baks = kevers.wits else: diff --git a/tests/test_kering.py b/tests/test_kering.py index 4041bced0..01a1f7e24 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -5,6 +5,7 @@ """ import re import json +from dataclasses import asdict, astuple import cbor2 as cbor import msgpack @@ -15,6 +16,7 @@ from keri.kering import Protocolage, Protocols from keri.kering import Serialage, Serials from keri.kering import Ilkage, Ilks +from keri.kering import TraitCodex, TraitDex from keri.kering import (Versionage, Version, MAXVERFULLSPAN, versify, deversify, Rever, Smellage, smell, VER1FULLSPAN, VER1TERM, VEREX1, @@ -27,7 +29,7 @@ codeB64ToB2, codeB2ToB64, Reb64, nabSextets) -def test_protos(): +def test_protocols(): """ Test protocols namedtuple instance Protocols """ @@ -42,6 +44,31 @@ def test_protos(): """End Test""" +def test_traitdex(): + """ + Test TraitDex instance of TraitCodex dataclass + """ + + assert isinstance(TraitDex, TraitCodex) + + assert asdict(TraitDex) == \ + { + 'EstOnly': 'EO', + 'DoNotDelegate': 'DND', + 'RegistrarBackers': 'RB', + 'NoBackers': 'NB', + 'NoRegistrarBackers': 'NRB', + 'DelegateIsDelegator': 'DID', + } + + assert 'EO' in TraitDex + assert 'DND' in TraitDex + assert 'RB' in TraitDex + assert 'NB' in TraitDex + assert 'NRB' in TraitDex + assert 'DID' in TraitDex + + """End Test""" def test_version_regex(): @@ -774,7 +801,8 @@ def test_ilks(): if __name__ == "__main__": - test_protos() + test_protocols() + test_traitdex() test_version_regex() test_smell() test_snuff() diff --git a/tests/vdr/test_eventing.py b/tests/vdr/test_eventing.py index 079acc328..3da4f055c 100644 --- a/tests/vdr/test_eventing.py +++ b/tests/vdr/test_eventing.py @@ -37,7 +37,7 @@ def test_incept(mockCoringRandomNonce): b'fpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') # no backers allowed - serder = eventing.incept(pre, baks=[], cnfg=[keventing.TraitDex.NoRegistrarBackers], code=MtrDex.Blake3_256) + serder = eventing.incept(pre, baks=[], cnfg=[keventing.TraitDex.NoBackers], code=MtrDex.Blake3_256) assert serder.raw == (b'{"v":"KERI10JSON000113_","t":"vcp","d":"EBoBPh3N5nr1tItAUCkXNx3vShB_Be6iiQPX' b'Bsg2LvxA","i":"EBoBPh3N5nr1tItAUCkXNx3vShB_Be6iiQPXBsg2LvxA","ii":"DAtNTPnDF' b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":["NB"],"bt":"0","b":[],"n":' @@ -45,7 +45,7 @@ def test_incept(mockCoringRandomNonce): # no backers allows, one attempted with pytest.raises(ValueError): - eventing.incept(pre, cnfg=[keventing.TraitDex.NoRegistrarBackers], + eventing.incept(pre, cnfg=[keventing.TraitDex.NoBackers], baks=[bak1]) # with backer dupes @@ -368,7 +368,7 @@ def test_prefixer(): t=Ilks.vcp, bt=0, b=[], - c=[keventing.TraitDex.NoRegistrarBackers], + c=[keventing.TraitDex.NoBackers], ) prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) assert prefixer.qb64 == 'EDz0QmMxf4Dk0C9uiP-y3okN-Bej2IAXSj8UwQgb3NsL' From 78c6bd3fa900c60a52d641a703a16e2495ca911f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 2 Apr 2024 09:39:00 -0600 Subject: [PATCH 103/418] some cleanup of code --- src/keri/core/coring.py | 119 +++++++++++++++++++++++++++++++++-- src/keri/core/eventing.py | 74 +--------------------- src/keri/core/serdering.py | 57 ++++++++++------- tests/core/test_coring.py | 106 +++++++++++++++++++++++++++++-- tests/core/test_serdering.py | 11 ++++ 5 files changed, 260 insertions(+), 107 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 30bfa061e..f72cee043 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -39,7 +39,8 @@ from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, VERRAWSIZE, VERFMT, MAXVERFULLSPAN, versify, deversify, Rever, smell) -from ..kering import Serials, Serialage, Protocols, Protocolage, Ilkage, Ilks +from ..kering import (Serials, Serialage, Protocols, Protocolage, Ilkage, Ilks, + TraitDex, ) from ..help import helping from ..help.helping import sceil, nonStringIterable, nonStringSequence @@ -181,12 +182,14 @@ def loads(raw, size=None, kind=Serials.json): return ked -# deprecated don't use anymore need to fix demo tests that use -# use with context instead + def generateSigners(salt=None, count=8, transferable=True): """ Returns list of Signers for Ed25519 + Use this when simply need valid AIDs but not when need valid controller + contexts. In the latter case use openHby or openHab which create databases. + Parameters: salt is bytes 16 byte long root cryptomatter from which seeds for Signers in list are derived @@ -2129,7 +2132,7 @@ def __init__(self, qb64b=None, qb64=None, qb2=None, tag='', ilk='', **kwa): tag (str | bytes): Base64 plain. Prepad is added as needed. Parameters: - + ilk (str): message type from Ilks of Ilkage """ if not (qb64b or qb64 or qb2): @@ -2143,7 +2146,7 @@ def __init__(self, qb64b=None, qb64=None, qb2=None, tag='', ilk='', **kwa): raise InvalidCodeError(f"Invalid code={self.code} for Ilker " f"{self.ilk=}.") if self.ilk not in Ilks: - raise InvalidSoftError("Ivalid ilk={self.ilk} for Ilker.") + raise InvalidSoftError(f"Ivalid ilk={self.ilk} for Ilker.") @@ -2158,6 +2161,110 @@ def ilk(self): return self.tag +class Traitor(Tagger): + """ + Traitor is subclass of Tagger, cryptographic material, for formatted + configuration traits for key events in Base64. Leverages Tagger support of + compact special fixed size primitives with non-empty soft part and empty raw part. + + Traitor provides a more compact representation than would be obtained by + converting the raw ASCII representation to Base64. + + Attributes: + + Inherited Properties: (See Tagger) + code (str): hard part of derivation code to indicate cypher suite + hard (str): hard part of derivation code. alias for code + soft (str): soft part of derivation code fs any. + Empty when ss = 0. + both (str): hard + soft parts of full text code + size (int | None): Number of quadlets/triplets of chars/bytes including + lead bytes of variable sized material (fs = None). + Converted value of the soft part (of len ss) of full + derivation code. + Otherwise None when not variably sized (fs != None) + fullSize (int): full size of primitive + raw (bytes): crypto material only. Not derivation code or lead bytes. + qb64 (str): Base64 fully qualified with derivation code + crypto mat + qb64b (bytes): Base64 fully qualified with derivation code + crypto mat + qb2 (bytes): binary with derivation code + crypto material + transferable (bool): True means transferable derivation code False otherwise + digestive (bool): True means digest derivation code False otherwise + prefixive (bool): True means identifier prefix derivation code False otherwise + special (bool): True when soft is special raw is empty and fixed size + composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip + tag (str): B64 primitive without prepad (strips prepad from soft) + + + Properties: + trait (str): configuration trait B64 from TraitDex + + Inherited Hidden: (See Tagger) + _code (str): value for .code property + _soft (str): soft value of full code + _raw (bytes): value for .raw property + _rawSize(): + _leadSize(): + _special(): + _infil(): creates qb64b from .raw and .code (fully qualified Base64) + _binfil(): creates qb2 from .raw and .code (fully qualified Base2) + _exfil(): extracts .code and .raw from qb64b (fully qualified Base64) + _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) + + Hidden: + + + Methods: + + """ + + + def __init__(self, qb64b=None, qb64=None, qb2=None, tag='', trait='', **kwa): + """ + Inherited Parameters: (see Tagger) + raw (bytes | bytearray | None): unqualified crypto material usable + for crypto operations. + code (str): stable (hard) part of derivation code + soft (str | bytes): soft part for special codes + rize (int | None): raw size in bytes when variable sized material not + including lead bytes if any + Otherwise None + qb64b (bytes | None): fully qualified crypto material Base64 + qb64 (str | bytes | None): fully qualified crypto material Base64 + qb2 (bytes | None): fully qualified crypto material Base2 + strip (bool): True means strip (delete) matter from input stream + bytearray after parsing qb64b or qb2. False means do not strip + tag (str | bytes): Base64 plain. Prepad is added as needed. + + Parameters: + trait (str): configuration trait B64 from TraitDex + + """ + if not (qb64b or qb64 or qb2): + if trait: + tag = trait + + + super(Traitor, self).__init__(qb64b=qb64b, qb64=qb64, qb2=qb2, tag=tag, **kwa) + + + if self.trait not in TraitDex: + raise InvalidSoftError(f"Invalid trait={self.trait} for Traitor.") + + + + @property + def trait(self): + """Returns: + trait (str): B64 primitive without prepad (strips prepad from soft) + + Alias for self.tag + + """ + return self.tag + + + # Versage namedtuple # proto (str): protocol element of Protocols @@ -2264,7 +2371,7 @@ def __init__(self, qb64b=None, qb64=None, qb2=None, versage=None, if self.code not in (MtrDex.Tag7, MtrDex.Tag10, ): raise InvalidCodeError(f"Invalid code={self.code} for " - "Verser={self.tag}.") + f"Verser={self.tag}.") @property diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 9968694df..7fe908939 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -34,6 +34,7 @@ QueryNotFoundError, MisfitEventSourceError, MissingDelegableApprovalError) from ..kering import Version, Versionage, TraitCodex, TraitDex +from ..kering import Coldage, Colds, ColdDex from ..help import helping @@ -102,53 +103,6 @@ StateEvent = namedtuple("StateEvent", 's t d') -@dataclass(frozen=True) -class ColdCodex: - """ - ColdCodex is codex of cold stream start tritets of first byte - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - - First three bits: - 0o0 = 000 free - 0o1 = 001 cntcode B64 - 0o2 = 010 opcode B64 - 0o3 = 011 json - 0o4 = 100 mgpk - 0o5 = 101 cbor - 0o6 = 110 mgpk - 007 = 111 cntcode or opcode B2 - - status is one of ('evt', 'txt', 'bny' ) - 'evt' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) - 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) - 'bny' if tritet in (ColdDex.CtOpB2,) - - otherwise raise ColdStartError - - x = bytearray([0x2d, 0x5f]) - x == bytearray(b'-_') - x[0] >> 5 == 0o1 - True - """ - Free: int = 0o0 # not taken - CtB64: int = 0o1 # CountCode Base64 - OpB64: int = 0o2 # OpCode Base64 - JSON: int = 0o3 # JSON Map Event Start - MGPK1: int = 0o4 # MGPK Fixed Map Event Start - CBOR: int = 0o5 # CBOR Map Event Start - MGPK2: int = 0o6 # MGPK Big 16 or 32 Map Event Start - CtOpB2: int = 0o7 # CountCode or OpCode Base2 - - def __iter__(self): - return iter(astuple(self)) - - -ColdDex = ColdCodex() # Make instance - -Coldage = namedtuple("Coldage", 'msg txt bny') # stream cold start status -Colds = Coldage(msg='msg', txt='txt', bny='bny') - # Future make Cues dataclasses instead of dicts. Dataclasses so may be converted # to/from dicts easily example: dict(kin="receipt", serder=serder) @@ -799,32 +753,6 @@ def incept(keys, serder._verify() # raises error if fails verifications return serder - #if delpre is not None: # delegated inception with ilk = dip - #ked['di'] = delpre - #if code is None: - #code = MtrDex.Blake3_256 # force digestive - - #if delpre is None and code is None and len(keys) == 1: - #prefixer = Prefixer(qb64=keys[0]) # defaults to not digestive code - #if prefixer.digestive: - #raise ValueError("Invalid code, digestive={}, must be derived from" - #" ked.".format(prefixer.code)) - #else: # digestive - ## raises derivation error if non-empty nxt but ephemeral code - #prefixer = Prefixer(ked=ked, code=code) # Derive AID from ked and code - - #if delpre is not None: - #if not prefixer.digestive: - #raise ValueError(f"Invalid derivation code = {prefixer.code} " - #f"for delegation. Must be digestive") - - #ked["i"] = prefixer.qb64 # update pre element in ked with pre qb64 - #if prefixer.digestive: - #ked["d"] = prefixer.qb64 - #else: - #_, ked = coring.Saider.saidify(sad=ked) - - #return Serder(ked=ked) # return serialized ked def delcept(keys, delpre, **kwa): """ diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 89bf53807..cb237303e 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -30,7 +30,8 @@ from ..kering import Protocols, Serials, Rever, versify, deversify, Ilks from ..core import coring from .coring import MtrDex, DigDex, PreDex, Saids, Digestage -from .coring import Matter, Saider, Verfer, Diger, Number, Tholder, Verser +from .coring import (Matter, Saider, Verfer, Diger, Number, Tholder, Tagger, + Ilker, Traitor, Verser, ) from ..core import counting from ..core.counting import GenDex, AllTags, Counter @@ -1242,11 +1243,11 @@ def _dumps(self, sad): # should dispatch or use match instead of big if else match l: # label - case "v": # protocol+version + case "v": # protocol+version do not use version string itself val = Verser(proto=self.proto, vrsn=self.vrsn).qb64b - case "t": # message type - val = (MtrDex.Tag3 + ilk).encode("utf-8") # add code + case "t": # message type (ilk), already got ilk + val = Ilker(ilk=v).qb64b # assumes same case "d" | "i" | "p" | "di": # said or aid val = v.encode("utf-8") # already primitive qb64 make qb6b @@ -1270,30 +1271,39 @@ def _dumps(self, sad): case "c": # list of config traits strings frame = bytearray() for e in v: # list - pass - #frame.extend(e.encode("utf-8")) + frame.extend(Traitor(trait=e).qb64n) val = bytearray(Counter(tag=AllTags.GenericListGroup, - count=len(frame) % 4, - version=self.gvrsn).qb64b) + count=len(frame) % 4).qb64b) val.extend(frame) case "a": # list of seals or field map of attributes frame = bytearray() - if isinstance(v, Mapping): - for l, e in v.items(): - pass - val = bytearray(Counter(tag=AllTags.GenericMapGroup, - count=len(frame) % 4, - version=self.gvrsn).qb64b) - else: - for e in v: # list - pass - #frame.extend(e.encode("utf-8")) + for e in v: # list of seal dicts + pass + #if tuple(v) == eventing.SealEvent._fields: + #eseal = eventing.SealEvent(**v) # convert to namedtuple + #SealSourceCouples: str = '-Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. + #SealSourceTriples: str = '-R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. + #DigestSealSingles: str = '-V' # Digest Seal Single(s), dig of sealed data. + #MerkleRootSealSingles: str = '-W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. + #BackerRegistrarSealCouples: str = '-X' # Backer Registrar Seal Couple(s), brid+dig of sealed data. + + # SealMark == tuple of seal dict field names tuple(dict) + #d = dict(a=1, b=2) + #tuple(d) + #('a', 'b') + + #frame.extend(Anchor(seal=e).qb64b) + # else: generic seal no count type (v, Mapping): + #for l, e in v.items(): + #pass + #val = bytearray(Counter(tag=AllTags.GenericMapGroup, + # count=len(frame) % 4).qb64b) + #val.extend(mapframe) - val = bytearray(Counter(tag=AllTags.GenericListGroup, - count=len(frame) % 4, - version=self.gvrsn).qb64b) + val = bytearray(Counter(tag=AllTags.GenericListGroup, + count=len(frame) % 4).qb64b) val.extend(frame) @@ -1319,7 +1329,10 @@ def _dumps(self, sad): # prepend count code for message if fixed: - pass + + val = bytearray(Counter(tag=AllTags.FixedMessageBodyGroup, + count=len(raw) % 4).qb64b) + val.extend(raw) else: pass diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index c4b08ad58..1e4aaf6bf 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -30,13 +30,13 @@ InvalidVarRawSizeError, ConversionError, SoftMaterialError, InvalidSoftError, InvalidCodeError) from keri.kering import Version, Versionage, VersionError, Vrsn_1_0, Vrsn_2_0 +from keri.kering import Protocols, Protocolage, Ilkage, Ilks, TraitDex from keri.core import coring from keri.core import eventing -from keri.core.coring import (Ilkage, Ilks, Saids, Protocols, Protocolage, - Sadder, Tholder, Seqner, NumDex, Number, Siger, - Dater, Bexter, Texter, - TagDex, PadTagDex, Tagger, Ilker, +from keri.core.coring import (Saids, Sadder, Tholder, Seqner, NumDex, Number, + Siger, Dater, Bexter, Texter, + TagDex, PadTagDex, Tagger, Ilker, Traitor, Verser, Versage, ) from keri.core.coring import Serialage, Serials, Tiers from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, @@ -4256,12 +4256,11 @@ def test_tagger(): def test_ilker(): """ - Test Ilker version primitive subclass of Tagger + Test Ilker message type subclass of Tagger """ with pytest.raises(EmptyMaterialError): ilker = Ilker() # defaults - # Tag1 ilk = Ilks.rot tag = ilk code = MtrDex.Tag3 @@ -4349,6 +4348,100 @@ def test_ilker(): """End Test""" +def test_traitor(): + """ + Test Traitor configuration trait subclass of Tagger + """ + with pytest.raises(EmptyMaterialError): + traitor = Traitor() # defaults + + trait = TraitDex.EstOnly + tag = trait + code = MtrDex.Tag2 + soft = 'EO' + qb64 = '0KEO' + qb64b = qb64.encode("utf-8") + qb2 = decodeB64(qb64b) + raw = b'' + + traitor = Traitor(trait=trait) # defaults + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + traitor = Traitor(qb2=qb2) + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + traitor = Traitor(qb64=qb64) + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + traitor = Traitor(qb64b=qb64b) + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + traitor = Traitor(tag=tag) + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + # test error condition + with pytest.raises(InvalidSoftError): + traitor = Traitor(trait='bad') + + # ignores code + traitor = Traitor(trait=trait, code=MtrDex.Tag4) + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + # test error using soft and code + with pytest.raises(InvalidSoftError): + traitor = Traitor(soft='bady', code=MtrDex.Tag4) + + """End Test""" + + def test_verser(): """ Test Verser version primitive subclass of Matter @@ -7248,6 +7341,7 @@ def test_tholder(): test_matter_special() test_tagger() test_ilker() + test_traitor() test_verser() #test_texter() #test_counter() diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 421284148..f40f61277 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -23,6 +23,8 @@ from keri.core.serdering import (FieldDom, FieldDom, Serdery, Serder, SerderKERI, SerderACDC, ) +from keri.core.eventing import (incept, ) + def test_fielddom(): """Test FieldDom dataclass""" @@ -2637,6 +2639,14 @@ def test_serdery(): """End Test""" +def test_cesr_native_dumps(): + """Test Serder._dumps""" + + keys = ["EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf87Bc70J"] + serder = incept(keys, version=Vrsn_2_0) + + """End Test""" + if __name__ == "__main__": @@ -2659,4 +2669,5 @@ def test_serdery(): test_serderacdc() test_serder_v2() test_serdery() + test_cesr_native_dumps() From 1b132892371583f7001771aabc79225211ef4448 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Tue, 2 Apr 2024 11:22:50 -0700 Subject: [PATCH 104/418] Getting migrations to work across all scenarios. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/clean.py | 4 +- src/keri/app/cli/commands/init.py | 16 ++++- src/keri/app/cli/commands/migrate/list.py | 7 +- src/keri/app/cli/commands/migrate/run.py | 25 ++++--- src/keri/app/cli/commands/migrate/show.py | 4 +- src/keri/app/cli/commands/version.py | 23 +++++- src/keri/app/cli/common/existing.py | 3 +- src/keri/db/basing.py | 76 +++++++++++++++++++- src/keri/db/dbing.py | 88 ++++++++++++++++------- src/keri/db/migrating.py | 49 ------------- 10 files changed, 192 insertions(+), 103 deletions(-) delete mode 100644 src/keri/db/migrating.py diff --git a/src/keri/app/cli/commands/clean.py b/src/keri/app/cli/commands/clean.py index a7b6e1226..7afc752bd 100644 --- a/src/keri/app/cli/commands/clean.py +++ b/src/keri/app/cli/commands/clean.py @@ -9,7 +9,6 @@ from hio.base import doing from keri.app.cli.common import existing -from keri.db import migrating logger = help.ogler.getLogger() @@ -52,8 +51,7 @@ def recur(self, tyme): hby = existing.setupHby(name=self.args.name, base=self.args.base, bran=self.args.bran, temp=self.args.temp) print("Migrating...") - migrator = migrating.Migrator(db=hby.db) - migrator.migrate() + hby.db.migrate() print("Finished") hby = existing.setupHby(name=self.args.name, base=self.args.base, diff --git a/src/keri/app/cli/commands/init.py b/src/keri/app/cli/commands/init.py index f66f936fb..58e11d796 100644 --- a/src/keri/app/cli/commands/init.py +++ b/src/keri/app/cli/commands/init.py @@ -5,14 +5,17 @@ """ import argparse import getpass +import os +import sys from hio import help from hio.base import doing import keri.app.oobiing -from keri.app import habbing, configing, oobiing, connecting +from keri import kering +from keri.app import habbing, configing, oobiing from keri.app.keeping import Algos -from keri.db import migrating +from keri.db import basing from keri.kering import ConfigurationError from keri.vdr import credentialing @@ -93,6 +96,15 @@ def initialize(self, tymth, tock=0.0): else: break + db = basing.Baser(name=name, + base=base, + temp=temp, + reopen=False) + + if db.exists(name=name, base=base, temp=temp): + print("Database already exists, exiting") + sys.exit(-1) + kwa = dict() kwa["salt"] = args.salt kwa["bran"] = bran diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py index ebe9722ae..198533fd9 100644 --- a/src/keri/app/cli/commands/migrate/list.py +++ b/src/keri/app/cli/commands/migrate/list.py @@ -10,7 +10,6 @@ from prettytable import PrettyTable from keri.app.cli.common import existing -from keri.db import migrating logger = help.ogler.getLogger() @@ -56,10 +55,10 @@ def recur(self, tyme): hby = existing.setupHby(name=self.args.name, base=self.args.base, bran=self.args.bran, temp=self.args.temp) - migrator = migrating.Migrator(db=hby.db) - for idx, (name, dater) in enumerate(migrator.complete()): + for idx, (name, dater) in enumerate(hby.db.complete()): + print(name, dater) date = dater.datetime.strftime("%Y-%m-%d %H:%M") if dater is not None else "Not Run" - tab.add_row((f"{idx+1}", f"{name}", date)) + tab.add_row((f"{idx + 1}", f"{name}", date)) print(tab) return True diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py index f4ae6e33c..b45f4e9aa 100644 --- a/src/keri/app/cli/commands/migrate/run.py +++ b/src/keri/app/cli/commands/migrate/run.py @@ -5,11 +5,13 @@ """ import argparse +import keri from hio import help from hio.base import doing +from keri import kering from keri.app.cli.common import existing -from keri.db import migrating +from keri.db import basing logger = help.ogler.getLogger() @@ -21,7 +23,7 @@ def handler(args): Args: args(Namespace): arguments object from command line """ - clean = CleanDoer(args) + clean = MigrateDoer(args) return [clean] @@ -41,18 +43,25 @@ def handler(args): dest="bran", default=None) -class CleanDoer(doing.Doer): +class MigrateDoer(doing.Doer): def __init__(self, args): self.args = args - super(CleanDoer, self).__init__() + super(MigrateDoer, self).__init__() def recur(self, tyme): - hby = existing.setupHby(name=self.args.name, base=self.args.base, - bran=self.args.bran, temp=self.args.temp) + db = basing.Baser(name=self.args.name, + base=self.args.base, + temp=self.args.temp, + reopen=False) + + try: + db.reopen() + except kering.DatabaseError: + pass + print("Migrating...") - migrator = migrating.Migrator(db=hby.db) - migrator.migrate() + db.migrate() print("Finished") return True diff --git a/src/keri/app/cli/commands/migrate/show.py b/src/keri/app/cli/commands/migrate/show.py index 3ce45c2ca..b8b9c6060 100644 --- a/src/keri/app/cli/commands/migrate/show.py +++ b/src/keri/app/cli/commands/migrate/show.py @@ -9,7 +9,6 @@ from hio.base import doing from keri.app.cli.common import existing -from keri.db import migrating logger = help.ogler.getLogger() @@ -53,8 +52,7 @@ def recur(self, tyme): hby = existing.setupHby(name=self.args.name, base=self.args.base, bran=self.args.bran, temp=self.args.temp) - migrator = migrating.Migrator(db=hby.db) - [(name, dater)] = migrator.complete(name=self.args.migration) + [(name, dater)] = hby.db.complete(name=self.args.migration) date = dater.datetime.strftime("%Y-%m-%d %H:%M") if dater is not None else "Not Run" print(f"{self.args.migration} -> {date}") diff --git a/src/keri/app/cli/commands/version.py b/src/keri/app/cli/commands/version.py index 5ea6b775a..98cc81c5a 100644 --- a/src/keri/app/cli/commands/version.py +++ b/src/keri/app/cli/commands/version.py @@ -8,18 +8,35 @@ from hio.base import doing import keri +from keri.app.cli.common import existing parser = argparse.ArgumentParser(description='Print version of KLI') parser.set_defaults(handler=lambda args: handler(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=False, + default=None) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran def handler(args): - return [doing.doify(version)] + kwa = dict(args=args) + return [doing.doify(version, **kwa)] -def version(tymth, tock=0.0): +def version(tymth, tock=0.0, **opts): """ Command line version handler """ _ = (yield tock) - print(keri.__version__) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + + print(f"Library version: {keri.__version__}") + + if name is not None: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + print(f"Database version: {hby.db.version}") diff --git a/src/keri/app/cli/common/existing.py b/src/keri/app/cli/common/existing.py index ec0f16e3c..dd730df95 100644 --- a/src/keri/app/cli/common/existing.py +++ b/src/keri/app/cli/common/existing.py @@ -47,7 +47,8 @@ def setupHby(name, base="", bran=None, cf=None, temp=False): retries += 1 hby = habbing.Habery(name=name, base=base, bran=bran, cf=cf, free=True) break - except (kering.AuthError, ValueError): + except (kering.AuthError, ValueError) as e: + print(e) if retries >= 3: raise kering.AuthError("too many attempts") print("Valid passcode required, try again...") diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 5bc4b2ac1..4a725fb7b 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -18,7 +18,7 @@ So only need to set dupsort first time opened each other opening does not need to call it """ - +import importlib import os import shutil from collections import namedtuple @@ -30,11 +30,13 @@ import cbor2 as cbor import msgpack import lmdb +import semver from ordered_set import OrderedSet as oset from hio.base import doing -from . import dbing, koming, subing, migrating +import keri +from . import dbing, koming, subing from .. import kering from ..core import coring, eventing, parsing, serdering @@ -46,6 +48,10 @@ logger = help.ogler.getLogger() +MIGRATIONS = [ + ("1.1.0", ["rekey_habs"]) +] + # ToDo XXXX maybe ''' @@ -1195,7 +1201,7 @@ def reload(self): """ # Check migrations to see if this database is up to date. Error otherwise - if not migrating.Migrator(db=self).current(self.version): + if not self.current: raise kering.DatabaseError("Database migrations must be run.") removes = [] @@ -1219,6 +1225,70 @@ def reload(self): for keys in removes: # remove bare .habs records self.habs.rem(keys=keys) + def migrate(self): + for (version, migrations) in MIGRATIONS: + # Check to see if this is for an older version + if self.version is not None and semver.compare(version, self.version) != 1: + continue + + for migration in migrations: + modName = f"keri.db.migrations.{migration}" + if self.migs.get(keys=(migration,)) is not None: + continue + + mod = importlib.import_module(modName) + try: + print(f"running migration {modName}") + mod.migrate(self) + except Exception as e: + print(f"\nAbandoning migration {migration} with error: {e}") + return + + self.migs.pin(keys=(migration,), val=coring.Dater()) + + self.version = keri.__version__ + + @property + def current(self): + """ Current property determines if we are at the current database migration state. + + If the database version matches the library version return True + If the current database version is behind the current library version, check for migrations + - If there are migrations to run, return False + - If there are no migrations to run, reset database version to library version and return True + If the current database version is ahead of the current library version, raise exception + + """ + if self.version == keri.__version__: + return True + + # If database version is ahead of library version, throw exception + if self.version is not None and semver.compare(self.version, keri.__version__) == 1: + raise kering.ConfigurationError( + f"Database version={self.version} is ahead of library version={keri.__version__}") + + last = MIGRATIONS[-1] + # If we aren't at latest version, but there are no outstanding migrations, reset version to latest + if self.migs.get(keys=(last[1][0],)) is not None: + return True + + # We have migrations to run + return False + + def complete(self, name=None): + migrations = [] + if not name: + for version, migs in MIGRATIONS: + for mig in migs: + dater = self.migs.get(keys=(mig,)) + migrations.append((mig, dater)) + else: + if name not in MIGRATIONS: + raise ValueError(f"No migration named {name}") + migrations.append((name, self.migs.get(keys=(name,)))) + + return migrations + def clean(self): """ Clean database by creating re-verified cleaned cloned copy diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 9150d9570..a0246f38e 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -53,19 +53,13 @@ from typing import Union import lmdb -from hio.help.helping import ocfn -from ordered_set import OrderedSet as oset - -from hio.base import filing - +from ordered_set import OrderedSet as oset from hio.base import filing import keri from ..kering import MaxON # maximum ordinal number for seqence or first seen - from ..help import helping -#MaxON = int("f"*32, 16) # largest possible ordinal number, sequence or first seen ProemSize = 32 # does not include trailing separator MaxProem = int("f"*(ProemSize), 16) SuffixSize = 32 # does not include trailing separator @@ -310,7 +304,6 @@ class LMDBer(filing.Filer): Perm = stat.S_ISVTX | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR # 0o1700==960 MaxNamedDBs = 96 - def __init__(self, readonly=False, **kwa): """ Setup main database directory at .dirpath. @@ -347,12 +340,12 @@ def __init__(self, readonly=False, **kwa): False means open database in read/write mode """ + self.env = None - self.version = None + self._version = None self.readonly = True if readonly else False super(LMDBer, self).__init__(**kwa) - def reopen(self, readonly=False, **kwa): """ Open if closed or close and reopen if opened or create and open if not @@ -378,6 +371,7 @@ def reopen(self, readonly=False, **kwa): readonly (bool): True means open database in readonly mode False means open database in read/write mode """ + exists = self.exists(name=self.name, base=self.base, temp=self.temp) opened = super(LMDBer, self).reopen(**kwa) if readonly is not None: self.readonly = readonly @@ -387,29 +381,56 @@ def reopen(self, readonly=False, **kwa): self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=104857600, mode=self.perm, readonly=self.readonly) - self.version = self.getVersion() - if self.version is None: + self.opened = True if opened and self.env else False + + if self.opened and not self.readonly and not exists: self.version = keri.__version__ - self.setVersion(keri.__version__) - self.opened = True if opened and self.env else False return self.opened - def getVersion(self): - with self.env.begin() as txn: - cursor = txn.cursor() - version = cursor.get(b'__version__') - return version.decode("utf-8") if version is not None else None + def exists(self, name="", base="", temp=None, headDirPath=None, clean=False, filed=False, fext=None): + temp = True if temp else False - def setVersion(self, val): - if hasattr(val, "encode"): - val = val.encode("utf-8") # convert str to bytes + if temp: + return False - with self.env.begin(write=True) as txn: - cursor = txn.cursor() - version = cursor.replace(b'__version__', val) - return version + # use class defaults here so can use makePath for other dirs and files + if headDirPath is None: + headDirPath = self.HeadDirPath + if fext is None: + fext = self.Fext + + tailDirPath = self.CleanTailDirPath if clean else self.TailDirPath + + if filed: + root, ext = os.path.splitext(name) + if not ext: + name = f"{name}.{fext}" + + path = os.path.abspath( + os.path.expanduser( + os.path.join(headDirPath, + tailDirPath, + base, + name))) + + return os.path.exists(path) + + @property + def version(self): + if self._version is None: + self._version = self.getVer() + + return self._version + + @version.setter + def version(self, val): + if hasattr(val, "decode"): + val = val.decode("utf-8") # convert bytes to str + + self._version = val + self.setVer(self._version) def close(self, clear=False): """ @@ -425,8 +446,21 @@ def close(self, clear=False): self.env = None - return(super(LMDBer, self).close(clear=clear)) + return super(LMDBer, self).close(clear=clear) + + def getVer(self): + with self.env.begin() as txn: + cursor = txn.cursor() + version = cursor.get(b'__version__') + return version.decode("utf-8") if version is not None else None + def setVer(self, val): + if hasattr(val, "encode"): + val = val.encode("utf-8") # convert str to bytes + + with self.env.begin(write=True) as txn: + cursor = txn.cursor() + cursor.replace(b'__version__', val) # For subdbs with no duplicate values allowed at each key. (dupsort==False) def putVal(self, db, key, val): diff --git a/src/keri/db/migrating.py b/src/keri/db/migrating.py deleted file mode 100644 index af9574f48..000000000 --- a/src/keri/db/migrating.py +++ /dev/null @@ -1,49 +0,0 @@ -import importlib -import sys - -import keri -from keri.core import coring - -MIGRATIONS = [ - ("1.1.0", ["rekey_habs"]) -] - - -class Migrator: - - def __init__(self, db): - self.db = db - - def migrate(self): - for migration in MIGRATIONS: - if self.db.migs.get(keys=(migration,)) is not None: - continue - - modName = f"keri.db.migrations.{migration}" - mod = importlib.import_module(modName) - try: - sys.stdout.write(f"Running migration {modName}... ") - mod.migrate(self.db) - print("done.") - except Exception as e: - print(f"\nAbandoning migratoin {migration} with error: {e}") - return - - self.db.migs.pin(keys=(migration,), val=coring.Dater()) - - def current(self, version): - return version == keri.__version__ - # return self.db.migs.get(MIGRATIONS[-1]) is not None - - def complete(self, name=None): - migrations = [] - if not name: - for mig in MIGRATIONS: - dater = self.db.migs.get(keys=(mig,)) - migrations.append((mig, dater)) - else: - if name not in MIGRATIONS: - raise ValueError(f"No migration named {name}") - migrations.append((name, self.db.migs.get(keys=(name,)))) - - return migrations From f35264789450a3c0358f9f00a24ef3370d871b59 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Wed, 3 Apr 2024 10:37:51 -0700 Subject: [PATCH 105/418] Update to latest hio, python and other out of date deps. Signed-off-by: pfeairheller --- setup.py | 9 +++++---- src/keri/app/cli/commands/init.py | 9 --------- src/keri/db/dbing.py | 33 ++----------------------------- 3 files changed, 7 insertions(+), 44 deletions(-) diff --git a/setup.py b/setup.py index c38653b49..ea3f57799 100644 --- a/setup.py +++ b/setup.py @@ -68,16 +68,16 @@ keywords=[ # eg: 'keyword1', 'keyword2', 'keyword3', ], - python_requires='>=3.12.1', + python_requires='>=3.12.2', install_requires=[ 'lmdb>=1.3.0', 'pysodium>=0.7.12', 'blake3>=0.3.1', 'msgpack>=1.0.4', - 'cbor2>=5.4.3', + 'cbor2>=5.6.2', 'multidict>=6.0.2', 'ordered-set>=4.1.0', - 'hio>=0.6.9', + 'hio>=0.6.11', 'multicommand>=1.0.0', 'jsonschema>=4.17.0', 'falcon>=3.1.0', @@ -87,7 +87,8 @@ 'mnemonic>=0.20', 'PrettyTable>=3.5.0', 'http_sfv>=0.9.8', - 'cryptography>=39.0.2' + 'cryptography>=42.0.5', + 'semver>=3.0.2' ], extras_require={ }, diff --git a/src/keri/app/cli/commands/init.py b/src/keri/app/cli/commands/init.py index 58e11d796..a4eedbefd 100644 --- a/src/keri/app/cli/commands/init.py +++ b/src/keri/app/cli/commands/init.py @@ -96,15 +96,6 @@ def initialize(self, tymth, tock=0.0): else: break - db = basing.Baser(name=name, - base=base, - temp=temp, - reopen=False) - - if db.exists(name=name, base=base, temp=temp): - print("Database already exists, exiting") - sys.exit(-1) - kwa = dict() kwa["salt"] = args.salt kwa["bran"] = bran diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index a0246f38e..2038bde7f 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -371,7 +371,7 @@ def reopen(self, readonly=False, **kwa): readonly (bool): True means open database in readonly mode False means open database in read/write mode """ - exists = self.exists(name=self.name, base=self.base, temp=self.temp) + exists = self.exists(name=self.name, base=self.base) opened = super(LMDBer, self).reopen(**kwa) if readonly is not None: self.readonly = readonly @@ -383,40 +383,11 @@ def reopen(self, readonly=False, **kwa): self.opened = True if opened and self.env else False - if self.opened and not self.readonly and not exists: + if self.opened and not self.readonly and (not exists or self.temp): self.version = keri.__version__ return self.opened - def exists(self, name="", base="", temp=None, headDirPath=None, clean=False, filed=False, fext=None): - temp = True if temp else False - - if temp: - return False - - # use class defaults here so can use makePath for other dirs and files - if headDirPath is None: - headDirPath = self.HeadDirPath - - if fext is None: - fext = self.Fext - - tailDirPath = self.CleanTailDirPath if clean else self.TailDirPath - - if filed: - root, ext = os.path.splitext(name) - if not ext: - name = f"{name}.{fext}" - - path = os.path.abspath( - os.path.expanduser( - os.path.join(headDirPath, - tailDirPath, - base, - name))) - - return os.path.exists(path) - @property def version(self): if self._version is None: From be0ebc6efa4e89a58814dcacf06447d0b3152de8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 3 Apr 2024 13:03:22 -0600 Subject: [PATCH 106/418] update setup.py with new dependencies. remove deprecated functions. add tests for Serder._dumps --- setup.py | 44 ++++++++++--------- src/keri/core/coring.py | 66 ++++++++++++++-------------- tests/core/test_coring.py | 13 +++--- tests/core/test_eventing.py | 8 ++-- tests/core/test_serdering.py | 83 ++++++++++++++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 62 deletions(-) diff --git a/setup.py b/setup.py index c38653b49..f35333b63 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,12 @@ $ twine upload dist/keri-0.0.1.tar.gz Create release git: +$ git tag # lists all tags +$ git tag -a v0.6.11 -m "new feature" +$ git show v0.6.11 +$ git push --tags # pushes tags to default remote +$ git push wot --tags # pushes tags to wot remote + $ git tag -a v0.4.2 -m "bump version" $ git push --tags $ git checkout -b release_0.4.2 @@ -68,32 +74,32 @@ keywords=[ # eg: 'keyword1', 'keyword2', 'keyword3', ], - python_requires='>=3.12.1', + python_requires='>=3.12.2', install_requires=[ - 'lmdb>=1.3.0', - 'pysodium>=0.7.12', - 'blake3>=0.3.1', - 'msgpack>=1.0.4', - 'cbor2>=5.4.3', - 'multidict>=6.0.2', + 'lmdb>=1.4.1', + 'pysodium>=0.7.17', + 'blake3>=0.4.1', + 'msgpack>=1.0.8', + 'cbor2>=5.6.2', + 'multidict>=6.0.5', 'ordered-set>=4.1.0', - 'hio>=0.6.9', + 'hio>=0.6.12', 'multicommand>=1.0.0', - 'jsonschema>=4.17.0', - 'falcon>=3.1.0', - 'hjson>=3.0.2', - 'PyYaml>=6.0', - 'apispec>=6.0.0', - 'mnemonic>=0.20', - 'PrettyTable>=3.5.0', - 'http_sfv>=0.9.8', - 'cryptography>=39.0.2' + 'jsonschema>=4.21.1', + 'falcon>=3.1.3', + 'hjson>=3.1.0', + 'PyYaml>=6..1', + 'apispec>=6.6.0', + 'mnemonic>=0.21', + 'PrettyTable>=3.10.0', + 'http_sfv>=0.9.9', + 'cryptography>=42.0.5' ], extras_require={ }, tests_require=[ - 'coverage>=6.5.0', - 'pytest>=7.2.0', + 'coverage>=7.4.4', + 'pytest>=8.1.1', 'pytest-shell>=0.3.2' ], setup_requires=[ diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index f72cee043..85bfc3378 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -183,23 +183,24 @@ def loads(raw, size=None, kind=Serials.json): return ked -def generateSigners(salt=None, count=8, transferable=True): - """ - Returns list of Signers for Ed25519 +def generateSigners(raw=None, count=8, transferable=True): + """Returns list of Signers for Ed25519 + + Deprecated, use Salter.signers instead. Use this when simply need valid AIDs but not when need valid controller contexts. In the latter case use openHby or openHab which create databases. Parameters: - salt is bytes 16 byte long root cryptomatter from which seeds for Signers - in list are derived + raw (bytes): 16 byte long salt cryptomatter from which seeds + for Signers in list are derived random salt created if not provided count is number of signers in list transferable is boolean true means signer.verfer code is transferable non-transferable otherwise """ - if not salt: - salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + if not raw: + raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) signers = [] for i in range(count): @@ -207,7 +208,7 @@ def generateSigners(salt=None, count=8, transferable=True): # algorithm default is argon2id seed = pysodium.crypto_pwhash(outlen=32, passwd=path, - salt=salt, + salt=raw, opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) @@ -217,34 +218,37 @@ def generateSigners(salt=None, count=8, transferable=True): return signers -def generatePrivates(salt=None, count=8): - """ - Returns list of fully qualified Base64 secret Ed25519 seeds i.e private keys +#def generatePrivates(raw=None, count=8): + #"""Returns list of fully qualified Base64 secret Ed25519 seeds i.e private keys - Parameters: - salt is bytes 16 byte long root cryptomatter from which seeds for Signers - in list are derived - random salt created if not provided - count is number of signers in list - """ - signers = generateSigners(salt=salt, count=count) + #Deprecated, use Salter.signers instead. - return [signer.qb64 for signer in signers] # fetch sigkey as private key + #Parameters: + #raw (bytes): 16 byte long salt cryptomatter from which seeds + #for Signers in list are derived + #random salt created if not provided + #count is number of signers in list + #""" + #signers = generateSigners(raw=raw, count=count) -def generatePublics(salt=None, count=8, transferable=True): - """ - Returns list of fully qualified Base64 secret seeds for Ed25519 private keys + #return [signer.qb64 for signer in signers] # fetch sigkey as private key - Parameters: - salt is bytes 16 byte long root cryptomatter from which seeds for Signers - in list are derived - random salt created if not provided - count is number of signers in list - """ - signers = generateSigners(salt=salt, count=count, transferable=transferable) - return [signer.verfer.qb64 for signer in signers] # fetch verkey as public key +#def generatePublics(raw=None, count=8, transferable=True): + #"""Returns list of fully qualified Base64 secret seeds for Ed25519 private keys + + #Deprecated, use Salter.signers instead. + + #Parameters: + #raw (bytes): 16 byte long salt cryptomatter from which seeds + #for Signers in list are derived + #random salt created if not provided + #count is number of signers in list + #""" + #signers = generateSigners(raw=raw, count=count, transferable=transferable) + + #return [signer.verfer.qb64 for signer in signers] # fetch verkey as public key # secret derivation security tier @@ -4755,7 +4759,7 @@ def __iter__(self): # hs is the hard size int number of chars in hard (stable) part of code # ss is the soft size int number of chars in soft (unstable) part of code # os is the other size int number of chars in other index part of soft -# ms = ss - os main index size computed +# ms = ss - os main index size computed # fs is the full size int number of chars in code plus appended material if any # ls is the lead size int number of bytes to pre-pad pre-converted raw binary Xizage = namedtuple("Xizage", "hs ss os fs ls") diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 1e4aaf6bf..ae13bca9c 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -45,7 +45,7 @@ from keri.core.coring import (Verfer, Cigar, Signer, Salter, Saider, DigDex, Diger, Prefixer, Cipher, Encrypter, Decrypter) from keri.core.coring import versify, deversify, Rever, MAXVERFULLSPAN -from keri.core.coring import generateSigners, generatePrivates +from keri.core.coring import generateSigners from keri.help import helping from keri.help.helping import (sceil, intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, @@ -5906,10 +5906,10 @@ def test_generatesigners(): for signer in signers: assert signer.verfer.code == MtrDex.Ed25519N - # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - assert len(salt) == 16 - signers = generateSigners(salt=salt, count=4) # default is transferable + # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) # raw salt + raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' + assert len(raw) == 16 + signers = generateSigners(raw=raw, count=4) # default is transferable assert len(signers) == 4 for signer in signers: assert signer.code == MtrDex.Ed25519_Seed @@ -5921,9 +5921,6 @@ def test_generatesigners(): 'AHMBU5PsIJN2U9m7j0SGyvs8YD8fkym2noELzxIrzfdG', 'AJZ7ZLd7unQ4IkMUwE69NXcvDO9rrmmRH_Xk3TPu9BpP'] - secrets = generatePrivates(salt=salt, count=4) - assert secrets == sigkeys - """ End Test """ diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index 785b83fca..d55f35e17 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -2451,7 +2451,7 @@ def test_keyeventsequence_0(): """ # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(salt=salt, count=8, transferable=True) + signers = generateSigners(raw=salt, count=8, transferable=True) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2721,7 +2721,7 @@ def test_keyeventsequence_1(): # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(salt=salt, count=8, transferable=True) + signers = generateSigners(raw=salt, count=8, transferable=True) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2818,7 +2818,7 @@ def test_multisig_digprefix(): # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(salt=salt, count=8, transferable=True) + signers = generateSigners(raw=salt, count=8, transferable=True) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2972,7 +2972,7 @@ def test_recovery(): """ # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(salt=salt, count=8, transferable=True) + signers = generateSigners(raw=salt, count=8, transferable=True) with openDB(name="controller") as conlgr, openDB(name="validator") as vallgr: event_digs = [] # list of event digs in sequence to verify against database diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index f40f61277..1ded13d64 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -25,6 +25,8 @@ from keri.core.eventing import (incept, ) +from keri.app import habbing + def test_fielddom(): """Test FieldDom dataclass""" @@ -2642,9 +2644,90 @@ def test_serdery(): def test_cesr_native_dumps(): """Test Serder._dumps""" + # use same salter for all but different path + # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' + salter = coring.Salter(raw=raw) + + csigners = coring.generateSigners(raw=salter.raw, count=3) + wsigners = coring.generateSigners(raw=salter.raw, count=3, transferable=False) + + keys = ["EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf87Bc70J"] serder = incept(keys, version=Vrsn_2_0) + assert serder.sad == \ + { + 'v': 'KERICAAJSONAAD8.', + 't': 'icp', + 'd': 'EF_SoHnCdQ0N9Kivxl54u3l1-sKwDL0gs729_REO6koi', + 'i': 'EF_SoHnCdQ0N9Kivxl54u3l1-sKwDL0gs729_REO6koi', + 's': '0', + 'kt': '1', + 'k': ['EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf87Bc70J'], + 'nt': '0', + 'n': [], + 'bt': '0', + 'b': [], + 'c': [], + 'a': [] + } + + salt = salter.qb64 + assert salt == '0AAFqo8tU5rp-lWcApybCEh1' + + # need to fix this so it uses different Kind and different Version + # makHab uses stem=name to make different names have differnt AID pre + with (habbing.openHby(name="wes", base="test", salt=salt) as wesHby, + habbing.openHby(name="wok", base="test", salt=salt) as wokHby, + habbing.openHby(name="wam", base="test", salt=salt) as wamHby, + habbing.openHby(name="cam", base="test", salt=salt) as camHby): + + # witnesses first so can setup inception event for tam + wsith = '1' + wesHab = wesHby.makeHab(name='wes', isith=wsith, icount=1, transferable=False) + wokHab = wokHby.makeHab(name='wok', isith=wsith, icount=1, transferable=False) + wamHab = wamHby.makeHab(name='wam', isith=wsith, icount=1, transferable=False) + + # setup Tam's habitat trans multisig + wits = [wesHab.pre, wokHab.pre, wamHab.pre] + tsith = '2' # hex str of threshold int + camHab = camHby.makeHab(name='cam', isith=tsith, icount=3, toad=2, wits=wits,) + + assert camHab.kever.prefixer.transferable + assert len(camHab.iserder.berfers) == len(wits) + for werfer in camHab.iserder.berfers: + assert werfer.qb64 in wits + assert camHab.kever.wits == wits + assert camHab.kever.toader.num == 2 + assert camHab.kever.sn == 0 + assert camHab.kever.tholder.thold == 2 == int(tsith, 16) + + serder, _, _ = camHab.getOwnEvent(sn=0) + + assert serder.sad == \ + { + 'v': 'KERI10JSON000273_', + 't': 'icp', + 'd': 'ED7ek7qhzr9SzqmV8IBxgHHWfsNcbWd-CKHG4-mHua6e', + 'i': 'ED7ek7qhzr9SzqmV8IBxgHHWfsNcbWd-CKHG4-mHua6e', + 's': '0', + 'kt': '2', + 'k': ['DJV4r5kpA-DuQGmDr3owzHvcWreg9fWetlS_hoznje4Q', + 'DHjp2Ewj88Url6d23i6myE-c3bSjOuNgjkZKnF8LkH7C', + 'DDjY_8DygjZg6F5-qWfZahKwPHjs1gSjzGU6nqikn1g0'], + 'nt': '2', + 'n': ['EHY_zOKIFva_iS1bGuu2etyuQOuq3tOrjaRIYHknRSSz', + 'ENlS_9WEDDgjpVRmux37ITU4O6UW8hOif-Gwa3Ch0I6t', + 'EJ67YtK72WQBmGSLS1ibDIVGM4hHtf2HrTPd1Mn51iWV'], + 'bt': '2', + 'b': ['BBVDlgWic_rAf-m_v7vz_VvIYAUPErvZgLTfXGNrFRom', + 'BKVb58uITf48YoMPz8SBOTVwLgTO9BY4oEXRPoYIOErX', + 'BByq5Nfi0KgohEaJ8h9JrLqbhX_waySFSXKsgumxEYQp'], + 'c': [], + 'a': [] + } + """End Test""" From db4d5e73ca2a4ff663c538db8e8546d330def84c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 3 Apr 2024 13:45:47 -0600 Subject: [PATCH 107/418] fixed typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f35333b63..1f766e230 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ 'jsonschema>=4.21.1', 'falcon>=3.1.3', 'hjson>=3.1.0', - 'PyYaml>=6..1', + 'PyYaml>=6.0.1', 'apispec>=6.6.0', 'mnemonic>=0.21', 'PrettyTable>=3.10.0', From def5b6a492f2f12a0d0ab1cbe4861a11616119d0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 3 Apr 2024 14:59:24 -0600 Subject: [PATCH 108/418] some clean up of kering and its tests remove commented out code --- src/keri/core/coring.py | 33 -------- src/keri/kering.py | 78 ++----------------- tests/test_kering.py | 164 +++++++++++++++++++++++++++++++++------- 3 files changed, 140 insertions(+), 135 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 85bfc3378..045ee8675 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -218,39 +218,6 @@ def generateSigners(raw=None, count=8, transferable=True): return signers -#def generatePrivates(raw=None, count=8): - #"""Returns list of fully qualified Base64 secret Ed25519 seeds i.e private keys - - #Deprecated, use Salter.signers instead. - - - #Parameters: - #raw (bytes): 16 byte long salt cryptomatter from which seeds - #for Signers in list are derived - #random salt created if not provided - #count is number of signers in list - #""" - #signers = generateSigners(raw=raw, count=count) - - #return [signer.qb64 for signer in signers] # fetch sigkey as private key - - -#def generatePublics(raw=None, count=8, transferable=True): - #"""Returns list of fully qualified Base64 secret seeds for Ed25519 private keys - - #Deprecated, use Salter.signers instead. - - #Parameters: - #raw (bytes): 16 byte long salt cryptomatter from which seeds - #for Signers in list are derived - #random salt created if not provided - #count is number of signers in list - #""" - #signers = generateSigners(raw=raw, count=count, transferable=transferable) - - #return [signer.verfer.qb64 for signer in signers] # fetch verkey as public key - - # secret derivation security tier Tierage = namedtuple("Tierage", 'low med high') diff --git a/src/keri/kering.py b/src/keri/kering.py index 089fc53ab..aee3481a7 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -57,12 +57,6 @@ SMELLSIZE = MAXVSOFFSET + MAXVERFULLSPAN # min buffer size to inhale -# version field in CESR native serialization -VFFULLSPAN = 12 # number of characters in full version string -VFREX = b'0N(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})' - -Revfer = re.compile(VFREX) # compile is faster - """ Smellage (results of smelling a version string such as in a Serder) @@ -209,69 +203,6 @@ def smell(raw): return rematch(match) -def snatch(match, size=0): - """ Returns: - smellage (Smellage): named tuple extracted from version string regex match - (protocol, version, kind, size) - - Parameters: - match (re.Match): instance of Match class - size (int): provided size to substitute when missing - - Notes: - regular expressions work with memoryview objects not just bytes or - bytearrays - """ - full = match.group() # full matched version string - if len(full) == VFFULLSPAN: - proto, major, minor, gmajor, gminor = match.group("proto0", - "major0", - "minor0", - "gmajor0", - "gminor0") - proto = proto.decode("utf-8") - if proto not in Protocols: - raise ProtocolError(f"Invalid protocol type = {proto}.") - vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) - if vrsn.major < 2: # version2 vs but major < 2 - raise VersionError(f"Incompatible {vrsn=} with version string.") - - gvrsn = Versionage(major=b64ToInt(gmajor), minor=b64ToInt(gminor)) - if gvrsn.major < 2: # version2 vs but major < 2 - raise VersionError(f"Incompatible {gvrsn=} with CESR native version" - f"field.") - kind = Serials.cesr - size = size - else: - raise VersionError(f"Bad snatch.") - - return Smellage(proto=proto, vrsn=vrsn, kind=kind, size=size, gvrsn=gvrsn) - - -def snuff(raw, size=0): - """Extract and return instance of Smellage from version string inside - raw serialization. - - Returns: - smellage (Smellage): named Tuple of (protocol, version, kind, size) - - Parameters: - raw (bytearray) of serialized incoming message stream. Assumes start - of stream is JSON, CBOR, or MGPK field map with first field - is labeled 'v' and value is version string. - size (int): provided size to substitute when missing - - """ - if len(raw) < SMELLSIZE: - raise ShortageError(f"Need more raw bytes to smell full version string.") - - match = Rever.search(raw) # Rever regex takes bytes/bytearray not str - if not match or match.start() > MAXVSOFFSET: - raise VersionError(f"Invalid version string from smelled raw = " - f"{raw[: SMELLSIZE]}.") - - return snatch(match, size=size) - @dataclass(frozen=True) class ColdCodex: @@ -281,19 +212,20 @@ class ColdCodex: Undefined are left out so that inclusion(exclusion) via 'in' operator works. First three bits: - 0o0 = 000 free + 0o0 = 000 annotated B64 (exhaustive) 0o1 = 001 cntcode B64 0o2 = 010 opcode B64 0o3 = 011 json - 0o4 = 100 mgpk + 0o4 = 100 mgpk1 0o5 = 101 cbor - 0o6 = 110 mgpk + 0o6 = 110 mgpk2 007 = 111 cntcode or opcode B2 status is one of ('evt', 'txt', 'bny' ) 'evt' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) 'bny' if tritet in (ColdDex.CtOpB2,) + 'ann' if trited in (ColdDex.AnB64) otherwise raise ColdStartError @@ -302,7 +234,7 @@ class ColdCodex: x[0] >> 5 == 0o1 True """ - Anno: int = 0o0 # Annotated CESR + AnB64: int = 0o0 # Annotated CESR CtB64: int = 0o1 # CountCode Base64 OpB64: int = 0o2 # OpCode Base64 JSON: int = 0o3 # JSON Map Event Start diff --git a/tests/test_kering.py b/tests/test_kering.py index 01a1f7e24..76b343456 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -16,13 +16,13 @@ from keri.kering import Protocolage, Protocols from keri.kering import Serialage, Serials from keri.kering import Ilkage, Ilks -from keri.kering import TraitCodex, TraitDex +from keri.kering import ColdCodex, ColdDex, TraitCodex, TraitDex from keri.kering import (Versionage, Version, MAXVERFULLSPAN, versify, deversify, Rever, Smellage, smell, VER1FULLSPAN, VER1TERM, VEREX1, VER2FULLSPAN, VER2TERM, VEREX2, VEREX) -from keri.kering import VFFULLSPAN, VFREX, Revfer + from keri.kering import VersionError, ProtocolError, KindError from keri.help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, @@ -44,31 +44,6 @@ def test_protocols(): """End Test""" -def test_traitdex(): - """ - Test TraitDex instance of TraitCodex dataclass - """ - - assert isinstance(TraitDex, TraitCodex) - - assert asdict(TraitDex) == \ - { - 'EstOnly': 'EO', - 'DoNotDelegate': 'DND', - 'RegistrarBackers': 'RB', - 'NoBackers': 'NB', - 'NoRegistrarBackers': 'NRB', - 'DelegateIsDelegator': 'DID', - } - - assert 'EO' in TraitDex - assert 'DND' in TraitDex - assert 'RB' in TraitDex - assert 'NB' in TraitDex - assert 'NRB' in TraitDex - assert 'DID' in TraitDex - - """End Test""" def test_version_regex(): @@ -234,6 +209,83 @@ def test_snuff(): """ + + # version field in CESR native serialization + VFFULLSPAN = 12 # number of characters in full version string + VFREX = b'0N(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})' + + Revfer = re.compile(VFREX) # compile is faster + + + def snatch(match, size=0): + """ Returns: + smellage (Smellage): named tuple extracted from version string regex match + (protocol, version, kind, size) + + Parameters: + match (re.Match): instance of Match class + size (int): provided size to substitute when missing + + Notes: + regular expressions work with memoryview objects not just bytes or + bytearrays + """ + full = match.group() # full matched version string + if len(full) == VFFULLSPAN: + proto, major, minor, gmajor, gminor = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + proto = proto.decode("utf-8") + if proto not in Protocols: + raise ProtocolError(f"Invalid protocol type = {proto}.") + vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) + if vrsn.major < 2: # version2 vs but major < 2 + raise VersionError(f"Incompatible {vrsn=} with version string.") + + gvrsn = Versionage(major=b64ToInt(gmajor), minor=b64ToInt(gminor)) + if gvrsn.major < 2: # version2 vs but major < 2 + raise VersionError(f"Incompatible {gvrsn=} with CESR native version" + f"field.") + kind = Serials.cesr + size = size + else: + raise VersionError(f"Bad snatch.") + + return Smellage(proto=proto, vrsn=vrsn, kind=kind, size=size, gvrsn=gvrsn) + + + def snuff(raw, size=0): + """Extract and return instance of Smellage from version string inside + raw serialization. + + Returns: + smellage (Smellage): named Tuple of (protocol, version, kind, size) + + Parameters: + raw (bytearray) of serialized incoming message stream. Assumes start + of stream is JSON, CBOR, or MGPK field map with first field + is labeled 'v' and value is version string. + size (int): provided size to substitute when missing + + """ + if len(raw) < SMELLSIZE: + raise ShortageError(f"Need more raw bytes to smell full version string.") + + match = Rever.search(raw) # Rever regex takes bytes/bytearray not str + if not match or match.start() > MAXVSOFFSET: + raise VersionError(f"Invalid version string from smelled raw = " + f"{raw[: SMELLSIZE]}.") + + return snatch(match, size=size) + + + + + + + #pattern = re.compile(VFREX) # compile is faster pattern = Revfer @@ -717,7 +769,35 @@ def test_versify_v2(): """End Test""" +def test_colddex(): + """ + Test ColdDex instance of ColdCodex dataclass + """ + assert isinstance(ColdDex, ColdCodex) + + assert asdict(ColdDex) == \ + { + 'AnB64': 0, + 'CtB64': 1, + 'OpB64': 2, + 'JSON': 3, + 'MGPK1': 4, + 'CBOR': 5, + 'MGPK2': 6, + 'CtOpB2': 7 + } + + assert 0o0 in ColdDex + assert 0o1 in ColdDex + assert 0o2 in ColdDex + assert 0o3 in ColdDex + assert 0o4 in ColdDex + assert 0o5 in ColdDex + assert 0o6 in ColdDex + assert 0o7 in ColdDex + + """End Test""" def test_ilks(): @@ -797,12 +877,37 @@ def test_ilks(): """End Test """ +def test_traitdex(): + """ + Test TraitDex instance of TraitCodex dataclass + """ + + assert isinstance(TraitDex, TraitCodex) + + assert asdict(TraitDex) == \ + { + 'EstOnly': 'EO', + 'DoNotDelegate': 'DND', + 'RegistrarBackers': 'RB', + 'NoBackers': 'NB', + 'NoRegistrarBackers': 'NRB', + 'DelegateIsDelegator': 'DID', + } + + assert 'EO' in TraitDex + assert 'DND' in TraitDex + assert 'RB' in TraitDex + assert 'NB' in TraitDex + assert 'NRB' in TraitDex + assert 'DID' in TraitDex + + """End Test""" if __name__ == "__main__": test_protocols() - test_traitdex() + test_version_regex() test_smell() test_snuff() @@ -810,4 +915,5 @@ def test_ilks(): test_versify_v1() test_versify_v2() test_ilks() - + test_colddex() + test_traitdex() From 333084d73e0719c3bd6ab34d2b7230344100d454 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Wed, 3 Apr 2024 15:08:32 -0700 Subject: [PATCH 109/418] Updated doc strings Signed-off-by: pfeairheller --- src/keri/db/basing.py | 20 +++++++++++++++++++- src/keri/db/dbing.py | 26 ++++++++++++++++++++++++++ src/keri/db/migrations/rekey_habs.py | 21 ++++++++++++++++----- 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 4a725fb7b..7b324194c 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1226,6 +1226,15 @@ def reload(self): self.habs.rem(keys=keys) def migrate(self): + """ Run all migrations required + + Run all migrations that are required from the current version of database up to the current version + of the software that have not already been run. + + Sets the version of the database to the current version of the software after successful completion + of required migrations + + """ for (version, migrations) in MIGRATIONS: # Check to see if this is for an older version if self.version is not None and semver.compare(version, self.version) != 1: @@ -1276,6 +1285,15 @@ def current(self): return False def complete(self, name=None): + """ Returns list of tuples of migrations completed with date of completion + + Parameters: + name(str): optional name of migration to check completeness + + Returns: + list: tuples of migration,date of completed migration names and the date of completion + + """ migrations = [] if not name: for version, migs in MIGRATIONS: @@ -1283,7 +1301,7 @@ def complete(self, name=None): dater = self.migs.get(keys=(mig,)) migrations.append((mig, dater)) else: - if name not in MIGRATIONS: + if name not in MIGRATIONS or not self.migs.get(keys=(name,)): raise ValueError(f"No migration named {name}") migrations.append((name, self.migs.get(keys=(name,)))) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 2038bde7f..ddb0ce4e0 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -390,6 +390,14 @@ def reopen(self, readonly=False, **kwa): @property def version(self): + """ Return the version of database stored in __version__ key. + + This value is read through cached in memory + + Returns: + str: the version of the database or None if not set in the database + + """ if self._version is None: self._version = self.getVer() @@ -397,6 +405,12 @@ def version(self): @version.setter def version(self, val): + """ Set the version of the database in memory and in the __version__ key + + Parameters: + val (str): The new semver formatted version of the database + + """ if hasattr(val, "decode"): val = val.decode("utf-8") # convert bytes to str @@ -420,12 +434,24 @@ def close(self, clear=False): return super(LMDBer, self).close(clear=clear) def getVer(self): + """ Returns the value of the the semver formatted version in the __version__ key in this database + + Returns: + str: semver formatted version of the database + + """ with self.env.begin() as txn: cursor = txn.cursor() version = cursor.get(b'__version__') return version.decode("utf-8") if version is not None else None def setVer(self, val): + """ Set the version of the database in the __version__ key + + Parameters: + val (str): The new semver formatted version of the database + + """ if hasattr(val, "encode"): val = val.encode("utf-8") # convert str to bytes diff --git a/src/keri/db/migrations/rekey_habs.py b/src/keri/db/migrations/rekey_habs.py index 9275a25e2..e21dfdb9f 100644 --- a/src/keri/db/migrations/rekey_habs.py +++ b/src/keri/db/migrations/rekey_habs.py @@ -26,6 +26,18 @@ class OldHabitatRecord: # baser.habs def migrate(db): + """ Re-key habs migration for changing the key for .habs and introducing the .names database + + This migrations performs the following: + 1. Rekey .habs from name (alias) to the AID of the Hab + 2. Add Name and domain to the HabitatRecord for all Habs + 3. Populate the .names index as (ns, name) -> AID + 4. Remove the .nmsp namespaced Habs database (replaced within .habs and .names now) + + Parameters: + db(Baser): Baser database object on which to run the migration + + """ habs = koming.Komer(db=db, subkey='habs.', schema=OldHabitatRecord, ) @@ -36,6 +48,7 @@ def migrate(db): schema=OldHabitatRecord, ) habords = dict() + # Update Hab records from .habs with name for name, habord in habs.getItemIter(): name = ".".join(name) # detupleize the database key name nhabord = basing.HabitatRecord(**asdict(habord)) @@ -44,6 +57,7 @@ def migrate(db): habs.trim() + # Update Hab records from .nmsp with name and domain (ns) for keys, habord in nmsp.getItemIter(): ns = keys[0] name = ".".join(keys[1:]) # detupleize the database key name @@ -52,13 +66,10 @@ def migrate(db): nhabord.domain = ns habords[habord.hid] = nhabord - nmsp.trim() + nmsp.trim() # remove existing records + # Rekey .habs and create .names index for pre, habord in habords.items(): - print(pre) - print(habord) db.habs.pin(keys=(pre,), val=habord) ns = "" if habord.domain is None else habord.domain - print(ns) - print(habord.name) db.names.pin(keys=(ns, habord.name), val=pre) From bfc3ca76cdd313eaa8acabbc688f0bf044999e73 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Wed, 3 Apr 2024 15:18:14 -0700 Subject: [PATCH 110/418] Update docker version in workflow. Signed-off-by: pfeairheller --- .github/workflows/python-app-ci.yml | 12 ++++++------ scripts/demo/basic/multisig.sh | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-app-ci.yml b/.github/workflows/python-app-ci.yml index 98751e757..1534a51c7 100644 --- a/.github/workflows/python-app-ci.yml +++ b/.github/workflows/python-app-ci.yml @@ -21,10 +21,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.12.1 + - name: Set up Python 3.12.2 uses: actions/setup-python@v2 with: - python-version: 3.12.1 + python-version: 3.12.2 - name: Install dependencies run: | python -m pip install --upgrade pip @@ -47,10 +47,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.12.1 + - name: Set up Python 3.12.2 uses: actions/setup-python@v2 with: - python-version: 3.12.1 + python-version: 3.12.2 - name: Install dependencies run: | python -m pip install --upgrade pip @@ -68,10 +68,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.12.1 + - name: Set up Python 3.12.2 uses: actions/setup-python@v2 with: - python-version: 3.12.1 + python-version: 3.12.2 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/scripts/demo/basic/multisig.sh b/scripts/demo/basic/multisig.sh index e6c6a8000..78006672d 100755 --- a/scripts/demo/basic/multisig.sh +++ b/scripts/demo/basic/multisig.sh @@ -6,11 +6,11 @@ kli init --name multisig1 --base "${KERI_TEMP_DIR}" --salt 0ACDEyMzQ1Njc4OWxtbm9aBc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis kli incept --name multisig1 --base "${KERI_TEMP_DIR}" --alias multisig1 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-1-sample.json -kli ends add --name multisig1 --base "${KERI_TEMP_DIR}" --alias multisig1 --eid BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM --role mailbox +#kli ends add --name multisig1 --base "${KERI_TEMP_DIR}" --alias multisig1 --eid BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM --role mailbox kli init --name multisig2 --base "${KERI_TEMP_DIR}" --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis kli incept --name multisig2 --base "${KERI_TEMP_DIR}" --alias multisig2 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-2-sample.json -kli ends add --name multisig2 --base "${KERI_TEMP_DIR}" --alias multisig2 --eid BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX --role mailbox +#kli ends add --name multisig2 --base "${KERI_TEMP_DIR}" --alias multisig2 --eid BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX --role mailbox kli oobi resolve --name multisig1 --base "${KERI_TEMP_DIR}" --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 kli oobi resolve --name multisig2 --base "${KERI_TEMP_DIR}" --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 From 569ff247fcd653f040599c1633e58bf7fefa0185 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 3 Apr 2024 18:29:14 -0600 Subject: [PATCH 111/418] fixed merge fix typo --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f766e230..ee4d6997a 100644 --- a/setup.py +++ b/setup.py @@ -93,7 +93,8 @@ 'mnemonic>=0.21', 'PrettyTable>=3.10.0', 'http_sfv>=0.9.9', - 'cryptography>=42.0.5' + 'cryptography>=42.0.5', + 'semver>=3.0.2' ], extras_require={ }, From ac0e17922e2c849a64450fa945a00f5a2a9e873b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 3 Apr 2024 19:24:24 -0600 Subject: [PATCH 112/418] Refactor Sealer to Anchorer Fix lint problems --- src/keri/app/cli/commands/incept.py | 2 +- src/keri/app/cli/commands/rotate.py | 2 +- src/keri/app/delegating.py | 6 +++--- src/keri/app/grouping.py | 2 +- tests/app/test_delegating.py | 8 ++++---- tests/test_kering.py | 9 ++++++--- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/keri/app/cli/commands/incept.py b/src/keri/app/cli/commands/incept.py index 29ff39cf6..084d6315c 100644 --- a/src/keri/app/cli/commands/incept.py +++ b/src/keri/app/cli/commands/incept.py @@ -139,7 +139,7 @@ def __init__(self, name, base, alias, bran, endpoint, proxy=None, cnfg=None, **k self.proxy = proxy hby = existing.setupHby(name=name, base=base, bran=bran, cf=cf) self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer - self.swain = delegating.Sealer(hby=hby) + self.swain = delegating.Anchorer(hby=hby) self.postman = forwarding.Poster(hby=hby) self.mbx = indirecting.MailboxDirector(hby=hby, topics=['/receipt', "/replay", "/reply"]) doers = [self.hbyDoer, self.postman, self.mbx, self.swain, doing.doify(self.inceptDo)] diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index 73d2cda2b..56092c617 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -147,7 +147,7 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer - self.swain = delegating.Sealer(hby=self.hby) + self.swain = delegating.Anchorer(hby=self.hby) self.postman = forwarding.Poster(hby=self.hby) self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=['/receipt', "/replay", "/reply"]) doers = [self.hbyDoer, self.mbx, self.swain, self.postman, doing.doify(self.rotateDo)] diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 475b027f3..0823af1b1 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -19,8 +19,8 @@ logger = help.ogler.getLogger() -class Sealer(doing.DoDoer): - """ +class Anchorer(doing.DoDoer): + """Anchorer subclass of DoDoer Sends messages to Delegator of an identifier and wait for the anchoring event to be processed to ensure the inception or rotation event has been approved by the delegator. @@ -46,7 +46,7 @@ def __init__(self, hby, proxy=None, **kwa): self.witDoer = agenting.Receiptor(hby=self.hby) self.proxy = proxy - super(Sealer, self).__init__(doers=[self.witq, self.witDoer, self.postman, doing.doify(self.escrowDo)], + super(Anchorer, self).__init__(doers=[self.witq, self.witDoer, self.postman, doing.doify(self.escrowDo)], **kwa) def delegation(self, pre, sn=None, proxy=None): diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 0495ce9f3..643aa7222 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -25,7 +25,7 @@ class Counselor(doing.DoDoer): def __init__(self, hby, swain=None, proxy=None, **kwa): self.hby = hby - self.swain = swain if swain is not None else delegating.Sealer(hby=self.hby) + self.swain = swain if swain is not None else delegating.Anchorer(hby=self.hby) self.proxy = proxy self.witDoer = agenting.Receiptor(hby=self.hby) self.witq = agenting.WitnessInquisitor(hby=hby) diff --git a/tests/app/test_delegating.py b/tests/app/test_delegating.py index d66bb6dfd..e94680e2e 100644 --- a/tests/app/test_delegating.py +++ b/tests/app/test_delegating.py @@ -12,14 +12,14 @@ from keri.db import dbing -def test_sealer(seeder): +def test_anchorer(seeder): with habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby, \ habbing.openHby(name="del", salt=coring.Salter(raw=b'0123456789ghijkl').qb64) as delHby: wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, tcpPort=5634, httpPort=5644) witDoer = agenting.Receiptor(hby=palHby) - bts = delegating.Sealer(hby=delHby) + bts = delegating.Anchorer(hby=delHby) wesHab = wesHby.habByName(name="wes") seeder.seedWitEnds(palHby.db, witHabs=[wesHab], protocols=[kering.Schemes.http]) @@ -33,7 +33,7 @@ def test_sealer(seeder): bts=bts ) - doers = wesDoers + [witDoer, bts, doing.doify(sealer_test_do, **opts)] + doers = wesDoers + [witDoer, bts, doing.doify(anchorer_test_do, **opts)] limit = 1.0 tock = 0.03125 @@ -60,7 +60,7 @@ def test_sealer(seeder): assert bytes(delHby.db.getAes(dgkey)) == couple -def sealer_test_do(tymth=None, tock=0.0, **opts): +def anchorer_test_do(tymth=None, tock=0.0, **opts): yield tock # enter context wesHab = opts["wesHab"] diff --git a/tests/test_kering.py b/tests/test_kering.py index 76b343456..6adbbc04e 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -216,6 +216,9 @@ def test_snuff(): Revfer = re.compile(VFREX) # compile is faster + MAXVFOFFSET = 12 + + SNUFFSIZE = MAXVFOFFSET + VFFULLSPAN def snatch(match, size=0): """ Returns: @@ -270,13 +273,13 @@ def snuff(raw, size=0): size (int): provided size to substitute when missing """ - if len(raw) < SMELLSIZE: + if len(raw) < SNUFFSIZE: raise ShortageError(f"Need more raw bytes to smell full version string.") match = Rever.search(raw) # Rever regex takes bytes/bytearray not str - if not match or match.start() > MAXVSOFFSET: + if not match or match.start() > MAXVFOFFSET: raise VersionError(f"Invalid version string from smelled raw = " - f"{raw[: SMELLSIZE]}.") + f"{raw[: SNUFFSIZE]}.") return snatch(match, size=size) From 9739c93a7680c25449bd08f5399352da70eba482 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 3 Apr 2024 20:23:56 -0600 Subject: [PATCH 113/418] fixed lint errors --- tests/test_kering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_kering.py b/tests/test_kering.py index 6adbbc04e..aeb86e717 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -274,11 +274,11 @@ def snuff(raw, size=0): """ if len(raw) < SNUFFSIZE: - raise ShortageError(f"Need more raw bytes to smell full version string.") + raise kering.ShortageError(f"Need more raw bytes to smell full version string.") match = Rever.search(raw) # Rever regex takes bytes/bytearray not str if not match or match.start() > MAXVFOFFSET: - raise VersionError(f"Invalid version string from smelled raw = " + raise kering.VersionError(f"Invalid version string from smelled raw = " f"{raw[: SNUFFSIZE]}.") return snatch(match, size=size) From 22a7e1438dd20b84805b96041c715faeec5293ef Mon Sep 17 00:00:00 2001 From: Shoeb Ahmed Tanjim <39959228+s-a-tanjim@users.noreply.github.com> Date: Thu, 4 Apr 2024 20:07:48 +0600 Subject: [PATCH 114/418] Writing logs into file is configurable from cli argument (#737) --- src/keri/app/cli/commands/witness/start.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/keri/app/cli/commands/witness/start.py b/src/keri/app/cli/commands/witness/start.py index 848f48e03..3ea02fe6e 100644 --- a/src/keri/app/cli/commands/witness/start.py +++ b/src/keri/app/cli/commands/witness/start.py @@ -49,11 +49,15 @@ parser.add_argument("--cafilepath", action="store", required=False, default=None) parser.add_argument("--loglevel", action="store", required=False, default="CRITICAL", help="Set log level to DEBUG | INFO | WARNING | ERROR | CRITICAL. Default is CRITICAL") +parser.add_argument("--logfile", action="store", required=False, default=None, + help="path of the log file. If not defined, logs will not be written to the file.") def launch(args): help.ogler.level = logging.getLevelName(args.loglevel) - help.ogler.reopen(name=args.name, temp=True, clear=True) # need to configure for logging persistent file + if(args.logfile != None): + help.ogler.headDirPath = args.logfile + help.ogler.reopen(name=args.name, temp=False, clear=True) logger = help.ogler.getLogger() logger.info("\n******* Starting Witness for %s listening: http/%s, tcp/%s " From 96c99ceda063a2bdb9263fd48bf6b61f50e2dcaa Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Thu, 4 Apr 2024 10:08:04 -0400 Subject: [PATCH 115/418] updates codecov adds secret (#736) Signed-off-by: Kevin Griffin --- .github/workflows/python-app-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app-ci.yml b/.github/workflows/python-app-ci.yml index 2df15d859..4fb0f8039 100644 --- a/.github/workflows/python-app-ci.yml +++ b/.github/workflows/python-app-ci.yml @@ -60,9 +60,9 @@ jobs: run: | pytest --cov=./ --cov-report=xml - name: Upload - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: - token: 5eb1c60e-8b43-45f1-a003-357b240061cd + token: ${{ secrets.CODECOV_TOKEN }} scripts: runs-on: ubuntu-latest From e66582619ba734430b966b665e6a86696441171e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 4 Apr 2024 12:19:26 -0600 Subject: [PATCH 116/418] clean up imports --- src/keri/core/eventing.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index ca93cab29..123194eab 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -13,18 +13,8 @@ from ordered_set import OrderedSet as oset from hio.help import decking -from . import coring, serdering -from .coring import (versify, Serials, Ilks, MtrDex, PreDex, DigDex, - NonTransDex, CtrDex, Counter, - Number, Seqner, Siger, Cigar, Dater, Indexer, IdrDex, - Verfer, Diger, Prefixer, Tholder, Saider) -from . import serdering -from .. import help -from .. import kering -from ..db import basing, dbing -from ..db.basing import KeyStateRecord, StateEERecord -from ..db.dbing import dgKey, snKey, fnKey, splitKeySN, splitKey +from .. import kering from ..kering import (MissingEntryError, ValidationError, DerivationError, MissingSignatureError, MissingWitnessSignatureError, UnverifiedReplyError, @@ -36,8 +26,22 @@ from ..kering import Version, Versionage, TraitCodex, TraitDex from ..kering import Coldage, Colds, ColdDex +from .. import help from ..help import helping +from . import coring +from .coring import (versify, Serials, Ilks, MtrDex, PreDex, DigDex, + NonTransDex, CtrDex, Counter, + Number, Seqner, Siger, Cigar, Dater, Indexer, IdrDex, + Verfer, Diger, Prefixer, Tholder, Saider) + +from . import serdering + +from ..db import basing, dbing +from ..db.basing import KeyStateRecord, StateEERecord +from ..db.dbing import dgKey, snKey, fnKey, splitKeySN, splitKey + + logger = help.ogler.getLogger() EscrowTimeoutPS = 3600 # seconds for partial signed escrow timeout From 5b9b8fd51430aa9d407f9a07621981fc1fd9cbb0 Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Fri, 5 Apr 2024 13:39:27 -0400 Subject: [PATCH 117/418] Updated docker images to use python 3.12 (#740) --- images/keripy.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/keripy.dockerfile b/images/keripy.dockerfile index ca12837fc..f04bb811d 100644 --- a/images/keripy.dockerfile +++ b/images/keripy.dockerfile @@ -1,5 +1,5 @@ # Builder layer -FROM python:3.10-alpine as builder +FROM python:3.12-alpine as builder # Install compilation dependencies RUN apk --no-cache add \ @@ -32,7 +32,7 @@ RUN . ${HOME}/.cargo/env && \ pip install -r requirements.txt # Runtime layer -FROM python:3.10.13-alpine3.18 +FROM python:3.12.2-alpine3.18 RUN apk --no-cache add \ bash \ From 0d7dd7a3d8035ff9516d31d881724858a548d674 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 5 Apr 2024 13:39:15 -0600 Subject: [PATCH 118/418] Moved Indexer to its own module --- src/keri/app/agenting.py | 4 +- src/keri/app/cli/commands/verify.py | 4 +- src/keri/app/grouping.py | 6 +- src/keri/app/habbing.py | 6 +- src/keri/core/__init__.py | 11 +- src/keri/core/coring.py | 2613 ++++++++++----------------- src/keri/core/counting.py | 21 +- src/keri/core/eventing.py | 11 +- src/keri/core/indexing.py | 785 ++++++++ src/keri/core/parsing.py | 3 +- src/keri/core/serdering.py | 3 +- src/keri/db/basing.py | 6 +- src/keri/db/escrowing.py | 12 +- src/keri/end/ending.py | 16 +- src/keri/vdr/eventing.py | 6 +- src/keri/vdr/viring.py | 4 +- tests/app/test_keeping.py | 17 +- tests/core/test_coring.py | 754 +------- tests/core/test_counting.py | 142 +- tests/core/test_eventing.py | 7 +- tests/core/test_indexing.py | 781 ++++++++ tests/core/test_replay.py | 8 +- tests/core/test_witness.py | 8 +- tests/db/test_subing.py | 37 +- tests/vc/test_proving.py | 5 +- tests/vdr/test_verifying.py | 8 +- 26 files changed, 2686 insertions(+), 2592 deletions(-) create mode 100644 src/keri/core/indexing.py create mode 100644 tests/core/test_indexing.py diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 7f90b41c5..e51ddc7f3 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -17,7 +17,7 @@ from . import httping, forwarding from .. import help from .. import kering -from ..core import eventing, parsing, coring, serdering +from ..core import eventing, parsing, coring, serdering, indexing from ..core.coring import CtrDex from ..db import dbing from ..kering import Roles @@ -360,7 +360,7 @@ def receiptDo(self, tymth=None, tock=0.0): continue # generate all rct msgs to send to all witnesses - awigers = [coring.Siger(qb64b=bytes(wig)) for wig in wigs] + awigers = [indexing.Siger(qb64b=bytes(wig)) for wig in wigs] # make sure all witnesses have fully receipted KERL and know about each other for witer in witers: diff --git a/src/keri/app/cli/commands/verify.py b/src/keri/app/cli/commands/verify.py index aa75a1c56..1de4fa25b 100644 --- a/src/keri/app/cli/commands/verify.py +++ b/src/keri/app/cli/commands/verify.py @@ -9,7 +9,7 @@ from keri import kering from keri.app.cli.common import existing -from keri.core import coring +from keri.core import coring, indexing parser = argparse.ArgumentParser(description='Verify signature(s) on arbitrary data') parser.set_defaults(handler=lambda args: handler(args)) @@ -49,7 +49,7 @@ def verify(tymth, tock=0.0, **opts): base = args.base bran = args.bran - sigers = [coring.Siger(qb64=sig) for sig in args.signature] + sigers = [indexing.Siger(qb64=sig) for sig in args.signature] try: with existing.existingHab(name=name, alias=alias, base=base, bran=bran) as (_, hab): diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 643aa7222..2f4bd3e82 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -12,7 +12,7 @@ from .. import kering from .. import help from ..app import delegating, agenting -from ..core import coring, routing, eventing, parsing, serdering +from ..core import coring, routing, eventing, parsing, serdering, indexing from ..db import dbing from ..db.dbing import snKey from ..peer import exchanging @@ -124,7 +124,7 @@ def processPartialSignedEscrow(self): if not sigs: # otherwise its a list of sigs continue - sigers = [coring.Siger(qb64b=bytes(sig)) for sig in sigs] + sigers = [indexing.Siger(qb64b=bytes(sig)) for sig in sigs] windex = min([siger.index for siger in sigers]) # True if Elected to perform delegation and witnessing @@ -516,7 +516,7 @@ def getEscrowedEvent(db, pre, sn): sigs = [] for sig in db.getSigsIter(key): - sigs.append(coring.Siger(qb64b=bytes(sig))) + sigs.append(indexing.Siger(qb64b=bytes(sig))) couple = db.getAes(key) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index eb81326c8..8da4000c7 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -16,7 +16,7 @@ from . import keeping, configing from .. import help from .. import kering -from ..core import coring, eventing, parsing, routing, serdering +from ..core import coring, eventing, parsing, routing, serdering, indexing from ..db import dbing, basing from ..kering import MissingSignatureError, Roles @@ -2011,7 +2011,7 @@ def getOwnEvent(self, sn, allowPartiallySigned=False): sigs = [] for sig in self.db.getSigsIter(key): - sigs.append(coring.Siger(qb64b=bytes(sig))) + sigs.append(indexing.Siger(qb64b=bytes(sig))) couple = self.db.getAes(key) @@ -2890,7 +2890,7 @@ def witnesser(self): if not sigs: # otherwise its a list of sigs return False - sigers = [coring.Siger(qb64b=bytes(sig)) for sig in sigs] + sigers = [indexing.Siger(qb64b=bytes(sig)) for sig in sigs] windex = min([siger.index for siger in sigers]) # True if Elected to perform delegation and witnessing diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index 02761a160..7b4d30b1a 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -4,4 +4,13 @@ keri.core Package """ -__all__ = ["coring", "eventing", "parsing", "scheming"] +#__all__ = ["coring", "eventing", "parsing", "scheming"] + +# Matter class and its subclasses +from .coring import (Matter, MtrDex, Number, NumDex, Dater,Texter, + Bexter, Pather, Verfer, Cigar, Signer, Salter, + Cipher, Encrypter, Decrypter, Diger, DigDex, + Prefixer, PreDex, ) + +from .coring import Tholder +from .indexing import Siger, IdrDex, IdxSigDex diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 045ee8675..24b1f6991 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -47,7 +47,7 @@ from ..help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, codeB64ToB2, codeB2ToB64, Reb64, nabSextets) - +from . import indexing DSS_SIG_MODE = "fips-186-3" @@ -218,6 +218,24 @@ def generateSigners(raw=None, count=8, transferable=True): return signers +# ToDo: nonces only need 128 bits of entropy. a Salt is enough +# Just use Salter().qb64. +# Deprecated + +def randomNonce(): + """ Generate a random ed25519 seed and encode as qb64 + + Returns: + str: qb64 encoded ed25519 random seed + """ + preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + seedqb64 = Matter(raw=preseed, code=MtrDex.Ed25519_Seed).qb64 + return seedqb64 + + + + + # secret derivation security tier Tierage = namedtuple("Tierage", 'low med high') @@ -648,6 +666,37 @@ def __iter__(self): PadTagDex = PadTagCodex() # Make instance +@dataclass(frozen=True) +class PreCodex: + """ + PreCodex is codex all identifier prefix derivation codes. + This is needed to verify valid inception events. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. + Ed25519: str = 'D' # Ed25519 verification key basic derivation + Blake3_256: str = 'E' # Blake3 256 bit digest self-addressing derivation. + Blake2b_256: str = 'F' # Blake2b 256 bit digest self-addressing derivation. + Blake2s_256: str = 'G' # Blake2s 256 bit digest self-addressing derivation. + SHA3_256: str = 'H' # SHA3 256 bit digest self-addressing derivation. + SHA2_256: str = 'I' # SHA2 256 bit digest self-addressing derivation. + Blake3_512: str = '0D' # Blake3 512 bit digest self-addressing derivation. + Blake2b_512: str = '0E' # Blake2b 512 bit digest self-addressing derivation. + SHA3_512: str = '0F' # SHA3 512 bit digest self-addressing derivation. + SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. + ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. + ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation + ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. + ECDSA_256r1: str = "1AAJ" # ECDSA secp256r1 verification or encryption key, basic derivation + + def __iter__(self): + return iter(astuple(self)) + + +PreDex = PreCodex() # Make instance + + # namedtuple for size entries in Matter and Counter derivation code tables # hs is the hard size int number of chars in hard (stable) part of code # ss is the soft size int number of chars in soft (unstable) part of code @@ -2428,74 +2477,6 @@ def b64ToVer(b64, *, texted=False): -class Streamer: - """ - Streamer is CESR sniffable stream class - - - Has the following public properties: - - Properties: - - - Methods: - - - Hidden: - - - - """ - - def __init__(self, stream): - """Initialize instance - - - Parameters: - stream (bytes | bytearray): sniffable CESR stream - - - """ - self._stream = bytes(stream) - - - @property - def stream(self): - """stream property getter - """ - return self._stream - - @property - def text(self): - """expanded stream as qb64 text - Returns: - stream (bytes): expanded text qb64 version of stream - - """ - return self._stream - - @property - def binary(self): - """compacted stream as qb2 binary - Returns: - stream (bytes): compacted binary qb2 version of stream - - """ - return self._stream - - @property - def texter(self): - """expanded stream as Texter instance - Returns: - texter (Texter): Texter primitive of stream suitable wrapping - - """ - return self._stream - - - - - class Texter(Matter): """ @@ -3265,22 +3246,22 @@ def _ed25519(ser, seed, verfer, index, only=False, ondex=None, **kwa): if only: # only main index ondex not used ondex = None if index <= 63: # (64 ** ms - 1) where ms is main index size - code = IdrDex.Ed25519_Crt_Sig # use small current only + code = indexing.IdrDex.Ed25519_Crt_Sig # use small current only else: - code = IdrDex.Ed25519_Big_Crt_Sig # use big current only + code = indexing.IdrDex.Ed25519_Big_Crt_Sig # use big current only else: # both if ondex == None: ondex = index # enable default to be same if ondex == index and index <= 63: # both same and small - code = IdrDex.Ed25519_Sig # use small both same + code = indexing.IdrDex.Ed25519_Sig # use small both same else: # otherwise big or both not same so use big both - code = IdrDex.Ed25519_Big_Sig # use use big both + code = indexing.IdrDex.Ed25519_Big_Sig # use use big both - return Siger(raw=sig, - code=code, - index=index, - ondex=ondex, - verfer=verfer,) + return indexing.Siger(raw=sig, + code=code, + index=index, + ondex=ondex, + verfer=verfer,) @staticmethod def _secp256r1(ser, seed, verfer, index, only=False, ondex=None, **kwa): @@ -3324,22 +3305,22 @@ def _secp256r1(ser, seed, verfer, index, only=False, ondex=None, **kwa): if only: # only main index ondex not used ondex = None if index <= 63: # (64 ** ms - 1) where ms is main index size - code = IdrDex.ECDSA_256r1_Crt_Sig # use small current only + code = indexing.IdrDex.ECDSA_256r1_Crt_Sig # use small current only else: - code = IdrDex.ECDSA_256r1_Big_Crt_Sig # use big current only + code = indexing.IdrDex.ECDSA_256r1_Big_Crt_Sig # use big current only else: # both if ondex == None: ondex = index # enable default to be same if ondex == index and index <= 63: # both same and small - code = IdrDex.ECDSA_256r1_Sig # use small both same + code = indexing.IdrDex.ECDSA_256r1_Sig # use small both same else: # otherwise big or both not same so use big both - code = IdrDex.ECDSA_256r1_Big_Sig # use use big both + code = indexing.IdrDex.ECDSA_256r1_Big_Sig # use use big both - return Siger(raw=sig, - code=code, - index=index, - ondex=ondex, - verfer=verfer,) + return indexing.Siger(raw=sig, + code=code, + index=index, + ondex=ondex, + verfer=verfer,) @staticmethod def _secp256k1(ser, seed, verfer, index, only=False, ondex=None, **kwa): @@ -3383,22 +3364,22 @@ def _secp256k1(ser, seed, verfer, index, only=False, ondex=None, **kwa): if only: # only main index ondex not used ondex = None if index <= 63: # (64 ** ms - 1) where ms is main index size - code = IdrDex.ECDSA_256k1_Crt_Sig # use small current only + code = indexing.IdrDex.ECDSA_256k1_Crt_Sig # use small current only else: - code = IdrDex.ECDSA_256k1_Big_Crt_Sig # use big current only + code = indexing.IdrDex.ECDSA_256k1_Big_Crt_Sig # use big current only else: # both if ondex == None: ondex = index # enable default to be same if ondex == index and index <= 63: # both same and small - code = IdrDex.ECDSA_256k1_Sig # use small both same + code = indexing.IdrDex.ECDSA_256k1_Sig # use small both same else: # otherwise big or both not same so use big both - code = IdrDex.ECDSA_256k1_Big_Sig # use use big both + code = indexing.IdrDex.ECDSA_256k1_Big_Sig # use use big both - return Siger(raw=sig, - code=code, - index=index, - ondex=ondex, - verfer=verfer,) + return indexing.Siger(raw=sig, + code=code, + index=index, + ondex=ondex, + verfer=verfer,) class Salter(Matter): @@ -3998,38 +3979,6 @@ def _sha2_256(ser, raw): return (hashlib.sha256(ser).digest() == raw) - -@dataclass(frozen=True) -class PreCodex: - """ - PreCodex is codex all identifier prefix derivation codes. - This is needed to verify valid inception events. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. - Ed25519: str = 'D' # Ed25519 verification key basic derivation - Blake3_256: str = 'E' # Blake3 256 bit digest self-addressing derivation. - Blake2b_256: str = 'F' # Blake2b 256 bit digest self-addressing derivation. - Blake2s_256: str = 'G' # Blake2s 256 bit digest self-addressing derivation. - SHA3_256: str = 'H' # SHA3 256 bit digest self-addressing derivation. - SHA2_256: str = 'I' # SHA2 256 bit digest self-addressing derivation. - Blake3_512: str = '0D' # Blake3 512 bit digest self-addressing derivation. - Blake2b_512: str = '0E' # Blake2b 512 bit digest self-addressing derivation. - SHA3_512: str = '0F' # SHA3 512 bit digest self-addressing derivation. - SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. - ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. - ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation - ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. - ECDSA_256r1: str = "1AAJ" # ECDSA secp256r1 verification or encryption key, basic derivation - - def __iter__(self): - return iter(astuple(self)) - - -PreDex = PreCodex() # Make instance - - class Prefixer(Matter): """ Prefixer is Matter subclass for autonomic identifier prefix using @@ -4600,310 +4549,167 @@ def verify(self, sad, *, prefixed=False, versioned=True, code=None, @dataclass(frozen=True) -class IndexerCodex: - """ IndexerCodex is codex hard (stable) part of all indexer derivation codes. - - Codes indicate which list of keys, current and/or prior next, index is for: - - _Sig: Indices in code may appear in both current signing and - prior next key lists when event has both current and prior - next key lists. Two character code table has only one index - so must be the same for both lists. Other index if for - prior next. - The indices may be different in those code tables which - have two sets of indices. - - _Crt_Sig: Index in code for current signing key list only. - - _Big_: Big index values - - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. +class CounterCodex: """ - - Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. - Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. - ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list. - ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. - ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. - Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. - Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. - Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both lists. - Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. - ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. - ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. - ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. - Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. - Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. - TBD0: str = '0z' # Test of Var len label L=N*4 <= 4095 char quadlets includes code - TBD1: str = '1z' # Test of index sig lead 1 - TBD4: str = '4z' # Test of index sig lead 1 big - - def __iter__(self): - return iter(astuple(self)) # enables inclusion test with "in" - -IdrDex = IndexerCodex() - - -@dataclass(frozen=True) -class IndexedSigCodex: - """IndexedSigCodex is codex all indexed signature derivation codes. - + CounterCodex is codex hard (stable) part of all counter derivation codes. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. - Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. - ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list. - ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. - ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. - Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. - Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. - Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both lists. - Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. - ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. - ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. - ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. - Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. - Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. - - def __iter__(self): - return iter(astuple(self)) - -IdxSigDex = IndexedSigCodex() # Make instance - - -@dataclass(frozen=True) -class IndexedCurrentSigCodex: - """IndexedCurrentSigCodex is codex indexed signature codes for current list. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list only. - ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. - Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. - Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. - ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. - Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. + ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. + WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. + NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. + TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. + FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. + TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. + SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegator/issuer/transaction event + TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. + SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event + SadPathSigGroups: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content + RootSadPathSigGroups: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples + PathedMaterialGroup: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) + AttachmentGroup: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) + BigAttachmentGroup: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) + KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Protocol Stack CESR Version def __iter__(self): - return iter(astuple(self)) - -IdxCrtSigDex = IndexedCurrentSigCodex() # Make instance - + return iter(astuple(self)) # enables inclusion test with "in" +CtrDex = CounterCodex() -@dataclass(frozen=True) -class IndexedBothSigCodex: - """IndexedBothSigCodex is codex indexed signature codes for both lists. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. +class Counter: """ - Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. - ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. - ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. - Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. - Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both listsy. - ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. - ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. - Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. - - def __iter__(self): - return iter(astuple(self)) - -IdxBthSigDex = IndexedBothSigCodex() # Make instance - -# namedtuple for size entries in Incexer derivation code tables -# hs is the hard size int number of chars in hard (stable) part of code -# ss is the soft size int number of chars in soft (unstable) part of code -# os is the other size int number of chars in other index part of soft -# ms = ss - os main index size computed -# fs is the full size int number of chars in code plus appended material if any -# ls is the lead size int number of bytes to pre-pad pre-converted raw binary -Xizage = namedtuple("Xizage", "hs ss os fs ls") - -class Indexer: - """ Indexer is fully qualified cryptographic material primitive base class for - indexed primitives. In special cases some codes in the Index code table - may be of variable length (i.e. not indexed) when the full size table entry - is None. In that case the index is used instread as the length. + Counter is fully qualified cryptographic material primitive base class for + counter primitives (framing composition grouping count codes). Sub classes are derivation code and key event element context specific. Includes the following attributes and properties: + Class Attributes: + Attributes: Properties: - code is str of stable (hard) part of derivation code - raw (bytes): unqualified crypto material usable for crypto operations - index (int): main index offset into list or length of material - ondex (int | None): other index offset into list or length of material - qb64b (bytes): fully qualified Base64 crypto material - qb64 (str | bytes): fully qualified Base64 crypto material - qb2 (bytes): fully qualified binary crypto material + .code is str derivation code to indicate cypher suite + .raw is bytes crypto material only without code + .pad is int number of pad chars given raw + .count is int count of grouped following material (not part of counter) + .qb64 is str in Base64 fully qualified with derivation code + crypto mat + .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat + .qb2 is bytes in binary with derivation code + crypto material Hidden: - ._code (str): value for .code property - ._raw (bytes): value for .raw property - ._index (int): value for .index property - ._ondex (int): value for .ondex property + ._code is str value for .code property + ._raw is bytes value for .raw property + ._pad is method to compute .pad property + ._count is int value for .count property ._infil is method to compute fully qualified Base64 from .raw and .code - ._binfil is method to compute fully qualified Base2 from .raw and .code ._exfil is method to extract .code and .raw from fully qualified Base64 - ._bexfil is method to extract .code and .raw from fully qualified Base2 """ - Codex = IdrDex - # Hards table maps from bytes Base64 first code char to int of hard size, hs, - # (stable) of code. The soft size, ss, (unstable) is always > 0 for Indexer. - Hards = ({chr(c): 1 for c in range(65, 65 + 26)}) - Hards.update({chr(c): 1 for c in range(97, 97 + 26)}) - Hards.update([('0', 2), ('1', 2), ('2', 2), ('3', 2), ('4', 2)]) - # Sizes table maps hs chars of code to Xizage namedtuple of (hs, ss, os, fs, ls) - # where hs is hard size, ss is soft size, os is other index size, - # and fs is full size, ls is lead size. - # where ss includes os, so main index size ms = ss - os - # soft size, ss, should always be > 0 for Indexer - Sizes = { - 'A': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'B': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'C': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'D': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'E': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'F': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - '0A': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), - '0B': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), - '2A': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2B': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2C': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2D': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2E': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2F': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '3A': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), - '3B': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), - '0z': Xizage(hs=2, ss=2, os=0, fs=None, ls=0), - '1z': Xizage(hs=2, ss=2, os=1, fs=76, ls=1), - '4z': Xizage(hs=2, ss=6, os=3, fs=80, ls=1), - } + # Hards table maps from bytes Base64 first two code chars to int of + # hard size, hs,(stable) of code. The soft size, ss, (unstable) for Counter + # is always > 0 and hs + ss = fs always + Hards = ({('-' + chr(c)): 2 for c in range(65, 65 + 26)}) + Hards.update({('-' + chr(c)): 2 for c in range(97, 97 + 26)}) + Hards.update([('-0', 3)]) + Hards.update([('--', 5)]) + # Bards table maps to hard size, hs, of code from bytes holding sextets - # converted from first code char. Used for ._bexfil. + # converted from first two code char. Used for ._bexfil. Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) - def __init__(self, raw=None, code=IdrDex.Ed25519_Sig, index=0, ondex=None, - qb64b=None, qb64=None, qb2=None, strip=False): - """ - Validate as fully qualified - Parameters: - raw (bytes): unqualified crypto material usable for crypto operations - code is str of stable (hard) part of derivation code - index (int): main index offset into list or length of material - ondex (int | None): other index offset into list or length of material - qb64b (bytes): fully qualified Base64 crypto material - qb64 (str | bytes): fully qualified Base64 crypto material - qb2 (bytes): fully qualified binary crypto material - strip (bool): True means strip counter contents from input stream - bytearray after parsing qb64b or qb2. False means do not strip - - Needs either (raw and code and index) or qb64b or qb64 or qb2 - Otherwise raises EmptyMaterialError - When raw and code provided then validate that code is correct - for length of raw and assign .raw - Else when qb64b or qb64 or qb2 provided extract and assign - .raw, .code, .index, .ondex. - - """ - if raw is not None: # raw provided - if not code: - raise EmptyMaterialError("Improper initialization need either " - "(raw and code) or qb64b or qb64 or qb2.") - if not isinstance(raw, (bytes, bytearray)): - raise TypeError(f"Not a bytes or bytearray, raw={raw}.") - - if code not in self.Sizes: - raise UnexpectedCodeError(f"Unsupported code={code}.") - - hs, ss, os, fs, ls = self.Sizes[code] # get sizes for code - cs = hs + ss # both hard + soft code size - ms = ss - os - - if not isinstance(index, int) or index < 0 or index > (64 ** ms - 1): - raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") + # Sizes table maps hs chars of code to Sizage namedtuple of (hs, ss, fs) + # where hs is hard size, ss is soft size, and fs is full size + # soft size, ss, should always be > 0 and hs+ss=fs for Counter + Sizes = { + '-A': Sizage(hs=2, ss=2, fs=4, ls=0), + '-B': Sizage(hs=2, ss=2, fs=4, ls=0), + '-C': Sizage(hs=2, ss=2, fs=4, ls=0), + '-D': Sizage(hs=2, ss=2, fs=4, ls=0), + '-E': Sizage(hs=2, ss=2, fs=4, ls=0), + '-F': Sizage(hs=2, ss=2, fs=4, ls=0), + '-G': Sizage(hs=2, ss=2, fs=4, ls=0), + '-H': Sizage(hs=2, ss=2, fs=4, ls=0), + '-I': Sizage(hs=2, ss=2, fs=4, ls=0), + '-J': Sizage(hs=2, ss=2, fs=4, ls=0), + '-K': Sizage(hs=2, ss=2, fs=4, ls=0), + '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + } - if isinstance(ondex, int) and os and not (ondex >= 0 and ondex <= (64 ** os - 1)): - raise InvalidVarIndexError(f"Invalid ondex={ondex} for code={code}.") + Codex = CtrDex - if code in IdxCrtSigDex and ondex is not None: - raise InvalidVarIndexError(f"Non None ondex={ondex} for code={code}.") - if code in IdxBthSigDex: - if ondex is None: # set default - ondex = index # when not provided make ondex match index - else: - if ondex != index and os == 0: # must match if os == 0 - raise InvalidVarIndexError(f"Non matching ondex={ondex}" - f" and index={index} for " - f"code={code}.") + def __init__(self, code=None, count=None, countB64=None, + qb64b=None, qb64=None, qb2=None, strip=False): + """ + Validate as fully qualified + Parameters: + code (str | None): stable (hard) part of derivation code + count (int | None): count for composition. + Count may represent quadlets/triplet, groups, primitives or + other numericy + When both count and countB64 are None then count defaults to 1 + countB64 (str | None): count for composition as Base64 + countB64 may represent quadlets/triplet, groups, primitives or + other numericy + qb64b (bytes | bytearray | None): fully qualified crypto material text domain + qb64 (str | None) fully qualified crypto material text domain + qb2 (bytes | bytearray | None) fully qualified crypto material binary domain + strip (bool): True means strip counter contents from input stream + bytearray after parsing qb64b or qb2. False means do not strip. + default False + + + Needs either code or qb64b or qb64 or qb2 + Otherwise raises EmptyMaterialError + When code and count provided then validate that code and count are correct + Else when qb64b or qb64 or qb2 provided extract and assign + .code and .count + """ + if code is not None: # code provided + if code not in self.Sizes: + raise InvalidCodeError("Unsupported code={}.".format(code)) - if not fs: # compute fs from index - if cs % 4: - raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if os != 0: - raise InvalidCodeSizeError(f"Non-zero other index size for " - f"variable length material. os={os}.") - fs = (index * 4) + cs + hs, ss, fs, ls = self.Sizes[code] # get sizes for code + cs = hs + ss # both hard + soft code size + if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes + raise InvalidCodeSizeError("Whole code size not full size or not " + "multiple of 4. cs={} fs={}.".format(cs, fs)) - rawsize = (fs - cs) * 3 // 4 + if count is None: + count = 1 if countB64 is None else b64ToInt(countB64) - raw = raw[:rawsize] # copy rawsize from stream, may be less - if len(raw) != rawsize: # forbids shorter - raise RawMaterialError(f"Not enougth raw bytes for code={code}" - f"and index={index} ,expected {rawsize} " - f"got {len(raw)}.") + if count < 0 or count > (64 ** ss - 1): + raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) self._code = code - self._index = index - self._ondex = ondex - self._raw = bytes(raw) # crypto ops require bytes not bytearray + self._count = count elif qb64b is not None: self._exfil(qb64b) if strip: # assumes bytearray - del qb64b[:len(self.qb64b)] # may be variable length fs + del qb64b[:self.Sizes[self.code].fs] elif qb64 is not None: self._exfil(qb64) - elif qb2 is not None: + elif qb2 is not None: # rewrite to use direct binary exfiltration self._bexfil(qb2) if strip: # assumes bytearray - del qb2[:len(self.qb2)] # may be variable length fs + del qb2[:self.Sizes[self.code].fs * 3 // 4] else: raise EmptyMaterialError("Improper initialization need either " - "(raw and code and index) or qb64b or " + "(code and count) or qb64b or " "qb64 or qb2.") - @classmethod - def _rawSize(cls, code): - """ - Returns expected raw size in bytes for a given code. Not applicable to - codes with fs = None - """ - hs, ss, os, fs, ls = cls.Sizes[code] # get sizes - return ((fs - (hs + ss)) * 3 // 4) - @property def code(self): """ @@ -4912,29 +4718,15 @@ def code(self): """ return self._code - @property - def raw(self): - """ - Returns ._raw - Makes .raw read only - """ - return self._raw @property - def index(self): + def count(self): """ - Returns ._index - Makes .index read only + Returns ._count + Makes ._count read only """ - return self._index + return self._count - @property - def ondex(self): - """ - Returns ._ondex - Makes .ondex read only - """ - return self._ondex @property def qb64b(self): @@ -4945,6 +4737,7 @@ def qb64b(self): """ return self._infil() + @property def qb64(self): """ @@ -4954,6 +4747,7 @@ def qb64(self): """ return self.qb64b.decode("utf-8") + @property def qb2(self): """ @@ -4962,1569 +4756,1039 @@ def qb2(self): """ return self._binfil() - def _infil(self): + + def countToB64(self, l=None): + """ Returns count as Base64 left padded with "A"s + Parameters: + l (int | None): minimum number characters including left padding + When not provided use the softsize of .code + """ - Returns fully qualified attached sig base64 bytes computed from - self.raw, self.code and self.index. + if l is None: + _, ss, _, _ = self.Sizes[self.code] + l = ss + return (intToB64(self.count, l=l)) - cs = hs + ss - os = ss - ms (main index size) - when fs None then size computed & fs = size * 4 + cs + + @staticmethod + def semVerToB64(version="", major=0, minor=0, patch=0): + """ Converts semantic version to Base64 representation of countB64 + suitable for CESR protocol genus and version + + Returns: + countB64 (str): suitable for input to Counter + example: Counter(countB64=semVerToB64(version = "1.0.0")) + + Parameters: + version (str | None): dot separated semantic version string of format + "major.minor.patch" + major (int): When version is None or empty then use major,minor, patch + minor (int): When version is None or empty then use major,minor, patch + patch (int): When version is None or empty then use major,minor, patch + + each of major, minor, patch must be in range [0,63] for represenation as + three Base64 characters """ - code = self.code # codex value chars hard code - index = self.index # main index value - ondex = self.ondex # other index value - raw = self.raw # bytes or bytearray + parts = [major, minor, patch] + if version: + splits = version.split(".", maxsplit=3) + splits = [(int(s) if s else 0) for s in splits] + for i in range(3-len(splits),0, -1): + splits.append(parts[-i]) + parts = splits - ps = (3 - (len(raw) % 3)) % 3 # if lead then same pad size chars & lead size bytes - hs, ss, os, fs, ls = self.Sizes[code] - cs = hs + ss - ms = ss - os - - if not fs: # compute fs from index - if cs % 4: - raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if os != 0: - raise InvalidCodeSizeError(f"Non-zero other index size for " - f"variable length material. os={os}.") - fs = (index * 4) + cs - - if index < 0 or index > (64 ** ms - 1): - raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") - - if (isinstance(ondex, int) and os and - not (ondex >= 0 and ondex <= (64 ** os - 1))): - raise InvalidVarIndexError(f"Invalid ondex={ondex} for os={os} and " - f"code={code}.") - - # both is hard code + converted index + converted ondex - both = (f"{code}{intToB64(index, l=ms)}" - f"{intToB64(ondex if ondex is not None else 0, l=os)}") - - # check valid pad size for whole code size, assumes ls is zero - if len(both) != cs: - raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." - .format(cs, len(both))) + for p in parts: + if p < 0 or p > 63: + raise ValueError(f"Out of bounds semantic version. " + f"Part={p} is < 0 or > 63.") + return ("".join(intToB64(p, l=1) for p in parts)) - if (cs % 4) != ps - ls: # adjusted pad given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted" - f" raw pad size={ps}.") - # prepend pad bytes, convert, then replace pad chars with full derivation - # code including index, - full = both.encode("utf-8") + encodeB64(bytes([0] * ps) + raw)[ps - ls:] + def _infil(self): + """ + Returns fully qualified attached sig base64 bytes computed from + self.code and self.count. + """ + code = self.code # codex value chars hard code + count = self.count # index value int used for soft + + hs, ss, fs, ls = self.Sizes[code] + cs = hs + ss # both hard + soft size + if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes + raise InvalidCodeSizeError("Whole code size not full size or not " + "multiple of 4. cs={} fs={}.".format(cs, fs)) + if count < 0 or count > (64 ** ss - 1): + raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) - if len(full) != fs: # invalid size - raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") + # both is hard code + converted count + both = "{}{}".format(code, intToB64(count, l=ss)) - return full + # check valid pad size for whole code size + if len(both) % 4: # no pad + raise InvalidCodeSizeError("Invalid size = {} of {} not a multiple of 4." + .format(len(both), both)) + # prepending full derivation code with index and strip off trailing pad characters + return (both.encode("utf-8")) def _binfil(self): """ Returns bytes of fully qualified base2 bytes, that is .qb2 - self.code and self.index converted to Base2 + self.raw left shifted - with pad bits equivalent of Base64 decode of .qb64 into .qb2 + self.code converted to Base2 left shifted with pad bits + equivalent of Base64 decode of .qb64 into .qb2 """ code = self.code # codex chars hard code - index = self.index # main index value - ondex = self.ondex # other index value - raw = self.raw # bytes or bytearray + count = self.count # index value int used for soft - ps = (3 - (len(raw) % 3)) % 3 # same pad size chars & lead size bytes - hs, ss, os, fs, ls = self.Sizes[code] + hs, ss, fs, ls = self.Sizes[code] cs = hs + ss - ms = ss - os - - if index < 0 or index > (64 ** ss - 1): - raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") - - if (isinstance(ondex, int) and os and - not (ondex >= 0 and ondex <= (64 ** os - 1))): - raise InvalidVarIndexError(f"Invalid ondex={ondex} for os={os} and " - f"code={code}.") - - if not fs: # compute fs from index - if cs % 4: - raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if os != 0: - raise InvalidCodeSizeError(f"Non-zero other index size for " - f"variable length material. os={os}.") - fs = (index * 4) + cs + if fs != cs or cs % 4: # fs must be cs and multiple of 4 for count codes + raise InvalidCodeSizeError("Whole code size not full size or not " + "multiple of 4. cs={} fs={}.".format(cs, fs)) - # both is hard code + converted index - both = (f"{code}{intToB64(index, l=ms)}" - f"{intToB64(ondex if ondex is not None else 0, l=os)}") + if count < 0 or count > (64 ** ss - 1): + raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) + # both is hard code + converted count + both = "{}{}".format(code, intToB64(count, l=ss)) if len(both) != cs: raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." .format(cs, len(both))) - if (cs % 4) != ps - ls: # adjusted pad given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted" - f" raw pad size={ps}.") - - n = sceil(cs * 3 / 4) # number of b2 bytes to hold b64 code + index - # convert code both to right align b2 int then left shift in pad bits - # then convert to bytes - bcode = (b64ToInt(both) << (2 * (ps - ls))).to_bytes(n, 'big') - full = bcode + bytes([0] * ls) + raw - - bfs = len(full) # binary full size - if bfs % 3 or (bfs * 4 // 3) != fs: # invalid size - raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") - - return full + return (codeB64ToB2(both)) # convert to b2 left shift if any def _exfil(self, qb64b): """ - Extracts self.code, self.index, and self.raw from qualified base64 bytes qb64b - - cs = hs + ss - ms = ss - os (main index size) - when fs None then size computed & fs = size * 4 + cs + Extracts self.code and self.count from qualified base64 bytes qb64b """ if not qb64b: # empty need more bytes - raise ShortageError("Empty material.") + raise ShortageError("Empty material, Need more characters.") - first = qb64b[:1] # extract first char code selector + first = qb64b[:2] # extract first two char code selector if hasattr(first, "decode"): first = first.decode("utf-8") if first not in self.Hards: - if first[0] == '-': - raise UnexpectedCountCodeError("Unexpected count code start" - "while extracing Indexer.") - elif first[0] == '_': - raise UnexpectedOpCodeError("Unexpected op code start" - "while extracing Indexer.") + if first[0] == '_': + raise UnexpectedOpCodeError("Unexpected op code start" + "while extracing Counter.") else: - raise UnexpectedCodeError(f"Unsupported code start char={first}.") + raise UnexpectedCodeError("Unsupported code start ={}.".format(first)) hs = self.Hards[first] # get hard code size if len(qb64b) < hs: # need more bytes - raise ShortageError(f"Need {hs - len(qb64b)} more characters.") + raise ShortageError("Need {} more characters.".format(hs - len(qb64b))) hard = qb64b[:hs] # get hard code if hasattr(hard, "decode"): - hard = hard.decode("utf-8") - if hard not in self.Sizes: - raise UnexpectedCodeError(f"Unsupported code ={hard}.") + hard = hard.decode("utf-8") # decode converts bytearray/bytes to str + if hard not in self.Sizes: # Sizes needs str not bytes + raise UnexpectedCodeError("Unsupported code ={}.".format(hard)) - hs, ss, os, fs, ls = self.Sizes[hard] # assumes hs in both tables consistent + hs, ss, fs, ls = self.Sizes[hard] # assumes hs consistent in both tables cs = hs + ss # both hard + soft code size - ms = ss - os - # assumes that unit tests on Indexer and IndexerCodex ensure that + + # assumes that unit tests on Counter and CounterCodex ensure that # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and (fs >= hs + ss if fs is not None else True) - # assumes no variable length indexed codes so fs is not None + # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 if len(qb64b) < cs: # need more bytes - raise ShortageError(f"Need {cs - len(qb64b)} more characters.") - - index = qb64b[hs:hs+ms] # extract index/size chars - if hasattr(index, "decode"): - index = index.decode("utf-8") - index = b64ToInt(index) # compute int index - - ondex = qb64b[hs+ms:hs+ms+os] # extract ondex chars - if hasattr(ondex, "decode"): - ondex = ondex.decode("utf-8") - - if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 - ondex = b64ToInt(ondex) if os else None # compute ondex from code - if ondex: # not zero or None so error - raise ValueError(f"Invalid ondex={ondex} for code={hard}.") - else: - ondex = None # zero so set to None when current only - else: - ondex = b64ToInt(ondex) if os else index - - # index is index for some codes and variable length for others - if not fs: # compute fs from index which means variable length - if cs % 4: - raise ValidationError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if os != 0: - raise ValidationError(f"Non-zero other index size for " - f"variable length material. os={os}.") - fs = (index * 4) + cs - - if len(qb64b) < fs: # need more bytes - raise ShortageError(f"Need {fs - len(qb64b)} more chars.") - - qb64b = qb64b[:fs] # fully qualified primitive code plus material - if hasattr(qb64b, "encode"): # only convert extracted chars from stream - qb64b = qb64b.encode("utf-8") - - # strip off prepended code and append pad characters - #ps = cs % 4 # pad size ps = cs mod 4, same pad chars and lead bytes - #base = ps * b'A' + qb64b[cs:] # replace prepend code with prepad zeros - #raw = decodeB64(base)[ps+ls:] # decode and strip off ps+ls prepad bytes - - # check for non-zeroed pad bits or lead bytes - ps = cs % 4 # code pad size ps = cs mod 4 - pbs = 2 * (ps if ps else ls) # pad bit size in bits - if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls - base = ps * b'A' + qb64b[cs:] # replace pre code with prepad chars of zero - paw = decodeB64(base) # decode base to leave prepadded raw - pi = (int.from_bytes(paw[:ps], "big")) # prepad as int - if pi & (2 ** pbs - 1 ): # masked pad bits non-zero - raise ValueError(f"Non zeroed prepad bits = " - f"{pi & (2 ** pbs - 1 ):<06b} in {qb64b[cs:cs+1]}.") - raw = paw[ps:] # strip off ps prepad paw bytes - else: # not ps. IF not ps THEN may or may not be ls (lead) - base = qb64b[cs:] # strip off code leaving lead chars if any and value - # decode lead chars + val leaving lead bytes + raw bytes - # then strip off ls lead bytes leaving raw - paw = decodeB64(base) # decode base to leave prepadded paw bytes - li = int.from_bytes(paw[:ls], "big") # lead as int - if li: # pre pad lead bytes must be zero - if ls == 1: - raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") - else: - raise ValueError(f"Non zeroed lead bytes = 0x{li:04x}.") - - raw = paw[ls:] + raise ShortageError("Need {} more characters.".format(cs - len(qb64b))) - if len(raw) != (len(qb64b) - cs) * 3 // 4: # exact lengths - raise ConversionError(f"Improperly qualified material = {qb64b}") + count = qb64b[hs:hs + ss] # extract count chars + if hasattr(count, "decode"): + count = count.decode("utf-8") + count = b64ToInt(count) # compute int count self._code = hard - self._index = index - self._ondex = ondex - self._raw = raw # must be bytes for crpto opts and immutable not bytearray - + self._count = count def _bexfil(self, qb2): """ - Extracts self.code, self.index, and self.raw from qualified base2 bytes qb2 - - cs = hs + ss - ms = ss - os (main index size) - when fs None then size computed & fs = size * 4 + cs + Extracts self.code and self.count from qualified base2 bytes qb2 """ if not qb2: # empty need more bytes raise ShortageError("Empty material, Need more bytes.") - first = nabSextets(qb2, 1) # extract first sextet as code selector + first = nabSextets(qb2, 2) # extract first two sextets as code selector if first not in self.Bards: - if first[0] == b'\xf8': # b64ToB2('-') - raise UnexpectedCountCodeError("Unexpected count code start" - "while extracing Matter.") - elif first[0] == b'\xfc': # b64ToB2('_') + if first[0] == b'\xfc': # b64ToB2('_') raise UnexpectedOpCodeError("Unexpected op code start" "while extracing Matter.") else: - raise UnexpectedCodeError(f"Unsupported code start sextet={first}.") + raise UnexpectedCodeError("Unsupported code start sextet={}.".format(first)) hs = self.Bards[first] # get code hard size equvalent sextets bhs = sceil(hs * 3 / 4) # bhs is min bytes to hold hs sextets if len(qb2) < bhs: # need more bytes - raise ShortageError(f"Need {bhs - len(qb2)} more bytes.") + raise ShortageError("Need {} more bytes.".format(bhs - len(qb2))) hard = codeB2ToB64(qb2, hs) # extract and convert hard part of code if hard not in self.Sizes: - raise UnexpectedCodeError(f"Unsupported code ={hard}.") + raise UnexpectedCodeError("Unsupported code ={}.".format(hard)) - hs, ss, os, fs, ls = self.Sizes[hard] + hs, ss, fs, ls = self.Sizes[hard] cs = hs + ss # both hs and ss - ms = ss - os - # assumes that unit tests on Indexer and IndexerCodex ensure that + # assumes that unit tests on Counter and CounterCodex ensure that # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and (fs >= hs + ss if fs is not None else True) + # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets if len(qb2) < bcs: # need more bytes raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code - index = b64ToInt(both[hs:hs+ms]) # compute index + count = b64ToInt(both[hs:hs + ss]) # get count - if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 - ondex = b64ToInt(both[hs+ms:hs+ms+os]) if os else None # compute ondex from code - if ondex: # not zero or None so error - raise ValueError(f"Invalid ondex={ondex} for code={hard}.") - else: - ondex = None # zero so set to None when current only - else: - ondex = b64ToInt(both[hs+ms:hs+ms+os]) if os else index + self._code = hard + self._count = count - if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 - if ondex: # not zero so error - raise ValueError(f"Invalid ondex={ondex} for code={hard}.") - else: # zero so set to None - ondex = None - if not fs: # compute fs from size chars in ss part of code - if cs % 4: - raise ValidationError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if os != 0: - raise ValidationError(f"Non-zero other index size for " - f"variable length material. os={os}.") - fs = (index * 4) + cs - bfs = sceil(fs * 3 / 4) # bfs is min bytes to hold fs sextets - if len(qb2) < bfs: # need more bytes - raise ShortageError("Need {} more bytes.".format(bfs - len(qb2))) +class Tholder: + """ + Tholder is KERI Signing Threshold Satisfaction class + .satisfy method evaluates satisfaction based on ordered list of indices of + verified signatures where indices correspond to offsets in key list of + associated signatures. - qb2 = qb2[:bfs] # extract qb2 fully qualified primitive code plus material + Has the following public properties: - # check for non-zeroed prepad bits or lead bytes - ps = cs % 4 # code pad size ps = cs mod 4 - pbs = 2 * (ps if ps else ls) # pad bit size in bits - if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls - # convert last byte of code bytes in which are pad bits to int - pi = (int.from_bytes(qb2[bcs-1:bcs], "big")) - if pi & (2 ** pbs - 1 ): # masked pad bits non-zero - raise ValueError(f"Non zeroed pad bits = " - f"{pi & (2 ** pbs - 1 ):>08b} in 0x{pi:02x}.") - else: # not ps. IF not ps THEN may or may not be ls (lead) - li = int.from_bytes(qb2[bcs:bcs+ls], "big") # lead as int - if li: # pre pad lead bytes must be zero - if ls == 1: - raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") - else: - raise ValueError(f"Non zeroed lead bytes = 0x{li:02x}.") + Properties: + .weighted is Boolean True if fractional weighted threshold False if numeric + .size is int of minimum size of keys list + when weighted is size of keys list + when unweighted is size of int thold since don't have anyway + to know size of keys list in this case + + .limen is qualified b64 signing threshold suitable for CESR serialization. + either Number.qb64b or Bexter.qb64b. + The b64 portion of limen with code stripped (Bexter.bext) of + [["1/2", "1/2", "1/4", "1/4", "1/4"], ["1", "1"]] + is '1s2c1s2c1s4c1s4c1s4a1c1' basically slash is 's', comma is 'c', + ANDed clauses are delimited by 'a'. + Each clause top level weight may be optionally a weighted set of weights + delimited by 'k' for the weight on the set and 'v' for the weights in + the set. + [[{'1/3': ['1/2', '1/2', '1/2']}, '1/2', {'1/2': ['1', '1']}], + ['1/2', {'1/2': ['1', '1']}]] + b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1' - raw = qb2[(bcs + ls):] # strip code and leader bytes from qb2 to get raw + .sith is original signing threshold suitable for value to be serialized + as json, cbor, mgpk in key event message as either: + non-negative hex number str or + list of str rational number fractions >= 0 and <= 1 or + list of list of str rational number fractions >= 0 and <= 1 + list of list of weighted map of weights - if len(raw) != (len(qb2) - bcs - ls): # exact lengths - raise ConversionError(f"Improperly qualified material = {qb2}") + .thold is parsed signing threshold suitable for calculating satisfaction. + either as int or list of Fractions - self._code = hard - self._index = index - self._ondex = ondex - self._raw = bytes(raw) # must be bytes for crypto ops and not bytearray mutable + .num is int signing threshold when not ._weighted + + Methods: + .satisfy returns bool, True means list of verified signature key indices + satisfies the threshold, False otherwise. + + Static Methods: + weight (str): converts weight str expression into either int or Fraction + else raises ValueError must satisfy 0 <= w <= 1 + Ensures strict proper rational number fraction of ints or + 0 or 1 + + Hidden: + ._weighted is Boolean, True if fractional weighted threshold False if numeric + ._size is int minimum size of of keys list + ._sith is signing threshold for .sith property + ._thold is signing threshold for .thold propery + ._bexter is Bexter instance of weighted signing threshold or None + ._number is Number instance of integer threshold or None + ._satisfy is method reference of threshold specified verification method + ._satisfy_numeric is numeric threshold verification method + ._satisfy_weighted is fractional weighted threshold verification method -class Siger(Indexer): """ - Siger is subclass of Indexer, indexed signature material, - Adds .verfer property which is instance of Verfer that provides - associated signature verifier. + def __init__(self, *, thold=None , limen=None, sith=None, **kwa): + """ + Accepts signing threshold in various forms so that may output correct + forms for serialization and/or calculation of satisfaction. - See Indexer for inherited attributes and properties: + The thold representation is meant to accept thresholds from computable + expressions for satisfaction of a threshold - Attributes: + The limen representation is meant to parse threshold expressions from + CESR serializations of key event message fields or attachments. - Properties: - verfer (Verfer): instance if any provides public verification key + The sith representation is meant to parse threhold expressions from + deserializations of JSON, CBOR, or MGPK key event message fields or + the command line or configuration files. - Methods: - Hidden: - _verfer (Verfer): value for .verfer property + Parameters: + thold is signing threshold (current or next) is suitable for computing + the satisfaction of a threshold and is expressed as either: + int of threshold number (M of N) + fractional weight clauses which may be expressed as either: + sequence of either Fractions or tuples of Fraction and + sequence of Fractions + sequence of sequence of either Fractions or tuples of + Fraction and sequence of Fractions - """ + limen is qualified signing threshold (current or next) expressed as either: + Number.qb64 or .qb64b of integer threshold or + Bexter.qb64 or .qb64b of fractional weight clauses which may be either: + Base64 delimited clauses of fractions + Base64 delimited clauses of fractions - def __init__(self, verfer=None, **kwa): - """Initialze instance + sith is signing threshold (current or next) expressed as either: + non-negative int of threshold number (M-of-N threshold) + next threshold may be zero + non-negative hex string of threshold number (M-of-N threshold) + next threshold may be zero + fractional weight clauses which may be expressed as either: + sequence of rational number fraction strings >= 0 and <= 1 + sequence of either rational number fraction strings >= 0 and <= 1 or + map with key rational number string and value as sequence + of rational number fraction strings + rational number fraction string + sequence of sequences of rational number fraction strings >= 0 and <= 1 + sequence of sequnces of either rational number fraction strings or + map with key rational number fraction string with value sequence of + rationaly number fraction strings + JSON serialized str of the above: - Parameters: See Matter for inherted parameters - verfer (Verfer): instance if any provides public verification key """ - super(Siger, self).__init__(**kwa) - if self.code not in IdxSigDex: - raise ValidationError("Invalid code = {} for Siger." - "".format(self.code)) - self.verfer = verfer + if thold is not None: + self._processThold(thold=thold) - @property - def verfer(self): - """ - Property verfer: - Returns Verfer instance - Assumes ._verfer is correctly assigned - """ - return self._verfer + elif limen is not None: + self._processLimen(limen=limen, **kwa) # kwa for strip - @verfer.setter - def verfer(self, verfer): - """ verfer property setter """ - self._verfer = verfer + elif sith is not None: + if isinstance(sith, str) and not sith: # empty str + raise EmptyMaterialError("Empty threshold expression.") + self._processSith(sith=sith) -@dataclass(frozen=True) -class CounterCodex: - """ - CounterCodex is codex hard (stable) part of all counter derivation codes. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ + else: + raise EmptyMaterialError("Missing threshold expression.") - ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. - WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. - NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. - TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. - FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. - TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. - SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegator/issuer/transaction event - TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. - SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event - SadPathSigGroups: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content - RootSadPathSigGroups: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples - PathedMaterialGroup: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) - AttachmentGroup: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) - BigAttachmentGroup: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) - KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Protocol Stack CESR Version - def __iter__(self): - return iter(astuple(self)) # enables inclusion test with "in" + @property + def weighted(self): + """ weighted property getter """ + return self._weighted -CtrDex = CounterCodex() + @property + def thold(self): + """ thold property getter """ + return self._thold + @property + def size(self): + """ size property getter """ + return self._size -class Counter: - """ - Counter is fully qualified cryptographic material primitive base class for - counter primitives (framing composition grouping count codes). + @property + def limen(self): + """ limen property getter """ + return self._bexter.qb64b if self._weighted else self._number.qb64b - Sub classes are derivation code and key event element context specific. + @property + def sith(self): + """ sith property getter """ + # make sith expression of thold + if self.weighted: + sith = [] + for c in self.thold: + clause = [] + for e in c: + if isinstance(e, tuple): + f = e[0] + k = f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" + v = [f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" + for f in e[1]] + clause.append({k: v}) + else: + f = e + clause.append(f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}") + sith.append(clause) - Includes the following attributes and properties: + #sith = [[f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" + #for f in clause] + #for clause in self.thold] + if len(sith) == 1: + sith = sith[0] # simplify list of one clause to clause + else: + sith = f"{self.thold:x}" - Class Attributes: + return sith - Attributes: + @property + def json(self): + """Returns json serialization of sith expression - Properties: - .code is str derivation code to indicate cypher suite - .raw is bytes crypto material only without code - .pad is int number of pad chars given raw - .count is int count of grouped following material (not part of counter) - .qb64 is str in Base64 fully qualified with derivation code + crypto mat - .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat - .qb2 is bytes in binary with derivation code + crypto material + Essentially JSON list of lists of strings + """ + return json.dumps(self.sith) - Hidden: - ._code is str value for .code property - ._raw is bytes value for .raw property - ._pad is method to compute .pad property - ._count is int value for .count property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 - """ - # Hards table maps from bytes Base64 first two code chars to int of - # hard size, hs,(stable) of code. The soft size, ss, (unstable) for Counter - # is always > 0 and hs + ss = fs always - Hards = ({('-' + chr(c)): 2 for c in range(65, 65 + 26)}) - Hards.update({('-' + chr(c)): 2 for c in range(97, 97 + 26)}) - Hards.update([('-0', 3)]) - Hards.update([('--', 5)]) + @property + def num(self): + """ sith property getter """ + return self.thold if not self._weighted else None - # Bards table maps to hard size, hs, of code from bytes holding sextets - # converted from first two code char. Used for ._bexfil. - Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) - # Sizes table maps hs chars of code to Sizage namedtuple of (hs, ss, fs) - # where hs is hard size, ss is soft size, and fs is full size - # soft size, ss, should always be > 0 and hs+ss=fs for Counter - Sizes = { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), - } - - Codex = CtrDex + def _processThold(self, thold: int | Sequence): + """Process thold input - def __init__(self, code=None, count=None, countB64=None, - qb64b=None, qb64=None, qb2=None, strip=False): - """ - Validate as fully qualified Parameters: - code (str | None): stable (hard) part of derivation code - count (int | None): count for composition. - Count may represent quadlets/triplet, groups, primitives or - other numericy - When both count and countB64 are None then count defaults to 1 - countB64 (str | None): count for composition as Base64 - countB64 may represent quadlets/triplet, groups, primitives or - other numericy - qb64b (bytes | bytearray | None): fully qualified crypto material text domain - qb64 (str | None) fully qualified crypto material text domain - qb2 (bytes | bytearray | None) fully qualified crypto material binary domain - strip (bool): True means strip counter contents from input stream - bytearray after parsing qb64b or qb2. False means do not strip. - default False - - - Needs either code or qb64b or qb64 or qb2 - Otherwise raises EmptyMaterialError - When code and count provided then validate that code and count are correct - Else when qb64b or qb64 or qb2 provided extract and assign - .code and .count - + thold (int | Sequence): computable thold expression """ - if code is not None: # code provided - if code not in self.Sizes: - raise InvalidCodeError("Unsupported code={}.".format(code)) + if isinstance(thold, int): + self._processUnweighted(thold=thold) - hs, ss, fs, ls = self.Sizes[code] # get sizes for code - cs = hs + ss # both hard + soft code size - if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes - raise InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) + else: + self._processWeighted(thold=thold) - if count is None: - count = 1 if countB64 is None else b64ToInt(countB64) - if count < 0 or count > (64 ** ss - 1): - raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) + def _processLimen(self, limen: str | bytes, **kwa): + """Process limen input - self._code = code - self._count = count + Parameters: + limen (str): CESR encoded qb64 threshold (weighted or unweighted) + """ + matter = Matter(qb64b=limen, **kwa) # kwa for strip of stream + if matter.code in NumDex: + number = Number(raw=matter.raw, code=matter.code, **kwa) + self._processUnweighted(thold=number.num) - elif qb64b is not None: - self._exfil(qb64b) - if strip: # assumes bytearray - del qb64b[:self.Sizes[self.code].fs] + elif matter.code in BexDex: + # Convert to fractional thold expression + bexter = Bexter(raw=matter.raw, code=matter.code, **kwa) + t = bexter.bext.replace('s', '/') + # get clauses + clauses = [clause.split('c') for clause in t.split('a')] - elif qb64 is not None: - self._exfil(qb64) + thold = [] + for c in clauses: + clause = [] + for e in c: + k, s, v = e.partition("k") + if s: #not empty + clause.append((self.weight(k), [self.weight(w) for w in v.split("v")])) + else: + clause.append(self.weight(k)) - elif qb2 is not None: # rewrite to use direct binary exfiltration - self._bexfil(qb2) - if strip: # assumes bytearray - del qb2[:self.Sizes[self.code].fs * 3 // 4] + thold.append(clause) - else: - raise EmptyMaterialError("Improper initialization need either " - "(code and count) or qb64b or " - "qb64 or qb2.") + self._processWeighted(thold=thold) - @property - def code(self): - """ - Returns ._code - Makes .code read only - """ - return self._code + else: + raise InvalidCodeError(f"Invalid code for limen = {matter.code}.") - @property - def count(self): - """ - Returns ._count - Makes ._count read only + def _processSith(self, sith: int | str | Sequence): """ - return self._count + Process attributes for fractionall weighted threshold sith + Parameters: + sith is signing threshold (current or next) expressed as either: + non-negative int of threshold number (M-of-N threshold) + next threshold may be zero + non-negative hex string of threshold number (M-of-N threshold) + next threshold may be zero + fractional weight clauses which may be expressed as either: + an sequence of rational number fraction weight str or int str + each denoted w where 0 <= w <= 1 + an sequence of sequences of rational number fraction weight + or int str + each denoted w where 0 <= w <= 1>= 0 + JSON serialized str of either: + list of rational number fraction weight strings + each denoted w where 0 <= w <= 1 + list of lists of rational number fraction weight strings + each denoted w where 0 <= w <= 1 - @property - def qb64b(self): - """ - Property qb64b: - Returns Fully Qualified Base64 Version encoded as bytes - Assumes self.raw and self.code are correctly populated + when any w is 0 or 1 then representation is 0 or 1 not 0/1 or 1/1 """ - return self._infil() + if isinstance(sith, int): + self._processUnweighted(thold=sith) + elif isinstance(sith, str) and '[' not in sith: + self._processUnweighted(thold=int(sith, 16)) - @property - def qb64(self): - """ - Property qb64: - Returns Fully Qualified Base64 Version - Assumes self.raw and self.code are correctly populated - """ - return self.qb64b.decode("utf-8") + else: # assumes sequence of weights or sequence of sequence of weights + if isinstance(sith, str): # json of weighted sith from cli + sith = json.loads(sith) # deserialize + if not sith: # empty or None + raise ValueError(f"Empty weight list = {sith}.") - @property - def qb2(self): - """ - Property qb2: - Returns Fully Qualified Binary Version Bytes - """ - return self._binfil() + # is it non str sequence of sequences? or non str sequnce of strs? + # must test for emply mask because all([]) == True + mask = [nonStringSequence(c) for c in sith] # check each element + if mask and not all(mask): # not empty and not sequence of sequenes + sith = [sith] # attempt to make sequnce of sequqnces of strs + for c in sith: # get each clause + # each element of a clause must be a str or dict + mask = [(isinstance(w, str) or isinstance(w, Mapping)) for w in c] + if mask and not all(mask): # not empty and not sequence of str or dicts + raise ValueError(f"Invalid sith = {sith} some weights in" + f"clause {c} are non string.") - def countToB64(self, l=None): - """ Returns count as Base64 left padded with "A"s - Parameters: - l (int | None): minimum number characters including left padding - When not provided use the softsize of .code + # replace weight str expression, int str or fractional strings with + # int or fraction as appropriate. + thold = [] + for c in sith: # convert string fractions to Fractions + # append list of where each element is either bare weight or + # single key map with value as list of weights + # each weight is converted from its str expression + clause = [] + for e in c: # each element of clause c + if isinstance(e, Mapping): + if len(e) != 1: + raise ValueError(f"Invalid sith = {sith} nested " + f"weight map {e} in clause {c} " + f" not single key value.") + k = list(e)[0] # zeroth key is used + # convert to tuple of (weight, [list of weights]) + clause.append((self.weight(k), [self.weight(w) for w in e[k]])) + else: + clause.append(self.weight(e)) - """ - if l is None: - _, ss, _, _ = self.Sizes[self.code] - l = ss - return (intToB64(self.count, l=l)) + thold.append(clause) + self._processWeighted(thold=thold) - @staticmethod - def semVerToB64(version="", major=0, minor=0, patch=0): - """ Converts semantic version to Base64 representation of countB64 - suitable for CESR protocol genus and version - Returns: - countB64 (str): suitable for input to Counter - example: Counter(countB64=semVerToB64(version = "1.0.0")) + def _processUnweighted(self, thold=0): + """ + Process attributes for unweighted (numeric) threshold thold Parameters: - version (str | None): dot separated semantic version string of format - "major.minor.patch" - major (int): When version is None or empty then use major,minor, patch - minor (int): When version is None or empty then use major,minor, patch - patch (int): When version is None or empty then use major,minor, patch - - each of major, minor, patch must be in range [0,63] for represenation as - three Base64 characters + thold (int): non-negative threshold number M-of-N threshold """ - parts = [major, minor, patch] - if version: - splits = version.split(".", maxsplit=3) - splits = [(int(s) if s else 0) for s in splits] - for i in range(3-len(splits),0, -1): - splits.append(parts[-i]) - parts = splits - - for p in parts: - if p < 0 or p > 63: - raise ValueError(f"Out of bounds semantic version. " - f"Part={p} is < 0 or > 63.") - return ("".join(intToB64(p, l=1) for p in parts)) + if thold < 0: + raise ValueError(f"Non-positive int threshold = {thold}.") + self._thold = thold + self._weighted = False + self._size = self._thold # used to verify that keys list size is at least size + self._satisfy = self._satisfy_numeric + self._number = Number(num=thold) + self._bexter = None - def _infil(self): + def _processWeighted(self, thold=[]): """ - Returns fully qualified attached sig base64 bytes computed from - self.code and self.count. - """ - code = self.code # codex value chars hard code - count = self.count # index value int used for soft - - hs, ss, fs, ls = self.Sizes[code] - cs = hs + ss # both hard + soft size - if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes - raise InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) - if count < 0 or count > (64 ** ss - 1): - raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) - - # both is hard code + converted count - both = "{}{}".format(code, intToB64(count, l=ss)) - - # check valid pad size for whole code size - if len(both) % 4: # no pad - raise InvalidCodeSizeError("Invalid size = {} of {} not a multiple of 4." - .format(len(both), both)) - # prepending full derivation code with index and strip off trailing pad characters - return (both.encode("utf-8")) - - - def _binfil(self): - """ - Returns bytes of fully qualified base2 bytes, that is .qb2 - self.code converted to Base2 left shifted with pad bits - equivalent of Base64 decode of .qb64 into .qb2 - """ - code = self.code # codex chars hard code - count = self.count # index value int used for soft - - hs, ss, fs, ls = self.Sizes[code] - cs = hs + ss - if fs != cs or cs % 4: # fs must be cs and multiple of 4 for count codes - raise InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) - - if count < 0 or count > (64 ** ss - 1): - raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) - - # both is hard code + converted count - both = "{}{}".format(code, intToB64(count, l=ss)) - if len(both) != cs: - raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." - .format(cs, len(both))) - - return (codeB64ToB2(both)) # convert to b2 left shift if any - - - def _exfil(self, qb64b): - """ - Extracts self.code and self.count from qualified base64 bytes qb64b - """ - if not qb64b: # empty need more bytes - raise ShortageError("Empty material, Need more characters.") - - first = qb64b[:2] # extract first two char code selector - if hasattr(first, "decode"): - first = first.decode("utf-8") - if first not in self.Hards: - if first[0] == '_': - raise UnexpectedOpCodeError("Unexpected op code start" - "while extracing Counter.") - else: - raise UnexpectedCodeError("Unsupported code start ={}.".format(first)) - - hs = self.Hards[first] # get hard code size - if len(qb64b) < hs: # need more bytes - raise ShortageError("Need {} more characters.".format(hs - len(qb64b))) - - hard = qb64b[:hs] # get hard code - if hasattr(hard, "decode"): - hard = hard.decode("utf-8") # decode converts bytearray/bytes to str - if hard not in self.Sizes: # Sizes needs str not bytes - raise UnexpectedCodeError("Unsupported code ={}.".format(hard)) - - hs, ss, fs, ls = self.Sizes[hard] # assumes hs consistent in both tables - cs = hs + ss # both hard + soft code size - - # assumes that unit tests on Counter and CounterCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 - - if len(qb64b) < cs: # need more bytes - raise ShortageError("Need {} more characters.".format(cs - len(qb64b))) - - count = qb64b[hs:hs + ss] # extract count chars - if hasattr(count, "decode"): - count = count.decode("utf-8") - count = b64ToInt(count) # compute int count - - self._code = hard - self._count = count - - - def _bexfil(self, qb2): - """ - Extracts self.code and self.count from qualified base2 bytes qb2 - """ - if not qb2: # empty need more bytes - raise ShortageError("Empty material, Need more bytes.") - - first = nabSextets(qb2, 2) # extract first two sextets as code selector - if first not in self.Bards: - if first[0] == b'\xfc': # b64ToB2('_') - raise UnexpectedOpCodeError("Unexpected op code start" - "while extracing Matter.") - else: - raise UnexpectedCodeError("Unsupported code start sextet={}.".format(first)) - - hs = self.Bards[first] # get code hard size equvalent sextets - bhs = sceil(hs * 3 / 4) # bhs is min bytes to hold hs sextets - if len(qb2) < bhs: # need more bytes - raise ShortageError("Need {} more bytes.".format(bhs - len(qb2))) - - hard = codeB2ToB64(qb2, hs) # extract and convert hard part of code - if hard not in self.Sizes: - raise UnexpectedCodeError("Unsupported code ={}.".format(hard)) - - hs, ss, fs, ls = self.Sizes[hard] - cs = hs + ss # both hs and ss - # assumes that unit tests on Counter and CounterCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 - - bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets - if len(qb2) < bcs: # need more bytes - raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) - - both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code - count = b64ToInt(both[hs:hs + ss]) # get count - - self._code = hard - self._count = count - - -class Sadder: - """ - Sadder is self addressed data (SAD) serializer-deserializer class - - Instance creation of a Sadder does not verifiy it .said property it merely - extracts it. In order to ensure Sadder instance has a verified .said then - must call .saider.verify(sad=self.ked) - - Has the following public properties: - - Properties: - raw (bytes): of serialized event only - ked (dict): self addressed data dict - kind (str): serialization kind coring.Serials such as JSON, CBOR, MGPK, CESR - size (int): number of bytes in serialization - version (Versionage): protocol version (Major, Minor) - proto (str): Protocolage value as protocol identifier such as KERI, ACDC - label (str): Saidage value as said field label - saider (Saider): of SAID of this SAD .ked['d'] if present - said (str): SAID of .saider qb64 - saidb (bytes): SAID of .saider qb64b - pretty (str): Pretty JSON of this SAD - - Hidden Attributes: - ._raw is bytes of serialized event only - ._ked is key event dict - ._kind is serialization kind string value (see namedtuple coring.Serials) - supported kinds are 'json', 'cbor', 'msgpack', 'binary' - ._size is int of number of bytes in serialed event only - ._version is Versionage instance of event version - ._proto (str): Protocolage value as protocol type identifier - ._saider (Saider): instance for this Sadder's SAID - - Note: - loads and jumps of json use str whereas cbor and msgpack use bytes - - """ - MaxVSOffset = 12 - SmellSize = MaxVSOffset + MAXVERFULLSPAN # min buffer size to inhale - - def __init__(self, raw=b'', ked=None, sad=None, kind=None, saidify=False, - code=MtrDex.Blake3_256): - """ - Deserialize if raw provided does not verify assumes embedded said is valid - Serialize if ked provided but not raw verifies if verify is True? - When serializing if kind provided then use kind instead of field in ked - - Parameters: - raw (bytes): serialized event - ked is key event dict or None - if None its deserialized from raw - kind is serialization kind string value or None (see namedtuple coring.Serials) - supported kinds are 'json', 'cbor', 'msgpack', 'binary' - if kind is None then its extracted from ked or raw - saidify (bool): True means compute said for ked - code is .diger default digest code for computing said .saider - - """ - self._code = code # need default code for .saider - if raw: # deserialize raw using property setter - self.raw = raw # raw property setter does the deserialization - elif ked: # serialize ked using property setter - #ToDo when pass in ked and saidify True then compute said - self._kind = kind - self.ked = ked # ked property setter does the serialization - elif sad: - # ToDo do we need this or should we be using ked above with saidify flag - self._clone(sad=sad) # copy fields from sad - else: - raise ValueError("Improper initialization need sad, raw or ked.") - - - def _clone(self, sad): - """ copy hidden attributes from sad """ - self._raw = sad.raw - self._ked = sad.ked - self._kind = sad.kind - self._size = sad.size - self._version = sad.version - self._proto = sad.proto - self._saider = sad.saider - - - def _inhale(self, raw): - """ - Parses serilized event ser of serialization kind and assigns to - instance attributes. - - Parameters: - raw is bytes of serialized event - kind is str of raw serialization kind (see namedtuple Serials) - size is int size of raw to be deserialized - - Note: - loads and jumps of json use str whereas cbor and msgpack use bytes - - """ - proto, vrsn, kind, size, _ = smell(raw) - if vrsn != Version: - raise VersionError("Unsupported version = {}.{}, expected {}." - "".format(vrsn.major, vrsn.minor, Version)) - - ked = loads(raw=raw, size=size, kind=kind) - - return ked, proto, kind, vrsn, size - - - def _exhale(self, ked, kind=None): - """ - Returns sizeify(ked, kind) - - From sizeify - Returns tuple of (raw, proto, kind, ked, version) where: - raw (str): serialized event as bytes of kind - proto (str): protocol type as value of Protocolage - kind (str): serialzation kind as value of Serialage - ked (dict): key event dict or sad dict - version (Versionage): instance - - Parameters: - ked (dict): key event dict or sad dict - kind (str): value of Serials serialization kind. - When not provided use - - Assumes only supports Version - """ - return sizeify(ked=ked, kind=kind) - - - def compare(self, said=None): - """ - Returns True if said and either .saider.qb64 or .saider.qb64b match - via string equality == - - Convenience method to allow comparison of own .saider digest self.raw - with some other purported said of self.raw + Process attributes for fractionall weighted threshold thold Parameters: - said is qb64b or qb64 SAID of ser to compare with .said + thold (iterable): iterable or iterable or iterables of + rational number fraction strings >= 0 and <= 1 """ + for clause in thold: # sum of top level weights in clause must be >= 1 + # When element is dict then sum of value's weights must be >= 1 + top = [] # top level weights + for e in clause: + if isinstance(e, tuple): + top.append(e[0]) + if not (sum(e[1]) >= 1): + raise ValueError(f"Invalid sith clause = {clause}, " + f"element = {e}. All nested clause " + f"weight sums must be >= 1.") + else: + top.append(e) + if not (sum(top) >= 1): + raise ValueError(f"Invalid sith clause = {clause}, all top level" + f"clause weight sums must be >= 1.") - if said is not None: - if hasattr(said, "encode"): - said = said.encode('utf-8') # makes bytes - - return said == self.saidb # matching - - else: - raise ValueError("Both said and saider may not be None.") - - - @property - def raw(self): - """ raw property getter """ - return self._raw - - @raw.setter - def raw(self, raw): - """ raw property setter """ - ked, proto, kind, version, size = self._inhale(raw=raw) - self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray - self._ked = ked - self._proto = proto - self._kind = kind - self._version = version - self._size = size - self._saider = Saider(qb64=ked["d"], code=self._code) - - @property - def ked(self): - """ ked property getter""" - return self._ked - - @ked.setter - def ked(self, ked): - """ ked property setter assumes ._kind """ - raw, proto, kind, ked, version = self._exhale(ked=ked, kind=self._kind) - size = len(raw) - self._raw = raw[:size] - self._ked = ked - self._proto = proto - self._kind = kind - self._size = size - self._version = version - self._saider = Saider(qb64=ked["d"], code=self._code) - - @property - def kind(self): - """ kind property getter""" - return self._kind + self._thold = thold + self._weighted = True + #self._size = sum(len(clause) for clause in thold) + s = 0 + for clause in thold: + for e in clause: + if isinstance(e, tuple): + s += len(e[1]) + else: + s += 1 + self._size = s - @kind.setter - def kind(self, kind): - """ kind property setter Assumes ._ked. Serialization kind. """ - raw, proto, kind, ked, version = self._exhale(ked=self._ked, kind=kind) - size = len(raw) - self._raw = raw[:size] - self._proto = proto - self._ked = ked - self._kind = kind - self._size = size - self._version = version - self._saider = Saider(qb64=ked["d"], code=self._code) + self._satisfy = self._satisfy_weighted + # make bext str of thold for .bexter for limen + ta = [] # list of list of fractions and/or single element map of fractions + for c in thold: + bc = [] # list of fractions and/or single element map of fractions + for e in c: + if isinstance(e, tuple): + f = e[0] + k = f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" + v = "v".join([f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" for f in e[1]]) + kv = "k".join([k, v]) + bc.append(kv) + else: + bc.append(f"{e.numerator}s{e.denominator}" if (0 < e < 1) else f"{int(e)}") + ta.append(bc) - @property - def size(self): - """ size property getter""" - return self._size + bext = "a".join(["c".join(bc) for bc in ta]) + self._number = None + self._bexter = Bexter(bext=bext) - @property - def version(self): - """ - version property getter + @staticmethod + def weight(w: str) -> Fraction: + """Returns valid weight from w else raises error (ValueError or TypeError). + w expression must evaluate to 0, 1, or strict proper rational fraction. + w expression must be 0 <= w <= 1 Else raises ValueError + w must not be float else raises TypeError + When not int w must be ratio of integers n/d else raise ValueError. - Returns: - (Versionage): + Parameters: + w (str): threshold weight expression """ - return self._version - + try: # float str or ratio str raises ValueError + if int(float(w)) != float(w): # float str + raise TypeError("Invalid weight str got float w={w}.") + w = int(w) # expression is int str + except TypeError as ex: + raise ValueError(str(ex)) from ex - @property - def proto(self): - """ proto property getter - protocol identifier type value of Protocolage such as 'KERI' or 'ACDC' + except ValueError as ex: # not float str or int str so try ration str + w = Fraction(w) - Returns: - (str): Protocolage value as protocol type - """ - return self._proto + if not 0 <= w <= 1: + raise ValueError(f"Invalid weight not 0 <= {w} <= 1.") + return w - @property - def saider(self): - """ - Returns Diger of digest of self.raw - diger (digest material) property getter + def satisfy(self, indices): """ - return self._saider + Returns True if indices list of verified signature key indices satisfies + threshold, False otherwise. - @property - def said(self): - """ - Returns str qb64 of .ked["d"] (said when ked is SAD) - said (self-addressing identifier) property getter + Parameters: + indices is list of non-negative indices (offsets into key list) + of verified signatures. the indices may be in any order, they + are normalized herein """ - return self.saider.qb64 + return (self._satisfy(indices=indices)) - @property - def saidb(self): - """ - Returns bytes qb64b of .ked["d"] (said when ked is SAD) - said (self-addressing identifier) property getter - """ - return self.saider.qb64b - def pretty(self, *, size=1024): + def _satisfy_numeric(self, indices): """ - Returns str JSON of .ked with pretty formatting + Returns True if satisfies numeric threshold False otherwise - ToDo: add default size limit on pretty when used for syslog UDP MCU - like 1024 for ogler.logger + Parameters: + indices is list of indices (offsets into key list) of verified signatures """ - return json.dumps(self.ked, indent=1)[:size if size is not None else None] + try: + if self.thold > 0 and len(indices) >= self.thold: # at least one + return True + except Exception as ex: + return False + return False -class Tholder: - """ - Tholder is KERI Signing Threshold Satisfaction class - .satisfy method evaluates satisfaction based on ordered list of indices of - verified signatures where indices correspond to offsets in key list of - associated signatures. - Has the following public properties: + def _satisfy_weighted(self, indices): + """ + Returns True if satifies fractional weighted threshold False otherwise - Properties: - .weighted is Boolean True if fractional weighted threshold False if numeric - .size is int of minimum size of keys list - when weighted is size of keys list - when unweighted is size of int thold since don't have anyway - to know size of keys list in this case - .limen is qualified b64 signing threshold suitable for CESR serialization. - either Number.qb64b or Bexter.qb64b. - The b64 portion of limen with code stripped (Bexter.bext) of - [["1/2", "1/2", "1/4", "1/4", "1/4"], ["1", "1"]] - is '1s2c1s2c1s4c1s4c1s4a1c1' basically slash is 's', comma is 'c', - ANDed clauses are delimited by 'a'. - Each clause top level weight may be optionally a weighted set of weights - delimited by 'k' for the weight on the set and 'v' for the weights in - the set. - [[{'1/3': ['1/2', '1/2', '1/2']}, '1/2', {'1/2': ['1', '1']}], - ['1/2', {'1/2': ['1', '1']}]] - b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1' + Parameters: + indices is list of non-negative indices (offsets into key list) + of verified signatures. the indices may be in any order, they + are normalized herein + """ + try: + if not indices: # empty indices + return False - .sith is original signing threshold suitable for value to be serialized - as json, cbor, mgpk in key event message as either: - non-negative hex number str or - list of str rational number fractions >= 0 and <= 1 or - list of list of str rational number fractions >= 0 and <= 1 - list of list of weighted map of weights + # remove duplicates with set, sort low to high + indices = sorted(set(indices)) + sats = [False] * self.size # default all satifactions to False + for idx in indices: + sats[idx] = True # set verified signature index to True - .thold is parsed signing threshold suitable for calculating satisfaction. - either as int or list of Fractions + wio = 0 # weight index offset + for clause in self.thold: + cw = 0 # init clause weight + for e in clause: + if isinstance(e, tuple): + vw = 0 # init element value weight + for w in e[1]: # sum weights of value + if sats[wio]: + vw += w + wio += 1 + if vw >= 1: # element true + cw += e[0] # add element key weight to clause weight + else: + w = e + if sats[wio]: # verified signature so weight applies + cw += w + wio += 1 + if cw < 1: # each clause must sum to at least 1 + return False - .num is int signing threshold when not ._weighted + return True # all clauses have cw >= 1 including final one, AND true - Methods: - .satisfy returns bool, True means list of verified signature key indices - satisfies the threshold, False otherwise. + except Exception as ex: + return False - Static Methods: - weight (str): converts weight str expression into either int or Fraction - else raises ValueError must satisfy 0 <= w <= 1 - Ensures strict proper rational number fraction of ints or - 0 or 1 + return False - Hidden: - ._weighted is Boolean, True if fractional weighted threshold False if numeric - ._size is int minimum size of of keys list - ._sith is signing threshold for .sith property - ._thold is signing threshold for .thold propery - ._bexter is Bexter instance of weighted signing threshold or None - ._number is Number instance of integer threshold or None - ._satisfy is method reference of threshold specified verification method - ._satisfy_numeric is numeric threshold verification method - ._satisfy_weighted is fractional weighted threshold verification method +class Streamer: """ + Streamer is CESR sniffable stream class - def __init__(self, *, thold=None , limen=None, sith=None, **kwa): - """ - Accepts signing threshold in various forms so that may output correct - forms for serialization and/or calculation of satisfaction. - - The thold representation is meant to accept thresholds from computable - expressions for satisfaction of a threshold - - The limen representation is meant to parse threshold expressions from - CESR serializations of key event message fields or attachments. - - The sith representation is meant to parse threhold expressions from - deserializations of JSON, CBOR, or MGPK key event message fields or - the command line or configuration files. - - - Parameters: - - thold is signing threshold (current or next) is suitable for computing - the satisfaction of a threshold and is expressed as either: - int of threshold number (M of N) - fractional weight clauses which may be expressed as either: - sequence of either Fractions or tuples of Fraction and - sequence of Fractions - sequence of sequence of either Fractions or tuples of - Fraction and sequence of Fractions - limen is qualified signing threshold (current or next) expressed as either: - Number.qb64 or .qb64b of integer threshold or - Bexter.qb64 or .qb64b of fractional weight clauses which may be either: - Base64 delimited clauses of fractions - Base64 delimited clauses of fractions + Has the following public properties: - sith is signing threshold (current or next) expressed as either: - non-negative int of threshold number (M-of-N threshold) - next threshold may be zero - non-negative hex string of threshold number (M-of-N threshold) - next threshold may be zero - fractional weight clauses which may be expressed as either: - sequence of rational number fraction strings >= 0 and <= 1 - sequence of either rational number fraction strings >= 0 and <= 1 or - map with key rational number string and value as sequence - of rational number fraction strings - rational number fraction string - sequence of sequences of rational number fraction strings >= 0 and <= 1 - sequence of sequnces of either rational number fraction strings or - map with key rational number fraction string with value sequence of - rationaly number fraction strings - JSON serialized str of the above: + Properties: - """ - if thold is not None: - self._processThold(thold=thold) + Methods: - elif limen is not None: - self._processLimen(limen=limen, **kwa) # kwa for strip - elif sith is not None: - if isinstance(sith, str) and not sith: # empty str - raise EmptyMaterialError("Empty threshold expression.") + Hidden: - self._processSith(sith=sith) - else: - raise EmptyMaterialError("Missing threshold expression.") + """ - @property - def weighted(self): - """ weighted property getter """ - return self._weighted + def __init__(self, stream): + """Initialize instance - @property - def thold(self): - """ thold property getter """ - return self._thold - @property - def size(self): - """ size property getter """ - return self._size + Parameters: + stream (bytes | bytearray): sniffable CESR stream - @property - def limen(self): - """ limen property getter """ - return self._bexter.qb64b if self._weighted else self._number.qb64b - @property - def sith(self): - """ sith property getter """ - # make sith expression of thold - if self.weighted: - sith = [] - for c in self.thold: - clause = [] - for e in c: - if isinstance(e, tuple): - f = e[0] - k = f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" - v = [f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" - for f in e[1]] - clause.append({k: v}) - else: - f = e - clause.append(f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}") - sith.append(clause) + """ + self._stream = bytes(stream) - #sith = [[f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" - #for f in clause] - #for clause in self.thold] - if len(sith) == 1: - sith = sith[0] # simplify list of one clause to clause - else: - sith = f"{self.thold:x}" - return sith + @property + def stream(self): + """stream property getter + """ + return self._stream @property - def json(self): - """Returns json serialization of sith expression + def text(self): + """expanded stream as qb64 text + Returns: + stream (bytes): expanded text qb64 version of stream - Essentially JSON list of lists of strings """ - return json.dumps(self.sith) - + return self._stream @property - def num(self): - """ sith property getter """ - return self.thold if not self._weighted else None - + def binary(self): + """compacted stream as qb2 binary + Returns: + stream (bytes): compacted binary qb2 version of stream + """ + return self._stream - def _processThold(self, thold: int | Sequence): - """Process thold input + @property + def texter(self): + """expanded stream as Texter instance + Returns: + texter (Texter): Texter primitive of stream suitable wrapping - Parameters: - thold (int | Sequence): computable thold expression """ - if isinstance(thold, int): - self._processUnweighted(thold=thold) + return self._stream - else: - self._processWeighted(thold=thold) - def _processLimen(self, limen: str | bytes, **kwa): - """Process limen input - Parameters: - limen (str): CESR encoded qb64 threshold (weighted or unweighted) - """ - matter = Matter(qb64b=limen, **kwa) # kwa for strip of stream - if matter.code in NumDex: - number = Number(raw=matter.raw, code=matter.code, **kwa) - self._processUnweighted(thold=number.num) +class Sadder: + """ + Sadder is self addressed data (SAD) serializer-deserializer class - elif matter.code in BexDex: - # Convert to fractional thold expression - bexter = Bexter(raw=matter.raw, code=matter.code, **kwa) - t = bexter.bext.replace('s', '/') - # get clauses - clauses = [clause.split('c') for clause in t.split('a')] + Instance creation of a Sadder does not verifiy it .said property it merely + extracts it. In order to ensure Sadder instance has a verified .said then + must call .saider.verify(sad=self.ked) - thold = [] - for c in clauses: - clause = [] - for e in c: - k, s, v = e.partition("k") - if s: #not empty - clause.append((self.weight(k), [self.weight(w) for w in v.split("v")])) - else: - clause.append(self.weight(k)) + Has the following public properties: - thold.append(clause) + Properties: + raw (bytes): of serialized event only + ked (dict): self addressed data dict + kind (str): serialization kind coring.Serials such as JSON, CBOR, MGPK, CESR + size (int): number of bytes in serialization + version (Versionage): protocol version (Major, Minor) + proto (str): Protocolage value as protocol identifier such as KERI, ACDC + label (str): Saidage value as said field label + saider (Saider): of SAID of this SAD .ked['d'] if present + said (str): SAID of .saider qb64 + saidb (bytes): SAID of .saider qb64b + pretty (str): Pretty JSON of this SAD - self._processWeighted(thold=thold) + Hidden Attributes: + ._raw is bytes of serialized event only + ._ked is key event dict + ._kind is serialization kind string value (see namedtuple coring.Serials) + supported kinds are 'json', 'cbor', 'msgpack', 'binary' + ._size is int of number of bytes in serialed event only + ._version is Versionage instance of event version + ._proto (str): Protocolage value as protocol type identifier + ._saider (Saider): instance for this Sadder's SAID - else: - raise InvalidCodeError(f"Invalid code for limen = {matter.code}.") + Note: + loads and jumps of json use str whereas cbor and msgpack use bytes + """ + MaxVSOffset = 12 + SmellSize = MaxVSOffset + MAXVERFULLSPAN # min buffer size to inhale - def _processSith(self, sith: int | str | Sequence): + def __init__(self, raw=b'', ked=None, sad=None, kind=None, saidify=False, + code=MtrDex.Blake3_256): """ - Process attributes for fractionall weighted threshold sith + Deserialize if raw provided does not verify assumes embedded said is valid + Serialize if ked provided but not raw verifies if verify is True? + When serializing if kind provided then use kind instead of field in ked Parameters: - sith is signing threshold (current or next) expressed as either: - non-negative int of threshold number (M-of-N threshold) - next threshold may be zero - non-negative hex string of threshold number (M-of-N threshold) - next threshold may be zero - fractional weight clauses which may be expressed as either: - an sequence of rational number fraction weight str or int str - each denoted w where 0 <= w <= 1 - an sequence of sequences of rational number fraction weight - or int str - each denoted w where 0 <= w <= 1>= 0 - JSON serialized str of either: - list of rational number fraction weight strings - each denoted w where 0 <= w <= 1 - list of lists of rational number fraction weight strings - each denoted w where 0 <= w <= 1 + raw (bytes): serialized event + ked is key event dict or None + if None its deserialized from raw + kind is serialization kind string value or None (see namedtuple coring.Serials) + supported kinds are 'json', 'cbor', 'msgpack', 'binary' + if kind is None then its extracted from ked or raw + saidify (bool): True means compute said for ked + code is .diger default digest code for computing said .saider - when any w is 0 or 1 then representation is 0 or 1 not 0/1 or 1/1 """ - if isinstance(sith, int): - self._processUnweighted(thold=sith) + self._code = code # need default code for .saider + if raw: # deserialize raw using property setter + self.raw = raw # raw property setter does the deserialization + elif ked: # serialize ked using property setter + #ToDo when pass in ked and saidify True then compute said + self._kind = kind + self.ked = ked # ked property setter does the serialization + elif sad: + # ToDo do we need this or should we be using ked above with saidify flag + self._clone(sad=sad) # copy fields from sad + else: + raise ValueError("Improper initialization need sad, raw or ked.") - elif isinstance(sith, str) and '[' not in sith: - self._processUnweighted(thold=int(sith, 16)) - else: # assumes sequence of weights or sequence of sequence of weights - if isinstance(sith, str): # json of weighted sith from cli - sith = json.loads(sith) # deserialize + def _clone(self, sad): + """ copy hidden attributes from sad """ + self._raw = sad.raw + self._ked = sad.ked + self._kind = sad.kind + self._size = sad.size + self._version = sad.version + self._proto = sad.proto + self._saider = sad.saider - if not sith: # empty or None - raise ValueError(f"Empty weight list = {sith}.") - # is it non str sequence of sequences? or non str sequnce of strs? - # must test for emply mask because all([]) == True - mask = [nonStringSequence(c) for c in sith] # check each element - if mask and not all(mask): # not empty and not sequence of sequenes - sith = [sith] # attempt to make sequnce of sequqnces of strs + def _inhale(self, raw): + """ + Parses serilized event ser of serialization kind and assigns to + instance attributes. - for c in sith: # get each clause - # each element of a clause must be a str or dict - mask = [(isinstance(w, str) or isinstance(w, Mapping)) for w in c] - if mask and not all(mask): # not empty and not sequence of str or dicts - raise ValueError(f"Invalid sith = {sith} some weights in" - f"clause {c} are non string.") + Parameters: + raw is bytes of serialized event + kind is str of raw serialization kind (see namedtuple Serials) + size is int size of raw to be deserialized - # replace weight str expression, int str or fractional strings with - # int or fraction as appropriate. - thold = [] - for c in sith: # convert string fractions to Fractions - # append list of where each element is either bare weight or - # single key map with value as list of weights - # each weight is converted from its str expression - clause = [] - for e in c: # each element of clause c - if isinstance(e, Mapping): - if len(e) != 1: - raise ValueError(f"Invalid sith = {sith} nested " - f"weight map {e} in clause {c} " - f" not single key value.") - k = list(e)[0] # zeroth key is used - # convert to tuple of (weight, [list of weights]) - clause.append((self.weight(k), [self.weight(w) for w in e[k]])) - else: - clause.append(self.weight(e)) + Note: + loads and jumps of json use str whereas cbor and msgpack use bytes - thold.append(clause) + """ + proto, vrsn, kind, size, _ = smell(raw) + if vrsn != Version: + raise VersionError("Unsupported version = {}.{}, expected {}." + "".format(vrsn.major, vrsn.minor, Version)) - self._processWeighted(thold=thold) + ked = loads(raw=raw, size=size, kind=kind) + return ked, proto, kind, vrsn, size - def _processUnweighted(self, thold=0): + + def _exhale(self, ked, kind=None): """ - Process attributes for unweighted (numeric) threshold thold + Returns sizeify(ked, kind) + + From sizeify + Returns tuple of (raw, proto, kind, ked, version) where: + raw (str): serialized event as bytes of kind + proto (str): protocol type as value of Protocolage + kind (str): serialzation kind as value of Serialage + ked (dict): key event dict or sad dict + version (Versionage): instance Parameters: - thold (int): non-negative threshold number M-of-N threshold + ked (dict): key event dict or sad dict + kind (str): value of Serials serialization kind. + When not provided use + Assumes only supports Version """ - if thold < 0: - raise ValueError(f"Non-positive int threshold = {thold}.") - self._thold = thold - self._weighted = False - self._size = self._thold # used to verify that keys list size is at least size - self._satisfy = self._satisfy_numeric - self._number = Number(num=thold) - self._bexter = None + return sizeify(ked=ked, kind=kind) - def _processWeighted(self, thold=[]): + def compare(self, said=None): """ - Process attributes for fractionall weighted threshold thold + Returns True if said and either .saider.qb64 or .saider.qb64b match + via string equality == + + Convenience method to allow comparison of own .saider digest self.raw + with some other purported said of self.raw Parameters: - thold (iterable): iterable or iterable or iterables of - rational number fraction strings >= 0 and <= 1 + said is qb64b or qb64 SAID of ser to compare with .said """ - for clause in thold: # sum of top level weights in clause must be >= 1 - # When element is dict then sum of value's weights must be >= 1 - top = [] # top level weights - for e in clause: - if isinstance(e, tuple): - top.append(e[0]) - if not (sum(e[1]) >= 1): - raise ValueError(f"Invalid sith clause = {clause}, " - f"element = {e}. All nested clause " - f"weight sums must be >= 1.") - else: - top.append(e) - if not (sum(top) >= 1): - raise ValueError(f"Invalid sith clause = {clause}, all top level" - f"clause weight sums must be >= 1.") - self._thold = thold - self._weighted = True - #self._size = sum(len(clause) for clause in thold) - s = 0 - for clause in thold: - for e in clause: - if isinstance(e, tuple): - s += len(e[1]) - else: - s += 1 - self._size = s + if said is not None: + if hasattr(said, "encode"): + said = said.encode('utf-8') # makes bytes - self._satisfy = self._satisfy_weighted - # make bext str of thold for .bexter for limen - ta = [] # list of list of fractions and/or single element map of fractions - for c in thold: - bc = [] # list of fractions and/or single element map of fractions - for e in c: - if isinstance(e, tuple): - f = e[0] - k = f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" - v = "v".join([f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" for f in e[1]]) - kv = "k".join([k, v]) - bc.append(kv) - else: - bc.append(f"{e.numerator}s{e.denominator}" if (0 < e < 1) else f"{int(e)}") + return said == self.saidb # matching - ta.append(bc) + else: + raise ValueError("Both said and saider may not be None.") - bext = "a".join(["c".join(bc) for bc in ta]) - self._number = None - self._bexter = Bexter(bext=bext) + @property + def raw(self): + """ raw property getter """ + return self._raw - @staticmethod - def weight(w: str) -> Fraction: - """Returns valid weight from w else raises error (ValueError or TypeError). - w expression must evaluate to 0, 1, or strict proper rational fraction. - w expression must be 0 <= w <= 1 Else raises ValueError - w must not be float else raises TypeError - When not int w must be ratio of integers n/d else raise ValueError. + @raw.setter + def raw(self, raw): + """ raw property setter """ + ked, proto, kind, version, size = self._inhale(raw=raw) + self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray + self._ked = ked + self._proto = proto + self._kind = kind + self._version = version + self._size = size + self._saider = Saider(qb64=ked["d"], code=self._code) - Parameters: - w (str): threshold weight expression - """ - try: # float str or ratio str raises ValueError - if int(float(w)) != float(w): # float str - raise TypeError("Invalid weight str got float w={w}.") - w = int(w) # expression is int str - except TypeError as ex: - raise ValueError(str(ex)) from ex + @property + def ked(self): + """ ked property getter""" + return self._ked - except ValueError as ex: # not float str or int str so try ration str - w = Fraction(w) + @ked.setter + def ked(self, ked): + """ ked property setter assumes ._kind """ + raw, proto, kind, ked, version = self._exhale(ked=ked, kind=self._kind) + size = len(raw) + self._raw = raw[:size] + self._ked = ked + self._proto = proto + self._kind = kind + self._size = size + self._version = version + self._saider = Saider(qb64=ked["d"], code=self._code) - if not 0 <= w <= 1: - raise ValueError(f"Invalid weight not 0 <= {w} <= 1.") - return w + @property + def kind(self): + """ kind property getter""" + return self._kind + @kind.setter + def kind(self, kind): + """ kind property setter Assumes ._ked. Serialization kind. """ + raw, proto, kind, ked, version = self._exhale(ked=self._ked, kind=kind) + size = len(raw) + self._raw = raw[:size] + self._proto = proto + self._ked = ked + self._kind = kind + self._size = size + self._version = version + self._saider = Saider(qb64=ked["d"], code=self._code) - def satisfy(self, indices): - """ - Returns True if indices list of verified signature key indices satisfies - threshold, False otherwise. - Parameters: - indices is list of non-negative indices (offsets into key list) - of verified signatures. the indices may be in any order, they - are normalized herein - """ - return (self._satisfy(indices=indices)) + @property + def size(self): + """ size property getter""" + return self._size - def _satisfy_numeric(self, indices): + @property + def version(self): """ - Returns True if satisfies numeric threshold False otherwise + version property getter - Parameters: - indices is list of indices (offsets into key list) of verified signatures + Returns: + (Versionage): """ - try: - if self.thold > 0 and len(indices) >= self.thold: # at least one - return True - - except Exception as ex: - return False + return self._version - return False + @property + def proto(self): + """ proto property getter + protocol identifier type value of Protocolage such as 'KERI' or 'ACDC' - def _satisfy_weighted(self, indices): + Returns: + (str): Protocolage value as protocol type """ - Returns True if satifies fractional weighted threshold False otherwise + return self._proto - Parameters: - indices is list of non-negative indices (offsets into key list) - of verified signatures. the indices may be in any order, they - are normalized herein + @property + def saider(self): + """ + Returns Diger of digest of self.raw + diger (digest material) property getter + """ + return self._saider + @property + def said(self): """ - try: - if not indices: # empty indices - return False + Returns str qb64 of .ked["d"] (said when ked is SAD) + said (self-addressing identifier) property getter + """ + return self.saider.qb64 - # remove duplicates with set, sort low to high - indices = sorted(set(indices)) - sats = [False] * self.size # default all satifactions to False - for idx in indices: - sats[idx] = True # set verified signature index to True + @property + def saidb(self): + """ + Returns bytes qb64b of .ked["d"] (said when ked is SAD) + said (self-addressing identifier) property getter + """ + return self.saider.qb64b - wio = 0 # weight index offset - for clause in self.thold: - cw = 0 # init clause weight - for e in clause: - if isinstance(e, tuple): - vw = 0 # init element value weight - for w in e[1]: # sum weights of value - if sats[wio]: - vw += w - wio += 1 - if vw >= 1: # element true - cw += e[0] # add element key weight to clause weight - else: - w = e - if sats[wio]: # verified signature so weight applies - cw += w - wio += 1 - if cw < 1: # each clause must sum to at least 1 - return False + def pretty(self, *, size=1024): + """ + Returns str JSON of .ked with pretty formatting - return True # all clauses have cw >= 1 including final one, AND true + ToDo: add default size limit on pretty when used for syslog UDP MCU + like 1024 for ogler.logger + """ + return json.dumps(self.ked, indent=1)[:size if size is not None else None] - except Exception as ex: - return False - return False class Dicter: @@ -6605,12 +5869,3 @@ def pretty(self, *, size=1024): return json.dumps(self.pad, indent=1)[:size if size is not None else None] -def randomNonce(): - """ Generate a random ed25519 seed and encode as qb64 - - Returns: - str: qb64 encoded ed25519 random seed - """ - preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - seedqb64 = Matter(raw=preseed, code=MtrDex.Ed25519_Seed).qb64 - return seedqb64 diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 21792db6c..b2f33c2ae 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -433,7 +433,7 @@ class Counter: def __init__(self, tag=None, *, code = None, count=None, countB64=None, - qb64b=None, qb64=None, qb2=None, strip=False, version=Vrsn_2_0): + qb64b=None, qb64=None, qb2=None, strip=False, gvrsn=Vrsn_2_0): """ Validate as fully qualified Parameters: @@ -459,8 +459,7 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, strip (bool): True means strip counter contents from input stream bytearray after parsing qb64b or qb2. False means do not strip. default False - version (Versionage): instance of version of code tables to use - provides protocol genera version + gvrsn (Versionage): instance of genera version of CESR code tables Needs either code or qb64b or qb64 or qb2 @@ -470,19 +469,19 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, .code and .count """ - if version.major not in self.Sizes: + if gvrsn.major not in self.Sizes: raise kering.InvalidVersionError(f"Unsupported major version=" - f"{version.major}.") + f"{gvrsn.major}.") - latest = list(self.Sizes[version.major])[0] # get latest minor version - if version.minor > latest: - raise kering.InvalidVersionError(f"Minor version={version.minor} " + latest = list(self.Sizes[gvrsn.major])[0] # get latest minor version + if gvrsn.minor > latest: + raise kering.InvalidVersionError(f"Minor version={gvrsn.minor} " f" exceeds latest supported minor" f" version={latest}.") - self._codes = self.Codes[version.major][latest] # use latest supported version codes - self._sizes = self.Sizes[version.major][latest] # use latest supported version sizes - self._version = version # provided version may be earlier than supported version + self._codes = self.Codes[gvrsn.major][latest] # use latest supported version codes + self._sizes = self.Sizes[gvrsn.major][latest] # use latest supported version sizes + self._version = gvrsn # provided version may be earlier than supported version if tag: diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 123194eab..3fded9595 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -32,9 +32,12 @@ from . import coring from .coring import (versify, Serials, Ilks, MtrDex, PreDex, DigDex, NonTransDex, CtrDex, Counter, - Number, Seqner, Siger, Cigar, Dater, Indexer, IdrDex, + Number, Seqner, Cigar, Dater, Verfer, Diger, Prefixer, Tholder, Saider) +from . import indexing +from .indexing import (IdrDex, Indexer, Siger) + from . import serdering from ..db import basing, dbing @@ -6315,7 +6318,7 @@ def loadEvent(db, preb, dig): sigs = db.getSigs(key=dgkey) dsigs = [] for s in sigs: - sig = coring.Siger(qb64b=bytes(s)) + sig = indexing.Siger(qb64b=bytes(s)) dsigs.append(dict(index=sig.index, signature=sig.qb64)) event["signatures"] = dsigs @@ -6327,7 +6330,7 @@ def loadEvent(db, preb, dig): dwigs = [] if wigs := db.getWigs(key=dgkey): for w in wigs: - sig = coring.Siger(qb64b=bytes(w)) + sig = indexing.Siger(qb64b=bytes(w)) dwigs.append(dict(index=sig.index, signature=sig.qb64)) event["witness_signatures"] = dwigs @@ -6349,7 +6352,7 @@ def loadEvent(db, preb, dig): prefix=coring.Prefixer(qb64b=raw, strip=True).qb64, sequence=coring.Seqner(qb64b=raw, strip=True).qb64, said=coring.Saider(qb64b=raw, strip=True).qb64, - signature=coring.Siger(qb64b=raw, strip=True).qb64, + signature=indexing.Siger(qb64b=raw, strip=True).qb64, )) receipts["transferable"] = trans diff --git a/src/keri/core/indexing.py b/src/keri/core/indexing.py new file mode 100644 index 000000000..6cb192a35 --- /dev/null +++ b/src/keri/core/indexing.py @@ -0,0 +1,785 @@ +# -*- coding: utf-8 -*- +""" +keri.core.indexing module + +Provides versioning support for Indexer classes and codes +""" +from collections import namedtuple, deque +from dataclasses import dataclass, astuple, asdict +from base64 import urlsafe_b64encode as encodeB64 +from base64 import urlsafe_b64decode as decodeB64 + +import pysodium + + +from ..kering import (EmptyMaterialError, RawMaterialError, SoftMaterialError, + InvalidCodeError, InvalidSoftError, + InvalidSizeError, + InvalidCodeSizeError, InvalidVarIndexError, + InvalidVarSizeError, InvalidVarRawSizeError, + ConversionError, InvalidValueError, InvalidTypeError, + ValidationError, VersionError, DerivationError, + EmptyListError, + ShortageError, UnexpectedCodeError, DeserializeError, + UnexpectedCountCodeError, UnexpectedOpCodeError) + +from ..help import helping +from ..help.helping import (sceil, intToB64, b64ToInt, + codeB64ToB2, codeB2ToB64, nabSextets) + + +@dataclass(frozen=True) +class IndexerCodex: + """ IndexerCodex is codex hard (stable) part of all indexer derivation codes. + + Codes indicate which list of keys, current and/or prior next, index is for: + + _Sig: Indices in code may appear in both current signing and + prior next key lists when event has both current and prior + next key lists. Two character code table has only one index + so must be the same for both lists. Other index if for + prior next. + The indices may be different in those code tables which + have two sets of indices. + + _Crt_Sig: Index in code for current signing key list only. + + _Big_: Big index values + + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + + Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. + Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. + ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list. + ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. + ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. + Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. + Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. + Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both lists. + Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. + ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. + ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. + ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. + Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. + Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. + TBD0: str = '0z' # Test of Var len label L=N*4 <= 4095 char quadlets includes code + TBD1: str = '1z' # Test of index sig lead 1 + TBD4: str = '4z' # Test of index sig lead 1 big + + def __iter__(self): + return iter(astuple(self)) # enables inclusion test with "in" + +IdrDex = IndexerCodex() + + +@dataclass(frozen=True) +class IndexedSigCodex: + """IndexedSigCodex is codex all indexed signature derivation codes. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. + Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. + ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list. + ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. + ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. + Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. + Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. + Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both lists. + Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. + ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. + ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. + ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. + Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. + Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. + + def __iter__(self): + return iter(astuple(self)) + +IdxSigDex = IndexedSigCodex() # Make instance + + +@dataclass(frozen=True) +class IndexedCurrentSigCodex: + """IndexedCurrentSigCodex is codex indexed signature codes for current list. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list only. + ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. + Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. + Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. + ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. + Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. + + def __iter__(self): + return iter(astuple(self)) + +IdxCrtSigDex = IndexedCurrentSigCodex() # Make instance + + + +@dataclass(frozen=True) +class IndexedBothSigCodex: + """IndexedBothSigCodex is codex indexed signature codes for both lists. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. + ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. + ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. + Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. + Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both listsy. + ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. + ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. + Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. + + def __iter__(self): + return iter(astuple(self)) + +IdxBthSigDex = IndexedBothSigCodex() # Make instance + +# namedtuple for size entries in Incexer derivation code tables +# hs is the hard size int number of chars in hard (stable) part of code +# ss is the soft size int number of chars in soft (unstable) part of code +# os is the other size int number of chars in other index part of soft +# ms = ss - os main index size computed +# fs is the full size int number of chars in code plus appended material if any +# ls is the lead size int number of bytes to pre-pad pre-converted raw binary +Xizage = namedtuple("Xizage", "hs ss os fs ls") + +class Indexer: + """ Indexer is fully qualified cryptographic material primitive base class for + indexed primitives. In special cases some codes in the Index code table + may be of variable length (i.e. not indexed) when the full size table entry + is None. In that case the index is used instread as the length. + + Sub classes are derivation code and key event element context specific. + + Includes the following attributes and properties: + + Attributes: + + Properties: + code is str of stable (hard) part of derivation code + raw (bytes): unqualified crypto material usable for crypto operations + index (int): main index offset into list or length of material + ondex (int | None): other index offset into list or length of material + qb64b (bytes): fully qualified Base64 crypto material + qb64 (str | bytes): fully qualified Base64 crypto material + qb2 (bytes): fully qualified binary crypto material + + Hidden: + ._code (str): value for .code property + ._raw (bytes): value for .raw property + ._index (int): value for .index property + ._ondex (int): value for .ondex property + ._infil is method to compute fully qualified Base64 from .raw and .code + ._binfil is method to compute fully qualified Base2 from .raw and .code + ._exfil is method to extract .code and .raw from fully qualified Base64 + ._bexfil is method to extract .code and .raw from fully qualified Base2 + + """ + Codex = IdrDex + # Hards table maps from bytes Base64 first code char to int of hard size, hs, + # (stable) of code. The soft size, ss, (unstable) is always > 0 for Indexer. + Hards = ({chr(c): 1 for c in range(65, 65 + 26)}) + Hards.update({chr(c): 1 for c in range(97, 97 + 26)}) + Hards.update([('0', 2), ('1', 2), ('2', 2), ('3', 2), ('4', 2)]) + # Sizes table maps hs chars of code to Xizage namedtuple of (hs, ss, os, fs, ls) + # where hs is hard size, ss is soft size, os is other index size, + # and fs is full size, ls is lead size. + # where ss includes os, so main index size ms = ss - os + # soft size, ss, should always be > 0 for Indexer + Sizes = { + 'A': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'B': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'C': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'D': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'E': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'F': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + '0A': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), + '0B': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), + '2A': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2B': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2C': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2D': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2E': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2F': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '3A': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), + '3B': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), + '0z': Xizage(hs=2, ss=2, os=0, fs=None, ls=0), + '1z': Xizage(hs=2, ss=2, os=1, fs=76, ls=1), + '4z': Xizage(hs=2, ss=6, os=3, fs=80, ls=1), + } + # Bards table maps to hard size, hs, of code from bytes holding sextets + # converted from first code char. Used for ._bexfil. + Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) + + def __init__(self, raw=None, code=IdrDex.Ed25519_Sig, index=0, ondex=None, + qb64b=None, qb64=None, qb2=None, strip=False): + """ + Validate as fully qualified + Parameters: + raw (bytes): unqualified crypto material usable for crypto operations + code is str of stable (hard) part of derivation code + index (int): main index offset into list or length of material + ondex (int | None): other index offset into list or length of material + qb64b (bytes): fully qualified Base64 crypto material + qb64 (str | bytes): fully qualified Base64 crypto material + qb2 (bytes): fully qualified binary crypto material + strip (bool): True means strip counter contents from input stream + bytearray after parsing qb64b or qb2. False means do not strip + + Needs either (raw and code and index) or qb64b or qb64 or qb2 + Otherwise raises EmptyMaterialError + When raw and code provided then validate that code is correct + for length of raw and assign .raw + Else when qb64b or qb64 or qb2 provided extract and assign + .raw, .code, .index, .ondex. + + """ + if raw is not None: # raw provided + if not code: + raise EmptyMaterialError("Improper initialization need either " + "(raw and code) or qb64b or qb64 or qb2.") + if not isinstance(raw, (bytes, bytearray)): + raise TypeError(f"Not a bytes or bytearray, raw={raw}.") + + if code not in self.Sizes: + raise UnexpectedCodeError(f"Unsupported code={code}.") + + hs, ss, os, fs, ls = self.Sizes[code] # get sizes for code + cs = hs + ss # both hard + soft code size + ms = ss - os + + if not isinstance(index, int) or index < 0 or index > (64 ** ms - 1): + raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") + + if isinstance(ondex, int) and os and not (ondex >= 0 and ondex <= (64 ** os - 1)): + raise InvalidVarIndexError(f"Invalid ondex={ondex} for code={code}.") + + if code in IdxCrtSigDex and ondex is not None: + raise InvalidVarIndexError(f"Non None ondex={ondex} for code={code}.") + + if code in IdxBthSigDex: + if ondex is None: # set default + ondex = index # when not provided make ondex match index + else: + if ondex != index and os == 0: # must match if os == 0 + raise InvalidVarIndexError(f"Non matching ondex={ondex}" + f" and index={index} for " + f"code={code}.") + + + if not fs: # compute fs from index + if cs % 4: + raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " + f"variable length material. cs={cs}.") + if os != 0: + raise InvalidCodeSizeError(f"Non-zero other index size for " + f"variable length material. os={os}.") + fs = (index * 4) + cs + + rawsize = (fs - cs) * 3 // 4 + + raw = raw[:rawsize] # copy rawsize from stream, may be less + if len(raw) != rawsize: # forbids shorter + raise RawMaterialError(f"Not enougth raw bytes for code={code}" + f"and index={index} ,expected {rawsize} " + f"got {len(raw)}.") + + self._code = code + self._index = index + self._ondex = ondex + self._raw = bytes(raw) # crypto ops require bytes not bytearray + + elif qb64b is not None: + self._exfil(qb64b) + if strip: # assumes bytearray + del qb64b[:len(self.qb64b)] # may be variable length fs + + elif qb64 is not None: + self._exfil(qb64) + + elif qb2 is not None: + self._bexfil(qb2) + if strip: # assumes bytearray + del qb2[:len(self.qb2)] # may be variable length fs + + else: + raise EmptyMaterialError("Improper initialization need either " + "(raw and code and index) or qb64b or " + "qb64 or qb2.") + + @classmethod + def _rawSize(cls, code): + """ + Returns expected raw size in bytes for a given code. Not applicable to + codes with fs = None + """ + hs, ss, os, fs, ls = cls.Sizes[code] # get sizes + return ((fs - (hs + ss)) * 3 // 4) + + @property + def code(self): + """ + Returns ._code + Makes .code read only + """ + return self._code + + @property + def raw(self): + """ + Returns ._raw + Makes .raw read only + """ + return self._raw + + @property + def index(self): + """ + Returns ._index + Makes .index read only + """ + return self._index + + @property + def ondex(self): + """ + Returns ._ondex + Makes .ondex read only + """ + return self._ondex + + @property + def qb64b(self): + """ + Property qb64b: + Returns Fully Qualified Base64 Version encoded as bytes + Assumes self.raw and self.code are correctly populated + """ + return self._infil() + + @property + def qb64(self): + """ + Property qb64: + Returns Fully Qualified Base64 Version + Assumes self.raw and self.code are correctly populated + """ + return self.qb64b.decode("utf-8") + + @property + def qb2(self): + """ + Property qb2: + Returns Fully Qualified Binary Version Bytes + """ + return self._binfil() + + def _infil(self): + """ + Returns fully qualified attached sig base64 bytes computed from + self.raw, self.code and self.index. + + cs = hs + ss + os = ss - ms (main index size) + when fs None then size computed & fs = size * 4 + cs + + """ + code = self.code # codex value chars hard code + index = self.index # main index value + ondex = self.ondex # other index value + raw = self.raw # bytes or bytearray + + ps = (3 - (len(raw) % 3)) % 3 # if lead then same pad size chars & lead size bytes + hs, ss, os, fs, ls = self.Sizes[code] + cs = hs + ss + ms = ss - os + + if not fs: # compute fs from index + if cs % 4: + raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " + f"variable length material. cs={cs}.") + if os != 0: + raise InvalidCodeSizeError(f"Non-zero other index size for " + f"variable length material. os={os}.") + fs = (index * 4) + cs + + if index < 0 or index > (64 ** ms - 1): + raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") + + if (isinstance(ondex, int) and os and + not (ondex >= 0 and ondex <= (64 ** os - 1))): + raise InvalidVarIndexError(f"Invalid ondex={ondex} for os={os} and " + f"code={code}.") + + # both is hard code + converted index + converted ondex + both = (f"{code}{intToB64(index, l=ms)}" + f"{intToB64(ondex if ondex is not None else 0, l=os)}") + + # check valid pad size for whole code size, assumes ls is zero + if len(both) != cs: + raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." + .format(cs, len(both))) + + if (cs % 4) != ps - ls: # adjusted pad given lead bytes + raise InvalidCodeSizeError(f"Invalid code={both} for converted" + f" raw pad size={ps}.") + + # prepend pad bytes, convert, then replace pad chars with full derivation + # code including index, + full = both.encode("utf-8") + encodeB64(bytes([0] * ps) + raw)[ps - ls:] + + if len(full) != fs: # invalid size + raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") + + return full + + + def _binfil(self): + """ + Returns bytes of fully qualified base2 bytes, that is .qb2 + self.code and self.index converted to Base2 + self.raw left shifted + with pad bits equivalent of Base64 decode of .qb64 into .qb2 + """ + code = self.code # codex chars hard code + index = self.index # main index value + ondex = self.ondex # other index value + raw = self.raw # bytes or bytearray + + ps = (3 - (len(raw) % 3)) % 3 # same pad size chars & lead size bytes + hs, ss, os, fs, ls = self.Sizes[code] + cs = hs + ss + ms = ss - os + + if index < 0 or index > (64 ** ss - 1): + raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") + + if (isinstance(ondex, int) and os and + not (ondex >= 0 and ondex <= (64 ** os - 1))): + raise InvalidVarIndexError(f"Invalid ondex={ondex} for os={os} and " + f"code={code}.") + + if not fs: # compute fs from index + if cs % 4: + raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " + f"variable length material. cs={cs}.") + if os != 0: + raise InvalidCodeSizeError(f"Non-zero other index size for " + f"variable length material. os={os}.") + fs = (index * 4) + cs + + # both is hard code + converted index + both = (f"{code}{intToB64(index, l=ms)}" + f"{intToB64(ondex if ondex is not None else 0, l=os)}") + + if len(both) != cs: + raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." + .format(cs, len(both))) + + if (cs % 4) != ps - ls: # adjusted pad given lead bytes + raise InvalidCodeSizeError(f"Invalid code={both} for converted" + f" raw pad size={ps}.") + + n = sceil(cs * 3 / 4) # number of b2 bytes to hold b64 code + index + # convert code both to right align b2 int then left shift in pad bits + # then convert to bytes + bcode = (b64ToInt(both) << (2 * (ps - ls))).to_bytes(n, 'big') + full = bcode + bytes([0] * ls) + raw + + bfs = len(full) # binary full size + if bfs % 3 or (bfs * 4 // 3) != fs: # invalid size + raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") + + return full + + + def _exfil(self, qb64b): + """ + Extracts self.code, self.index, and self.raw from qualified base64 bytes qb64b + + cs = hs + ss + ms = ss - os (main index size) + when fs None then size computed & fs = size * 4 + cs + """ + if not qb64b: # empty need more bytes + raise ShortageError("Empty material.") + + first = qb64b[:1] # extract first char code selector + if hasattr(first, "decode"): + first = first.decode("utf-8") + if first not in self.Hards: + if first[0] == '-': + raise UnexpectedCountCodeError("Unexpected count code start" + "while extracing Indexer.") + elif first[0] == '_': + raise UnexpectedOpCodeError("Unexpected op code start" + "while extracing Indexer.") + else: + raise UnexpectedCodeError(f"Unsupported code start char={first}.") + + hs = self.Hards[first] # get hard code size + if len(qb64b) < hs: # need more bytes + raise ShortageError(f"Need {hs - len(qb64b)} more characters.") + + hard = qb64b[:hs] # get hard code + if hasattr(hard, "decode"): + hard = hard.decode("utf-8") + if hard not in self.Sizes: + raise UnexpectedCodeError(f"Unsupported code ={hard}.") + + hs, ss, os, fs, ls = self.Sizes[hard] # assumes hs in both tables consistent + cs = hs + ss # both hard + soft code size + ms = ss - os + # assumes that unit tests on Indexer and IndexerCodex ensure that + # .Codes and .Sizes are well formed. + # hs consistent and hs > 0 and ss > 0 and (fs >= hs + ss if fs is not None else True) + # assumes no variable length indexed codes so fs is not None + + if len(qb64b) < cs: # need more bytes + raise ShortageError(f"Need {cs - len(qb64b)} more characters.") + + index = qb64b[hs:hs+ms] # extract index/size chars + if hasattr(index, "decode"): + index = index.decode("utf-8") + index = b64ToInt(index) # compute int index + + ondex = qb64b[hs+ms:hs+ms+os] # extract ondex chars + if hasattr(ondex, "decode"): + ondex = ondex.decode("utf-8") + + if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 + ondex = b64ToInt(ondex) if os else None # compute ondex from code + if ondex: # not zero or None so error + raise ValueError(f"Invalid ondex={ondex} for code={hard}.") + else: + ondex = None # zero so set to None when current only + else: + ondex = b64ToInt(ondex) if os else index + + # index is index for some codes and variable length for others + if not fs: # compute fs from index which means variable length + if cs % 4: + raise ValidationError(f"Whole code size not multiple of 4 for " + f"variable length material. cs={cs}.") + if os != 0: + raise ValidationError(f"Non-zero other index size for " + f"variable length material. os={os}.") + fs = (index * 4) + cs + + if len(qb64b) < fs: # need more bytes + raise ShortageError(f"Need {fs - len(qb64b)} more chars.") + + qb64b = qb64b[:fs] # fully qualified primitive code plus material + if hasattr(qb64b, "encode"): # only convert extracted chars from stream + qb64b = qb64b.encode("utf-8") + + # strip off prepended code and append pad characters + #ps = cs % 4 # pad size ps = cs mod 4, same pad chars and lead bytes + #base = ps * b'A' + qb64b[cs:] # replace prepend code with prepad zeros + #raw = decodeB64(base)[ps+ls:] # decode and strip off ps+ls prepad bytes + + # check for non-zeroed pad bits or lead bytes + ps = cs % 4 # code pad size ps = cs mod 4 + pbs = 2 * (ps if ps else ls) # pad bit size in bits + if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls + base = ps * b'A' + qb64b[cs:] # replace pre code with prepad chars of zero + paw = decodeB64(base) # decode base to leave prepadded raw + pi = (int.from_bytes(paw[:ps], "big")) # prepad as int + if pi & (2 ** pbs - 1 ): # masked pad bits non-zero + raise ValueError(f"Non zeroed prepad bits = " + f"{pi & (2 ** pbs - 1 ):<06b} in {qb64b[cs:cs+1]}.") + raw = paw[ps:] # strip off ps prepad paw bytes + else: # not ps. IF not ps THEN may or may not be ls (lead) + base = qb64b[cs:] # strip off code leaving lead chars if any and value + # decode lead chars + val leaving lead bytes + raw bytes + # then strip off ls lead bytes leaving raw + paw = decodeB64(base) # decode base to leave prepadded paw bytes + li = int.from_bytes(paw[:ls], "big") # lead as int + if li: # pre pad lead bytes must be zero + if ls == 1: + raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") + else: + raise ValueError(f"Non zeroed lead bytes = 0x{li:04x}.") + + raw = paw[ls:] + + if len(raw) != (len(qb64b) - cs) * 3 // 4: # exact lengths + raise ConversionError(f"Improperly qualified material = {qb64b}") + + self._code = hard + self._index = index + self._ondex = ondex + self._raw = raw # must be bytes for crpto opts and immutable not bytearray + + + + def _bexfil(self, qb2): + """ + Extracts self.code, self.index, and self.raw from qualified base2 bytes qb2 + + cs = hs + ss + ms = ss - os (main index size) + when fs None then size computed & fs = size * 4 + cs + """ + if not qb2: # empty need more bytes + raise ShortageError("Empty material, Need more bytes.") + + first = nabSextets(qb2, 1) # extract first sextet as code selector + if first not in self.Bards: + if first[0] == b'\xf8': # b64ToB2('-') + raise UnexpectedCountCodeError("Unexpected count code start" + "while extracing Matter.") + elif first[0] == b'\xfc': # b64ToB2('_') + raise UnexpectedOpCodeError("Unexpected op code start" + "while extracing Matter.") + else: + raise UnexpectedCodeError(f"Unsupported code start sextet={first}.") + + hs = self.Bards[first] # get code hard size equvalent sextets + bhs = sceil(hs * 3 / 4) # bhs is min bytes to hold hs sextets + if len(qb2) < bhs: # need more bytes + raise ShortageError(f"Need {bhs - len(qb2)} more bytes.") + + hard = codeB2ToB64(qb2, hs) # extract and convert hard part of code + if hard not in self.Sizes: + raise UnexpectedCodeError(f"Unsupported code ={hard}.") + + hs, ss, os, fs, ls = self.Sizes[hard] + cs = hs + ss # both hs and ss + ms = ss - os + # assumes that unit tests on Indexer and IndexerCodex ensure that + # .Codes and .Sizes are well formed. + # hs consistent and hs > 0 and ss > 0 and (fs >= hs + ss if fs is not None else True) + + bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets + if len(qb2) < bcs: # need more bytes + raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) + + both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code + index = b64ToInt(both[hs:hs+ms]) # compute index + + if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 + ondex = b64ToInt(both[hs+ms:hs+ms+os]) if os else None # compute ondex from code + if ondex: # not zero or None so error + raise ValueError(f"Invalid ondex={ondex} for code={hard}.") + else: + ondex = None # zero so set to None when current only + else: + ondex = b64ToInt(both[hs+ms:hs+ms+os]) if os else index + + if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 + if ondex: # not zero so error + raise ValueError(f"Invalid ondex={ondex} for code={hard}.") + else: # zero so set to None + ondex = None + + if not fs: # compute fs from size chars in ss part of code + if cs % 4: + raise ValidationError(f"Whole code size not multiple of 4 for " + f"variable length material. cs={cs}.") + if os != 0: + raise ValidationError(f"Non-zero other index size for " + f"variable length material. os={os}.") + fs = (index * 4) + cs + + bfs = sceil(fs * 3 / 4) # bfs is min bytes to hold fs sextets + if len(qb2) < bfs: # need more bytes + raise ShortageError("Need {} more bytes.".format(bfs - len(qb2))) + + qb2 = qb2[:bfs] # extract qb2 fully qualified primitive code plus material + + # check for non-zeroed prepad bits or lead bytes + ps = cs % 4 # code pad size ps = cs mod 4 + pbs = 2 * (ps if ps else ls) # pad bit size in bits + if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls + # convert last byte of code bytes in which are pad bits to int + pi = (int.from_bytes(qb2[bcs-1:bcs], "big")) + if pi & (2 ** pbs - 1 ): # masked pad bits non-zero + raise ValueError(f"Non zeroed pad bits = " + f"{pi & (2 ** pbs - 1 ):>08b} in 0x{pi:02x}.") + else: # not ps. IF not ps THEN may or may not be ls (lead) + li = int.from_bytes(qb2[bcs:bcs+ls], "big") # lead as int + if li: # pre pad lead bytes must be zero + if ls == 1: + raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") + else: + raise ValueError(f"Non zeroed lead bytes = 0x{li:02x}.") + + + raw = qb2[(bcs + ls):] # strip code and leader bytes from qb2 to get raw + + if len(raw) != (len(qb2) - bcs - ls): # exact lengths + raise ConversionError(f"Improperly qualified material = {qb2}") + + self._code = hard + self._index = index + self._ondex = ondex + self._raw = bytes(raw) # must be bytes for crypto ops and not bytearray mutable + + +class Siger(Indexer): + """ + Siger is subclass of Indexer, indexed signature material, + + Adds .verfer property which is instance of Verfer that provides + associated signature verifier. + + See Indexer for inherited attributes and properties: + + Attributes: + + Properties: + verfer (Verfer): instance if any provides public verification key + + Methods: + + Hidden: + _verfer (Verfer): value for .verfer property + + + """ + + def __init__(self, verfer=None, **kwa): + """Initialze instance + + Parameters: See Matter for inherted parameters + verfer (Verfer): instance if any provides public verification key + + """ + super(Siger, self).__init__(**kwa) + if self.code not in IdxSigDex: + raise ValidationError("Invalid code = {} for Siger." + "".format(self.code)) + self.verfer = verfer + + @property + def verfer(self): + """ + Property verfer: + Returns Verfer instance + Assumes ._verfer is correctly assigned + """ + return self._verfer + + @verfer.setter + def verfer(self, verfer): + """ verfer property setter """ + self._verfer = verfer + + diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index aae8f64a4..88d572e4d 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -10,8 +10,9 @@ from collections import namedtuple from dataclasses import dataclass, astuple -from .coring import (Ilks, CtrDex, Counter, Seqner, Siger, Cigar, +from .coring import (Ilks, CtrDex, Counter, Seqner, Cigar, Dater, Verfer, Prefixer, Saider, Pather, Protocols ) +from .indexing import (Siger, ) from . import serdering from .. import help from .. import kering diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index cb237303e..8e029973f 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1264,8 +1264,7 @@ def _dumps(self, sad): frame.extend(e.encode("utf-8")) val = bytearray(Counter(tag=AllTags.GenericListGroup, - count=len(frame) % 4, - version=self.gvrsn).qb64b) + count=len(frame) % 4).qb64b) val.extend(frame) case "c": # list of config traits strings diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 7b324194c..7a3864b7c 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -39,7 +39,7 @@ from . import dbing, koming, subing from .. import kering -from ..core import coring, eventing, parsing, serdering +from ..core import coring, eventing, parsing, serdering, indexing from .. import help from ..help import helping @@ -976,7 +976,7 @@ def reopen(self, **kwa): # given by quadruple (saider.qb64, prefixer.qb64, seqner.q64, diger.qb64) # of reply and trans signer's key state est evt to val Siger for each # signature. - self.ssgs = subing.CesrIoSetSuber(db=self, subkey='ssgs.', klas=coring.Siger) + self.ssgs = subing.CesrIoSetSuber(db=self, subkey='ssgs.', klas=indexing.Siger) # all sad scgs (sad non-indexed signature serializations) maps SAD SAID # to couple (Verfer, Cigar) of nontrans signer of signature in Cigar @@ -1052,7 +1052,7 @@ def reopen(self, **kwa): # exchange message signatures # TODO: clean - self.esigs = subing.CesrIoSetSuber(db=self, subkey='esigs.', klas=coring.Siger) + self.esigs = subing.CesrIoSetSuber(db=self, subkey='esigs.', klas=indexing.Siger) # exchange message signatures # TODO: clean diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 03ea429ef..71fe06248 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -7,13 +7,15 @@ import logging from typing import Type -from keri.core import coring -from keri import help + from keri import kering -from keri.core import eventing -from keri.db import subing +from keri import help from keri.help import helping +from keri.core import coring, eventing, indexing +from keri.db import subing + + logger = help.ogler.getLogger() @@ -42,7 +44,7 @@ def __init__(self, db, subkey, timeout=3600): # given by quadruple (saider.qb64, subkeyer.qb64, seqner.q64, diger.qb64) # of reply and trans signer's key state est evt to val Siger for each # signature. - self.tigerdb = subing.CesrIoSetSuber(db=self.db, subkey=subkey + '-sgs.', klas=coring.Siger) + self.tigerdb = subing.CesrIoSetSuber(db=self.db, subkey=subkey + '-sgs.', klas=indexing.Siger) # all key state kcgs (ksn non-indexed signature serializations) maps ksn SAID # to couple (Verfer, Cigar) of nontrans signer of signature in Cigar diff --git a/src/keri/end/ending.py b/src/keri/end/ending.py index d775930fb..624a615d7 100644 --- a/src/keri/end/ending.py +++ b/src/keri/end/ending.py @@ -22,7 +22,7 @@ from .. import help from .. import kering from ..app import habbing -from ..core import coring +from ..core import coring, indexing from ..help import helping logger = help.ogler.getLogger() @@ -82,12 +82,12 @@ def signature(signages): where: markers (Union[list, dict]): When dict each item (key, val) has key as str identifier of marker and has val as instance of - either coring.Siger or coring.Cigar. - When list each item is instance of either coring.Siger or + either indexing.Siger or coring.Cigar. + When list each item is instance of either indexing.Siger or coring.Cigar. All markers must be of same class indexed (bool): True means marker values are indexed signatures - using coring.Siger. False means marker values are unindexed + using indexing.Siger. False means marker values are unindexed signatures using coring.Cigar. None means auto detect from first marker value class. All markers must be of same class. signer (str): optional identifier of signage. May be a @@ -188,12 +188,12 @@ def designature(value): where: markers (Union[list, dict]): When dict each item (key, val) has key as str identifier of marker and has val as instance of - either coring.Siger or coring.Cigar. - When list each item is instance of either coring.Siger or + either indexing.Siger or coring.Cigar. + When list each item is instance of either indexing.Siger or coring.Cigar. All markers must be of same class indexed (bool): True means marker values are indexed signatures - using coring.Siger. False means marker values are unindexed + using indexing.Siger. False means marker values are unindexed signatures using coring.Cigar. None means auto detect from first marker value class. All markers must be of same class. signer (str): optional identifier of signage. May be a @@ -250,7 +250,7 @@ def designature(value): if kind == "CESR": # convert to Siger or Cigar instances for key, val in items.items(): if indexed: - items[key] = coring.Siger(qb64=val) + items[key] = indexing.Siger(qb64=val) else: items[key] = coring.Cigar(qb64=val) diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index baa6eed41..3bd5ab55f 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -17,7 +17,7 @@ from keri import kering from .. import core from .. import help -from ..core import serdering, coring +from ..core import serdering, coring, indexing from ..core.coring import (MtrDex, Serials, versify, Prefixer, Ilks, Seqner, Verfer, Number) from ..core.eventing import SealEvent, ample, TraitDex, verifySigs @@ -2059,7 +2059,7 @@ def processEscrowOutOfOrders(self): bigers = None if tibs := self.reger.getTibs(key=dgkey): - bigers = [coring.Siger(qb64b=tib) for tib in tibs] + bigers = [indexing.Siger(qb64b=tib) for tib in tibs] couple = self.reger.getAnc(dgkey) if couple is None: @@ -2126,7 +2126,7 @@ def processEscrowAnchorless(self): bigers = None if tibs := self.reger.getTibs(key=dgkey): - bigers = [coring.Siger(qb64b=tib) for tib in tibs] + bigers = [indexing.Siger(qb64b=tib) for tib in tibs] couple = self.reger.getAnc(dgkey) if couple is None: diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 91b11317e..a65c62d57 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -15,7 +15,7 @@ from .. import kering from ..app import signing -from ..core import coring, serdering +from ..core import coring, serdering, indexing from ..db import dbing, basing from ..db.dbing import snKey from ..help import helping @@ -328,7 +328,7 @@ def reopen(self, **kwa): # given by quintuple (saider.qb64, path, prefixer.qb64, seqner.q64, diger.qb64) # of credential and trans signer's key state est evt to val Siger for each # signature. - self.spsgs = subing.CesrIoSetSuber(db=self, subkey='ssgs.', klas=coring.Siger) + self.spsgs = subing.CesrIoSetSuber(db=self, subkey='ssgs.', klas=indexing.Siger) # all sad path scgs (sad pathed non-indexed signature serializations) maps # couple (SAD SAID, path) to couple (Verfer, Cigar) of nontrans signer of signature in Cigar diff --git a/tests/app/test_keeping.py b/tests/app/test_keeping.py index 4f289a0d9..29d7e8044 100644 --- a/tests/app/test_keeping.py +++ b/tests/app/test_keeping.py @@ -18,8 +18,9 @@ from keri import kering from keri.help import helping -from keri.core import coring -from keri.core.coring import IdrDex + +from keri.core import coring, indexing +from keri.core.indexing import IdrDex from keri.app import keeping @@ -774,7 +775,7 @@ def test_manager(): psigers = manager.sign(ser=ser, pubs=ps.new.pubs) for siger in psigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) vsigers = manager.sign(ser=ser, verfers=verfers) psigs = [siger.qb64 for siger in psigers] vsigs = [siger.qb64 for siger in vsigers] @@ -787,7 +788,7 @@ def test_manager(): # Test with pubs list psigers = manager.sign(ser=ser, pubs=ps.new.pubs, indices=indices) for siger in psigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) assert psigers[0].index == indices[0] psigs = [siger.qb64 for siger in psigers] assert psigs == ['ADAa70b4QnTOtGOsMqcezMtVzCFuRJHGeIMkWYHZ5ZxGIXM0XDVAzkYdCeadfPfzlKC6dkfiwuJ0IzLOElaanUgH'] @@ -795,7 +796,7 @@ def test_manager(): # Test with verfers list vsigers = manager.sign(ser=ser, verfers=verfers, indices=indices) for siger in vsigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) assert psigers[0].index == indices[0] vsigs = [siger.qb64 for siger in vsigers] assert vsigs == psigs @@ -1626,7 +1627,7 @@ def test_manager_with_aeid(): psigers = manager.sign(ser=ser, pubs=ps.new.pubs) for siger in psigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) vsigers = manager.sign(ser=ser, verfers=verfers) psigs = [siger.qb64 for siger in psigers] vsigs = [siger.qb64 for siger in vsigers] @@ -1639,7 +1640,7 @@ def test_manager_with_aeid(): # Test with pubs list psigers = manager.sign(ser=ser, pubs=ps.new.pubs, indices=indices) for siger in psigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) assert psigers[0].index == indices[0] psigs = [siger.qb64 for siger in psigers] assert psigs == ['ADAa70b4QnTOtGOsMqcezMtVzCFuRJHGeIMkWYHZ5ZxGIXM0XDVAzkYdCeadfPfzlKC6dkfiwuJ0IzLOElaanUgH'] @@ -1647,7 +1648,7 @@ def test_manager_with_aeid(): # Test with verfers list vsigers = manager.sign(ser=ser, verfers=verfers, indices=indices) for siger in vsigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) assert psigers[0].index == indices[0] vsigs = [siger.qb64 for siger in vsigers] assert vsigs == psigs diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index ae13bca9c..7b9b67f4f 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -35,17 +35,17 @@ from keri.core import coring from keri.core import eventing from keri.core.coring import (Saids, Sadder, Tholder, Seqner, NumDex, Number, - Siger, Dater, Bexter, Texter, + Dater, Bexter, Texter, TagDex, PadTagDex, Tagger, Ilker, Traitor, Verser, Versage, ) from keri.core.coring import Serialage, Serials, Tiers -from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, - IdxCrtSigDex, IdxBthSigDex, Indexer, - CtrDex, Counter) +from keri.core.coring import (Sizage, MtrDex, Matter, CtrDex, Counter) from keri.core.coring import (Verfer, Cigar, Signer, Salter, Saider, DigDex, Diger, Prefixer, Cipher, Encrypter, Decrypter) from keri.core.coring import versify, deversify, Rever, MAXVERFULLSPAN from keri.core.coring import generateSigners +from keri.core.indexing import (Siger, Xizage, IdrDex, IdxSigDex, + IdxCrtSigDex, IdxBthSigDex, Indexer) from keri.help import helping from keri.help.helping import (sceil, intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, @@ -1918,693 +1918,6 @@ def test_matter_special(): """ Done Test """ -def test_indexer(): - """ - Test Indexer class - """ - assert Indexer.Codex == IdrDex - - assert asdict(IdrDex) == { - 'Ed25519_Sig': 'A', - 'Ed25519_Crt_Sig': 'B', - 'ECDSA_256k1_Sig': 'C', - 'ECDSA_256k1_Crt_Sig': 'D', - 'ECDSA_256r1_Sig': 'E', - 'ECDSA_256r1_Crt_Sig': 'F', - 'Ed448_Sig': '0A', - 'Ed448_Crt_Sig': '0B', - 'Ed25519_Big_Sig': '2A', - 'Ed25519_Big_Crt_Sig': '2B', - 'ECDSA_256k1_Big_Sig': '2C', - 'ECDSA_256k1_Big_Crt_Sig': '2D', - 'ECDSA_256r1_Big_Sig': '2E', - 'ECDSA_256r1_Big_Crt_Sig': '2F', - 'Ed448_Big_Sig': '3A', - 'Ed448_Big_Crt_Sig': '3B', - 'TBD0': '0z', - 'TBD1': '1z', - 'TBD4': '4z', - } - - assert IdrDex.Ed25519_Sig == 'A' - assert IdrDex.Ed25519_Crt_Sig == 'B' - assert IdrDex.ECDSA_256k1_Sig == 'C' - assert IdrDex.ECDSA_256k1_Crt_Sig == 'D' - assert IdrDex.ECDSA_256r1_Sig == 'E' - assert IdrDex.ECDSA_256r1_Crt_Sig == 'F' - assert IdrDex.Ed448_Sig == '0A' - assert IdrDex.Ed448_Crt_Sig == '0B' - assert IdrDex.Ed25519_Big_Sig == '2A' - assert IdrDex.Ed25519_Big_Crt_Sig == '2B' - assert IdrDex.ECDSA_256k1_Big_Sig == '2C' - assert IdrDex.ECDSA_256k1_Big_Crt_Sig == '2D' - assert IdrDex.ECDSA_256r1_Big_Sig == '2E' - assert IdrDex.ECDSA_256r1_Big_Crt_Sig == '2F' - assert IdrDex.Ed448_Big_Sig == '3A' - assert IdrDex.Ed448_Big_Crt_Sig == '3B' - assert IdrDex.TBD0 == '0z' - assert IdrDex.TBD1 == '1z' - assert IdrDex.TBD4 == '4z' - - assert asdict(IdxSigDex) == { - 'Ed25519_Sig': 'A', - 'Ed25519_Crt_Sig': 'B', - 'ECDSA_256k1_Sig': 'C', - 'ECDSA_256k1_Crt_Sig': 'D', - 'ECDSA_256r1_Sig': 'E', - 'ECDSA_256r1_Crt_Sig': 'F', - 'Ed448_Sig': '0A', - 'Ed448_Crt_Sig': '0B', - 'Ed25519_Big_Sig': '2A', - 'Ed25519_Big_Crt_Sig': '2B', - 'ECDSA_256k1_Big_Sig': '2C', - 'ECDSA_256k1_Big_Crt_Sig': '2D', - 'ECDSA_256r1_Big_Sig': '2E', - 'ECDSA_256r1_Big_Crt_Sig': '2F', - 'Ed448_Big_Sig': '3A', - 'Ed448_Big_Crt_Sig': '3B', - } - - assert IdxSigDex.Ed25519_Sig == 'A' - assert IdxSigDex.Ed25519_Crt_Sig == 'B' - assert IdxSigDex.ECDSA_256k1_Sig == 'C' - assert IdxSigDex.ECDSA_256k1_Crt_Sig == 'D' - assert IdxSigDex.ECDSA_256r1_Sig == 'E' - assert IdxSigDex.ECDSA_256r1_Crt_Sig == 'F' - assert IdxSigDex.Ed448_Sig == '0A' - assert IdxSigDex.Ed448_Crt_Sig == '0B' - assert IdxSigDex.Ed25519_Big_Sig == '2A' - assert IdxSigDex.Ed25519_Big_Crt_Sig == '2B' - assert IdxSigDex.ECDSA_256k1_Big_Sig == '2C' - assert IdxSigDex.ECDSA_256k1_Big_Crt_Sig == '2D' - assert IdxSigDex.ECDSA_256r1_Big_Sig == '2E' - assert IdxSigDex.ECDSA_256r1_Big_Crt_Sig == '2F' - assert IdxSigDex.Ed448_Big_Sig == '3A' - assert IdxSigDex.Ed448_Big_Crt_Sig == '3B' - - - assert asdict(IdxCrtSigDex) == { - 'Ed25519_Crt_Sig': 'B', - 'ECDSA_256k1_Crt_Sig': 'D', - 'ECDSA_256r1_Crt_Sig': 'F', - 'Ed448_Crt_Sig': '0B', - 'Ed25519_Big_Crt_Sig': '2B', - 'ECDSA_256k1_Big_Crt_Sig': '2D', - 'ECDSA_256r1_Big_Crt_Sig': '2F', - 'Ed448_Big_Crt_Sig': '3B', - } - - assert IdxCrtSigDex.Ed25519_Crt_Sig == 'B' - assert IdxCrtSigDex.ECDSA_256k1_Crt_Sig == 'D' - assert IdxCrtSigDex.ECDSA_256r1_Crt_Sig == 'F' - assert IdxCrtSigDex.Ed448_Crt_Sig == '0B' - assert IdxCrtSigDex.Ed25519_Big_Crt_Sig == '2B' - assert IdxCrtSigDex.ECDSA_256k1_Big_Crt_Sig == '2D' - assert IdxCrtSigDex.ECDSA_256r1_Big_Crt_Sig == '2F' - assert IdxCrtSigDex.Ed448_Big_Crt_Sig == '3B' - - - assert asdict(IdxBthSigDex) == { - 'Ed25519_Sig': 'A', - 'ECDSA_256k1_Sig': 'C', - 'ECDSA_256r1_Sig': 'E', - 'Ed448_Sig': '0A', - 'Ed25519_Big_Sig': '2A', - 'ECDSA_256k1_Big_Sig': '2C', - 'ECDSA_256r1_Big_Sig': '2E', - 'Ed448_Big_Sig': '3A', - } - - assert IdxBthSigDex.Ed25519_Sig == 'A' - assert IdxBthSigDex.ECDSA_256k1_Sig == 'C' - assert IdxBthSigDex.ECDSA_256r1_Sig == 'E' - assert IdxBthSigDex.Ed448_Sig == '0A' - assert IdxBthSigDex.Ed25519_Big_Sig == '2A' - assert IdxBthSigDex.ECDSA_256k1_Big_Sig == '2C' - assert IdxBthSigDex.ECDSA_256r1_Big_Sig == '2E' - assert IdxBthSigDex.Ed448_Big_Sig == '3A' - - - # first character of code with hard size of code - assert Indexer.Hards == { - 'A': 1, 'B': 1, 'C': 1, 'D': 1, 'E': 1, 'F': 1, 'G': 1, 'H': 1, 'I': 1, - 'J': 1, 'K': 1, 'L': 1, 'M': 1, 'N': 1, 'O': 1, 'P': 1, 'Q': 1, 'R': 1, - 'S': 1, 'T': 1, 'U': 1, 'V': 1, 'W': 1, 'X': 1, 'Y': 1, 'Z': 1, - 'a': 1, 'b': 1, 'c': 1, 'd': 1, 'e': 1, 'f': 1, 'g': 1, 'h': 1, 'i': 1, - 'j': 1, 'k': 1, 'l': 1, 'm': 1, 'n': 1, 'o': 1, 'p': 1, 'q': 1, 'r': 1, - 's': 1, 't': 1, 'u': 1, 'v': 1, 'w': 1, 'x': 1, 'y': 1, 'z': 1, - '0': 2, '1': 2, '2': 2, '3': 2, '4': 2, - } - - # Codes table with sizes of code (hard) and full primitive material - assert Indexer.Sizes == { - 'A': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'B': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'C': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'D': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'E': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'F': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - '0A': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), - '0B': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), - '2A': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2B': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2C': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2D': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2E': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2F': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '3A': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), - '3B': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), - '0z': Xizage(hs=2, ss=2, os=0, fs=None, ls=0), - '1z': Xizage(hs=2, ss=2, os=1, fs=76, ls=1), - '4z': Xizage(hs=2, ss=6, os=3, fs=80, ls=1), - } - - assert Indexer.Sizes['A'].hs == 1 # hard size - assert Indexer.Sizes['A'].ss == 1 # soft size - assert Indexer.Sizes['A'].os == 0 # other size - assert Indexer.Sizes['A'].fs == 88 # full size - assert Indexer.Sizes['A'].ls == 0 # lead size - - # verify first hs Sizes matches hs in Codes for same first char - for ckey in Indexer.Sizes.keys(): - assert Indexer.Hards[ckey[0]] == Indexer.Sizes[ckey].hs - - # verify all Codes have hs > 0 and ss > 0 and fs >= hs + ss if fs is not None - # verify os is part of ss - for val in Indexer.Sizes.values(): - assert val.hs > 0 and val.ss > 0 - assert val.os >= 0 and val.os < val.ss - if val.os: - assert val.os == val.ss // 2 - if val.fs is not None: - assert val.fs >= val.hs + val.ss - assert val.fs % 4 == 0 - - # Bizes maps bytes of sextet of decoded first character of code with hard size of code - # verify equivalents of items for Sizes and Bizes - for skey, sval in Indexer.Hards.items(): - ckey = codeB64ToB2(skey) - assert Indexer.Bards[ckey] == sval - - with pytest.raises(EmptyMaterialError): - indexer = Indexer() - - # Test signatures - sig = (b"\x99\xd2<9$$0\x9fk\xfb\x18\xa0\x8c@r\x122.k\xb2\xc7\x1fp\x0e'm\x8f@" - b'\xaa\xa5\x8c\xc8n\x85\xc8!\xf6q\x91p\xa9\xec\xcf\x92\xaf)\xde\xca' - b'\xfc\x7f~\xd7o|\x17\x82\x1d\xd4 0 and ss > 0 and fs >= hs + ss if fs is not None + # verify os is part of ss + for val in Indexer.Sizes.values(): + assert val.hs > 0 and val.ss > 0 + assert val.os >= 0 and val.os < val.ss + if val.os: + assert val.os == val.ss // 2 + if val.fs is not None: + assert val.fs >= val.hs + val.ss + assert val.fs % 4 == 0 + + # Bizes maps bytes of sextet of decoded first character of code with hard size of code + # verify equivalents of items for Sizes and Bizes + for skey, sval in Indexer.Hards.items(): + ckey = codeB64ToB2(skey) + assert Indexer.Bards[ckey] == sval + + with pytest.raises(EmptyMaterialError): + indexer = Indexer() + + # Test signatures + sig = (b"\x99\xd2<9$$0\x9fk\xfb\x18\xa0\x8c@r\x122.k\xb2\xc7\x1fp\x0e'm\x8f@" + b'\xaa\xa5\x8c\xc8n\x85\xc8!\xf6q\x91p\xa9\xec\xcf\x92\xaf)\xde\xca' + b'\xfc\x7f~\xd7o|\x17\x82\x1d\xd4 Date: Fri, 5 Apr 2024 14:35:17 -0700 Subject: [PATCH 119/418] Final merge of development branch into main before deleting development. Signed-off-by: pfeairheller --- scripts/demo/test_scripts.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/demo/test_scripts.sh b/scripts/demo/test_scripts.sh index 48a07efda..8d7a3aad0 100755 --- a/scripts/demo/test_scripts.sh +++ b/scripts/demo/test_scripts.sh @@ -65,7 +65,7 @@ printf "\n************************************\n" isSuccess printf "\n************************************\n" -printf "Running multisig-join.sh" +printf "Skipping multisig-join.sh" printf "\n************************************\n" -"${script_dir}/basic/multisig-join.sh" -isSuccess +#"${script_dir}/basic/multisig-join.sh" +#isSuccess From ce618b0968af54a139f7b2047fd53c2c1a8d7067 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Fri, 5 Apr 2024 14:54:10 -0700 Subject: [PATCH 120/418] Update to contact list to account for non-backward compatible changes and to multisig update to account for KeyStateRecord changes. (#742) Remove development branch from CI Signed-off-by: pfeairheller --- .github/workflows/python-app-ci.yml | 1 - src/keri/app/cli/commands/contacts/list.py | 11 +++++++++-- src/keri/app/cli/commands/multisig/update.py | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-app-ci.yml b/.github/workflows/python-app-ci.yml index 75d96fa82..359f8f64f 100644 --- a/.github/workflows/python-app-ci.yml +++ b/.github/workflows/python-app-ci.yml @@ -7,7 +7,6 @@ on: push: branches: - 'main' - - 'development' pull_request: workflow_dispatch: diff --git a/src/keri/app/cli/commands/contacts/list.py b/src/keri/app/cli/commands/contacts/list.py index fa7527d79..1ba574483 100644 --- a/src/keri/app/cli/commands/contacts/list.py +++ b/src/keri/app/cli/commands/contacts/list.py @@ -9,6 +9,7 @@ from hio import help from hio.base import doing +from keri import kering from keri.app import connecting from keri.app.cli.common import existing @@ -53,8 +54,14 @@ def list(tymth, tock=0.0, **opts): challenges = [] for said in valid: - exn = hby.db.exns.get(keys=(said,)) - challenges.append(dict(dt=exn.ked['dt'], words=exn.ked['a']['words'])) + try: + exn = hby.db.exns.get(keys=(said,)) + except kering.ValidationError: + val = hby.db.getVal(db=hby.db.exns.sdb, key=hby.db.exns._tokey((said,))) + d = json.loads(bytes(val).decode("utf-8")) + challenges.append(dict(dt=d['dt'], words=d['a']['words'])) + else: + challenges.append(dict(dt=exn.ked['dt'], words=exn.ked['a']['words'])) c["challenges"] = challenges diff --git a/src/keri/app/cli/commands/multisig/update.py b/src/keri/app/cli/commands/multisig/update.py index 5d7284ea8..4ef7d19fc 100644 --- a/src/keri/app/cli/commands/multisig/update.py +++ b/src/keri/app/cli/commands/multisig/update.py @@ -119,8 +119,8 @@ def updateDo(self, tymth, tock=0.0, **opts): print("") witstate = self.hab.db.ksns.get((saider.qb64,)) - if witstate.sn != self.sn and witstate.ked['d'] != self.said: - print(f"Witness state ({witstate.sn}, {witstate.ked['d']}) does not match requested state.") + if int(witstate.s, 16) != self.sn and witstate.d != self.said: + print(f"Witness state ({witstate.s}, {witstate.d}) does not match requested state.") self.remove(self.toRemove) return From 0f81b0fe5622b9a229a4781695fe8ba77baf679d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 5 Apr 2024 16:29:13 -0600 Subject: [PATCH 121/418] Added structing.py for Structor and subclasses --- src/keri/core/counting.py | 12 +-- src/keri/core/structing.py | 169 +++++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 src/keri/core/structing.py diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index b2f33c2ae..11de4f2fe 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -165,18 +165,18 @@ class CounterCodex_2_0(MapCodex): BigTransReceiptQuadruples: str = '-0M' # Big Trans Receipt Quadruple(s), pre+snu+dig+sig. FirstSeenReplayCouples: str = '-N' # First Seen Replay Couple(s), fnu+dts. BigFirstSeenReplayCouples: str = '-0N' # First Seen Replay Couple(s), fnu+dts. - TransIdxSigGroups: str = '-O' # Trans Indexed Signature Group(s), pre+snu+dig+ControllerIdxSigs of qb64. - TransIdxSigGroups: str = '-0O' # Big Trans Indexed Signature Group(s), pre+snu+dig+ControllerIdxSigs of qb64. - TransLastIdxSigGroups: str = '-P' # Trans Last Est Evt Indexed Signature Group(s), pre+ControllerIdxSigs of qb64. - BigTransLastIdxSigGroups: str = '-0P' # Big Trans Last Est Evt Indexed Signature Group(s), pre+ControllerIdxSigs of qb64. + TransIdxSigGroups: str = '-O' # Trans Indexed Signature Group(s), pre+snu+dig+CtrControllerIdxSigs of qb64. + TransIdxSigGroups: str = '-0O' # Big Trans Indexed Signature Group(s), pre+snu+dig+CtrControllerIdxSigs of qb64. + TransLastIdxSigGroups: str = '-P' # Trans Last Est Evt Indexed Signature Group(s), pre+CtrControllerIdxSigs of qb64. + BigTransLastIdxSigGroups: str = '-0P' # Big Trans Last Est Evt Indexed Signature Group(s), pre+CtrControllerIdxSigs of qb64. SealSourceCouples: str = '-Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. BigSealSourceCouples: str = '-0Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. SealSourceTriples: str = '-R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. BigSealSourceTriples: str = '-0R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. PathedMaterialGroup: str = '-S' # Pathed Material Group. BigPathedMaterialGroup: str = '-0S' # Big Pathed Material Group. - SadPathSigGroups: str = '-T' # SAD Path Group(s) sadpath+TransIdxSigGroup(s) of SAID qb64 of content. - BigSadPathSigGroups: str = '-0T' # Big SAD Path Group(s) sadpath+TransIdxSigGroup(s) of SAID qb64 of content. + SadPathSigGroups: str = '-T' # SAD Path Group(s) sadpath+CtrTransIdxSigGroup(s) of SAID qb64 of content. + BigSadPathSigGroups: str = '-0T' # Big SAD Path Group(s) sadpath+CtrTransIdxSigGroup(s) of SAID qb64 of content. RootSadPathSigGroups: str = '-U' # Root Path SAD Path Group(s), rootpath+SadPathGroup(s). BigRootSadPathSigGroups: str = '-0U' # Big Root Path SAD Path Group(s), rootpath+SadPathGroup(s). DigestSealSingles: str = '-V' # Digest Seal Single(s), dig of sealed data. diff --git a/src/keri/core/structing.py b/src/keri/core/structing.py new file mode 100644 index 000000000..044245d4d --- /dev/null +++ b/src/keri/core/structing.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- +""" +keri.core.indexing module + +Provides versioning support for Indexer classes and codes +""" +from collections import namedtuple + + +# for the following Seal namedtuples use the ._asdict() method to convert to dict +# when using in events +# to convert seal namedtuple to dict use namedtuple._asdict() +# seal == SealEvent(i="abc",s="1",d="efg") +# sealdict =seal._asdict() +# to convet dict to namedtuple use ** unpacking as in seal = SealDigest(**sealdict) +# to check if dict of seal matches fields of associted namedtuple +# if tuple(sealdict) == SealEvent._fields: + +# Digest Seal: uniple (d,) +# d = digest qb64 of data (usually SAID) +SealDigest = namedtuple("SealDigest", 'd') + +# Root Seal: uniple (rd,) +# rd = Merkle tree root digest qb64 digest of anchored (sealed) data in Merkle tree +SealRoot = namedtuple("SealRoot", 'rd') + +# Backer Seal: couple (bi, d) +# bi = pre qb64 backer nontrans identifier prefix +# d = digest qb64 of backer metadata anchored to event usually SAID of data +SealBacker = namedtuple("SealBacker", 'bi d') + +# Event Seal: triple (i, s, d) +# i = pre is qb64 of identifier prefix of KEL for event, +# s = sn of event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of event +SealEvent = namedtuple("SealEvent", 'i s d') + +# Last Estalishment Event Seal: uniple (i,) +# i = pre is qb64 of identifier prefix of KEL from which to get last est, event +# used to indicate to get the latest keys available from KEL for 'i' +SealLast = namedtuple("SealLast", 'i') + +# State Establishment Event (latest current) : quadruple (s, d, br, ba) +# s = sn of latest est event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of latest establishment event +# br = backer (witness) remove list (cuts) from latest est event +# ba = backer (witness) add list (adds) from latest est event +StateEstEvent = namedtuple("StateEstEvent", 's d br ba') + +# Transaction Event Seal for Transaction Event: duple (s, d) +# s = sn of transaction event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of transaction event +# the pre is provided in the 'i' field qb64 of identifier prefix of KEL +# key event that this seal appears. +# use SealSourceCouples count code for attachment +SealTrans = namedtuple("SealTrans", 's d') + + +class Structor: + """Structor class holds a namedtuple .fields whose values are primitive + (Matter subclass) instances. + These instances can be serialized and deserialized as a concatenation of + their qb64 or qb2 representations. + A Structor can also construct a dict version of its .fields suitable for + serialization by using each field name as item key and each named value's + qb64 as item value. Structor may have only one field. + + Has the following public properties: + + Properties: + + + Methods: + + + Hidden: + + + + """ + + def __init__(self, fields): + """Initialize instance + + + Parameters: + stream (bytes | bytearray): sniffable CESR stream + + + """ + self._stream = bytes(stream) + + + @property + def qb64(self): + """ + """ + return + + @property + def qb2(self): + """ + + """ + return + + @property + def asdict(self): + """ + + """ + return + + +class Sealer(Structor): + """Sealer is Structor subclass that holds a KERI namedtuple representation + of a KERI seal where its values are primitive (Matter subclass) instances. + + Its primitives can be serialized and deserialized as a concatenation of + their qb64 or qb2 representations. + A Structor can also construct a dict version of its .fields suitable for + serialization by using each field name as item key and each named value's + qb64 as item value. Structor may have only one field. + + Has the following public properties: + + Properties: + + + Methods: + + + Hidden: + + + + """ + + def __init__(self, fields): + """Initialize instance + + + Parameters: + stream (bytes | bytearray): sniffable CESR stream + + + """ + self._stream = bytes(stream) + + + @property + def qb64(self): + """ + """ + return + + @property + def qb2(self): + """ + + """ + return + + @property + def asdict(self): + """ + + """ + return From 4abd0db84a37b272f8490a699c88f9ae6e87b085 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Sat, 6 Apr 2024 20:25:59 -0700 Subject: [PATCH 122/418] Updating main version to next development release Signed-off-by: pfeairheller --- Makefile | 4 ++-- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 6c1cbcd0c..e16e41364 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.1.10 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.1.10-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev0 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev0-arm64 . .PHONY: build-witness-demo build-witness-demo: diff --git a/setup.py b/setup.py index 0b684bf8d..c280298d3 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.1.10', # also change in src/keri/__init__.py + version='1.2.0-dev0', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 6430d86e9..dcacb989d 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.1.10' # also change in setup.py +__version__ = '1.2.0-dev0' # also change in setup.py From 043d1ce67596fa829bee28b47d5d8ac5bfa172d8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 7 Apr 2024 18:47:08 -0600 Subject: [PATCH 123/418] some refactoring of core and help to expose external api more development of Structor with some unit tests make consistent exception class for Serdering and Structing --- src/keri/core/__init__.py | 2 +- src/keri/core/counting.py | 10 +- src/keri/core/serdering.py | 7 +- src/keri/core/structing.py | 356 ++++++++++++++++++++++++++++++++--- src/keri/help/__init__.py | 3 +- tests/core/test_serdering.py | 8 +- tests/core/test_structing.py | 159 ++++++++++++++++ tests/vc/test_proving.py | 3 +- 8 files changed, 509 insertions(+), 39 deletions(-) create mode 100644 tests/core/test_structing.py diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index 7b4d30b1a..aa271e140 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -7,7 +7,7 @@ #__all__ = ["coring", "eventing", "parsing", "scheming"] # Matter class and its subclasses -from .coring import (Matter, MtrDex, Number, NumDex, Dater,Texter, +from .coring import (Matter, MtrDex, Number, NumDex, Dater, Texter, Bexter, Pather, Verfer, Cigar, Signer, Salter, Cipher, Encrypter, Decrypter, Diger, DigDex, Prefixer, PreDex, ) diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 11de4f2fe..24b9f1ebc 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -77,6 +77,8 @@ class MapCodex: Adds support for dunder methods for map syntax dc[name]. Converts exceptions from attribute syntax to raise map syntax when using map syntax. + + Enables Mapping item syntas for dataclasses """ def __getitem__(self, name): @@ -107,6 +109,9 @@ class CounterCodex_1_0(MapCodex): CounterCodex is codex hard (stable) part of all counter derivation codes. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get codes with item syntax using tag variables. + Example: codex[tag] """ ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. @@ -136,6 +141,9 @@ class CounterCodex_2_0(MapCodex): CounterCodex is codex hard (stable) part of all counter derivation codes. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get codes with item syntax using tag variables. + Example: codex[tag] """ GenericGroup: str = '-A' # Generic Group (Universal with Override). BigGenericGroup: str = '-0A' # Big Generic Group (Universal with Override). @@ -189,8 +197,6 @@ class CounterCodex_2_0(MapCodex): BigESSRPayloadGroup: str = '-0Z' # Big ESSR Payload Group. KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Stack CESR Protocol Genus Version (Universal) - - def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 8e029973f..f010cbca1 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -19,7 +19,7 @@ from .. import kering from ..kering import (ValidationError, MissingFieldError, ExtraFieldError, - AlternateFieldError, + AlternateFieldError, InvalidValueError, ShortageError, VersionError, ProtocolError, KindError, DeserializeError, FieldError, SerializeError) from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, @@ -642,7 +642,8 @@ def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, f"{self._sad}.") from ex else: - raise ValueError("Improper initialization need raw or sad or makify.") + raise InvalidValueError("Improper initialization need raw or sad " + f"or makify.") @@ -840,7 +841,7 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, if sad and 'v' in sad: # attempt to get from vs in sad try: # extract version string elements as defaults if provided sproto, svrsn, skind, _, _ = deversify(sad["v"]) - except ValueError as ex: + except VersionError as ex: pass else: silk = sad.get('t') # if 't' not in sad .get returns None which may be valid diff --git a/src/keri/core/structing.py b/src/keri/core/structing.py index 044245d4d..2425b22f2 100644 --- a/src/keri/core/structing.py +++ b/src/keri/core/structing.py @@ -4,8 +4,26 @@ Provides versioning support for Indexer classes and codes """ + + +from typing import NamedTuple from collections import namedtuple +from collections.abc import Mapping + +from ..kering import (EmptyMaterialError, InvalidValueError,) + +from .. import help +from ..help import nonStringSequence +from . import coring +from .coring import (Matter, ) + + +# ToDo Change seal namedtuple definitions to NamedTuple subclasses so can +# use typehints on field values which type hints are the primitive types. Use +# union | on type hints to allow qb64, qb2, primitive instance, primitive class +# as acceptable values This provides more clarity in documentation. Actually +# enforcing types is harder with union | but can still be accomplished. # for the following Seal namedtuples use the ._asdict() method to convert to dict # when using in events @@ -35,18 +53,11 @@ # d = SAID digest qb64 of event SealEvent = namedtuple("SealEvent", 'i s d') -# Last Estalishment Event Seal: uniple (i,) +# Last Establishment Event Seal: uniple (i,) # i = pre is qb64 of identifier prefix of KEL from which to get last est, event # used to indicate to get the latest keys available from KEL for 'i' SealLast = namedtuple("SealLast", 'i') -# State Establishment Event (latest current) : quadruple (s, d, br, ba) -# s = sn of latest est event as lowercase hex string no leading zeros, -# d = SAID digest qb64 of latest establishment event -# br = backer (witness) remove list (cuts) from latest est event -# ba = backer (witness) add list (adds) from latest est event -StateEstEvent = namedtuple("StateEstEvent", 's d br ba') - # Transaction Event Seal for Transaction Event: duple (s, d) # s = sn of transaction event as lowercase hex string no leading zeros, # d = SAID digest qb64 of transaction event @@ -55,66 +66,357 @@ # use SealSourceCouples count code for attachment SealTrans = namedtuple("SealTrans", 's d') +# State Establishment Event (latest current) : quadruple (s, d, br, ba) +# s = sn of latest est event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of latest establishment event +# br = backer (witness) remove list (cuts) from latest est event +# ba = backer (witness) add list (adds) from latest est event +StateEstEvent = namedtuple("StateEstEvent", 's d br ba') + + class Structor: - """Structor class holds a namedtuple .fields whose values are primitive - (Matter subclass) instances. - These instances can be serialized and deserialized as a concatenation of - their qb64 or qb2 representations. - A Structor can also construct a dict version of its .fields suitable for - serialization by using each field name as item key and each named value's - qb64 as item value. Structor may have only one field. + """Structor class each instance holds a namedtuple .data of named values. + Each value is a primitive instance of CESR primitive subclass that supports + text (ab64) and binary (qb2) domains. + Structor instances can be serialized to or deserialized from concatenation + of the qb64 or qb2 representations of the data values. Creation requires + input of a ordered named classes for creating the named instances from the + input data. Smart data format input are supported to accomodate the many ways + the named data may appear in messages and or databases. + + Instance Creation Patterns: + + Structor(data): + + Structor(clan, cast, crew): + Structor(clan, cast, qb64): + Structor(clan, cast, qb2): + + Structor(cast, crew): + Structor(cast, qb64): + Structor(cast, qb2): + + Structor(clan, crew): when known cast in .Casts for clan + Structor(clan, qb64): when known cast in .Casts for clan + Structor(clan, qb2): when known cast in .Casts for clan + + Structor(crew): when known cast in .Casts for crew + + + Class Attributes: + Clans (type[Namedtuple]): each value is known NamedTuple class keyed + by its own field names (tuple). Enables easy query of its values() to + find known data types given field names tuple. + + Casts (NamedTuple): each value is primitive class of cast keyed by fields + names of the associated NamedTuple class in .Clans. Enables finding + known primitive classes given NamedTuple class of clan or instance + of cast or crew. + + When known casts or provided in .Clans/.Casts then more flexible creation + is supported for different types of provided cast and crew. + When no clan is provided and an unknown cast and/or crew are provided as + Mappings then Structor may create custom clan from the names given by the + cast and/or crew keys(). Subclasses may override this behavior by raising + an exception for unknown or custom clans. - Has the following public properties: Properties: + data (NamedTuple): fields are named instances of CESR primitives + clan (type[NamedTuple]): class reference of .data's class + cast (NamedTuple): CESR primitive class references of .data's primitive + instances + qb64 (str): concatenated data values as qb64 str of data's primitives + qb64b (bytes): concatenated data values as qb64b of data's primitives + qb2 (bytes): concatenated data values as qb2 bytes of data's primitives Methods: Hidden: + _data (NamedTuple): named CESR primitive instances + + + if not nonStringIterable(val): # not iterable + val = (val, ) # make iterable + return (b''.join(obj.qb64b for obj in val)) + + + if not isinstance(val, bytearray): # is memoryview or bytes + val = bytearray(val) # convert so may strip + return tuple(klas(qb64b=val, strip=True) for klas in self.klas) + + >>> T = namedtuple("a_b_c", ["a", "b", "c"]) + >>> T + """ + Clans = {} # each value is known namedtuple class keyed by own fields (tuple) + Casts = {} # each value is cast primitive class for its .Clans keyed by fields - def __init__(self, fields): - """Initialize instance + def __init__(self, data=None, clan=None, cast=None, crew=None, + qb64=None, qb2=None, strip=False): + """Initialize instance Parameters: - stream (bytes | bytearray): sniffable CESR stream + data (NamedTuple): fields are named primitive instances for .data + Given data can derive clan, cast, crew, qb64, and qb2 + clan (type[NamedTuple]): provides class reference for generated .data + when data missing. + cast (NamedTuple | dict | Iterable): each value provides CESR + primitive subclass reference used to create primitive instances + for generating .data. Can be used to infer namedtuple type of + .data when data and clan missing. Takes precendence over crew. + crew (NamedTuple | dict | Iterable): each value provides qb64 value + of primitive for generating .data with .cast when data missing. + Can be used to infer namedtuple type of .data when data and clan + missing. + qb64 (str | bytes | bytearray): concatenation of qb64 data values to + generate .data with data and crew missing. + qb2 (bytes | bytearray): concatenation of qb2 data values to generate + .data when data and crew and qb64 missing. + strip (bool): False means do not strip each value from qb64 or qb2. + Default is False. + True means if qb64 or qb2 are bytearray then strip + contained concatenated data values. Enables parser + to extract data fields from front of CESR stream. + + + from collections import namedtuple + T = namedtuple("Test", "a b c") + issubclass(T, tuple) + True + hasattr(T, "_fields") + True + T._fields + ('a', 'b', 'c') + T.__name__ + 'Test' + t = T(a=1, b=2, c=3) + t + Test(a=1, b=2, c=3) + t.__class__ + + issubclass(t.__class__, T) + True + + + Test for namedtuple subclass + + issubclass(T, tuple) and hasattr(T, "_fields") + T._feilds + + def FuncA(arg: type[CustomClass]): + + """ + if data: + if not (isinstance(data, tuple) and hasattr(data, "_fields")): + raise InvalidValueError(f"Not namedtuple subclass {data=}.") + + for val in data: # check for primitive interface + if not (hasattr(val, "qb64") and hasattr(val, "qb2")): + raise InvalidValueError(f"Non-primitive data member={val}.") + + + else: + if not clan: # attempt to get from cast and/or crew + if cast and isinstance(cast, tuple) and hasattr(cast, "_fields"): + clan = cast.__class__ + + if not clan and crew: + if isinstance(crew, tuple) and hasattr(crew, "_fields"): + clan = crew.__class__ + + if not clan and isinstance(cast, Mapping): # get clan from cast + names = tuple(cast) # create custom clan based on cast + if names in self.Clans: # get known clan and cast + clan = self.Clans[names] + cast = self.Casts[names] # same keys as self.Clans + + else: # create custom clan from cast + clan = namedtuple("_".join(names), names) # custom clan from cast keys + cast = clan(**cast) # convert to clan + + if not clan and isinstance(crew, Mapping): # get clan from crew + names = tuple(crew) # create custom clan based on cast + if names in self.Clans: # get known clan and cast + clan = self.Clans[names] + crew = clan(**crew) # convert to clan + + else: # create custom clan from crew + clan = namedtuple("_".join(names), names) # custom clan from cast keys + crew = clan(**crew) # convert to clan + + if clan: + if not (issubclass(clan, tuple) and hasattr(clan, "_fields")): + raise InvalidValueError(f"Not namedtuple subclass {clan=}.") + else: + raise InvalidValueError(f"Missing or unobtainable clan.") + # have clan but may not have cast + if cast: + if isinstance(cast, tuple) and hasattr(cast, "_fields"): + if cast._fields != clan._fields: + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and cast=" + f"{cast._fields}.") + if not isinstance(cast, clan): + cast = clan(**cast._asdict()) # convert to clan + + elif isinstance(cast, Mapping): + if tuple(cast) != clan._fields: + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and keys cast=" + f"{tuple(cast)}.") + + cast = clan(**cast) # convert to clan + + elif isinstance(cast, nonStringSequence): + cast = clan(*cast) # convert to clan assumes elements in correct order + + else: + raise InvalidValueError(f"Invalid {cast=}.") + + else: # get cast from .Casts if possible + if clan._fields in self.Casts: # same keys for self.Clans + cast = self.Casts[clan._fields] # get known cast + else: # cast missing or unobtainable + raise InvalidValueError(f"Missing or unobtainable " + f"{cast=}.") + # have cast now + for klas in cast: + if not (hasattr(klas, "qb64") and hasattr(klas, "qb2")): + raise InvalidValueError(f"Cast member {klas=} not CESR" + " Primitive.") + + # have clan and cast but may not have crew + if crew: + if isinstance(crew, tuple) and hasattr(crew, "_fields"): + if crew._fields != clan._fields: + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and crew=" + f"{crew._fields}.") + + if not isinstance(crew, clan): + crew = clan(**crew._asdict()) # convert to clan + + elif isinstance(crew, Mapping): + if tuple(crew) != clan._fields: + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and keys crew=" + f"{tuple(crew)}.") + + crew = clan(**crew) # convert to clan + + elif isinstance(crew, nonStringSequence): + crew = clan(*crew) # convert to clan assumes elements in correct order + + else: + raise InvalidValueError(f"Invalid {crew=}.") + + data = clan(*(klas(qb64=val) for klas, val in zip(cast, crew))) + + elif qb64: + if hasattr(qb64, "encode"): + qb64 = qb64.encode() + + + + + # if strip then make bytearray if not and strip + + if not isinstance(qb64, bytearray): + qb64 = bytearray(qb64) + + data = clan(*(klas(qb64=qb64, strip=strip) for klas in cast)) + + # if not strip then must count and offset qb64 + #data = clan(*(klas(qb64=qb64, strip=strip) for klas in cast)) + + elif qb2: + if not isinstance(qb2, bytearray): + qb64 = bytearray(qb2) + + data = clan(*(klas(qb2=qb2, strip=strip) for klas in cast)) + + else: + raise EmptyMaterialError("Need crew or qb64 or qb2.") + + self._data = data + + + @property + def data(self): + """Returns: + data (NamedTuple): ._data namedtuple of primitive instances + + Getter for ._data makes it read only """ - self._stream = bytes(stream) + return self._data @property - def qb64(self): + def clan(self): + """Returns: + clan (type[NamedTuple]): class of .data + """ + return self.data.__class__ + + @property + def cast(self): + """Return: + cast (NamedTuple): named primitive classes in .data + """ - return + return self.clan(*(val.__class__ for val in self.data)) + @property - def qb2(self): + def asdict(self): + """Returns: + map (dict): .data as a dictionary + """ + return self.data._asdict() + + @property + def qb64(self): + """Returns: + qb64 (str): concatenated qb64 of each primitive in .data """ - return + return (''.join(val.qb64 for val in self.data)) + @property - def asdict(self): + def qb64b(self): + """Returns: + qb64b (bytes): concatenated qb64b of each primitive in .data """ + return (b''.join(val.qb64b for val in self.data)) + + + @property + def qb2(self): + """Returns: + qb2 (bytes): concatenated qb2 of each primitive in .data """ - return + return (b''.join(val.qb2 for val in self.data)) + + class Sealer(Structor): """Sealer is Structor subclass that holds a KERI namedtuple representation - of a KERI seal where its values are primitive (Matter subclass) instances. + of a KERI seal where its values are CESR primitive instances. Its primitives can be serialized and deserialized as a concatenation of their qb64 or qb2 representations. diff --git a/src/keri/help/__init__.py b/src/keri/help/__init__.py index 72f5f27eb..2223582c8 100644 --- a/src/keri/help/__init__.py +++ b/src/keri/help/__init__.py @@ -16,4 +16,5 @@ # want help.ogler always defined by default ogler = ogling.initOgler(prefix='keri', syslogged=False) # inits once only on first import -from .helping import nowIso8601, toIso8601, fromIso8601 +from .helping import (nowIso8601, toIso8601, fromIso8601, + nonStringSequence, nonStringIterable) diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 1ded13d64..8151f3944 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -16,7 +16,7 @@ from keri import kering from keri.kering import (Protocols, Versionage, Version, Vrsn_1_0, Vrsn_2_0, VERRAWSIZE, VERFMT, - MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN) + MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN,) from keri.core import coring @@ -100,7 +100,7 @@ def test_serder(): assert set(fields.saids) <= set(fields.alls) - with pytest.raises(ValueError): + with pytest.raises(kering.InvalidValueError): serder = Serder() #Test Serder bare makify bootstrap for ACDC JSON @@ -2403,7 +2403,7 @@ def test_serderkeri_vcp(): def test_serderacdc(): """Test SerderACDC""" - with pytest.raises(ValueError): + with pytest.raises(kering.InvalidValueError): serder = SerderACDC() serder = SerderACDC(makify=True, proto=Protocols.acdc, verify=False) # make defaults for ACDC @@ -2487,7 +2487,7 @@ def test_serder_v2(): - with pytest.raises(ValueError): + with pytest.raises(kering.InvalidValueError): serder = Serder() #Test Serder bare makify bootstrap for ACDC JSON diff --git a/tests/core/test_structing.py b/tests/core/test_structing.py new file mode 100644 index 000000000..0b71bc142 --- /dev/null +++ b/tests/core/test_structing.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +""" +tests.core.test_structing module + +""" +from dataclasses import dataclass, astuple, asdict +from typing import NamedTuple +from collections import namedtuple +from collections.abc import Mapping + + +import pytest + + +from keri import kering + +from keri.help import helping + +from keri.core import (Matter, Diger, Verfer, Number) + + +from keri.core import structing +from keri.core.structing import (SealDigest, SealRoot, SealBacker, SealEvent, + SealLast, SealTrans) +from keri.core.structing import Structor + + +def test_structor(): + """ + test Structor instance + """ + + with pytest.raises(kering.InvalidValueError): + structor = Structor() # test default + + dig = 'ELC5L3iBVD77d_MYbYGGCUQgqQBju1o4x1Ud-z2sL-ux' + diger = Diger(qb64=dig) + aid = "" + #verfer = Verfer() + num = 14 + number = Number(num=num) + + + qb64 = diger.qb64 + qb2 = diger.qb2 + + data = SealDigest(d=diger) + clan = SealDigest + cast = SealDigest(d=Diger) + crew = SealDigest(d=dig) + + dcast = cast._asdict() + dcrew = crew._asdict() + + assert data._fields == SealDigest._fields + klas = data.__class__ + assert klas == clan + + # Test data + structor = Structor(data=data) + assert structor.data == data + assert structor.clan == clan + assert structor.cast == cast + assert structor.asdict == data._asdict() + assert structor.asdict == {'d': diger} + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test cast + structor = Structor(cast=cast, crew=crew) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64.encode()) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb2=qb2) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test clan and cast + structor = Structor(clan=clan, cast=cast, crew=crew) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb64=qb64) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb64=qb64.encode()) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb2=qb2) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test clan with cast and crew as dicts + + + # Test no clan but with one or the other of cast and crew as dict or namedtuple + + + # repeat all tests with multiple field clan so can test strip and not strip + + + """End Test""" + + +if __name__ == "__main__": + test_structor() + + diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index 1a612d242..8675fa1ca 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -5,6 +5,7 @@ """ import pytest +from keri import kering from keri.app import habbing from keri.core import coring, scheming, parsing, serdering, indexing from keri.core.coring import Serials, Counter, CtrDex, Prefixer, Seqner, Diger @@ -109,7 +110,7 @@ def test_proving(mockHelpingNowIso8601): def test_credentialer(): """Test SerderACDC as credential""" - with pytest.raises(ValueError): + with pytest.raises(kering.InvalidValueError): serdering.SerderACDC() # Creder() sub = dict(a=123, b="abc", issuanceDate="2021-06-27T21:26:21.233257+00:00") From 5ec12713f764d6b7650ede24107a1cc7442eb954 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 8 Apr 2024 10:40:01 -0600 Subject: [PATCH 124/418] added support for memoryview to Matter.exfil to support Structor More work on Structor --- src/keri/core/coring.py | 30 ++++++++----- src/keri/core/structing.py | 88 ++++++++++++++++++++++++-------------- src/keri/help/helping.py | 2 +- 3 files changed, 78 insertions(+), 42 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 24b1f6991..0bcee0720 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -1336,19 +1336,23 @@ def _binfil(self): def _exfil(self, qb64b): """ - Extracts self.code and self.raw from qualified base64 str or bytes qb64b - Detects is str and converts to bytes + Extracts self.code and self.raw from qualified base64 qb64b of type + str or bytes or bytearray or memoryview + + Detects if str and converts to bytes Parameters: - qb64b (str | bytes | bytearray): fully qualified base64 from stream + qb64b (str | bytes | bytearray | memoryview): fully qualified base64 from stream """ if not qb64b: # empty need more bytes raise ShortageError("Empty material.") first = qb64b[:1] # extract first char code selector + if isinstance(first, memoryview): + first = bytes(first) if hasattr(first, "decode"): - first = first.decode("utf-8") + first = first.decode() # converts bytes/bytearray to str if first not in self.Hards: if first[0] == '-': raise UnexpectedCountCodeError("Unexpected count code start" @@ -1364,8 +1368,10 @@ def _exfil(self, qb64b): raise ShortageError(f"Need {hs - len(qb64b)} more characters.") hard = qb64b[:hs] # extract hard code + if isinstance(hard, memoryview): + hard = bytes(hard) if hasattr(hard, "decode"): - hard = hard.decode("utf-8") # converts bytes/bytearray to str + hard = hard.decode() # converts bytes/bytearray to str if hard not in self.Sizes: raise UnexpectedCodeError(f"Unsupported code ={hard}.") @@ -1376,8 +1382,10 @@ def _exfil(self, qb64b): # when fs is None then ss > 0 otherwise fs > hs + ss when ss > 0 soft = qb64b[hs:hs + ss] # extract soft chars, empty when ss==0 + if isinstance(soft, memoryview): + soft = bytes(soft) if hasattr(soft, "decode"): - soft = soft.decode("utf-8") + soft = soft.decode() # converts bytes/bytearray to str if not fs: # compute fs from soft from ss part which provides size B64 # compute variable size as int may have value 0 @@ -1387,8 +1395,10 @@ def _exfil(self, qb64b): raise ShortageError(f"Need {fs - len(qb64b)} more chars.") qb64b = qb64b[:fs] # fully qualified primitive code plus material + if isinstance(qb64b, memoryview): + qb64b = bytes(qb64b) if hasattr(qb64b, "encode"): # only convert extracted chars from stream - qb64b = qb64b.encode("utf-8") + qb64b = qb64b.encode() # converts str to bytes # check for non-zeroed pad bits and/or lead bytes # net prepad ps == cs % 4 (remainer). Assumes ps != 3 i.e ps in (0,1,2) @@ -1407,8 +1417,8 @@ def _exfil(self, qb64b): if len(raw) != ((len(qb64b) - cs) * 3 // 4) - ls: # exact lengths raise ConversionError(f"Improperly qualified material = {qb64b}") - self._code = hard # hard only - self._soft = soft # soft only + self._code = hard # hard only str + self._soft = soft # soft only str self._raw = raw # ensure bytes for crypto ops, may be empty @@ -1417,7 +1427,7 @@ def _bexfil(self, qb2): Extracts self.code and self.raw from qualified base2 qb2 Parameters: - qb2 (bytes | bytearray): fully qualified base2 from stream + qb2 (bytes | bytearray | memoryview): fully qualified base2 from stream """ if not qb2: # empty need more bytes raise ShortageError("Empty material, Need more bytes.") diff --git a/src/keri/core/structing.py b/src/keri/core/structing.py index 2425b22f2..614e470f1 100644 --- a/src/keri/core/structing.py +++ b/src/keri/core/structing.py @@ -139,21 +139,6 @@ class Structor: _data (NamedTuple): named CESR primitive instances - if not nonStringIterable(val): # not iterable - val = (val, ) # make iterable - return (b''.join(obj.qb64b for obj in val)) - - - if not isinstance(val, bytearray): # is memoryview or bytes - val = bytearray(val) # convert so may strip - return tuple(klas(qb64b=val, strip=True) for klas in self.klas) - - >>> T = namedtuple("a_b_c", ["a", "b", "c"]) - >>> T - - - - """ Clans = {} # each value is known namedtuple class keyed by own fields (tuple) Casts = {} # each value is cast primitive class for its .Clans keyed by fields @@ -187,6 +172,21 @@ def __init__(self, data=None, clan=None, cast=None, crew=None, to extract data fields from front of CESR stream. + if not nonStringIterable(val): # not iterable + val = (val, ) # make iterable + return (b''.join(obj.qb64b for obj in val)) + + + if not isinstance(val, bytearray): # is memoryview or bytes + val = bytearray(val) # convert so may strip + return tuple(klas(qb64b=val, strip=True) for klas in self.klas) + + >>> T = namedtuple("a_b_c", ["a", "b", "c"]) + >>> T + + + + from collections import namedtuple T = namedtuple("Test", "a b c") issubclass(T, tuple) @@ -213,14 +213,26 @@ def __init__(self, data=None, clan=None, cast=None, crew=None, def FuncA(arg: type[CustomClass]): + + ba = bytearray(b'abcdefg') + mv = memoryview(ba) + mv[0:2] == ba[0:2] + True + b = bytes(ba) + b[0:2] == ba[0:2] + True + b[0:2] == mv[0:2] + True + + """ if data: if not (isinstance(data, tuple) and hasattr(data, "_fields")): raise InvalidValueError(f"Not namedtuple subclass {data=}.") - for val in data: # check for primitive interface - if not (hasattr(val, "qb64") and hasattr(val, "qb2")): - raise InvalidValueError(f"Non-primitive data member={val}.") + for pi in data: # check for primitive interface + if not (hasattr(pi, "qb64") and hasattr(pi, "qb2")): + raise InvalidValueError(f"Non-primitive data member={pi}.") else: @@ -326,24 +338,38 @@ def FuncA(arg: type[CustomClass]): if hasattr(qb64, "encode"): qb64 = qb64.encode() + if strip: + if not isinstance(qb64, bytearray): + qb64 = bytearray(qb64) + data = clan(*(klas(qb64b=qb64, strip=strip) for klas in cast)) - - # if strip then make bytearray if not and strip - - if not isinstance(qb64, bytearray): - qb64 = bytearray(qb64) - - data = clan(*(klas(qb64=qb64, strip=strip) for klas in cast)) - - # if not strip then must count and offset qb64 - #data = clan(*(klas(qb64=qb64, strip=strip) for klas in cast)) + else: + o = 0 # offset into memoryview of qb64 + pis = [] # primitive instances + mv = memoryview(qb64) + for klas in cast: + pi = klas(qb64b=mv[o:]) + pis.append(pi) + o += len(pi.qb64b) + data = clan(*pis) elif qb2: - if not isinstance(qb2, bytearray): - qb64 = bytearray(qb2) + if strip: + if not isinstance(qb2, bytearray): + qb2 = bytearray(qb2) + + data = clan(*(klas(qb2=qb2, strip=strip) for klas in cast)) - data = clan(*(klas(qb2=qb2, strip=strip) for klas in cast)) + else: + o = 0 # offset into memoryview of qb2 + pis = [] # primitive instances + mv = memoryview(qb2) + for klas in cast: + pi = klas(qb2=mv[o:]) + pis.append(pi) + o += len(pi.qb2) + data = clan(*pis) else: raise EmptyMaterialError("Need crew or qb64 or qb2.") diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index 401d23db2..7e3b8717a 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -354,7 +354,7 @@ def nabSextets(b, l): b is bytes or str """ if hasattr(b, 'encode'): - b = b.encode("utf-8") # convert to bytes + b = b.encode() # convert to bytes n = sceil(l * 3 / 4) # number of bytes needed for l sextets if n > len(b): raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) From 3c6ca5b1a70e041e454fe5243f6f76f4a36b1697 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 8 Apr 2024 10:47:24 -0600 Subject: [PATCH 125/418] added tests of strip flag for Structor --- tests/core/test_structing.py | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/core/test_structing.py b/tests/core/test_structing.py index 0b71bc142..a16e984f4 100644 --- a/tests/core/test_structing.py +++ b/tests/core/test_structing.py @@ -95,6 +95,26 @@ def test_structor(): assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 + structor = Structor(cast=cast, qb64=qb64.encode(), strip=True) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + ba = bytearray(qb64.encode()) + structor = Structor(cast=cast, qb64=ba, strip=True) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + assert not ba # stripped so empty + structor = Structor(cast=cast, qb2=qb2) #assert structor.data == data # not same diger object so no == assert structor.clan == clan @@ -104,6 +124,26 @@ def test_structor(): assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 + structor = Structor(cast=cast, qb2=qb2, strip=True) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + ba = bytearray(qb2) + structor = Structor(cast=cast, qb2=ba, strip=True) + #assert structor.data == data # not same diger object so no == + assert structor.clan == clan + assert structor.cast == cast + # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + assert not ba # stripped so empty + # Test clan and cast structor = Structor(clan=clan, cast=cast, crew=crew) #assert structor.data == data # not same diger object so no == From 649348b675adac2ded82471dc9375d772b4d56f4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 8 Apr 2024 12:01:32 -0600 Subject: [PATCH 126/418] some refactor to move MapDom MapCodex to coring updated comments on bran --- src/keri/app/habbing.py | 14 +-- src/keri/core/coring.py | 62 ++++++++++++++ src/keri/core/counting.py | 61 +------------ tests/core/test_coring.py | 160 +++++++++++++++++++++++++++++++++-- tests/core/test_counting.py | 148 +------------------------------- tests/core/test_structing.py | 140 ++++++++++++++++++++++++++++-- 6 files changed, 363 insertions(+), 222 deletions(-) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 8da4000c7..6da9ed2dc 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -57,9 +57,10 @@ def openHby(*, name="test", base="", temp=True, salt=None, **kwa): all secrets are re-encrypted using new aeid. In this case the provided prikey must not be empty. A change in aeid should require a second authentication mechanism besides the prikey. - bran (str): Base64 22 char string that is used as base material for - seed. bran allows alphanumeric passcodes generated by key managers - like 1password to be key store for seed. + bran (str): Base64 char string of which first 21 chars are used as + base material for seed. bran allows alphanumeric passcodes + generated by key managers like 1password to be key store for + bran as salt base material for seed. pidx (int): Initial prefix index for vacuous ks algo (str): algorithm (randy or salty) for creating key pairs default is root algo which defaults to salty @@ -184,9 +185,10 @@ def __init__(self, *, name='test', base="", temp=False, all secrets Haberyare re-encrypted using new aeid. In this case the provided prikey must not be empty. A change in aeid should require a second authentication mechanism besides the prikey. - bran (str): Base64 22 char string that is used as base material for - seed. bran allows alphanumeric passcodes generated by key managers - like 1password to be key store for seed. + bran (str): Base64 char string of which first 21 chars are used as + base material for seed. bran allows alphanumeric passcodes + generated by key managers like 1password to be key store for + bran as salt base material for seed. pidx (int): Initial prefix index for vacuous ks algo (str): algorithm (randy or salty) for creating key pairs default is root algo which defaults to salty diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 0bcee0720..197489966 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -242,6 +242,68 @@ def randomNonce(): Tiers = Tierage(low='low', med='med', high='high') + +@dataclass +class MapDom: + """Base class for dataclasses that support map syntax + Adds support for dunder methods for map syntax dc[name]. + Converts exceptions from attribute syntax to raise map syntax when using + map syntax. + """ + + def __getitem__(self, name): + try: + return getattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + def __setitem__(self, name, value): + try: + return setattr(self, name, value) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + def __delitem__(self, name): + try: + return delattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + +@dataclass(frozen=True) +class MapCodex: + """Base class for frozen dataclasses (codexes) that support map syntax + Adds support for dunder methods for map syntax dc[name]. + Converts exceptions from attribute syntax to raise map syntax when using + map syntax. + + Enables Mapping item syntas for dataclasses + """ + + def __getitem__(self, name): + try: + return getattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + def __setitem__(self, name, value): + try: + return setattr(self, name, value) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + def __delitem__(self, name): + try: + return delattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + @dataclass(frozen=True) class MatterCodex: """ diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 24b9f1ebc..7419301b8 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -16,7 +16,7 @@ from .. import kering from ..kering import (Versionage, Vrsn_1_0, Vrsn_2_0) -from ..core.coring import Sizage +from ..core.coring import Sizage, MapCodex @@ -42,65 +42,6 @@ def __iter__(self): -@dataclass -class MapDom: - """Base class for dataclasses that support map syntax - Adds support for dunder methods for map syntax dc[name]. - Converts exceptions from attribute syntax to raise map syntax when using - map syntax. - """ - - def __getitem__(self, name): - try: - return getattr(self, name) - except AttributeError as ex: - raise IndexError(ex.args) from ex - - - def __setitem__(self, name, value): - try: - return setattr(self, name, value) - except AttributeError as ex: - raise IndexError(ex.args) from ex - - - def __delitem__(self, name): - try: - return delattr(self, name) - except AttributeError as ex: - raise IndexError(ex.args) from ex - - -@dataclass(frozen=True) -class MapCodex: - """Base class for frozen dataclasses (codexes) that support map syntax - Adds support for dunder methods for map syntax dc[name]. - Converts exceptions from attribute syntax to raise map syntax when using - map syntax. - - Enables Mapping item syntas for dataclasses - """ - - def __getitem__(self, name): - try: - return getattr(self, name) - except AttributeError as ex: - raise IndexError(ex.args) from ex - - - def __setitem__(self, name, value): - try: - return setattr(self, name, value) - except AttributeError as ex: - raise IndexError(ex.args) from ex - - - def __delitem__(self, name): - try: - return delattr(self, name) - except AttributeError as ex: - raise IndexError(ex.args) from ex - @dataclass(frozen=True) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 7b9b67f4f..5f3df476a 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -3,7 +3,7 @@ tests.core.test_coring module """ -from dataclasses import asdict, astuple +from dataclasses import dataclass, asdict, astuple import hashlib import json from base64 import urlsafe_b64decode as decodeB64 @@ -32,6 +32,11 @@ from keri.kering import Version, Versionage, VersionError, Vrsn_1_0, Vrsn_2_0 from keri.kering import Protocols, Protocolage, Ilkage, Ilks, TraitDex +from keri.help import helping +from keri.help.helping import (sceil, intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, + B64_CHARS, Reb64, nabSextets) + + from keri.core import coring from keri.core import eventing from keri.core.coring import (Saids, Sadder, Tholder, Seqner, NumDex, Number, @@ -47,10 +52,153 @@ from keri.core.indexing import (Siger, Xizage, IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, Indexer) -from keri.help import helping -from keri.help.helping import (sceil, intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, - B64_CHARS, Reb64, nabSextets) +from keri.core.coring import MapDom, MapCodex + + + +def test_mapdom(): + """Test MapDom base dataclass""" + + @dataclass + class TestMapDom(MapDom): + """ + + """ + xray: str = 'X' + yankee: str = 'Y' + zulu: str = 'Z' + + def __iter__(self): # so value in dataclass not key in dataclass + return iter(astuple(self)) + + tmd = TestMapDom() + + assert 'X' in tmd + assert 'Y' in tmd + assert 'Z' in tmd + + assert tmd["xray"] == tmd.xray == 'X' + assert tmd["yankee"] == tmd.yankee == 'Y' + assert tmd["zulu"] == tmd.zulu == 'Z' + + + tmd["xray"] = "x" + assert tmd.xray == tmd["xray"] == "x" + + tmd["yankee"] = "y" + assert tmd.yankee == tmd["yankee"] == "y" + + tmd["zulu"] = "z" + assert tmd.zulu == tmd["zulu"] == "z" + + delattr(tmd, "zulu") # deletes instance attribute + assert tmd.zulu == "Z" # so returns so class attribute default value + + tmd["zulu"] = "z" + assert tmd["zulu"] == "z" + + del tmd["zulu"] # deletes instance attribute + assert tmd.zulu == "Z" # so returns so class attribute default value + + # create dynamic attribute + with pytest.raises(AttributeError): + assert tmd.alpha == None + + with pytest.raises(IndexError): + assert tmd["alpha"] == None + + tmd["alpha"] = "A" # add new attribute but without default + assert tmd.alpha == tmd["alpha"] == "A" + + del tmd["alpha"] # deletes instance attribute and no class default + + with pytest.raises(AttributeError): + assert tmd.alpha == "A" + + with pytest.raises(IndexError): + assert tmd["alpha"] == "A" + + # another dynamic attribut but delattr instead of del + with pytest.raises(AttributeError): + assert tmd.beta == None + + with pytest.raises(IndexError): + assert tmd["beta"] == None + + tmd["beta"] = "B" # add new attribute but without default + assert tmd.beta == tmd["beta"] == "B" + + delattr(tmd, "beta") # deletes instance attribute and no class default + + with pytest.raises(AttributeError): + assert tmd.beta == "B" + + with pytest.raises(IndexError): + assert tmd["beta"] == "B" + + # attempt to delete non-existing + with pytest.raises(IndexError): + del tmd["gamma"] + + with pytest.raises(AttributeError): + delattr(tmd, "gamma") + + """End Test""" + + +def test_mapcodex(): + """Test MapCodex base dataclass frozen""" + + + @dataclass(frozen=True) + class TestMapCodex(MapCodex): + """ + + """ + xray: str = 'X' + yankee: str = 'Y' + zulu: str = 'Z' + + def __iter__(self): # so value in dataclass not key in dataclass + return iter(astuple(self)) + + tmc = TestMapCodex() + + assert 'X' in tmc + assert 'Y' in tmc + assert 'Z' in tmc + + assert tmc.xray == tmc["xray"] == 'X' + assert tmc.yankee == tmc["yankee"] == 'Y' + assert tmc.zulu == tmc["zulu"] == 'Z' + + with pytest.raises(IndexError): + tmc["xray"] = "x" + + with pytest.raises(AttributeError): + tmc.xray = "x" + + with pytest.raises(IndexError): + del tmc["xray"] + + with pytest.raises(AttributeError): + delattr(tmc, "xray") + + with pytest.raises(IndexError): + tmc["alpha"] = "A" + + with pytest.raises(AttributeError): + tmc.alpha = "A" + + # attempt to delete non-existing + with pytest.raises(IndexError): + del tmc["gamma"] + + with pytest.raises(AttributeError): + delattr(tmc, "gamma") + + """End Test""" def test_matter_class(): @@ -4532,7 +4680,7 @@ def test_verfer(): def test_cigar(): """ - Test Cigar subclass of CryMat + Test Cigar subclass of Matter """ with pytest.raises(EmptyMaterialError): cigar = Cigar() @@ -6590,6 +6738,8 @@ def test_tholder(): if __name__ == "__main__": + test_mapdom() + test_mapcodex() test_matter_class() test_matter() test_matter_special() diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 060643a5e..3a6b8177d 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -18,8 +18,10 @@ from keri.help.helping import (intToB64, b64ToInt, codeB64ToB2, codeB2ToB64, nabSextets) + + from keri.core import counting -from keri.core.counting import GenDex, Sizage, MapDom, MapCodex, Counter +from keri.core.counting import GenDex, Sizage, MapCodex, Counter from keri.core.counting import Versionage, Vrsn_1_0, Vrsn_2_0, AllTags @@ -54,150 +56,6 @@ def test_genus_codex(): -def test_mapdom(): - """Test MapDom base dataclass""" - - @dataclass - class TestMapDom(MapDom): - """ - - """ - xray: str = 'X' - yankee: str = 'Y' - zulu: str = 'Z' - - def __iter__(self): # so value in dataclass not key in dataclass - return iter(astuple(self)) - - tmd = TestMapDom() - - assert 'X' in tmd - assert 'Y' in tmd - assert 'Z' in tmd - - assert tmd["xray"] == tmd.xray == 'X' - assert tmd["yankee"] == tmd.yankee == 'Y' - assert tmd["zulu"] == tmd.zulu == 'Z' - - - tmd["xray"] = "x" - assert tmd.xray == tmd["xray"] == "x" - - tmd["yankee"] = "y" - assert tmd.yankee == tmd["yankee"] == "y" - - tmd["zulu"] = "z" - assert tmd.zulu == tmd["zulu"] == "z" - - delattr(tmd, "zulu") # deletes instance attribute - assert tmd.zulu == "Z" # so returns so class attribute default value - - tmd["zulu"] = "z" - assert tmd["zulu"] == "z" - - del tmd["zulu"] # deletes instance attribute - assert tmd.zulu == "Z" # so returns so class attribute default value - - # create dynamic attribute - with pytest.raises(AttributeError): - assert tmd.alpha == None - - with pytest.raises(IndexError): - assert tmd["alpha"] == None - - tmd["alpha"] = "A" # add new attribute but without default - assert tmd.alpha == tmd["alpha"] == "A" - - del tmd["alpha"] # deletes instance attribute and no class default - - with pytest.raises(AttributeError): - assert tmd.alpha == "A" - - with pytest.raises(IndexError): - assert tmd["alpha"] == "A" - - # another dynamic attribut but delattr instead of del - with pytest.raises(AttributeError): - assert tmd.beta == None - - with pytest.raises(IndexError): - assert tmd["beta"] == None - - tmd["beta"] = "B" # add new attribute but without default - assert tmd.beta == tmd["beta"] == "B" - - delattr(tmd, "beta") # deletes instance attribute and no class default - - with pytest.raises(AttributeError): - assert tmd.beta == "B" - - with pytest.raises(IndexError): - assert tmd["beta"] == "B" - - # attempt to delete non-existing - with pytest.raises(IndexError): - del tmd["gamma"] - - with pytest.raises(AttributeError): - delattr(tmd, "gamma") - - """End Test""" - - -def test_mapcodex(): - """Test MapCodex base dataclass frozen""" - - - @dataclass(frozen=True) - class TestMapCodex(MapCodex): - """ - - """ - xray: str = 'X' - yankee: str = 'Y' - zulu: str = 'Z' - - def __iter__(self): # so value in dataclass not key in dataclass - return iter(astuple(self)) - - tmc = TestMapCodex() - - assert 'X' in tmc - assert 'Y' in tmc - assert 'Z' in tmc - - assert tmc.xray == tmc["xray"] == 'X' - assert tmc.yankee == tmc["yankee"] == 'Y' - assert tmc.zulu == tmc["zulu"] == 'Z' - - with pytest.raises(IndexError): - tmc["xray"] = "x" - - with pytest.raises(AttributeError): - tmc.xray = "x" - - with pytest.raises(IndexError): - del tmc["xray"] - - with pytest.raises(AttributeError): - delattr(tmc, "xray") - - with pytest.raises(IndexError): - tmc["alpha"] = "A" - - with pytest.raises(AttributeError): - tmc.alpha = "A" - - # attempt to delete non-existing - with pytest.raises(IndexError): - del tmc["gamma"] - - with pytest.raises(AttributeError): - delattr(tmc, "gamma") - - """End Test""" - - def test_codexes_tags(): """ Test supporting module attributes diff --git a/tests/core/test_structing.py b/tests/core/test_structing.py index a16e984f4..b58cd968d 100644 --- a/tests/core/test_structing.py +++ b/tests/core/test_structing.py @@ -33,16 +33,17 @@ def test_structor(): with pytest.raises(kering.InvalidValueError): structor = Structor() # test default - dig = 'ELC5L3iBVD77d_MYbYGGCUQgqQBju1o4x1Ud-z2sL-ux' - diger = Diger(qb64=dig) - aid = "" - #verfer = Verfer() + + aid = 'BN5Lu0RqptmJC-iXEldMMrlEew7Q01te2fLgqlbqW9zR' + verfer = Verfer(qb64=aid) num = 14 number = Number(num=num) + snq = number.qb64 + dig = 'ELC5L3iBVD77d_MYbYGGCUQgqQBju1o4x1Ud-z2sL-ux' + diger = Diger(qb64=dig) - qb64 = diger.qb64 - qb2 = diger.qb2 + # Test with single field namedtuple for data data = SealDigest(d=diger) clan = SealDigest @@ -56,6 +57,9 @@ def test_structor(): klas = data.__class__ assert klas == clan + qb64 = diger.qb64 + qb2 = diger.qb2 + # Test data structor = Structor(data=data) assert structor.data == data @@ -181,6 +185,130 @@ def test_structor(): assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 + # Test with multiple field namedtuple for data + + data = SealEvent(i=verfer, s=number, d=diger) + clan = SealEvent + cast = SealEvent(i=Verfer, s=Number, d=Diger) + crew = SealEvent(i=aid, s=snq, d=dig) + + dcast = cast._asdict() + dcrew = crew._asdict() + + assert data._fields == SealEvent._fields + klas = data.__class__ + assert klas == clan + + qb64 = verfer.qb64 + number.qb64 + diger.qb64 # ''.join(crew) + qb2 = verfer.qb2 + number.qb2 + diger.qb2 + + # Test data + structor = Structor(data=data) + assert structor.data == data + assert structor.clan == clan + assert structor.cast == cast + assert structor.asdict == data._asdict() + assert structor.asdict == \ + { + 'i': verfer, + 's': number, + 'd': diger, + } + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test cast + structor = Structor(cast=cast, crew=crew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64.encode()) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64.encode(), strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + ba = bytearray(qb64.encode()) + structor = Structor(cast=cast, qb64=ba, strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + assert not ba # stripped so empty + + structor = Structor(cast=cast, qb2=qb2) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb2=qb2, strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + ba = bytearray(qb2) + structor = Structor(cast=cast, qb2=ba, strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + assert not ba # stripped so empty + + # Test clan and cast + structor = Structor(clan=clan, cast=cast, crew=crew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb64=qb64) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb64=qb64.encode()) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb2=qb2) + assert structor.clan == clan + assert structor.cast == cast + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test clan with cast and crew as dicts From b44bd38646a011be26c3d6a6412a45c46d07d51b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 8 Apr 2024 15:11:51 -0600 Subject: [PATCH 127/418] fixed tests functionality for Structor fixed typos in MatterDex --- src/keri/core/coring.py | 16 +-- src/keri/core/eventing.py | 18 +-- src/keri/core/structing.py | 225 +++++++++++++++++++++++++++-------- tests/core/test_coring.py | 12 +- tests/core/test_structing.py | 152 +++++++++++++++++------ 5 files changed, 319 insertions(+), 104 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 197489966..7caaa7e5d 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -249,6 +249,8 @@ class MapDom: Adds support for dunder methods for map syntax dc[name]. Converts exceptions from attribute syntax to raise map syntax when using map syntax. + + Enables dataclass instances to use Mapping item syntax """ def __getitem__(self, name): @@ -279,7 +281,7 @@ class MapCodex: Converts exceptions from attribute syntax to raise map syntax when using map syntax. - Enables Mapping item syntas for dataclasses + Enables dataclass instances to use Mapping item syntax """ def __getitem__(self, name): @@ -397,12 +399,12 @@ class MatterCodex: X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 - X25519_Cipher_QB2_L0: str = '4D' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 - X25519_Cipher_QB2_L1: str = '5D' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 - X25519_Cipher_QB2_L2: str = '6D' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 - X25519_Cipher_QB2_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 - X25519_Cipher_QB2_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 - X25519_Cipher_QB2_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 + X25519_Cipher_QB2_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 + X25519_Cipher_QB2_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 + X25519_Cipher_QB2_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 + X25519_Cipher_QB2_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 + X25519_Cipher_QB2_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 + X25519_Cipher_QB2_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 def __iter__(self): diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 3fded9595..772fd03ab 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -87,13 +87,6 @@ # used to indicate to get the latest keys available from KEL for 'i' SealLast = namedtuple("SealLast", 'i') -# State Establishment Event (latest current) : quadruple (s, d, br, ba) -# s = sn of latest est event as lowercase hex string no leading zeros, -# d = SAID digest qb64 of latest establishment event -# br = backer (witness) remove list (cuts) from latest est event -# ba = backer (witness) add list (adds) from latest est event -StateEstEvent = namedtuple("StateEstEvent", 's d br ba') - # Transaction Event Seal for Transaction Event: duple (s, d) # s = sn of transaction event as lowercase hex string no leading zeros, # d = SAID digest qb64 of transaction event @@ -102,7 +95,16 @@ # use SealSourceCouples count code for attachment SealTrans = namedtuple("SealTrans", 's d') -# not used should this be depricated +# Following are not seals only used in database + +# State Establishment Event (latest current) : quadruple (s, d, br, ba) +# s = sn of latest est event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of latest establishment event +# br = backer (witness) remove list (cuts) from latest est event +# ba = backer (witness) add list (adds) from latest est event +StateEstEvent = namedtuple("StateEstEvent", 's d br ba') + +# not used should this be depricated? # State Event (latest current) : triple (s, t, d) # s = sn of latest event as lowercase hex string no leading zeros, # t = message type of latest event (ilk) diff --git a/src/keri/core/structing.py b/src/keri/core/structing.py index 614e470f1..544c8b0b6 100644 --- a/src/keri/core/structing.py +++ b/src/keri/core/structing.py @@ -9,6 +9,7 @@ from typing import NamedTuple from collections import namedtuple from collections.abc import Mapping +from dataclasses import dataclass, astuple, asdict from ..kering import (EmptyMaterialError, InvalidValueError,) @@ -16,7 +17,7 @@ from ..help import nonStringSequence from . import coring -from .coring import (Matter, ) +from .coring import (MapCodex, Matter, Diger, Prefixer, Number) # ToDo Change seal namedtuple definitions to NamedTuple subclasses so can @@ -66,6 +67,9 @@ # use SealSourceCouples count code for attachment SealTrans = namedtuple("SealTrans", 's d') + +# Following are not seals only used in database + # State Establishment Event (latest current) : quadruple (s, d, br, ba) # s = sn of latest est event as lowercase hex string no leading zeros, # d = SAID digest qb64 of latest establishment event @@ -73,6 +77,113 @@ # ba = backer (witness) add list (adds) from latest est event StateEstEvent = namedtuple("StateEstEvent", 's d br ba') +# not used should this be depricated? +# State Event (latest current) : triple (s, t, d) +# s = sn of latest event as lowercase hex string no leading zeros, +# t = message type of latest event (ilk) +# d = SAID digest qb64 of latest event +StateEvent = namedtuple("StateEvent", 's t d') + + +@dataclass(frozen=True) +class EmptyClanCodex(MapCodex): + """ + SealClanDom is dataclass of namedtuple seal class references (clans) each + indexed by its class name. + + Only provide defined classes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get class reference with item syntax using + name variables. + + Example: EmptyClanDex[name] + """ + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +EmptyClanDex = EmptyClanCodex() # create instance + + +@dataclass(frozen=True) +class EmptyCastCodex(MapCodex): + """ + SealCastCodex is dataclass of namedtuple instances (seal casts) whose values + are named primitive class references + + indexed by its namedtuple class name. + + Only provide defined namedtuples casts. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get namedtuple instance with item syntax using + name variables. + + Example: EmptyCastDex[name] + """ + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +EmptyCastDex = EmptyCastCodex() # create instance + + +@dataclass(frozen=True) +class SealClanCodex(MapCodex): + """ + SealClanCodex is dataclass of namedtuple seal class references (clans) each + indexed by its class name. + + Only provide defined classes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get class reference with item syntax using + name variables. + + Example: SealClanDex[name] + """ + SealDigest: type[NamedTuple] = SealDigest # SealDigest class reference + SealRoot: type[NamedTuple] = SealRoot # SealRoot class reference + SealBacker: type[NamedTuple] = SealBacker # SealBacker class reference + SealEvent: type[NamedTuple] = SealEvent # SealEvent class reference + SealLast: type[NamedTuple] = SealLast # SealLast class reference + SealTrans: type[NamedTuple] = SealTrans # SealTrans class reference + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +SealClanDex = SealClanCodex() # create instance + + +@dataclass(frozen=True) +class SealCastCodex(MapCodex): + """ + SealCastCodex is dataclass of namedtuple instances (seal casts) whose values + are named primitive class references + + indexed by its namedtuple class name. + + Only provide defined namedtuples casts. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get namedtuple instance with item syntax using + name variables. + + Example: SealCastDom[name] + """ + SealDigest: NamedTuple = SealDigest(d=Diger) # SealDigest class reference + SealRoot: NamedTuple = SealRoot(rd=Diger) # SealRoot class reference + SealBacker: NamedTuple = SealBacker(bi=Prefixer, d=Diger) # SealBacker class reference + SealEvent: NamedTuple = SealEvent(i=Prefixer, s=Number, d=Diger) # SealEvent class reference + SealLast: NamedTuple = SealLast(i=Prefixer) # SealLast class reference + SealTrans: NamedTuple = SealTrans(s=Number, d=Diger) # SealTrans class reference + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +SealCastDex = SealCastCodex() # create instance + class Structor: @@ -127,6 +238,7 @@ class Structor: clan (type[NamedTuple]): class reference of .data's class cast (NamedTuple): CESR primitive class references of .data's primitive instances + crew (NamedTuple): named qb64 values of .data's primitive instances qb64 (str): concatenated data values as qb64 str of data's primitives qb64b (bytes): concatenated data values as qb64b of data's primitives qb2 (bytes): concatenated data values as qb2 bytes of data's primitives @@ -140,8 +252,12 @@ class Structor: """ - Clans = {} # each value is known namedtuple class keyed by own fields (tuple) - Casts = {} # each value is cast primitive class for its .Clans keyed by fields + Clans = EmptyClanDex # known namedtuple clans. Override in subclass with non-empty + Casts = EmptyCastDex # known namedtuple casts. Override in subclass with non-empty + # Create .Names dict that maps clan/cast fields names to its namedtuple + # class type name so can look up a know clan or cast given a matching set + # of either field names from a namedtuple or keys from a dict. + Names = {tuple(clan._fields): clan.__name__ for clan in Clans} def __init__(self, data=None, clan=None, cast=None, crew=None, @@ -246,23 +362,23 @@ def FuncA(arg: type[CustomClass]): if not clan and isinstance(cast, Mapping): # get clan from cast names = tuple(cast) # create custom clan based on cast - if names in self.Clans: # get known clan and cast - clan = self.Clans[names] - cast = self.Casts[names] # same keys as self.Clans + #if names in self.Clans: # get known clan and cast + if (cname := self.Names.get(names)): # get known else None + clan = self.Clans[cname] + cast = self.Casts[cname] else: # create custom clan from cast - clan = namedtuple("_".join(names), names) # custom clan from cast keys - cast = clan(**cast) # convert to clan + clan = namedtuple("_".join(names), names) # custom clan if not clan and isinstance(crew, Mapping): # get clan from crew names = tuple(crew) # create custom clan based on cast - if names in self.Clans: # get known clan and cast - clan = self.Clans[names] - crew = clan(**crew) # convert to clan + #if names in self.Clans: # get known clan and cast + if (cname := self.Names.get(names)): # get known else None + clan = self.Clans[cname] + cast = self.Casts[cname] else: # create custom clan from crew clan = namedtuple("_".join(names), names) # custom clan from cast keys - crew = clan(**crew) # convert to clan if clan: if not (issubclass(clan, tuple) and hasattr(clan, "_fields")): @@ -272,35 +388,38 @@ def FuncA(arg: type[CustomClass]): # have clan but may not have cast if cast: - if isinstance(cast, tuple) and hasattr(cast, "_fields"): - if cast._fields != clan._fields: - raise InvalidValueError(f"Mismatching fields clan=" - f"{clan._fields} and cast=" - f"{cast._fields}.") + if not isinstance(cast, clan): + if isinstance(cast, tuple) and hasattr(cast, "_fields"): + if cast._fields != clan._fields: + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and cast=" + f"{cast._fields}.") - if not isinstance(cast, clan): cast = clan(**cast._asdict()) # convert to clan - elif isinstance(cast, Mapping): - if tuple(cast) != clan._fields: - raise InvalidValueError(f"Mismatching fields clan=" - f"{clan._fields} and keys cast=" - f"{tuple(cast)}.") - cast = clan(**cast) # convert to clan - elif isinstance(cast, nonStringSequence): - cast = clan(*cast) # convert to clan assumes elements in correct order + elif isinstance(cast, Mapping): + if tuple(cast) != clan._fields: + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and keys cast=" + f"{tuple(cast)}.") - else: - raise InvalidValueError(f"Invalid {cast=}.") + cast = clan(**cast) # convert to clan + + elif isinstance(cast, nonStringSequence): + cast = clan(*cast) # convert to clan assumes elements in correct order - else: # get cast from .Casts if possible - if clan._fields in self.Casts: # same keys for self.Clans - cast = self.Casts[clan._fields] # get known cast + else: + raise InvalidValueError(f"Invalid {cast=}.") + + else: # get cast from known .Casts if possible + #if clan._fields in self.Casts: # same keys for self.Clans + if (cname := self.Names.get(clan._fields)): + cast = self.Casts[cname] # get known cast else: # cast missing or unobtainable - raise InvalidValueError(f"Missing or unobtainable " - f"{cast=}.") + raise InvalidValueError(f"Missing or unobtainable cast.") + # have cast now for klas in cast: if not (hasattr(klas, "qb64") and hasattr(klas, "qb2")): @@ -309,28 +428,28 @@ def FuncA(arg: type[CustomClass]): # have clan and cast but may not have crew if crew: - if isinstance(crew, tuple) and hasattr(crew, "_fields"): - if crew._fields != clan._fields: - raise InvalidValueError(f"Mismatching fields clan=" - f"{clan._fields} and crew=" - f"{crew._fields}.") + if not isinstance(crew, clan): + if isinstance(crew, tuple) and hasattr(crew, "_fields"): + if crew._fields != clan._fields: + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and crew=" + f"{crew._fields}.") - if not isinstance(crew, clan): crew = clan(**crew._asdict()) # convert to clan - elif isinstance(crew, Mapping): - if tuple(crew) != clan._fields: - raise InvalidValueError(f"Mismatching fields clan=" - f"{clan._fields} and keys crew=" - f"{tuple(crew)}.") + elif isinstance(crew, Mapping): + if tuple(crew) != clan._fields: + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and keys crew=" + f"{tuple(crew)}.") - crew = clan(**crew) # convert to clan + crew = clan(**crew) # convert to clan - elif isinstance(crew, nonStringSequence): - crew = clan(*crew) # convert to clan assumes elements in correct order + elif isinstance(crew, nonStringSequence): + crew = clan(*crew) # convert to clan assumes elements in correct order - else: - raise InvalidValueError(f"Invalid {crew=}.") + else: + raise InvalidValueError(f"Invalid {crew=}.") data = clan(*(klas(qb64=val) for klas, val in zip(cast, crew))) @@ -403,6 +522,14 @@ def cast(self): """ return self.clan(*(val.__class__ for val in self.data)) + @property + def crew(self): + """Return: + crew (NamedTuple): named qb64 field values from .data + + """ + return self.clan(*(val.qb64 for val in self.data)) + @property def asdict(self): diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 5f3df476a..552072c4b 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -293,12 +293,12 @@ def test_matter_class(): 'X25519_Cipher_QB64_Big_L0': '7AAD', 'X25519_Cipher_QB64_Big_L1': '8AAD', 'X25519_Cipher_QB64_Big_L2': '9AAD', - 'X25519_Cipher_QB2_L0': '4D', - 'X25519_Cipher_QB2_L1': '5D', - 'X25519_Cipher_QB2_L2': '6D', - 'X25519_Cipher_QB2_Big_L0': '7AAD', - 'X25519_Cipher_QB2_Big_L1': '8AAD', - 'X25519_Cipher_QB2_Big_L2': '9AAD' + 'X25519_Cipher_QB2_L0': '4E', + 'X25519_Cipher_QB2_L1': '5E', + 'X25519_Cipher_QB2_L2': '6E', + 'X25519_Cipher_QB2_Big_L0': '7AAE', + 'X25519_Cipher_QB2_Big_L1': '8AAE', + 'X25519_Cipher_QB2_Big_L2': '9AAE' } diff --git a/tests/core/test_structing.py b/tests/core/test_structing.py index b58cd968d..f55cdec6c 100644 --- a/tests/core/test_structing.py +++ b/tests/core/test_structing.py @@ -16,15 +16,25 @@ from keri.help import helping -from keri.core import (Matter, Diger, Verfer, Number) +from keri.core import (Matter, Diger, Prefixer, Number) from keri.core import structing from keri.core.structing import (SealDigest, SealRoot, SealBacker, SealEvent, SealLast, SealTrans) -from keri.core.structing import Structor +from keri.core.structing import Structor, EmptyClanDex, EmptyCastDex +def test_structor_class(): + """ + test Structor class variables etc + """ + assert Structor.Clans == EmptyClanDex + assert Structor.Casts == EmptyCastDex + assert Structor.Names == {} + + """End Test""" + def test_structor(): """ test Structor instance @@ -35,14 +45,13 @@ def test_structor(): aid = 'BN5Lu0RqptmJC-iXEldMMrlEew7Q01te2fLgqlbqW9zR' - verfer = Verfer(qb64=aid) + prefixer = Prefixer(qb64=aid) num = 14 number = Number(num=num) snq = number.qb64 dig = 'ELC5L3iBVD77d_MYbYGGCUQgqQBju1o4x1Ud-z2sL-ux' diger = Diger(qb64=dig) - # Test with single field namedtuple for data data = SealDigest(d=diger) @@ -65,6 +74,7 @@ def test_structor(): assert structor.data == data assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.asdict == data._asdict() assert structor.asdict == {'d': diger} assert structor.qb64 == qb64 @@ -73,76 +83,68 @@ def test_structor(): # Test cast structor = Structor(cast=cast, crew=crew) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 structor = Structor(cast=cast, qb64=qb64) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 structor = Structor(cast=cast, qb64=qb64.encode()) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 structor = Structor(cast=cast, qb64=qb64.encode(), strip=True) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 ba = bytearray(qb64.encode()) structor = Structor(cast=cast, qb64=ba, strip=True) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 assert not ba # stripped so empty structor = Structor(cast=cast, qb2=qb2) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 structor = Structor(cast=cast, qb2=qb2, strip=True) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 ba = bytearray(qb2) structor = Structor(cast=cast, qb2=ba, strip=True) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -150,46 +152,81 @@ def test_structor(): # Test clan and cast structor = Structor(clan=clan, cast=cast, crew=crew) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 structor = Structor(clan=clan, cast=cast, qb64=qb64) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 structor = Structor(clan=clan, cast=cast, qb64=qb64.encode()) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 structor = Structor(clan=clan, cast=cast, qb2=qb2) - #assert structor.data == data # not same diger object so no == assert structor.clan == clan assert structor.cast == cast - # assert structor.asdict == {'d': diger} # not same diger object so no == + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test clan with cast and crew as dicts + structor = Structor(clan=clan, cast=dcast, crew=dcrew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test no clan but with one or the other of cast and crew as dict or namedtuple + structor = Structor(cast=cast, crew=dcrew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=dcast, crew=crew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 + # creates custom clan since both cast and crew are dicts + structor = Structor(cast=dcast, crew=dcrew) + assert structor.data.__class__.__name__ == "d" + assert structor.clan != clan + assert structor.clan.__name__ == "d" + assert structor.cast == cast # tuple compare is by field value not type + assert structor.cast.__class__.__name__ == "d" + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test with multiple field namedtuple for data - data = SealEvent(i=verfer, s=number, d=diger) + data = SealEvent(i=prefixer, s=number, d=diger) clan = SealEvent - cast = SealEvent(i=Verfer, s=Number, d=Diger) + cast = SealEvent(i=Prefixer, s=Number, d=Diger) crew = SealEvent(i=aid, s=snq, d=dig) dcast = cast._asdict() @@ -199,18 +236,19 @@ def test_structor(): klas = data.__class__ assert klas == clan - qb64 = verfer.qb64 + number.qb64 + diger.qb64 # ''.join(crew) - qb2 = verfer.qb2 + number.qb2 + diger.qb2 + qb64 = prefixer.qb64 + number.qb64 + diger.qb64 # ''.join(crew) + qb2 = prefixer.qb2 + number.qb2 + diger.qb2 # Test data structor = Structor(data=data) assert structor.data == data assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.asdict == data._asdict() assert structor.asdict == \ { - 'i': verfer, + 'i': prefixer, 's': number, 'd': diger, } @@ -222,6 +260,7 @@ def test_structor(): structor = Structor(cast=cast, crew=crew) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -229,6 +268,7 @@ def test_structor(): structor = Structor(cast=cast, qb64=qb64) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -236,6 +276,7 @@ def test_structor(): structor = Structor(cast=cast, qb64=qb64.encode()) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -243,6 +284,7 @@ def test_structor(): structor = Structor(cast=cast, qb64=qb64.encode(), strip=True) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -251,6 +293,7 @@ def test_structor(): structor = Structor(cast=cast, qb64=ba, strip=True) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -259,6 +302,7 @@ def test_structor(): structor = Structor(cast=cast, qb2=qb2) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -266,6 +310,7 @@ def test_structor(): structor = Structor(cast=cast, qb2=qb2, strip=True) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -274,6 +319,7 @@ def test_structor(): structor = Structor(cast=cast, qb2=ba, strip=True) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -283,6 +329,7 @@ def test_structor(): structor = Structor(clan=clan, cast=cast, crew=crew) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -290,6 +337,7 @@ def test_structor(): structor = Structor(clan=clan, cast=cast, qb64=qb64) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -297,6 +345,7 @@ def test_structor(): structor = Structor(clan=clan, cast=cast, qb64=qb64.encode()) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -304,24 +353,59 @@ def test_structor(): structor = Structor(clan=clan, cast=cast, qb2=qb2) assert structor.clan == clan assert structor.cast == cast + assert structor.crew == crew assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 # Test clan with cast and crew as dicts + structor = Structor(clan=clan, cast=dcast, crew=dcrew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 # Test no clan but with one or the other of cast and crew as dict or namedtuple + structor = Structor(cast=cast, crew=dcrew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=dcast, crew=crew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # creates custom clan since both cast and crew are dicts + structor = Structor(cast=dcast, crew=dcrew) + assert structor.data.__class__.__name__ == "i_s_d" + assert structor.clan != clan + assert structor.clan.__name__ == "i_s_d" + assert structor.cast == cast # tuple compare is by field value not type + assert structor.cast.__class__.__name__ == "i_s_d" + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 - # repeat all tests with multiple field clan so can test strip and not strip """End Test""" if __name__ == "__main__": + test_structor_class() test_structor() From 4f7b982e17a1a17badf5c32ca8c141e7ba0e1767 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 8 Apr 2024 16:49:50 -0600 Subject: [PATCH 128/418] Sealer subclass of Structor with tests --- src/keri/core/structing.py | 154 ++++++++++++++--------------------- tests/core/test_structing.py | 101 ++++++++++++++++++++++- 2 files changed, 163 insertions(+), 92 deletions(-) diff --git a/src/keri/core/structing.py b/src/keri/core/structing.py index 544c8b0b6..5c1a77176 100644 --- a/src/keri/core/structing.py +++ b/src/keri/core/structing.py @@ -260,7 +260,7 @@ class Structor: Names = {tuple(clan._fields): clan.__name__ for clan in Clans} - def __init__(self, data=None, clan=None, cast=None, crew=None, + def __init__(self, data=None, *, clan=None, cast=None, crew=None, qb64=None, qb2=None, strip=False): """Initialize instance @@ -288,59 +288,6 @@ def __init__(self, data=None, clan=None, cast=None, crew=None, to extract data fields from front of CESR stream. - if not nonStringIterable(val): # not iterable - val = (val, ) # make iterable - return (b''.join(obj.qb64b for obj in val)) - - - if not isinstance(val, bytearray): # is memoryview or bytes - val = bytearray(val) # convert so may strip - return tuple(klas(qb64b=val, strip=True) for klas in self.klas) - - >>> T = namedtuple("a_b_c", ["a", "b", "c"]) - >>> T - - - - - from collections import namedtuple - T = namedtuple("Test", "a b c") - issubclass(T, tuple) - True - hasattr(T, "_fields") - True - T._fields - ('a', 'b', 'c') - T.__name__ - 'Test' - t = T(a=1, b=2, c=3) - t - Test(a=1, b=2, c=3) - t.__class__ - - issubclass(t.__class__, T) - True - - - Test for namedtuple subclass - - issubclass(T, tuple) and hasattr(T, "_fields") - T._feilds - - def FuncA(arg: type[CustomClass]): - - - ba = bytearray(b'abcdefg') - mv = memoryview(ba) - mv[0:2] == ba[0:2] - True - b = bytes(ba) - b[0:2] == ba[0:2] - True - b[0:2] == mv[0:2] - True - - """ if data: if not (isinstance(data, tuple) and hasattr(data, "_fields")): @@ -362,7 +309,6 @@ def FuncA(arg: type[CustomClass]): if not clan and isinstance(cast, Mapping): # get clan from cast names = tuple(cast) # create custom clan based on cast - #if names in self.Clans: # get known clan and cast if (cname := self.Names.get(names)): # get known else None clan = self.Clans[cname] cast = self.Casts[cname] @@ -372,7 +318,6 @@ def FuncA(arg: type[CustomClass]): if not clan and isinstance(crew, Mapping): # get clan from crew names = tuple(crew) # create custom clan based on cast - #if names in self.Clans: # get known clan and cast if (cname := self.Names.get(names)): # get known else None clan = self.Clans[cname] cast = self.Casts[cname] @@ -414,7 +359,6 @@ def FuncA(arg: type[CustomClass]): raise InvalidValueError(f"Invalid {cast=}.") else: # get cast from known .Casts if possible - #if clan._fields in self.Casts: # same keys for self.Clans if (cname := self.Names.get(clan._fields)): cast = self.Casts[cname] # get known cast else: # cast missing or unobtainable @@ -568,57 +512,85 @@ def qb2(self): class Sealer(Structor): - """Sealer is Structor subclass that holds a KERI namedtuple representation - of a KERI seal where its values are CESR primitive instances. + """Sealer is Structor subclass each instance holds a namedtuple .data of + named values belonging to KERI Seals for anchoring in messages or adding + to message attachments. - Its primitives can be serialized and deserialized as a concatenation of - their qb64 or qb2 representations. - A Structor can also construct a dict version of its .fields suitable for - serialization by using each field name as item key and each named value's - qb64 as item value. Structor may have only one field. - - Has the following public properties: - - Properties: + See Structor class for more details. - Methods: + Inherited Class Attributes: + Clans (type[Namedtuple]): each value is known NamedTuple class keyed + by its own field names (tuple). Enables easy query of its values() to + find known data types given field names tuple. + Casts (NamedTuple): each value is primitive class of cast keyed by fields + names of the associated NamedTuple class in .Clans. Enables finding + known primitive classes given NamedTuple class of clan or instance + of cast or crew. - Hidden: + When known casts or provided in .Clans/.Casts then more flexible creation + is supported for different types of provided cast and crew. + When no clan is provided and an unknown cast and/or crew are provided as + Mappings then Structor may create custom clan from the names given by the + cast and/or crew keys(). Subclasses may override this behavior by raising + an exception for unknown or custom clans. + Inherited Properties: + data (NamedTuple): fields are named instances of CESR primitives + clan (type[NamedTuple]): class reference of .data's class + cast (NamedTuple): CESR primitive class references of .data's primitive + instances + crew (NamedTuple): named qb64 values of .data's primitive instances + qb64 (str): concatenated data values as qb64 str of data's primitives + qb64b (bytes): concatenated data values as qb64b of data's primitives + qb2 (bytes): concatenated data values as qb2 bytes of data's primitives - """ - def __init__(self, fields): - """Initialize instance + Methods: - Parameters: - stream (bytes | bytearray): sniffable CESR stream + Hidden: + _data (NamedTuple): named CESR primitive instances - """ - self._stream = bytes(stream) + """ + Clans = SealClanDex # known namedtuple clans. Override in subclass with non-empty + Casts = SealCastDex # known namedtuple casts. Override in subclass with non-empty + # Create .Names dict that maps clan/cast fields names to its namedtuple + # class type name so can look up a know clan or cast given a matching set + # of either field names from a namedtuple or keys from a dict. + Names = {tuple(clan._fields): clan.__name__ for clan in Clans} - @property - def qb64(self): - """ - """ - return + def __init__(self, *pa, **kwa): + """Initialize instance - @property - def qb2(self): - """ - """ - return + Inherited Parameters: + data (NamedTuple): fields are named primitive instances for .data + Given data can derive clan, cast, crew, qb64, and qb2 + clan (type[NamedTuple]): provides class reference for generated .data + when data missing. + cast (NamedTuple | dict | Iterable): each value provides CESR + primitive subclass reference used to create primitive instances + for generating .data. Can be used to infer namedtuple type of + .data when data and clan missing. Takes precendence over crew. + crew (NamedTuple | dict | Iterable): each value provides qb64 value + of primitive for generating .data with .cast when data missing. + Can be used to infer namedtuple type of .data when data and clan + missing. + qb64 (str | bytes | bytearray): concatenation of qb64 data values to + generate .data with data and crew missing. + qb2 (bytes | bytearray): concatenation of qb2 data values to generate + .data when data and crew and qb64 missing. + strip (bool): False means do not strip each value from qb64 or qb2. + Default is False. + True means if qb64 or qb2 are bytearray then strip + contained concatenated data values. Enables parser + to extract data fields from front of CESR stream. - @property - def asdict(self): """ - """ - return + super(Sealer, self).__init__(*pa, **kwa) diff --git a/tests/core/test_structing.py b/tests/core/test_structing.py index f55cdec6c..f99ebed20 100644 --- a/tests/core/test_structing.py +++ b/tests/core/test_structing.py @@ -22,7 +22,8 @@ from keri.core import structing from keri.core.structing import (SealDigest, SealRoot, SealBacker, SealEvent, SealLast, SealTrans) -from keri.core.structing import Structor, EmptyClanDex, EmptyCastDex +from keri.core.structing import (Structor, EmptyClanDex, EmptyCastDex, + Sealer, SealClanDex, SealCastDex, ) def test_structor_class(): @@ -398,14 +399,112 @@ def test_structor(): assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 + # Test no clan and cast or crew as dict + with pytest.raises(kering.EmptyMaterialError): + structor = Structor(cast=dcast) # missing crew + with pytest.raises(kering.InvalidValueError): + structor = Structor(crew=crew) # missing cast + + """End Test""" +def test_sealer_class(): + """ + test sealer class variables etc + """ + assert Sealer.Clans == SealClanDex + assert Sealer.Casts == SealCastDex + assert Sealer.Names == \ + { + ('d',): 'SealDigest', + ('rd',): 'SealRoot', + ('bi', 'd'): 'SealBacker', + ('i', 's', 'd'): 'SealEvent', + ('i',): 'SealLast', + ('s', 'd'): 'SealTrans' + } + """End Test""" +def test_sealer(): + """ + test sealer instance + """ + + with pytest.raises(kering.InvalidValueError): + sealer = Sealer() # test default + + + aid = 'BN5Lu0RqptmJC-iXEldMMrlEew7Q01te2fLgqlbqW9zR' + prefixer = Prefixer(qb64=aid) + num = 14 + number = Number(num=num) + snq = number.qb64 + dig = 'ELC5L3iBVD77d_MYbYGGCUQgqQBju1o4x1Ud-z2sL-ux' + diger = Diger(qb64=dig) + + # Test with single field namedtuple for data + + data = SealDigest(d=diger) + clan = SealDigest + cast = SealDigest(d=Diger) + crew = SealDigest(d=dig) + + dcast = cast._asdict() + dcrew = crew._asdict() + + assert data._fields == SealDigest._fields + klas = data.__class__ + assert klas == clan + + qb64 = diger.qb64 + qb2 = diger.qb2 + + # Test data + sealer = Sealer(data=data) + assert sealer.data == data + assert sealer.clan == clan + assert sealer.cast == cast + assert sealer.crew == crew + assert sealer.asdict == data._asdict() + assert sealer.asdict == {'d': diger} + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + # Test no clan but with one or the other of cast and crew as dict or namedtuple + sealer = Sealer(crew=crew) # uses known cast + assert sealer.clan == clan + assert sealer.cast == cast + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + sealer = Sealer(crew=dcrew) # uses known cast + assert sealer.clan == clan + assert sealer.cast == cast + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + # uses known class + sealer = Sealer(cast=dcast, crew=dcrew) + assert sealer.clan == clan + assert sealer.cast == cast # tuple compare is by field value not type + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + if __name__ == "__main__": test_structor_class() test_structor() + test_sealer_class() + test_sealer() + From 8e2aed04f6fc1975dde394e9fc6e04780595ed75 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 8 Apr 2024 16:57:06 -0600 Subject: [PATCH 129/418] added test for Seal Codexes --- tests/core/test_structing.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/core/test_structing.py b/tests/core/test_structing.py index f99ebed20..a2ade17b8 100644 --- a/tests/core/test_structing.py +++ b/tests/core/test_structing.py @@ -408,6 +408,30 @@ def test_structor(): """End Test""" +def test_seal_dexes(): + """ + test Seal Codexes + """ + + assert asdict(SealClanDex) == \ + { + 'SealDigest': SealDigest, + 'SealRoot': SealRoot, + 'SealBacker': SealBacker, + 'SealEvent': SealEvent, + 'SealLast': SealLast, + 'SealTrans': SealTrans + } + + assert asdict(SealCastDex) == \ + { + 'SealDigest': SealDigest(d=Diger), + 'SealRoot': SealRoot(rd=Diger), + 'SealBacker': SealBacker(bi=Prefixer, d=Diger), + 'SealEvent': SealEvent(i=Prefixer, s=Number, d=Diger), + 'SealLast': SealLast(i=Prefixer), + 'SealTrans': SealTrans(s=Number, d=Diger) + } def test_sealer_class(): """ @@ -503,6 +527,7 @@ def test_sealer(): if __name__ == "__main__": test_structor_class() test_structor() + test_seal_dexes() test_sealer_class() test_sealer() From 505af328a043b523b796d849a71a00890218e00c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 8 Apr 2024 17:09:41 -0600 Subject: [PATCH 130/418] fix lint errors --- tests/core/test_counting.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 3a6b8177d..5556e9ef6 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -1242,8 +1242,6 @@ def test_counter_v2(): if __name__ == "__main__": test_genus_codex() - test_mapdom() - test_mapcodex() test_codexes_tags() test_counter_class() test_counter_v1() From 9d301f6ac893eab29177ebe5e02bb53084afb6fa Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 9 Apr 2024 09:27:57 -0600 Subject: [PATCH 131/418] fixed typo in counting Counter Codex table --- src/keri/core/counting.py | 2 +- tests/core/test_counting.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 7419301b8..62ea757ca 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -115,7 +115,7 @@ class CounterCodex_2_0(MapCodex): FirstSeenReplayCouples: str = '-N' # First Seen Replay Couple(s), fnu+dts. BigFirstSeenReplayCouples: str = '-0N' # First Seen Replay Couple(s), fnu+dts. TransIdxSigGroups: str = '-O' # Trans Indexed Signature Group(s), pre+snu+dig+CtrControllerIdxSigs of qb64. - TransIdxSigGroups: str = '-0O' # Big Trans Indexed Signature Group(s), pre+snu+dig+CtrControllerIdxSigs of qb64. + BigTransIdxSigGroups: str = '-0O' # Big Trans Indexed Signature Group(s), pre+snu+dig+CtrControllerIdxSigs of qb64. TransLastIdxSigGroups: str = '-P' # Trans Last Est Evt Indexed Signature Group(s), pre+CtrControllerIdxSigs of qb64. BigTransLastIdxSigGroups: str = '-0P' # Big Trans Last Est Evt Indexed Signature Group(s), pre+CtrControllerIdxSigs of qb64. SealSourceCouples: str = '-Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 5556e9ef6..da1c21aeb 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -110,7 +110,8 @@ def test_codexes_tags(): 'BigTransReceiptQuadruples': '-0M', 'FirstSeenReplayCouples': '-N', 'BigFirstSeenReplayCouples': '-0N', - 'TransIdxSigGroups': '-0O', + 'TransIdxSigGroups': '-O', + 'BigTransIdxSigGroups': '-0O', 'TransLastIdxSigGroups': '-P', 'BigTransLastIdxSigGroups': '-0P', 'SealSourceCouples': '-Q', @@ -186,6 +187,7 @@ def test_codexes_tags(): 'FirstSeenReplayCouples': 'FirstSeenReplayCouples', 'BigFirstSeenReplayCouples': 'BigFirstSeenReplayCouples', 'TransIdxSigGroups': 'TransIdxSigGroups', + 'BigTransIdxSigGroups': 'BigTransIdxSigGroups', 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', 'BigTransLastIdxSigGroups': 'BigTransLastIdxSigGroups', 'SealSourceCouples': 'SealSourceCouples', @@ -242,6 +244,7 @@ def test_codexes_tags(): 'FirstSeenReplayCouples': 'FirstSeenReplayCouples', 'BigFirstSeenReplayCouples': 'BigFirstSeenReplayCouples', 'TransIdxSigGroups': 'TransIdxSigGroups', + 'BigTransIdxSigGroups': 'BigTransIdxSigGroups', 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', 'BigTransLastIdxSigGroups': 'BigTransLastIdxSigGroups', 'SealSourceCouples': 'SealSourceCouples', From c98554c7baa0bc92e8fc2f46ee9d9e98967e39e7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 9 Apr 2024 09:57:49 -0600 Subject: [PATCH 132/418] started process of moving Signer to its own file signing so its dependencies on non Matter, Indexer and Siger are clean from an import perspecitive. Part of cleaning up and regularizing the keripy import policy moving forward. Including minimizing or eliminating circular imports. --- src/keri/core/signing.py | 320 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 src/keri/core/signing.py diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py new file mode 100644 index 000000000..ad483762b --- /dev/null +++ b/src/keri/core/signing.py @@ -0,0 +1,320 @@ +# -*- coding: utf-8 -*- +""" +keri.core.signing module + +Provides support Signer class +""" + +import pysodium + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat +from cryptography.hazmat.primitives.asymmetric import ec, utils + +from ..kering import (EmptyMaterialError,) + +from ..help import helping + +from .coring import (Matter, MtrDex, Verfer, Cigar) +from .indexing import IdrDex, Siger + + +DSS_SIG_MODE = "fips-186-3" +ECDSA_256r1_SEEDBYTES = 32 +ECDSA_256k1_SEEDBYTES = 32 + + + +class Signer(Matter): + """ + Signer is Matter subclass with method to create signature of serialization + using: + .raw as signing (private) key seed, + .code as cipher suite for signing + .verfer whose property .raw is public key for signing. + + If not provided .verfer is generated from private key seed using .code + as cipher suite for creating key-pair. + + + See Matter for inherited attributes and properties: + + Attributes: + + Properties: (inherited) + code (str): hard part of derivation code to indicate cypher suite + both (int): hard and soft parts of full text code + size (int): Number of triplets of bytes including lead bytes + (quadlets of chars) of variable sized material. Value of soft size, + ss, part of full text code. + Otherwise None. + rize (int): number of bytes of raw material not including + lead bytes + raw (bytes): private signing key crypto material only without code + qb64 (str): private signing key Base64 fully qualified with + derivation code + crypto mat + qb64b (bytes): private signing keyBase64 fully qualified with + derivation code + crypto mat + qb2 (bytes): private signing key binary with + derivation code + crypto material + transferable (bool): True means transferable derivation code False otherwise + digestive (bool): True means digest derivation code False otherwise + + Properties: + + .verfer is Verfer object instance of public key derived from private key + seed which is .raw + + Methods: + sign: create signature + + """ + + def __init__(self, raw=None, code=MtrDex.Ed25519_Seed, transferable=True, **kwa): + """ + Assign signing cipher suite function to ._sign + + Parameters: See Matter for inherted parameters + raw is bytes crypto material seed or private key + code is derivation code + transferable is Boolean True means make verifier code transferable + False make non-transferable + + """ + try: + super(Signer, self).__init__(raw=raw, code=code, **kwa) + except EmptyMaterialError as ex: + if code == MtrDex.Ed25519_Seed: + raw = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + super(Signer, self).__init__(raw=raw, code=code, **kwa) + elif code == MtrDex.ECDSA_256r1_Seed: + raw = pysodium.randombytes(ECDSA_256r1_SEEDBYTES) + super(Signer, self).__init__(raw=bytes(raw), code=code, **kwa) + elif code == MtrDex.ECDSA_256k1_Seed: + raw = pysodium.randombytes(ECDSA_256k1_SEEDBYTES) + super(Signer, self).__init__(raw=bytes(raw), code=code, **kwa) + + else: + raise ValueError("Unsupported signer code = {}.".format(code)) + + if self.code == MtrDex.Ed25519_Seed: + self._sign = self._ed25519 + verkey, sigkey = pysodium.crypto_sign_seed_keypair(self.raw) + verfer = Verfer(raw=verkey, + code=MtrDex.Ed25519 if transferable + else MtrDex.Ed25519N) + elif self.code == MtrDex.ECDSA_256r1_Seed: + self._sign = self._secp256r1 + d = int.from_bytes(self.raw, byteorder="big") + sigkey = ec.derive_private_key(d, ec.SECP256R1()) + verkey = sigkey.public_key().public_bytes(encoding=Encoding.X962, format=PublicFormat.CompressedPoint) + verfer = Verfer(raw=verkey, + code=MtrDex.ECDSA_256r1 if transferable + else MtrDex.ECDSA_256r1N) + elif self.code == MtrDex.ECDSA_256k1_Seed: + self._sign = self._secp256k1 + d = int.from_bytes(self.raw, byteorder="big") + sigkey = ec.derive_private_key(d, ec.SECP256K1()) + verkey = sigkey.public_key().public_bytes(encoding=Encoding.X962, format=PublicFormat.CompressedPoint) + verfer = Verfer(raw=verkey, + code=MtrDex.ECDSA_256k1 if transferable + else MtrDex.ECDSA_256k1N) + else: + raise ValueError("Unsupported signer code = {}.".format(self.code)) + + self._verfer = verfer + + @property + def verfer(self): + """ + Property verfer: + Returns Verfer instance + Assumes ._verfer is correctly assigned + """ + return self._verfer + + def sign(self, ser, index=None, only=False, ondex=None, **kwa): + """ + Returns either Cigar or Siger (indexed) instance of cryptographic + signature material on bytes serialization ser + + If index is None + return Cigar instance + Else + return Siger instance + + Parameters: + ser (bytes): serialization to be signed + index (int): main index of associated verifier key in event keys + only (bool): True means main index only list, ondex ignored + False means both index lists (default), ondex used + ondex (int | None): other index offset into list such as prior next + + """ + return (self._sign(ser=ser, + seed=self.raw, + verfer=self.verfer, + index=index, + only=only, + ondex=ondex, + **kwa)) + + @staticmethod + def _ed25519(ser, seed, verfer, index, only=False, ondex=None, **kwa): + """ + Returns signature as either Cigar or Siger instance as appropriate for + Ed25519 digital signatures given index and ondex values + + The seed's code determins the crypto key-pair algorithm and signing suite + The signature type, Cigar or Siger, and when indexed the Siger code + may be completely determined by the seed and index values (index, ondex) + by assuming that the index values are intentional. + Without the seed code its more difficult for Siger to + determine when for the Indexer code value should be changed from the + than the provided value with respect to provided but incompatible index + values versus error conditions. + + Parameters: + ser (bytes): serialization to be signed + seed (bytes): raw binary seed (private key) + verfer (Verfer): instance. verfer.raw is public key + index (int |None): main index offset into list such as current signing + None means return non-indexed Cigar + Not None means return indexed Siger with Indexer code derived + from index, conly, and ondex values + only (bool): True means main index only list, ondex ignored + False means both index lists (default), ondex used + ondex (int | None): other index offset into list such as prior next + """ + # compute raw signature sig using seed on serialization ser + sig = pysodium.crypto_sign_detached(ser, seed + verfer.raw) + + if index is None: # Must be Cigar i.e. non-indexed signature + return Cigar(raw=sig, code=MtrDex.Ed25519_Sig, verfer=verfer) + else: # Must be Siger i.e. indexed signature + # should add Indexer class method to get ms main index size for given code + if only: # only main index ondex not used + ondex = None + if index <= 63: # (64 ** ms - 1) where ms is main index size + code = IdrDex.Ed25519_Crt_Sig # use small current only + else: + code = IdrDex.Ed25519_Big_Crt_Sig # use big current only + else: # both + if ondex == None: + ondex = index # enable default to be same + if ondex == index and index <= 63: # both same and small + code = IdrDex.Ed25519_Sig # use small both same + else: # otherwise big or both not same so use big both + code = IdrDex.Ed25519_Big_Sig # use use big both + + return Siger(raw=sig, code=code, index=index, ondex=ondex, verfer=verfer) + + @staticmethod + def _secp256r1(ser, seed, verfer, index, only=False, ondex=None, **kwa): + """ + Returns signature as either Cigar or Siger instance as appropriate for + Ed25519 digital signatures given index and ondex values + + The seed's code determins the crypto key-pair algorithm and signing suite + The signature type, Cigar or Siger, and when indexed the Siger code + may be completely determined by the seed and index values (index, ondex) + by assuming that the index values are intentional. + Without the seed code its more difficult for Siger to + determine when for the Indexer code value should be changed from the + than the provided value with respect to provided but incompatible index + values versus error conditions. + + Parameters: + ser (bytes): serialization to be signed + seed (bytes): raw binary seed (private key) + verfer (Verfer): instance. verfer.raw is public key + index (int |None): main index offset into list such as current signing + None means return non-indexed Cigar + Not None means return indexed Siger with Indexer code derived + from index, conly, and ondex values + only (bool): True means main index only list, ondex ignored + False means both index lists (default), ondex used + ondex (int | None): other index offset into list such as prior next + """ + # compute raw signature sig using seed on serialization ser + d = int.from_bytes(seed, byteorder="big") + sigkey = ec.derive_private_key(d, ec.SECP256R1()) + der = sigkey.sign(ser, ec.ECDSA(hashes.SHA256())) + (r, s) = utils.decode_dss_signature(der) + sig = bytearray(r.to_bytes(32, "big")) + sig.extend(s.to_bytes(32, "big")) + + if index is None: # Must be Cigar i.e. non-indexed signature + return Cigar(raw=sig, code=MtrDex.ECDSA_256r1_Sig, verfer=verfer) + else: # Must be Siger i.e. indexed signature + # should add Indexer class method to get ms main index size for given code + if only: # only main index ondex not used + ondex = None + if index <= 63: # (64 ** ms - 1) where ms is main index size + code = IdrDex.ECDSA_256r1_Crt_Sig # use small current only + else: + code = IdrDex.ECDSA_256r1_Big_Crt_Sig # use big current only + else: # both + if ondex == None: + ondex = index # enable default to be same + if ondex == index and index <= 63: # both same and small + code = IdrDex.ECDSA_256r1_Sig # use small both same + else: # otherwise big or both not same so use big both + code = IdrDex.ECDSA_256r1_Big_Sig # use use big both + + return Siger(raw=sig, code=code, index=index, ondex=ondex, verfer=verfer,) + + @staticmethod + def _secp256k1(ser, seed, verfer, index, only=False, ondex=None, **kwa): + """ + Returns signature as either Cigar or Siger instance as appropriate for + secp256k1 digital signatures given index and ondex values + + The seed's code determins the crypto key-pair algorithm and signing suite + The signature type, Cigar or Siger, and when indexed the Siger code + may be completely determined by the seed and index values (index, ondex) + by assuming that the index values are intentional. + Without the seed code its more difficult for Siger to + determine when for the Indexer code value should be changed from the + than the provided value with respect to provided but incompatible index + values versus error conditions. + + Parameters: + ser (bytes): serialization to be signed + seed (bytes): raw binary seed (private key) + verfer (Verfer): instance. verfer.raw is public key + index (int |None): main index offset into list such as current signing + None means return non-indexed Cigar + Not None means return indexed Siger with Indexer code derived + from index, conly, and ondex values + only (bool): True means main index only list, ondex ignored + False means both index lists (default), ondex used + ondex (int | None): other index offset into list such as prior next + """ + # compute raw signature sig using seed on serialization ser + d = int.from_bytes(seed, byteorder="big") + sigkey = ec.derive_private_key(d, ec.SECP256K1()) + der = sigkey.sign(ser, ec.ECDSA(hashes.SHA256())) + (r, s) = utils.decode_dss_signature(der) + sig = bytearray(r.to_bytes(32, "big")) + sig.extend(s.to_bytes(32, "big")) + + if index is None: # Must be Cigar i.e. non-indexed signature + return Cigar(raw=sig, code=MtrDex.ECDSA_256k1_Sig, verfer=verfer) + else: # Must be Siger i.e. indexed signature + # should add Indexer class method to get ms main index size for given code + if only: # only main index ondex not used + ondex = None + if index <= 63: # (64 ** ms - 1) where ms is main index size + code = IdrDex.ECDSA_256k1_Crt_Sig # use small current only + else: + code = IdrDex.ECDSA_256k1_Big_Crt_Sig # use big current only + else: # both + if ondex == None: + ondex = index # enable default to be same + if ondex == index and index <= 63: # both same and small + code = IdrDex.ECDSA_256k1_Sig # use small both same + else: # otherwise big or both not same so use big both + code = IdrDex.ECDSA_256k1_Big_Sig # use use big both + + return Siger(raw=sig, code=code, index=index, ondex=ondex, verfer=verfer,) From b8871fa73a5b402b36bb4cd630cc0f04a8fbb9b8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 9 Apr 2024 17:51:38 -0600 Subject: [PATCH 133/418] Refactor move Signer and classes with hard cross dependencies on Signer --- src/keri/app/cli/commands/passcode/set.py | 14 +- src/keri/app/cli/commands/salt.py | 4 +- src/keri/app/cli/commands/witness/demo.py | 9 +- src/keri/app/habbing.py | 11 +- src/keri/app/keeping.py | 31 +- src/keri/core/__init__.py | 11 +- src/keri/core/coring.py | 756 +--------------------- src/keri/core/signing.py | 451 +++++++++++++ src/keri/db/subing.py | 23 +- src/keri/demo/demo_bob.py | 11 +- src/keri/demo/demo_eve.py | 10 +- src/keri/demo/demo_sam.py | 10 +- src/keri/vc/proving.py | 6 +- tests/app/cli/test_kli_commands.py | 16 +- tests/app/test_agenting.py | 29 +- tests/app/test_delegating.py | 12 +- tests/app/test_directing.py | 11 +- tests/app/test_forwarding.py | 7 +- tests/app/test_habbing.py | 48 +- tests/app/test_keeping.py | 93 +-- tests/app/test_signify.py | 10 +- tests/app/test_signing.py | 14 +- tests/comply/test_direct_mode.py | 14 +- tests/core/test_bare.py | 10 +- tests/core/test_coring.py | 695 +------------------- tests/core/test_delegating.py | 22 +- tests/core/test_escrow.py | 20 +- tests/core/test_eventing.py | 39 +- tests/core/test_indexing.py | 2 +- tests/core/test_kevery.py | 26 +- tests/core/test_keystate.py | 10 +- tests/core/test_parsing.py | 14 +- tests/core/test_parsing_pathed.py | 11 +- tests/core/test_partial_rotation.py | 6 +- tests/core/test_replay.py | 12 +- tests/core/test_reply.py | 9 +- tests/core/test_serdering.py | 16 +- tests/core/test_signing.py | 717 ++++++++++++++++++++ tests/core/test_weighted_threshold.py | 10 +- tests/core/test_witness.py | 12 +- tests/db/test_basing.py | 22 +- tests/db/test_escrowing.py | 8 +- tests/db/test_subing.py | 65 +- tests/demo/test_demo.py | 32 +- tests/end/test_ending.py | 14 +- tests/peer/test_exchanging.py | 7 +- tests/vc/test_protocoling.py | 10 +- tests/vc/test_proving.py | 13 +- tests/vc/test_walleting.py | 8 +- tests/vdr/__init__.py | 7 +- tests/vdr/test_eventing.py | 3 +- tests/vdr/test_txn_state.py | 19 +- 52 files changed, 1662 insertions(+), 1778 deletions(-) create mode 100644 tests/core/test_signing.py diff --git a/src/keri/app/cli/commands/passcode/set.py b/src/keri/app/cli/commands/passcode/set.py index a31f978bb..0bdaba9ce 100644 --- a/src/keri/app/cli/commands/passcode/set.py +++ b/src/keri/app/cli/commands/passcode/set.py @@ -7,13 +7,19 @@ import getpass from hio import help + from hio.base import doing -from keri.app.cli.common import existing -from keri.core import coring from keri.kering import ConfigurationError -logger = help.ogler.getLogger() +from keri import core +from keri.core import coring + +from keri.app.cli.common import existing + + + +logger = help.ogler.getLogger() # I think this should be keri.help not hio.help parser = argparse.ArgumentParser(description='Initialize a prefix') parser.set_defaults(handler=lambda args: set_passcode(args), @@ -61,7 +67,7 @@ def set_passcode(tymth, tock=0.0, **opts): break bran = coring.MtrDex.Salt_128 + newpasscode[:22] # qb64 salt for seed - signer = coring.Salter(qb64=bran).signer(transferable=False, + signer = core.Salter(qb64=bran).signer(transferable=False, temp=False) seed = signer.qb64 aeid = signer.verfer.qb64 diff --git a/src/keri/app/cli/commands/salt.py b/src/keri/app/cli/commands/salt.py index 274ccd5de..0818c6d27 100644 --- a/src/keri/app/cli/commands/salt.py +++ b/src/keri/app/cli/commands/salt.py @@ -8,7 +8,7 @@ import pysodium from hio.base import doing -from keri.core import coring +from keri import core parser = argparse.ArgumentParser(description='Print a new random passcode') parser.set_defaults(handler=lambda args: handler(args)) @@ -23,4 +23,4 @@ def passcode(tymth, tock=0.0): """ _ = (yield tock) - print(coring.Salter(raw=pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES)).qb64) + print(core.Salter(raw=pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES)).qb64) diff --git a/src/keri/app/cli/commands/witness/demo.py b/src/keri/app/cli/commands/witness/demo.py index dd83dc849..912fa0417 100644 --- a/src/keri/app/cli/commands/witness/demo.py +++ b/src/keri/app/cli/commands/witness/demo.py @@ -11,10 +11,13 @@ from hio.base import doing -from keri.app import habbing, indirecting, configing -from keri.core.coring import Salter from keri import help +from keri.app import habbing, indirecting, configing + +from keri.core import Salter + + parser = argparse.ArgumentParser(description="Run a demo collection of witnesses") parser.set_defaults(handler=lambda args: demo(args)) @@ -57,7 +60,7 @@ def __init__(self, wan, wil, wes, wit, wub, wyz): self.wit = wit self.wub = wub self.wyz = wyz - + super(InitDoer, self).__init__(doers=[doing.doify(self.initialize)]) def initialize(self, tymth, tock=0.0): diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 6da9ed2dc..95cbc8d51 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -16,6 +16,7 @@ from . import keeping, configing from .. import help from .. import kering +from .. import core from ..core import coring, eventing, parsing, routing, serdering, indexing from ..db import dbing, basing from ..kering import MissingSignatureError, Roles @@ -69,7 +70,7 @@ def openHby(*, name="test", base="", temp=True, salt=None, **kwa): """ habery = None - salt = salt if salt is not None else coring.Salter().qb64 + salt = salt if salt is not None else core.Salter().qb64 try: habery = Habery(name=name, base=base, temp=temp, salt=salt, **kwa) yield habery @@ -96,7 +97,7 @@ def openHab(name="test", base="", salt=None, temp=True, cf=None, **kwa): """ - salt = coring.Salter(raw=salt).qb64 + salt = core.Salter(raw=salt).qb64 with openHby(name=name, base=base, salt=salt, temp=temp, cf=cf) as hby: if (hab := hby.habByName(name)) is None: @@ -290,7 +291,7 @@ def setup(self, *, seed=None, aeid=None, bran=None, pidx=None, algo=None, if len(bran) < 21: raise ValueError(f"Bran (passcode seed material) too short.") bran = coring.MtrDex.Salt_128 + 'A' + bran[:21] # qb64 salt for seed - signer = coring.Salter(qb64=bran).signer(transferable=False, + signer = core.Salter(qb64=bran).signer(transferable=False, tier=tier, temp=temp) seed = signer.qb64 @@ -298,9 +299,9 @@ def setup(self, *, seed=None, aeid=None, bran=None, pidx=None, algo=None, aeid = signer.verfer.qb64 # lest it remove encryption if salt is None: # salt for signing keys not aeid seed - salt = coring.Salter().qb64 + salt = core.Salter().qb64 else: - salt = coring.Salter(qb64=salt).qb64 + salt = core.Salter(qb64=salt).qb64 try: self.mgr = keeping.Manager(ks=self.ks, seed=seed, aeid=aeid, pidx=pidx, diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index 085957fa5..3a626a622 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -30,6 +30,7 @@ from hio.base import doing from .. import kering +from .. import core from ..core import coring from ..db import dbing, subing, koming from ..help import helping @@ -267,10 +268,10 @@ def reopen(self, **kwa): self.pris = subing.CryptSignerSuber(db=self, subkey='pris.') self.prxs = subing.CesrSuber(db=self, subkey='prxs.', - klas=coring.Cipher) + klas=core.Cipher) self.nxts = subing.CesrSuber(db=self, subkey='nxts.', - klas=coring.Cipher) + klas=core.Cipher) self.smids = subing.CatCesrIoSetSuber(db=self, subkey='smids.', klas=(coring.Prefixer, coring.Seqner)) @@ -444,7 +445,7 @@ def create(self, codes=None, count=1, code=coring.MtrDex.Ed25519_Seed, codes = [code for i in range(count)] for code in codes: - signers.append(coring.Signer(code=code, transferable=transferable)) + signers.append(core.Signer(code=code, transferable=transferable)) return signers @@ -477,7 +478,7 @@ def __init__(self, salt=None, stem=None, tier=None, **kwa): """ super(SaltyCreator, self).__init__(**kwa) - self.salter = coring.Salter(qb64=salt, tier=tier) + self.salter = core.Salter(qb64=salt, tier=tier) self._stem = stem if stem is not None else '' @property @@ -596,9 +597,9 @@ class Manager: Attributes: ks (Keeper): key store LMDB database instance for storing public and private keys - encrypter (coring.Encrypter): instance for encrypting secrets. Public + encrypter (core.Encrypter): instance for encrypting secrets. Public encryption key is derived from aeid (public signing key) - decrypter (coring.Decrypter): instance for decrypting secrets. Private + decrypter (core.Decrypter): instance for decrypting secrets. Private decryption key is derived seed (private signing key seed) inited (bool): True means fully initialized wrt database. False means not yet fully initialized @@ -724,13 +725,13 @@ def setup(self, aeid=None, pidx=None, algo=None, salt=None, tier=None): if algo is None: algo = Algos.salty if salt is None: - salt = coring.Salter().qb64 + salt = core.Salter().qb64 else: - if coring.Salter(qb64=salt).qb64 != salt: + if core.Salter(qb64=salt).qb64 != salt: raise ValueError(f"Invalid qb64 for salt={salt}.") if tier is None: - tier = coring.Tiers.low + tier = core.Tiers.low # update database if never before initialized if self.pidx is None: # never before initialized @@ -749,13 +750,13 @@ def setup(self, aeid=None, pidx=None, algo=None, salt=None, tier=None): if not self.aeid: # never before initialized self.updateAeid(aeid, self.seed) else: - self.encrypter = coring.Encrypter(verkey=self.aeid) # derive encrypter from aeid + self.encrypter = core.Encrypter(verkey=self.aeid) # derive encrypter from aeid if not self.seed or not self.encrypter.verifySeed(self.seed): raise kering.AuthError("Last seed missing or provided last seed " "not associated with last aeid={}." "".format(self.aeid)) - self.decrypter = coring.Decrypter(seed=self.seed) + self.decrypter = core.Decrypter(seed=self.seed) self.inited = True @@ -781,7 +782,7 @@ def updateAeid(self, aeid, seed): if aeid: # aeid provided if aeid != self.aeid: # changing to a new aeid so update .encrypter - self.encrypter = coring.Encrypter(verkey=aeid) # derive encrypter from aeid + self.encrypter = core.Encrypter(verkey=aeid) # derive encrypter from aeid # verifies new seed belongs to new aeid if not seed or not self.encrypter.verifySeed(seed): raise kering.AuthError("Seed missing or provided seed not associated" @@ -817,7 +818,7 @@ def updateAeid(self, aeid, seed): self._seed = seed # set .seed in memory # update .decrypter - self.decrypter = coring.Decrypter(seed=seed) if seed else None + self.decrypter = core.Decrypter(seed=seed) if seed else None @property @@ -1185,7 +1186,7 @@ def rotate(self, pre, ncodes=None, ncount=1, raise kering.DecryptError("Unauthorized decryption. Aeid but no decrypter.") salt = self.decrypter.decrypt(ser=salt).qb64 else: - salt = coring.Salter(qb64=salt).qb64 # ensures salt was unencrypted + salt = core.Salter(qb64=salt).qb64 # ensures salt was unencrypted creator = Creatory(algo=pp.algo).make(salt=salt, stem=pp.stem, tier=pp.tier) @@ -1531,7 +1532,7 @@ def ingest(self, secrecies, iridx=0, ncount=1, ncode=coring.MtrDex.Ed25519_Seed, secrecies = deque(secrecies) while secrecies: csecrets = secrecies.popleft() # current - csigners = [coring.Signer(qb64=secret, transferable=transferable) + csigners = [core.Signer(qb64=secret, transferable=transferable) for secret in csecrets] csize = len(csigners) verferies.append([signer.verfer for signer in csigners]) diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index aa271e140..ea1b52de1 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -6,11 +6,16 @@ #__all__ = ["coring", "eventing", "parsing", "scheming"] + +# Constants etc +from .coring import (Tiers, ) + # Matter class and its subclasses from .coring import (Matter, MtrDex, Number, NumDex, Dater, Texter, - Bexter, Pather, Verfer, Cigar, Signer, Salter, - Cipher, Encrypter, Decrypter, Diger, DigDex, + Bexter, Pather, Verfer, Cigar, Diger, DigDex, Prefixer, PreDex, ) from .coring import Tholder -from .indexing import Siger, IdrDex, IdxSigDex +from .indexing import Indexer, Siger, IdrDex, IdxSigDex +from .signing import (Signer, Salter, Cipher, Encrypter, Decrypter, + generateSigners, ) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 7caaa7e5d..599b948d7 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -47,7 +47,8 @@ from ..help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, codeB64ToB2, codeB2ToB64, Reb64, nabSextets) -from . import indexing + + DSS_SIG_MODE = "fips-186-3" @@ -183,41 +184,6 @@ def loads(raw, size=None, kind=Serials.json): return ked -def generateSigners(raw=None, count=8, transferable=True): - """Returns list of Signers for Ed25519 - - Deprecated, use Salter.signers instead. - - Use this when simply need valid AIDs but not when need valid controller - contexts. In the latter case use openHby or openHab which create databases. - - Parameters: - raw (bytes): 16 byte long salt cryptomatter from which seeds - for Signers in list are derived - random salt created if not provided - count is number of signers in list - transferable is boolean true means signer.verfer code is transferable - non-transferable otherwise - """ - if not raw: - raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - - signers = [] - for i in range(count): - path = f"{i:x}" - # algorithm default is argon2id - seed = pysodium.crypto_pwhash(outlen=32, - passwd=path, - salt=raw, - opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, - memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, - alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) - - signers.append(Signer(raw=seed, transferable=transferable)) - - return signers - - # ToDo: nonces only need 128 bits of entropy. a Salt is enough # Just use Salter().qb64. # Deprecated @@ -3149,724 +3115,6 @@ def verfer(self, verfer): self._verfer = verfer -class Signer(Matter): - """ - Signer is Matter subclass with method to create signature of serialization - using: - .raw as signing (private) key seed, - .code as cipher suite for signing - .verfer whose property .raw is public key for signing. - - If not provided .verfer is generated from private key seed using .code - as cipher suite for creating key-pair. - - - See Matter for inherited attributes and properties: - - Attributes: - - Properties: (inherited) - code (str): hard part of derivation code to indicate cypher suite - both (int): hard and soft parts of full text code - size (int): Number of triplets of bytes including lead bytes - (quadlets of chars) of variable sized material. Value of soft size, - ss, part of full text code. - Otherwise None. - rize (int): number of bytes of raw material not including - lead bytes - raw (bytes): private signing key crypto material only without code - qb64 (str): private signing key Base64 fully qualified with - derivation code + crypto mat - qb64b (bytes): private signing keyBase64 fully qualified with - derivation code + crypto mat - qb2 (bytes): private signing key binary with - derivation code + crypto material - transferable (bool): True means transferable derivation code False otherwise - digestive (bool): True means digest derivation code False otherwise - - Properties: - - .verfer is Verfer object instance of public key derived from private key - seed which is .raw - - Methods: - sign: create signature - - """ - - def __init__(self, raw=None, code=MtrDex.Ed25519_Seed, transferable=True, **kwa): - """ - Assign signing cipher suite function to ._sign - - Parameters: See Matter for inherted parameters - raw is bytes crypto material seed or private key - code is derivation code - transferable is Boolean True means make verifier code transferable - False make non-transferable - - """ - try: - super(Signer, self).__init__(raw=raw, code=code, **kwa) - except EmptyMaterialError as ex: - if code == MtrDex.Ed25519_Seed: - raw = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - super(Signer, self).__init__(raw=raw, code=code, **kwa) - elif code == MtrDex.ECDSA_256r1_Seed: - raw = pysodium.randombytes(ECDSA_256r1_SEEDBYTES) - super(Signer, self).__init__(raw=bytes(raw), code=code, **kwa) - elif code == MtrDex.ECDSA_256k1_Seed: - raw = pysodium.randombytes(ECDSA_256k1_SEEDBYTES) - super(Signer, self).__init__(raw=bytes(raw), code=code, **kwa) - - else: - raise ValueError("Unsupported signer code = {}.".format(code)) - - if self.code == MtrDex.Ed25519_Seed: - self._sign = self._ed25519 - verkey, sigkey = pysodium.crypto_sign_seed_keypair(self.raw) - verfer = Verfer(raw=verkey, - code=MtrDex.Ed25519 if transferable - else MtrDex.Ed25519N) - elif self.code == MtrDex.ECDSA_256r1_Seed: - self._sign = self._secp256r1 - d = int.from_bytes(self.raw, byteorder="big") - sigkey = ec.derive_private_key(d, ec.SECP256R1()) - verkey = sigkey.public_key().public_bytes(encoding=Encoding.X962, format=PublicFormat.CompressedPoint) - verfer = Verfer(raw=verkey, - code=MtrDex.ECDSA_256r1 if transferable - else MtrDex.ECDSA_256r1N) - elif self.code == MtrDex.ECDSA_256k1_Seed: - self._sign = self._secp256k1 - d = int.from_bytes(self.raw, byteorder="big") - sigkey = ec.derive_private_key(d, ec.SECP256K1()) - verkey = sigkey.public_key().public_bytes(encoding=Encoding.X962, format=PublicFormat.CompressedPoint) - verfer = Verfer(raw=verkey, - code=MtrDex.ECDSA_256k1 if transferable - else MtrDex.ECDSA_256k1N) - else: - raise ValueError("Unsupported signer code = {}.".format(self.code)) - - self._verfer = verfer - - @property - def verfer(self): - """ - Property verfer: - Returns Verfer instance - Assumes ._verfer is correctly assigned - """ - return self._verfer - - def sign(self, ser, index=None, only=False, ondex=None, **kwa): - """ - Returns either Cigar or Siger (indexed) instance of cryptographic - signature material on bytes serialization ser - - If index is None - return Cigar instance - Else - return Siger instance - - Parameters: - ser (bytes): serialization to be signed - index (int): main index of associated verifier key in event keys - only (bool): True means main index only list, ondex ignored - False means both index lists (default), ondex used - ondex (int | None): other index offset into list such as prior next - - """ - return (self._sign(ser=ser, - seed=self.raw, - verfer=self.verfer, - index=index, - only=only, - ondex=ondex, - **kwa)) - - @staticmethod - def _ed25519(ser, seed, verfer, index, only=False, ondex=None, **kwa): - """ - Returns signature as either Cigar or Siger instance as appropriate for - Ed25519 digital signatures given index and ondex values - - The seed's code determins the crypto key-pair algorithm and signing suite - The signature type, Cigar or Siger, and when indexed the Siger code - may be completely determined by the seed and index values (index, ondex) - by assuming that the index values are intentional. - Without the seed code its more difficult for Siger to - determine when for the Indexer code value should be changed from the - than the provided value with respect to provided but incompatible index - values versus error conditions. - - Parameters: - ser (bytes): serialization to be signed - seed (bytes): raw binary seed (private key) - verfer (Verfer): instance. verfer.raw is public key - index (int |None): main index offset into list such as current signing - None means return non-indexed Cigar - Not None means return indexed Siger with Indexer code derived - from index, conly, and ondex values - only (bool): True means main index only list, ondex ignored - False means both index lists (default), ondex used - ondex (int | None): other index offset into list such as prior next - """ - # compute raw signature sig using seed on serialization ser - sig = pysodium.crypto_sign_detached(ser, seed + verfer.raw) - - if index is None: # Must be Cigar i.e. non-indexed signature - return Cigar(raw=sig, code=MtrDex.Ed25519_Sig, verfer=verfer) - else: # Must be Siger i.e. indexed signature - # should add Indexer class method to get ms main index size for given code - if only: # only main index ondex not used - ondex = None - if index <= 63: # (64 ** ms - 1) where ms is main index size - code = indexing.IdrDex.Ed25519_Crt_Sig # use small current only - else: - code = indexing.IdrDex.Ed25519_Big_Crt_Sig # use big current only - else: # both - if ondex == None: - ondex = index # enable default to be same - if ondex == index and index <= 63: # both same and small - code = indexing.IdrDex.Ed25519_Sig # use small both same - else: # otherwise big or both not same so use big both - code = indexing.IdrDex.Ed25519_Big_Sig # use use big both - - return indexing.Siger(raw=sig, - code=code, - index=index, - ondex=ondex, - verfer=verfer,) - - @staticmethod - def _secp256r1(ser, seed, verfer, index, only=False, ondex=None, **kwa): - """ - Returns signature as either Cigar or Siger instance as appropriate for - Ed25519 digital signatures given index and ondex values - - The seed's code determins the crypto key-pair algorithm and signing suite - The signature type, Cigar or Siger, and when indexed the Siger code - may be completely determined by the seed and index values (index, ondex) - by assuming that the index values are intentional. - Without the seed code its more difficult for Siger to - determine when for the Indexer code value should be changed from the - than the provided value with respect to provided but incompatible index - values versus error conditions. - - Parameters: - ser (bytes): serialization to be signed - seed (bytes): raw binary seed (private key) - verfer (Verfer): instance. verfer.raw is public key - index (int |None): main index offset into list such as current signing - None means return non-indexed Cigar - Not None means return indexed Siger with Indexer code derived - from index, conly, and ondex values - only (bool): True means main index only list, ondex ignored - False means both index lists (default), ondex used - ondex (int | None): other index offset into list such as prior next - """ - # compute raw signature sig using seed on serialization ser - d = int.from_bytes(seed, byteorder="big") - sigkey = ec.derive_private_key(d, ec.SECP256R1()) - der = sigkey.sign(ser, ec.ECDSA(hashes.SHA256())) - (r, s) = utils.decode_dss_signature(der) - sig = bytearray(r.to_bytes(32, "big")) - sig.extend(s.to_bytes(32, "big")) - - if index is None: # Must be Cigar i.e. non-indexed signature - return Cigar(raw=sig, code=MtrDex.ECDSA_256r1_Sig, verfer=verfer) - else: # Must be Siger i.e. indexed signature - # should add Indexer class method to get ms main index size for given code - if only: # only main index ondex not used - ondex = None - if index <= 63: # (64 ** ms - 1) where ms is main index size - code = indexing.IdrDex.ECDSA_256r1_Crt_Sig # use small current only - else: - code = indexing.IdrDex.ECDSA_256r1_Big_Crt_Sig # use big current only - else: # both - if ondex == None: - ondex = index # enable default to be same - if ondex == index and index <= 63: # both same and small - code = indexing.IdrDex.ECDSA_256r1_Sig # use small both same - else: # otherwise big or both not same so use big both - code = indexing.IdrDex.ECDSA_256r1_Big_Sig # use use big both - - return indexing.Siger(raw=sig, - code=code, - index=index, - ondex=ondex, - verfer=verfer,) - - @staticmethod - def _secp256k1(ser, seed, verfer, index, only=False, ondex=None, **kwa): - """ - Returns signature as either Cigar or Siger instance as appropriate for - secp256k1 digital signatures given index and ondex values - - The seed's code determins the crypto key-pair algorithm and signing suite - The signature type, Cigar or Siger, and when indexed the Siger code - may be completely determined by the seed and index values (index, ondex) - by assuming that the index values are intentional. - Without the seed code its more difficult for Siger to - determine when for the Indexer code value should be changed from the - than the provided value with respect to provided but incompatible index - values versus error conditions. - - Parameters: - ser (bytes): serialization to be signed - seed (bytes): raw binary seed (private key) - verfer (Verfer): instance. verfer.raw is public key - index (int |None): main index offset into list such as current signing - None means return non-indexed Cigar - Not None means return indexed Siger with Indexer code derived - from index, conly, and ondex values - only (bool): True means main index only list, ondex ignored - False means both index lists (default), ondex used - ondex (int | None): other index offset into list such as prior next - """ - # compute raw signature sig using seed on serialization ser - d = int.from_bytes(seed, byteorder="big") - sigkey = ec.derive_private_key(d, ec.SECP256K1()) - der = sigkey.sign(ser, ec.ECDSA(hashes.SHA256())) - (r, s) = utils.decode_dss_signature(der) - sig = bytearray(r.to_bytes(32, "big")) - sig.extend(s.to_bytes(32, "big")) - - if index is None: # Must be Cigar i.e. non-indexed signature - return Cigar(raw=sig, code=MtrDex.ECDSA_256k1_Sig, verfer=verfer) - else: # Must be Siger i.e. indexed signature - # should add Indexer class method to get ms main index size for given code - if only: # only main index ondex not used - ondex = None - if index <= 63: # (64 ** ms - 1) where ms is main index size - code = indexing.IdrDex.ECDSA_256k1_Crt_Sig # use small current only - else: - code = indexing.IdrDex.ECDSA_256k1_Big_Crt_Sig # use big current only - else: # both - if ondex == None: - ondex = index # enable default to be same - if ondex == index and index <= 63: # both same and small - code = indexing.IdrDex.ECDSA_256k1_Sig # use small both same - else: # otherwise big or both not same so use big both - code = indexing.IdrDex.ECDSA_256k1_Big_Sig # use use big both - - return indexing.Siger(raw=sig, - code=code, - index=index, - ondex=ondex, - verfer=verfer,) - - -class Salter(Matter): - """ - Salter is Matter subclass to maintain random salt for secrets (private keys) - Its .raw is random salt, .code as cipher suite for salt - - Attributes: - .level is str security level code. Provides default level - - Inherited Properties - .pad is int number of pad chars given raw - .code is str derivation code to indicate cypher suite - .raw is bytes crypto material only without code - .index is int count of attached crypto material by context (receipts) - .qb64 is str in Base64 fully qualified with derivation code + crypto mat - .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat - .qb2 is bytes in binary with derivation code + crypto material - .transferable is Boolean, True when transferable derivation code False otherwise - - Properties: - - Methods: - - Hidden: - ._pad is method to compute .pad property - ._code is str value for .code property - ._raw is bytes value for .raw property - ._index is int value for .index property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 - - """ - Tier = Tiers.low - - def __init__(self, raw=None, code=MtrDex.Salt_128, tier=None, **kwa): - """ - Initialize salter's raw and code - - Inherited Parameters: - raw is bytes of unqualified crypto material usable for crypto operations - qb64b is bytes of fully qualified crypto material - qb64 is str or bytes of fully qualified crypto material - qb2 is bytes of fully qualified crypto material - code is str of derivation code - index is int of count of attached receipts for CryCntDex codes - - Parameters: - - """ - try: - super(Salter, self).__init__(raw=raw, code=code, **kwa) - except EmptyMaterialError as ex: - if code == MtrDex.Salt_128: - raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - super(Salter, self).__init__(raw=raw, code=code, **kwa) - else: - raise ValueError("Unsupported salter code = {}.".format(code)) - - if self.code not in (MtrDex.Salt_128,): - raise ValueError("Unsupported salter code = {}.".format(self.code)) - - self.tier = tier if tier is not None else self.Tier - - def stretch(self, *, size=32, path="", tier=None, temp=False): - """ - Returns (bytes): raw binary seed (secret) derived from path and .raw - and stretched to size given by code using argon2d stretching algorithm. - - Parameters: - size (int): number of bytes in stretched seed - path (str): unique chars used in derivation of seed (secret) - tier (str): value from Tierage for security level of stretch - temp is Boolean, True means use quick method to stretch salt - for testing only, Otherwise use time set by tier to stretch - """ - tier = tier if tier is not None else self.tier - - if temp: - opslimit = 1 # pysodium.crypto_pwhash_OPSLIMIT_MIN - memlimit = 8192 # pysodium.crypto_pwhash_MEMLIMIT_MIN - else: - if tier == Tiers.low: - opslimit = 2 # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE - memlimit = 67108864 # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE - elif tier == Tiers.med: - opslimit = 3 # pysodium.crypto_pwhash_OPSLIMIT_MODERATE - memlimit = 268435456 # pysodium.crypto_pwhash_MEMLIMIT_MODERATE - elif tier == Tiers.high: - opslimit = 4 # pysodium.crypto_pwhash_OPSLIMIT_SENSITIVE - memlimit = 1073741824 # pysodium.crypto_pwhash_MEMLIMIT_SENSITIVE - else: - raise ValueError("Unsupported security tier = {}.".format(tier)) - - # stretch algorithm is argon2id - seed = pysodium.crypto_pwhash(outlen=size, - passwd=path, - salt=self.raw, - opslimit=opslimit, - memlimit=memlimit, - alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) - return (seed) - - def signer(self, *, code=MtrDex.Ed25519_Seed, transferable=True, path="", - tier=None, temp=False): - """ - Returns Signer instance whose .raw secret is derived from path and - salter's .raw and stretched to size given by code. The signers public key - for its .verfer is derived from code and transferable. - - Parameters: - code is str code of secret crypto suite - transferable is Boolean, True means use transferace code for public key - path is str of unique chars used in derivation of secret seed for signer - tier is str Tierage security level - temp is Boolean, True means use quick method to stretch salt - for testing only, Otherwise use more time to stretch - """ - seed = self.stretch(size=Matter._rawSize(code), path=path, tier=tier, - temp=temp) - - return (Signer(raw=seed, code=code, transferable=transferable)) - - - def signers(self, count=1, start=0, path="", **kwa): - """ - Returns list of count number of Signer instances with unique derivation - path made from path prefix and suffix of start plus offset for each count - value from 0 to count - 1. - - See .signer for parameters used to create each signer. - - """ - return [self.signer(path=f"{path}{i + start:x}", **kwa) for i in range(count)] - - -class Cipher(Matter): - """ - Cipher is Matter subclass holding a cipher text of a secret that may be - either a secret seed (private key) or secret salt with appropriate CESR code - to indicate which kind (which indicates size). The cipher text is created - with assymetric encryption using an unrelated (public, private) - encryption/decryption key pair. The public key is used for encryption the - private key for decryption. The default is to use X25519 sealed box encryption. - - The Cipher instances .raw is the raw binary encrypted cipher text and its - .code indicates what type of secret has been encrypted. The cipher suite used - for the encryption/decryption is implied by the context where the cipher is - used. - - See Matter for inherited attributes and properties - - """ - - def __init__(self, raw=None, code=None, **kwa): - """ - Parmeters: - raw (Union[bytes, str]): cipher text - code (str): cipher suite - """ - if raw is not None and code is None: - if len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Salt): - code = MtrDex.X25519_Cipher_Salt - elif len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Seed): - code = MtrDex.X25519_Cipher_Seed - - if hasattr(raw, "encode"): - raw = raw.encode("utf-8") # ensure bytes not str - - super(Cipher, self).__init__(raw=raw, code=code, **kwa) - - if self.code not in (MtrDex.X25519_Cipher_Salt, MtrDex.X25519_Cipher_Seed): - raise ValueError("Unsupported cipher code = {}.".format(self.code)) - - def decrypt(self, prikey=None, seed=None): - """ - Returns plain text as Matter instance (Signer or Salter) of cryptographic - cipher text material given by .raw. Encrypted plain text is fully - qualified (qb64) so derivaton code of plain text preserved through - encryption/decryption round trip. - - Uses either decryption key given by prikey or derives prikey from - signing key derived from private seed. - - Parameters: - prikey (Union[bytes, str]): qb64b or qb64 serialization of private - decryption key - seed (Union[bytes, str]): qb64b or qb64 serialization of private - signing key seed used to derive private decryption key - """ - decrypter = Decrypter(qb64b=prikey, seed=seed) - return decrypter.decrypt(ser=self.qb64b) - - -class Encrypter(Matter): - """ - Encrypter is Matter subclass with method to create a cipher text of a - fully qualified (qb64) private key/seed where private key/seed is the plain - text. Encrypter uses assymetric (public, private) key encryption of a - serialization (plain text). Using its .raw as the encrypting (public) key and - its .code to indicate the cipher suite for the encryption operation. - - For example .code == MtrDex.X25519 indicates that X25519 sealed box - encyrption is used. The encryption key may be derived from an Ed25519 - signing public key that associated with a nontransferable or basic derivation - self certifying identifier. This allows use of the self certifying identifier - to track or manage the encryption/decryption key pair. And could be used to - provide additional authentication operations for using the - encryption/decryption key pair. Support for this is provided at init time - with the verkey parameter which allows deriving the encryption public key from - the fully qualified verkey (signature verification key). - - See Matter for inherited attributes and properties: - - Methods: - encrypt: returns cipher text - - """ - - def __init__(self, raw=None, code=MtrDex.X25519, verkey=None, **kwa): - """ - Assign encrypting cipher suite function to ._encrypt - - Parameters: See Matter for inherted parameters such as qb64, qb64b - raw (bytes): public encryption key - qb64b (bytes): fully qualified public encryption key - qb64 (str): fully qualified public encryption key - code (str): derivation code for public encryption key - verkey (Union[bytes, str]): qb64b or qb64 of verkey used to derive raw - """ - if not raw and verkey: - verfer = Verfer(qb64b=verkey) - if verfer.code not in (MtrDex.Ed25519N, MtrDex.Ed25519): - raise ValueError("Unsupported verkey derivation code = {}." - "".format(verfer.code)) - # convert signing public key to encryption public key - raw = pysodium.crypto_sign_pk_to_box_pk(verfer.raw) - - super(Encrypter, self).__init__(raw=raw, code=code, **kwa) - - if self.code == MtrDex.X25519: - self._encrypt = self._x25519 - else: - raise ValueError("Unsupported encrypter code = {}.".format(self.code)) - - def verifySeed(self, seed): - """ - Returns: - Boolean: True means private signing key seed corresponds to public - signing key verkey used to derive encrypter's .raw public - encryption key. - - Parameters: - seed (Union(bytes,str)): qb64b or qb64 serialization of private - signing key seed - """ - signer = Signer(qb64b=seed) - verkey, sigkey = pysodium.crypto_sign_seed_keypair(signer.raw) - pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) - return (pubkey == self.raw) - - def encrypt(self, ser=None, matter=None): - """ - Returns: - Cipher instance of cipher text encryption of plain text serialization - provided by either ser or Matter instance when provided. - - Parameters: - ser (Union[bytes,str]): qb64b or qb64 serialization of plain text - matter (Matter): plain text as Matter instance of seed or salt to - be encrypted - """ - if not (ser or matter): - raise EmptyMaterialError("Neither ser or plain are provided.") - - if ser: - matter = Matter(qb64b=ser) - - if matter.code == MtrDex.Salt_128: # future other salt codes - code = MtrDex.X25519_Cipher_Salt - elif matter.code == MtrDex.Ed25519_Seed: # future other seed codes - code = MtrDex.X25519_Cipher_Seed - else: - raise ValueError("Unsupported plain text code = {}.".format(matter.code)) - - # encrypting fully qualified qb64 version of plain text ensures its - # derivation code round trips through eventual decryption - return (self._encrypt(ser=matter.qb64b, pubkey=self.raw, code=code)) - - @staticmethod - def _x25519(ser, pubkey, code): - """ - Returns cipher text as Cipher instance - Parameters: - ser (Union[bytes, str]): qb64b or qb64 serialization of seed or salt - to be encrypted. - pubkey (bytes): raw binary serialization of encryption public key - code (str): derivation code of serialized plain text seed or salt - """ - raw = pysodium.crypto_box_seal(ser, pubkey) - return Cipher(raw=raw, code=code) - - -class Decrypter(Matter): - """ - Decrypter is Matter subclass with method to decrypt the plain text from a - ciper text of a fully qualified (qb64) private key/seed where private - key/seed is the plain text. Decrypter uses assymetric (public, private) key - decryption of the cipher text using its .raw as the decrypting (private) key - and its .code to indicate the cipher suite for the decryption operation. - - For example .code == MtrDex.X25519 indicates that X25519 sealed box - decyrption is used. The decryption key may be derived from an Ed25519 - signing private key that is associated with a nontransferable or basic derivation - self certifying identifier. This allows use of the self certifying identifier - to track or manage the encryption/decryption key pair. And could be used to - provide additional authentication operations for using the - encryption/decryption key pair. Support for this is provided at init time - with the sigkey parameter which allows deriving the decryption private key - from the fully qualified sigkey (signing key). - - See Matter for inherited attributes and properties: - - Attributes: - - Properties: - - - Methods: - decrypt: create cipher text - - """ - - def __init__(self, code=MtrDex.X25519_Private, seed=None, **kwa): - """ - Assign decrypting cipher suite function to ._decrypt - - Parameters: See Matter for inheirted parameters - raw (bytes): private decryption key derived from seed (private signing key) - qb64b (bytes): fully qualified private decryption key - qb64 (str): fully qualified private decryption key - code (str): derivation code for private decryption key - seed (Union[bytes, str]): qb64b or qb64 of signing key seed used to - derive raw which is private decryption key - """ - try: - super(Decrypter, self).__init__(code=code, **kwa) - except EmptyMaterialError as ex: - if seed: - signer = Signer(qb64b=seed) - if signer.code not in (MtrDex.Ed25519_Seed,): - raise ValueError("Unsupported signing seed derivation code = {}." - "".format(signer.code)) - # verkey, sigkey = pysodium.crypto_sign_seed_keypair(signer.raw) - sigkey = signer.raw + signer.verfer.raw # sigkey is raw seed + raw verkey - raw = pysodium.crypto_sign_sk_to_box_sk(sigkey) # raw private encrypt key - super(Decrypter, self).__init__(raw=raw, code=code, **kwa) - else: - raise - - if self.code == MtrDex.X25519_Private: - self._decrypt = self._x25519 - else: - raise ValueError("Unsupported decrypter code = {}.".format(self.code)) - - def decrypt(self, ser=None, cipher=None, transferable=False): - """ - Returns: - Salter or Signer instance derived from plain text decrypted from - encrypted cipher text material given by ser or cipher. Plain text - that is orignally encrypt should always be fully qualified (qb64b) - so that derivaton code of plain text is preserved through - encryption/decryption round trip. - - Parameters: - ser (Union[bytes,str]): qb64b or qb64 serialization of cipher text - cipher (Cipher): optional Cipher instance when ser is None - transferable (bool): True means associated verfer of returned - signer is transferable. False means non-transferable - """ - if not (ser or cipher): - raise EmptyMaterialError("Neither ser or cipher are provided.") - - if ser: # create cipher to ensure valid derivation code of material in ser - cipher = Cipher(qb64b=ser) - - return (self._decrypt(cipher=cipher, - prikey=self.raw, - transferable=transferable)) - - @staticmethod - def _x25519(cipher, prikey, transferable=False): - """ - Returns plain text as Salter or Signer instance depending on the cipher - code and the embedded encrypted plain text derivation code. - - Parameters: - cipher (Cipher): instance of encrypted seed or salt - prikey (bytes): raw binary decryption private key derived from - signing seed or sigkey - transferable (bool): True means associated verfer of returned - signer is transferable. False means non-transferable - """ - pubkey = pysodium.crypto_scalarmult_curve25519_base(prikey) - plain = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) # qb64b - # ensure raw plain text is qb64b or qb64 so its derivation code is round tripped - if cipher.code == MtrDex.X25519_Cipher_Salt: - return Salter(qb64b=plain) - elif cipher.code == MtrDex.X25519_Cipher_Seed: - return Signer(qb64b=plain, transferable=transferable) - else: - raise ValueError("Unsupported cipher text code = {}.".format(cipher.code)) - class Diger(Matter): """ diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index ad483762b..f3571016f 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -15,6 +15,7 @@ from ..help import helping +from .coring import (Tiers, ) from .coring import (Matter, MtrDex, Verfer, Cigar) from .indexing import IdrDex, Siger @@ -24,6 +25,43 @@ ECDSA_256k1_SEEDBYTES = 32 +# deprecated use Salter.signers instead +def generateSigners(raw=None, count=8, transferable=True): + """Returns list of Signers for Ed25519 + + Deprecated, use Salter.signers instead. + + Use this when simply need valid AIDs but not when need valid controller + contexts. In the latter case use openHby or openHab which create databases. + + Parameters: + raw (bytes): 16 byte long salt cryptomatter from which seeds + for Signers in list are derived + random salt created if not provided + count is number of signers in list + transferable is boolean true means signer.verfer code is transferable + non-transferable otherwise + """ + if not raw: + raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + + signers = [] + for i in range(count): + path = f"{i:x}" + # algorithm default is argon2id + seed = pysodium.crypto_pwhash(outlen=32, + passwd=path, + salt=raw, + opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, + memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, + alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) + + signers.append(Signer(raw=seed, transferable=transferable)) + + return signers + + + class Signer(Matter): """ @@ -318,3 +356,416 @@ def _secp256k1(ser, seed, verfer, index, only=False, ondex=None, **kwa): code = IdrDex.ECDSA_256k1_Big_Sig # use use big both return Siger(raw=sig, code=code, index=index, ondex=ondex, verfer=verfer,) + + + +class Salter(Matter): + """ + Salter is Matter subclass to maintain random salt for secrets (private keys) + Its .raw is random salt, .code as cipher suite for salt + + Attributes: + .level is str security level code. Provides default level + + Inherited Properties + .pad is int number of pad chars given raw + .code is str derivation code to indicate cypher suite + .raw is bytes crypto material only without code + .index is int count of attached crypto material by context (receipts) + .qb64 is str in Base64 fully qualified with derivation code + crypto mat + .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat + .qb2 is bytes in binary with derivation code + crypto material + .transferable is Boolean, True when transferable derivation code False otherwise + + Properties: + + Methods: + + Hidden: + ._pad is method to compute .pad property + ._code is str value for .code property + ._raw is bytes value for .raw property + ._index is int value for .index property + ._infil is method to compute fully qualified Base64 from .raw and .code + ._exfil is method to extract .code and .raw from fully qualified Base64 + + """ + Tier = Tiers.low + + def __init__(self, raw=None, code=MtrDex.Salt_128, tier=None, **kwa): + """ + Initialize salter's raw and code + + Inherited Parameters: + raw is bytes of unqualified crypto material usable for crypto operations + qb64b is bytes of fully qualified crypto material + qb64 is str or bytes of fully qualified crypto material + qb2 is bytes of fully qualified crypto material + code is str of derivation code + index is int of count of attached receipts for CryCntDex codes + + Parameters: + + """ + try: + super(Salter, self).__init__(raw=raw, code=code, **kwa) + except EmptyMaterialError as ex: + if code == MtrDex.Salt_128: + raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + super(Salter, self).__init__(raw=raw, code=code, **kwa) + else: + raise ValueError("Unsupported salter code = {}.".format(code)) + + if self.code not in (MtrDex.Salt_128,): + raise ValueError("Unsupported salter code = {}.".format(self.code)) + + self.tier = tier if tier is not None else self.Tier + + def stretch(self, *, size=32, path="", tier=None, temp=False): + """ + Returns (bytes): raw binary seed (secret) derived from path and .raw + and stretched to size given by code using argon2d stretching algorithm. + + Parameters: + size (int): number of bytes in stretched seed + path (str): unique chars used in derivation of seed (secret) + tier (str): value from Tierage for security level of stretch + temp is Boolean, True means use quick method to stretch salt + for testing only, Otherwise use time set by tier to stretch + """ + tier = tier if tier is not None else self.tier + + if temp: + opslimit = 1 # pysodium.crypto_pwhash_OPSLIMIT_MIN + memlimit = 8192 # pysodium.crypto_pwhash_MEMLIMIT_MIN + else: + if tier == Tiers.low: + opslimit = 2 # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE + memlimit = 67108864 # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE + elif tier == Tiers.med: + opslimit = 3 # pysodium.crypto_pwhash_OPSLIMIT_MODERATE + memlimit = 268435456 # pysodium.crypto_pwhash_MEMLIMIT_MODERATE + elif tier == Tiers.high: + opslimit = 4 # pysodium.crypto_pwhash_OPSLIMIT_SENSITIVE + memlimit = 1073741824 # pysodium.crypto_pwhash_MEMLIMIT_SENSITIVE + else: + raise ValueError("Unsupported security tier = {}.".format(tier)) + + # stretch algorithm is argon2id + seed = pysodium.crypto_pwhash(outlen=size, + passwd=path, + salt=self.raw, + opslimit=opslimit, + memlimit=memlimit, + alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) + return (seed) + + def signer(self, *, code=MtrDex.Ed25519_Seed, transferable=True, path="", + tier=None, temp=False): + """ + Returns Signer instance whose .raw secret is derived from path and + salter's .raw and stretched to size given by code. The signers public key + for its .verfer is derived from code and transferable. + + Parameters: + code is str code of secret crypto suite + transferable is Boolean, True means use transferace code for public key + path is str of unique chars used in derivation of secret seed for signer + tier is str Tierage security level + temp is Boolean, True means use quick method to stretch salt + for testing only, Otherwise use more time to stretch + """ + seed = self.stretch(size=Matter._rawSize(code), path=path, tier=tier, + temp=temp) + + return (Signer(raw=seed, code=code, transferable=transferable)) + + + def signers(self, count=1, start=0, path="", **kwa): + """ + Returns list of count number of Signer instances with unique derivation + path made from path prefix and suffix of start plus offset for each count + value from 0 to count - 1. + + See .signer for parameters used to create each signer. + + """ + return [self.signer(path=f"{path}{i + start:x}", **kwa) for i in range(count)] + + +class Cipher(Matter): + """ + Cipher is Matter subclass holding a cipher text of a secret that may be + either a secret seed (private key) or secret salt with appropriate CESR code + to indicate which kind (which indicates size). The cipher text is created + with assymetric encryption using an unrelated (public, private) + encryption/decryption key pair. The public key is used for encryption the + private key for decryption. The default is to use X25519 sealed box encryption. + + The Cipher instances .raw is the raw binary encrypted cipher text and its + .code indicates what type of secret has been encrypted. The cipher suite used + for the encryption/decryption is implied by the context where the cipher is + used. + + See Matter for inherited attributes and properties + + """ + + def __init__(self, raw=None, code=None, **kwa): + """ + Parmeters: + raw (Union[bytes, str]): cipher text + code (str): cipher suite + """ + if raw is not None and code is None: + if len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Salt): + code = MtrDex.X25519_Cipher_Salt + elif len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Seed): + code = MtrDex.X25519_Cipher_Seed + + if hasattr(raw, "encode"): + raw = raw.encode("utf-8") # ensure bytes not str + + super(Cipher, self).__init__(raw=raw, code=code, **kwa) + + if self.code not in (MtrDex.X25519_Cipher_Salt, MtrDex.X25519_Cipher_Seed): + raise ValueError("Unsupported cipher code = {}.".format(self.code)) + + def decrypt(self, prikey=None, seed=None): + """ + Returns plain text as Matter instance (Signer or Salter) of cryptographic + cipher text material given by .raw. Encrypted plain text is fully + qualified (qb64) so derivaton code of plain text preserved through + encryption/decryption round trip. + + Uses either decryption key given by prikey or derives prikey from + signing key derived from private seed. + + Parameters: + prikey (Union[bytes, str]): qb64b or qb64 serialization of private + decryption key + seed (Union[bytes, str]): qb64b or qb64 serialization of private + signing key seed used to derive private decryption key + """ + decrypter = Decrypter(qb64b=prikey, seed=seed) + return decrypter.decrypt(ser=self.qb64b) + + +class Encrypter(Matter): + """ + Encrypter is Matter subclass with method to create a cipher text of a + fully qualified (qb64) private key/seed where private key/seed is the plain + text. Encrypter uses assymetric (public, private) key encryption of a + serialization (plain text). Using its .raw as the encrypting (public) key and + its .code to indicate the cipher suite for the encryption operation. + + For example .code == MtrDex.X25519 indicates that X25519 sealed box + encyrption is used. The encryption key may be derived from an Ed25519 + signing public key that associated with a nontransferable or basic derivation + self certifying identifier. This allows use of the self certifying identifier + to track or manage the encryption/decryption key pair. And could be used to + provide additional authentication operations for using the + encryption/decryption key pair. Support for this is provided at init time + with the verkey parameter which allows deriving the encryption public key from + the fully qualified verkey (signature verification key). + + See Matter for inherited attributes and properties: + + Methods: + encrypt: returns cipher text + + """ + + def __init__(self, raw=None, code=MtrDex.X25519, verkey=None, **kwa): + """ + Assign encrypting cipher suite function to ._encrypt + + Parameters: See Matter for inherted parameters such as qb64, qb64b + raw (bytes): public encryption key + qb64b (bytes): fully qualified public encryption key + qb64 (str): fully qualified public encryption key + code (str): derivation code for public encryption key + verkey (Union[bytes, str]): qb64b or qb64 of verkey used to derive raw + """ + if not raw and verkey: + verfer = Verfer(qb64b=verkey) + if verfer.code not in (MtrDex.Ed25519N, MtrDex.Ed25519): + raise ValueError("Unsupported verkey derivation code = {}." + "".format(verfer.code)) + # convert signing public key to encryption public key + raw = pysodium.crypto_sign_pk_to_box_pk(verfer.raw) + + super(Encrypter, self).__init__(raw=raw, code=code, **kwa) + + if self.code == MtrDex.X25519: + self._encrypt = self._x25519 + else: + raise ValueError("Unsupported encrypter code = {}.".format(self.code)) + + def verifySeed(self, seed): + """ + Returns: + Boolean: True means private signing key seed corresponds to public + signing key verkey used to derive encrypter's .raw public + encryption key. + + Parameters: + seed (Union(bytes,str)): qb64b or qb64 serialization of private + signing key seed + """ + signer = Signer(qb64b=seed) + verkey, sigkey = pysodium.crypto_sign_seed_keypair(signer.raw) + pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) + return (pubkey == self.raw) + + def encrypt(self, ser=None, matter=None): + """ + Returns: + Cipher instance of cipher text encryption of plain text serialization + provided by either ser or Matter instance when provided. + + Parameters: + ser (Union[bytes,str]): qb64b or qb64 serialization of plain text + matter (Matter): plain text as Matter instance of seed or salt to + be encrypted + """ + if not (ser or matter): + raise EmptyMaterialError("Neither ser or plain are provided.") + + if ser: + matter = Matter(qb64b=ser) + + if matter.code == MtrDex.Salt_128: # future other salt codes + code = MtrDex.X25519_Cipher_Salt + elif matter.code == MtrDex.Ed25519_Seed: # future other seed codes + code = MtrDex.X25519_Cipher_Seed + else: + raise ValueError("Unsupported plain text code = {}.".format(matter.code)) + + # encrypting fully qualified qb64 version of plain text ensures its + # derivation code round trips through eventual decryption + return (self._encrypt(ser=matter.qb64b, pubkey=self.raw, code=code)) + + @staticmethod + def _x25519(ser, pubkey, code): + """ + Returns cipher text as Cipher instance + Parameters: + ser (Union[bytes, str]): qb64b or qb64 serialization of seed or salt + to be encrypted. + pubkey (bytes): raw binary serialization of encryption public key + code (str): derivation code of serialized plain text seed or salt + """ + raw = pysodium.crypto_box_seal(ser, pubkey) + return Cipher(raw=raw, code=code) + + +class Decrypter(Matter): + """ + Decrypter is Matter subclass with method to decrypt the plain text from a + ciper text of a fully qualified (qb64) private key/seed where private + key/seed is the plain text. Decrypter uses assymetric (public, private) key + decryption of the cipher text using its .raw as the decrypting (private) key + and its .code to indicate the cipher suite for the decryption operation. + + For example .code == MtrDex.X25519 indicates that X25519 sealed box + decyrption is used. The decryption key may be derived from an Ed25519 + signing private key that is associated with a nontransferable or basic derivation + self certifying identifier. This allows use of the self certifying identifier + to track or manage the encryption/decryption key pair. And could be used to + provide additional authentication operations for using the + encryption/decryption key pair. Support for this is provided at init time + with the sigkey parameter which allows deriving the decryption private key + from the fully qualified sigkey (signing key). + + See Matter for inherited attributes and properties: + + Attributes: + + Properties: + + + Methods: + decrypt: create cipher text + + """ + + def __init__(self, code=MtrDex.X25519_Private, seed=None, **kwa): + """ + Assign decrypting cipher suite function to ._decrypt + + Parameters: See Matter for inheirted parameters + raw (bytes): private decryption key derived from seed (private signing key) + qb64b (bytes): fully qualified private decryption key + qb64 (str): fully qualified private decryption key + code (str): derivation code for private decryption key + seed (Union[bytes, str]): qb64b or qb64 of signing key seed used to + derive raw which is private decryption key + """ + try: + super(Decrypter, self).__init__(code=code, **kwa) + except EmptyMaterialError as ex: + if seed: + signer = Signer(qb64b=seed) + if signer.code not in (MtrDex.Ed25519_Seed,): + raise ValueError("Unsupported signing seed derivation code = {}." + "".format(signer.code)) + # verkey, sigkey = pysodium.crypto_sign_seed_keypair(signer.raw) + sigkey = signer.raw + signer.verfer.raw # sigkey is raw seed + raw verkey + raw = pysodium.crypto_sign_sk_to_box_sk(sigkey) # raw private encrypt key + super(Decrypter, self).__init__(raw=raw, code=code, **kwa) + else: + raise + + if self.code == MtrDex.X25519_Private: + self._decrypt = self._x25519 + else: + raise ValueError("Unsupported decrypter code = {}.".format(self.code)) + + def decrypt(self, ser=None, cipher=None, transferable=False): + """ + Returns: + Salter or Signer instance derived from plain text decrypted from + encrypted cipher text material given by ser or cipher. Plain text + that is orignally encrypt should always be fully qualified (qb64b) + so that derivaton code of plain text is preserved through + encryption/decryption round trip. + + Parameters: + ser (Union[bytes,str]): qb64b or qb64 serialization of cipher text + cipher (Cipher): optional Cipher instance when ser is None + transferable (bool): True means associated verfer of returned + signer is transferable. False means non-transferable + """ + if not (ser or cipher): + raise EmptyMaterialError("Neither ser or cipher are provided.") + + if ser: # create cipher to ensure valid derivation code of material in ser + cipher = Cipher(qb64b=ser) + + return (self._decrypt(cipher=cipher, + prikey=self.raw, + transferable=transferable)) + + @staticmethod + def _x25519(cipher, prikey, transferable=False): + """ + Returns plain text as Salter or Signer instance depending on the cipher + code and the embedded encrypted plain text derivation code. + + Parameters: + cipher (Cipher): instance of encrypted seed or salt + prikey (bytes): raw binary decryption private key derived from + signing seed or sigkey + transferable (bool): True means associated verfer of returned + signer is transferable. False means non-transferable + """ + pubkey = pysodium.crypto_scalarmult_curve25519_base(prikey) + plain = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) # qb64b + # ensure raw plain text is qb64b or qb64 so its derivation code is round tripped + if cipher.code == MtrDex.X25519_Cipher_Salt: + return Salter(qb64b=plain) + elif cipher.code == MtrDex.X25519_Cipher_Seed: + return Signer(qb64b=plain, transferable=transferable) + else: + raise ValueError("Unsupported cipher text code = {}.".format(cipher.code)) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 951a70c37..4fbfc8ff2 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -43,6 +43,7 @@ from .. import help from ..help.helping import nonStringIterable +from .. import core from ..core import coring, scheming, serdering from . import dbing @@ -855,16 +856,16 @@ class SignerSuber(CesrSuber): Signer instance to have its .transferable property set correctly. """ - def __init__(self, *pa, klas: Type[coring.Signer] = coring.Signer, **kwa): + def __init__(self, *pa, klas: Type[core.Signer] = core.Signer, **kwa): """ Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key klas (Type[coring.Matter]): Class reference to subclass of Matter """ - if not (issubclass(klas, coring.Signer)): + if not (issubclass(klas, core.Signer)): raise ValueError("Invalid klas type={}, expected {}." - "".format(klas, coring.Signer)) + "".format(klas, core.Signer)) super(SignerSuber, self).__init__(*pa, **kwa) self.klas = klas @@ -933,7 +934,7 @@ class CryptSignerSuber(SignerSuber): """ def put(self, keys: Union[str, Iterable], val: coring.Matter, - encrypter: coring.Encrypter = None): + encrypter: core.Encrypter = None): """ Puts qb64 of Matter instance val at key made from keys. Does not overwrite If encrypter provided then encrypts first @@ -941,7 +942,7 @@ def put(self, keys: Union[str, Iterable], val: coring.Matter, Parameters: keys (tuple): of key strs to be combined in order to form key val (Signer): instance of self.klas - encrypter (coring.Encrypter): optional + encrypter (core.Encrypter): optional Returns: result (bool): True If successful, False otherwise, such as key @@ -955,7 +956,7 @@ def put(self, keys: Union[str, Iterable], val: coring.Matter, def pin(self, keys: Union[str, Iterable], val: coring.Matter, - encrypter: coring.Encrypter = None): + encrypter: core.Encrypter = None): """ Pins (sets) qb64 of Matter instance val at key made from keys. Overwrites. If encrypter provided then encrypts first @@ -963,7 +964,7 @@ def pin(self, keys: Union[str, Iterable], val: coring.Matter, Parameters: keys (tuple): of key strs to be combined in order to form key val (Signer): instance of self.klas - encrypter (coring.Encrypter): optional + encrypter (core.Encrypter): optional Returns: result (bool): True If successful. False otherwise. @@ -976,7 +977,7 @@ def pin(self, keys: Union[str, Iterable], val: coring.Matter, - def get(self, keys: Union[str, Iterable], decrypter: coring.Decrypter = None): + def get(self, keys: Union[str, Iterable], decrypter: core.Decrypter = None): """ Gets Signer instance at keys. If decrypter then assumes value in db was encrypted and so decrypts value in db before converting to Signer. @@ -990,7 +991,7 @@ def get(self, keys: Union[str, Iterable], decrypter: coring.Decrypter = None): keys (Union[str, iterable]): key strs to be combined in order to form key. Last element of keys is verkey used to determin .transferable for Signer - decrypter (coring.Decrypter): optional. If provided assumes value in + decrypter (core.Decrypter): optional. If provided assumes value in db was encrypted and so decrypts before converting to Signer. Usage: @@ -1014,7 +1015,7 @@ def get(self, keys: Union[str, Iterable], decrypter: coring.Decrypter = None): def getItemIter(self, keys: Union[str, Iterable]=b"", - decrypter: coring.Decrypter = None): + decrypter: core.Decrypter = None): """ Returns: iterator (Iterator): of tuples (key, val) over the all the items in @@ -1022,7 +1023,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b"", prefix to return branches of key space. When keys is empty then returns all items in subdb - decrypter (coring.Decrypter): optional. If provided assumes value in + decrypter (core.Decrypter): optional. If provided assumes value in db was encrypted and so decrypts before converting to Signer. Parameters: diff --git a/src/keri/demo/demo_bob.py b/src/keri/demo/demo_bob.py index 07c7fccc7..aa812b2dd 100644 --- a/src/keri/demo/demo_bob.py +++ b/src/keri/demo/demo_bob.py @@ -10,10 +10,15 @@ import logging from keri import __version__ + +from keri import help + +from keri import core + from keri.app import directing from keri.demo import demoing -from keri.core import coring -from keri import help + + def runDemo(name="bob", remote=5621, local=5620, expire=0.0): @@ -24,7 +29,7 @@ def runDemo(name="bob", remote=5621, local=5620, expire=0.0): # create secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path=name, temp=True)] diff --git a/src/keri/demo/demo_eve.py b/src/keri/demo/demo_eve.py index 688dea03b..895e28dbf 100644 --- a/src/keri/demo/demo_eve.py +++ b/src/keri/demo/demo_eve.py @@ -10,10 +10,14 @@ import logging from keri import __version__ +from keri import help # logger support + +from keri import core + from keri.app import directing from keri.demo import demoing -from keri.core import coring -from keri import help # logger support + + def runDemo(name="eve", remote=5620, local=5621, expire=0.0): """ @@ -24,7 +28,7 @@ def runDemo(name="eve", remote=5620, local=5621, expire=0.0): # create secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path=name, temp=True)] diff --git a/src/keri/demo/demo_sam.py b/src/keri/demo/demo_sam.py index b31d2f053..48ef3411e 100644 --- a/src/keri/demo/demo_sam.py +++ b/src/keri/demo/demo_sam.py @@ -11,10 +11,14 @@ from keri import __version__ +from keri import help # logger support + +from keri import core + from keri.app import directing from keri.demo import demoing -from keri.core import coring -from keri import help # logger support + + def runDemo(name="sam", remote=5621, local=5620, expire=0.0): """ @@ -25,7 +29,7 @@ def runDemo(name="sam", remote=5621, local=5620, expire=0.0): # create secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path=name, temp=True)] diff --git a/src/keri/vc/proving.py b/src/keri/vc/proving.py index 67721ddfb..bc2fdbb28 100644 --- a/src/keri/vc/proving.py +++ b/src/keri/vc/proving.py @@ -8,6 +8,8 @@ from typing import Union from .. import help + +from .. import core from ..core import coring, serdering from ..core.coring import (Serials, versify) from ..db import subing @@ -62,8 +64,8 @@ def credential(schema, ) if private: - vc["u"] = salt if salt is not None else coring.Salter().qb64 - subject["u"] = salt if salt is not None else coring.Salter().qb64 + vc["u"] = salt if salt is not None else core.Salter().qb64 + subject["u"] = salt if salt is not None else core.Salter().qb64 if recipient is not None: subject['i'] = recipient diff --git a/tests/app/cli/test_kli_commands.py b/tests/app/cli/test_kli_commands.py index 1fcfd566f..94d960424 100644 --- a/tests/app/cli/test_kli_commands.py +++ b/tests/app/cli/test_kli_commands.py @@ -3,11 +3,17 @@ import multicommand import pytest -from keri.app import directing, habbing + +from keri.kering import ValidationError + +from keri import core +from keri.core import coring + +from keri.app import directing + from keri.app.cli import commands from keri.app.cli.common import existing -from keri.core import coring -from keri.kering import ValidationError + TEST_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -17,7 +23,7 @@ def test_standalone_kli_commands(helpers, capsys): assert os.path.isdir("/usr/local/var/keri/ks/test") is False parser = multicommand.create_parser(commands) - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = core.Salter(raw=b'0123456789abcdef').qb64 args = parser.parse_args(["init", "--name", "test", "--nopasscode", "--salt", salt]) assert args.handler is not None doers = args.handler(args) @@ -257,7 +263,7 @@ def test_incept_and_rotate_opts(helpers, capsys): assert os.path.isdir("/usr/local/var/keri/ks/test-opts") is False parser = multicommand.create_parser(commands) - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = core.Salter(raw=b'0123456789abcdef').qb64 args = parser.parse_args(["init", "--name", "test-opts", "--nopasscode", "--salt", salt]) assert args.handler is not None doers = args.handler(args) diff --git a/tests/app/test_agenting.py b/tests/app/test_agenting.py index 244549fd3..4b2022ddd 100644 --- a/tests/app/test_agenting.py +++ b/tests/app/test_agenting.py @@ -8,6 +8,9 @@ from hio.base import doing, tyming from keri import kering + +from keri import core + from keri.core import coring, serdering from keri.core.coring import Counter, CtrDex, Seqner from keri.help import nowIso8601 @@ -17,10 +20,10 @@ def test_withness_receiptor(seeder): - with habbing.openHby(name="wan", salt=coring.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ - habbing.openHby(name="wil", salt=coring.Salter(raw=b'will-the-witness').qb64) as wilHby, \ - habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ - habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby: + with habbing.openHby(name="wan", salt=core.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ + habbing.openHby(name="wil", salt=core.Salter(raw=b'will-the-witness').qb64) as wilHby, \ + habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby: wanDoers = indirecting.setupWitness(alias="wan", hby=wanHby, tcpPort=5632, httpPort=5642) wilDoers = indirecting.setupWitness(alias="wil", hby=wilHby, tcpPort=5633, httpPort=5643) @@ -113,10 +116,10 @@ def testDo(self, tymth, tock=0.0): def test_witness_sender(seeder): - with habbing.openHby(name="wan", salt=coring.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ - habbing.openHby(name="wil", salt=coring.Salter(raw=b'will-the-witness').qb64) as wilHby, \ - habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ - habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby: + with habbing.openHby(name="wan", salt=core.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ + habbing.openHby(name="wil", salt=core.Salter(raw=b'will-the-witness').qb64) as wilHby, \ + habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby: # looks like bad magic value in seeder is causing this to fail pdoer = PublishDoer(wanHby, wilHby, wesHby, palHby, seeder) @@ -183,11 +186,11 @@ def testDo(self, tymth, tock=0.0): def test_witness_inquisitor(mockHelpingNowUTC, seeder): - with habbing.openHby(name="wan", salt=coring.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ - habbing.openHby(name="wil", salt=coring.Salter(raw=b'will-the-witness').qb64) as wilHby, \ - habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ - habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby, \ - habbing.openHby(name="qin", salt=coring.Salter(raw=b'abcdef0123456789').qb64) as qinHby: + with habbing.openHby(name="wan", salt=core.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ + habbing.openHby(name="wil", salt=core.Salter(raw=b'will-the-witness').qb64) as wilHby, \ + habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby, \ + habbing.openHby(name="qin", salt=core.Salter(raw=b'abcdef0123456789').qb64) as qinHby: wanDoers = indirecting.setupWitness(alias="wan", hby=wanHby, tcpPort=5632, httpPort=5642) wilDoers = indirecting.setupWitness(alias="wil", hby=wilHby, tcpPort=5633, httpPort=5643) wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, tcpPort=5634, httpPort=5644) diff --git a/tests/app/test_delegating.py b/tests/app/test_delegating.py index e94680e2e..132a588c5 100644 --- a/tests/app/test_delegating.py +++ b/tests/app/test_delegating.py @@ -7,15 +7,19 @@ from hio.base import doing, tyming from keri import kering -from keri.app import habbing, delegating, indirecting, agenting, notifying + +from keri import core from keri.core import eventing, parsing, coring + +from keri.app import habbing, delegating, indirecting, agenting, notifying + from keri.db import dbing def test_anchorer(seeder): - with habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ - habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby, \ - habbing.openHby(name="del", salt=coring.Salter(raw=b'0123456789ghijkl').qb64) as delHby: + with habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby, \ + habbing.openHby(name="del", salt=core.Salter(raw=b'0123456789ghijkl').qb64) as delHby: wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, tcpPort=5634, httpPort=5644) witDoer = agenting.Receiptor(hby=palHby) diff --git a/tests/app/test_directing.py b/tests/app/test_directing.py index 2a86227b5..2c39484e4 100644 --- a/tests/app/test_directing.py +++ b/tests/app/test_directing.py @@ -11,8 +11,11 @@ from hio.core.tcp import clienting, serving from keri import help # logger support -from keri.app import habbing, directing +from keri import core from keri.core import eventing, coring + +from keri.app import habbing, directing + from keri.demo import demoing @@ -25,7 +28,7 @@ def test_directing_basic(): raw = b"raw salt to test" # create bob signers and secrecies - bobSigners = coring.Salter(raw=raw).signers(count=8, path="bob", temp=True) + bobSigners = core.Salter(raw=raw).signers(count=8, path="bob", temp=True) bobSecrecies = [[signer.qb64] for signer in bobSigners] # bob inception transferable (nxt digest not empty) @@ -38,7 +41,7 @@ def test_directing_basic(): # create eve signers and secrecies - eveSigners = coring.Salter(raw=raw).signers(count=8, path="eve", temp=True) + eveSigners = core.Salter(raw=raw).signers(count=8, path="eve", temp=True) eveSecrecies = [[signer.qb64] for signer in eveSigners] # eve inception transferable (nxt digest not empty) @@ -172,7 +175,7 @@ def test_runcontroller_demo(): # create secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path=name, temp=True)] diff --git a/tests/app/test_forwarding.py b/tests/app/test_forwarding.py index b5932af0e..6f3523cdb 100644 --- a/tests/app/test_forwarding.py +++ b/tests/app/test_forwarding.py @@ -8,14 +8,17 @@ from hio.base import doing, tyming -from keri.app import forwarding, habbing, indirecting, storing +from keri import core from keri.core import coring, eventing, parsing, serdering + +from keri.app import forwarding, habbing, indirecting, storing + from keri.peer import exchanging def test_postman(seeder): with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab), \ - habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64, temp=True) as wesHby, \ + habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64, temp=True) as wesHby, \ habbing.openHby(name="repTest", temp=True) as recpHby: mbx = storing.Mailboxer(name="wes", temp=True) diff --git a/tests/app/test_habbing.py b/tests/app/test_habbing.py index 9b23675d4..4fb896f87 100644 --- a/tests/app/test_habbing.py +++ b/tests/app/test_habbing.py @@ -11,11 +11,17 @@ from hio.base import doing from keri import kering + from keri import help +from keri.help import helping + +from keri import core +from keri.core import coring, eventing, parsing + from keri.app import habbing, keeping, configing from keri.db import basing -from keri.core import coring, eventing, parsing -from keri.help import helping + + def test_habery(): @@ -23,7 +29,7 @@ def test_habery(): Test Habery class """ # test default - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 hby = habbing.Habery(temp=True, salt=default_salt) assert hby.name == "test" assert hby.base == "" @@ -55,7 +61,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low hby.cf.close(clear=True) hby.db.close(clear=True) @@ -70,7 +76,7 @@ def test_habery(): assert len(bran) == 21 # these are the ones generated by Manager for the given bran # bran = coring.MtrDex.Salt_128 + 'A' + bran[:21] # qb64 salt for seed - # signer = coring.Salter(qb64=bran).signer(transferable=False, tier=tier, temp=temp) + # signer = core.Salter(qb64=bran).signer(transferable=False, tier=tier, temp=temp) # seed = signer.qb64 # aeid = signer.verfer.qb64 # lest it remove encryption seed4bran = 'ALQVu9AjGW3JrIzX0UHm2awGCbDXcsLzy-vAE649Fz1j' @@ -88,7 +94,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low assert hby.rtr.routes assert hby.rvy.rtr == hby.rtr @@ -155,7 +161,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low assert hby.rtr.routes assert hby.rvy.rtr == hby.rtr @@ -216,7 +222,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low assert hby.rtr.routes assert hby.rvy.rtr == hby.rtr @@ -234,7 +240,7 @@ def test_habery(): assert not os.path.exists(hby.db.path) assert not os.path.exists(hby.ks.path) - with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: assert hby.name == "test" assert hby.base == "" assert hby.temp @@ -264,7 +270,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low assert hby.rtr.routes assert hby.rvy.rtr == hby.rtr @@ -281,7 +287,7 @@ def test_habery(): assert not os.path.exists(hby.ks.path) bran = "MyPasscodeARealSecret" - with habbing.openHby(bran=bran, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(bran=bran, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: assert hby.name == "test" assert hby.base == "" assert hby.temp @@ -313,7 +319,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low assert hby.rtr.routes assert hby.rvy.rtr == hby.rtr @@ -342,7 +348,7 @@ def test_make_load_hab_with_habery(): name = "sue" suePre = 'ELF1S0jZkyQx8YtHaPLu-qyFmrkcykAiEW8twS-KPSO1' # with temp=True - with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True on openHab + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True on openHab hab = hby.makeHab(name=name) assert isinstance(hab, habbing.Hab) assert hab.pre in hby.habs @@ -386,7 +392,7 @@ def test_make_load_hab_with_habery(): suePre = 'EAxe215BJ4Iy9r0mfoMEGVmHW8A4Avk3RYBC1A1_DZam' # with temp=False bobPre = 'ENya5E5pvc6MVCe75huDK0QQhE4_64J55vCn4aKdXhR9' # with temp=False - with habbing.openHby(base=base, temp=False, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True + with habbing.openHby(base=base, temp=False, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True assert hby.cf.path.endswith("keri/cf/hold/test.json") assert hby.db.path.endswith("keri/db/hold/test") assert hby.ks.path.endswith('keri/ks/hold/test') @@ -560,7 +566,7 @@ def test_habery_reinitialization(): def test_habery_signatory(): - with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: signer = hby.signator assert signer is not None @@ -599,11 +605,11 @@ def test_habery_reconfigure(mockHelpingNowUTC): # use same salter but with different path from name for each # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) # raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - # salter = coring.Salter(raw=raw) + # salter = core.Salter(raw=raw) # salt = salter.qb64 # assert salt == '0ABaqPLVOa6fpVnAKcmwhIdQ' - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = core.Salter(raw=b'0123456789abcdef').qb64 cname = "tam" # controller name cbase = "main" # controller base shared @@ -714,7 +720,7 @@ def test_habery_reconfigure(mockHelpingNowUTC): def test_namespaced_habs(): - with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name="test") assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" @@ -788,7 +794,7 @@ def test_namespaced_habs(): def test_make_other_event(): - with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name="test") assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" @@ -847,8 +853,8 @@ def test_hab_by_pre(): def test_postman_endsfor(): - with habbing.openHby(name="test", temp=True, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby, \ - habbing.openHby(name="wes", temp=True, salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + with habbing.openHby(name="test", temp=True, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby, \ + habbing.openHby(name="wes", temp=True, salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ habbing.openHab(name="agent", temp=True, salt=b'0123456789abcdef') as (agentHby, agentHab): wesHab = wesHby.makeHab(name='wes', isith="1", icount=1, transferable=False) diff --git a/tests/app/test_keeping.py b/tests/app/test_keeping.py index 29d7e8044..0d638aef3 100644 --- a/tests/app/test_keeping.py +++ b/tests/app/test_keeping.py @@ -19,8 +19,11 @@ from keri import kering from keri.help import helping +from keri import core + from keri.core import coring, indexing from keri.core.indexing import IdrDex + from keri.app import keeping @@ -391,20 +394,20 @@ def test_keeper(): key = b'tier' assert keeper.gbls.get(key) == None assert keeper.gbls.rem(key) == False - assert keeper.gbls.put(key, val=coring.Tiers.low) == True - assert keeper.gbls.get(key) == coring.Tiers.low - assert keeper.gbls.put(key, val=coring.Tiers.med) == False - assert keeper.gbls.get(key) == coring.Tiers.low - assert keeper.gbls.pin(key, val=coring.Tiers.med) == True - assert keeper.gbls.get(key) == coring.Tiers.med + assert keeper.gbls.put(key, val=core.Tiers.low) == True + assert keeper.gbls.get(key) == core.Tiers.low + assert keeper.gbls.put(key, val=core.Tiers.med) == False + assert keeper.gbls.get(key) == core.Tiers.low + assert keeper.gbls.pin(key, val=core.Tiers.med) == True + assert keeper.gbls.get(key) == core.Tiers.med assert keeper.gbls.rem(key) == True assert keeper.gbls.get(key) == None # test .pris sub db methods key = puba - signera = coring.Signer(qb64b=pria) + signera = core.Signer(qb64b=pria) assert signera.qb64b == pria - signerb = coring.Signer(qb64b=prib) + signerb = core.Signer(qb64b=prib) assert signerb.qb64b == prib assert keeper.pris.get(key) == None assert keeper.pris.rem(key) == False @@ -603,7 +606,7 @@ def test_creator(): signers = creator.create() assert len(signers) == 1 signer = signers[0] - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.verfer.code == coring.MtrDex.Ed25519 assert signer.verfer.code not in coring.NonTransDex @@ -611,7 +614,7 @@ def test_creator(): signers = creator.create(count=2, transferable=False) assert len(signers) == 2 for signer in signers: - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.verfer.code == coring.MtrDex.Ed25519N assert signer.verfer.code in coring.NonTransDex @@ -619,7 +622,7 @@ def test_creator(): creator = keeping.SaltyCreator() assert isinstance(creator, keeping.SaltyCreator) assert isinstance(creator, keeping.Creator) - assert isinstance(creator.salter, coring.Salter) + assert isinstance(creator.salter, core.Salter) assert creator.salter.code == coring.MtrDex.Salt_128 assert creator.salt == creator.salter.qb64 assert creator.stem == '' @@ -627,7 +630,7 @@ def test_creator(): signers = creator.create() assert len(signers) == 1 signer = signers[0] - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.verfer.code == coring.MtrDex.Ed25519 assert signer.verfer.code not in coring.NonTransDex @@ -635,25 +638,25 @@ def test_creator(): signers = creator.create(count=2, transferable=False) assert len(signers) == 2 for signer in signers: - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.verfer.code == coring.MtrDex.Ed25519N assert signer.verfer.code in coring.NonTransDex raw = b'0123456789abcdef' - salt = coring.Salter(raw=raw).qb64 + salt = core.Salter(raw=raw).qb64 assert salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' creator = keeping.SaltyCreator(salt=salt) assert isinstance(creator, keeping.SaltyCreator) assert isinstance(creator, keeping.Creator) - assert isinstance(creator.salter, coring.Salter) + assert isinstance(creator.salter, core.Salter) assert creator.salter.code == coring.MtrDex.Salt_128 assert creator.salter.raw == raw assert creator.salter.qb64 == salt signers = creator.create() assert len(signers) == 1 signer = signers[0] - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.qb64 == 'APMJe0lwOpwnX9PkvX1mh26vlzGYl6RWgWGclc8CAQJ9' assert signer.verfer.code == coring.MtrDex.Ed25519 @@ -663,7 +666,7 @@ def test_creator(): signers = creator.create(count=1, transferable=False, temp=True) assert len(signers) == 1 signer = signers[0] - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.qb64 == 'AMGrAM0noxLpRteO9mxGT-yzYSrKFwJMuNI4KlmSk26e' assert signer.verfer.code == coring.MtrDex.Ed25519N @@ -694,7 +697,7 @@ def test_manager(): assert not manager.ks.opened raw = b'0123456789abcdef' - salt = coring.Salter(raw=raw).qb64 + salt = core.Salter(raw=raw).qb64 stem = "red" assert salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' @@ -717,7 +720,7 @@ def test_manager(): manager = keeping.Manager(ks=keeper, salt=salt) assert manager.ks.opened assert manager.pidx == 0 - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low assert manager.salt == salt assert manager.aeid == "" assert manager.seed == "" @@ -738,7 +741,7 @@ def test_manager(): assert pp.algo == keeping.Algos.salty assert pp.salt == salt assert pp.stem == '' - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(spre) assert ps.old.pubs == [] @@ -821,7 +824,7 @@ def test_manager(): assert pp.algo == keeping.Algos.salty assert pp.salt == salt assert pp.stem == '' - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(spre) assert ps.old.pubs == ['DFRtyHAjSuJaRX6TDPva35GN11VHAruaOXMc79ZYDKsT'] @@ -954,7 +957,7 @@ def test_manager(): assert pp.algo == keeping.Algos.salty assert pp.salt == salt assert pp.stem == stem == 'red' - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(spre) assert ps.old.pubs == [] @@ -1021,7 +1024,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 6 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 0 ipre, verferies = manager.ingest(secrecies=secrecies) # use default iridx assert ipre == 'DNsGfyf7JArtQgioD7BdVwRulGAsQk5REIKSTjFJqE0a' @@ -1095,7 +1098,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 7 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 3 ipre, verferies = manager.ingest(secrecies=secrecies, iridx=iridx) @@ -1169,7 +1172,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 8 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 7 ipre, verferies = manager.ingest(secrecies=secrecies, iridx=iridx) @@ -1251,7 +1254,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 9 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 0 ipre, verferies = manager.ingest(secrecies=secrecies, ncount=3) # default iridx @@ -1345,7 +1348,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 10 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 1 ipre, verferies = manager.ingest(secrecies=secrecies, iridx=iridx, ncount=3) assert ipre == 'DHc6ZvVpAjLq_digYYZMhq0OlEnnbCrgSvcxPJQY_oAE' @@ -1438,7 +1441,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 11 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 3 ipre, verferies = manager.ingest(secrecies=secrecies, iridx=iridx, ncount=3) assert ipre == 'DO1mU48dTzyFtHjqH744gl0QoEIMxphSC4qbgMoEuyq7' @@ -1518,7 +1521,7 @@ def test_manager_with_aeid(): """ # rawsalt =pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) rawsalt = b'0123456789abcdef' - salter = coring.Salter(raw=rawsalt) + salter = core.Salter(raw=rawsalt) salt = salter.qb64 assert salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' stem = "blue" @@ -1526,25 +1529,25 @@ def test_manager_with_aeid(): # cryptseed0 = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) cryptseed0 = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' - cryptsigner0 = coring.Signer(raw=cryptseed0, code=coring.MtrDex.Ed25519_Seed, + cryptsigner0 = core.Signer(raw=cryptseed0, code=coring.MtrDex.Ed25519_Seed, transferable=False) seed0 = cryptsigner0.qb64 aeid0 = cryptsigner0.verfer.qb64 assert aeid0 == 'BCa7mK96FwxkU0TdF54Yqg3qBDXUWpOhQ_Mtr7E77yZB' - decrypter0 = coring.Decrypter(seed=seed0) - encrypter0 = coring.Encrypter(verkey=aeid0) + decrypter0 = core.Decrypter(seed=seed0) + encrypter0 = core.Encrypter(verkey=aeid0) assert encrypter0.verifySeed(seed=seed0) # cryptseed1 = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) cryptseed1 = (b"\x89\xfe{\xd9'\xa7\xb3\x89#\x19\xbec\xee\xed\xc0\xf9\x97\xd0\x8f9\x1dyNI" b'I\x98\xbd\xa4\xf6\xfe\xbb\x03') - cryptsigner1 = coring.Signer(raw=cryptseed1, code=coring.MtrDex.Ed25519_Seed, + cryptsigner1 = core.Signer(raw=cryptseed1, code=coring.MtrDex.Ed25519_Seed, transferable=False) seed1 = cryptsigner1.qb64 aeid1 = cryptsigner1.verfer.qb64 assert aeid1 == 'BEcOrMrG_7r_NWaLl6h8UJapwIfQWIkjrIPXkCZm2fFM' - decrypter1 = coring.Decrypter(seed=seed1) - encrypter1 = coring.Encrypter(verkey=aeid1) + decrypter1 = core.Decrypter(seed=seed1) + encrypter1 = core.Encrypter(verkey=aeid1) assert encrypter1.verifySeed(seed=seed1) # something to sign doesn't matter what for testing purposes @@ -1572,8 +1575,8 @@ def test_manager_with_aeid(): assert manager.algo == keeping.Algos.salty assert manager.salt == salt # encrypted on disk but property decrypts if seed assert manager.pidx == 0 - assert manager.tier == coring.Tiers.low - saltCipher0 = coring.Cipher(qb64=manager.ks.gbls.get('salt')) + assert manager.tier == core.Tiers.low + saltCipher0 = core.Cipher(qb64=manager.ks.gbls.get('salt')) assert saltCipher0.decrypt(seed=seed0).qb64 == salt # salty algorithm incept @@ -1590,7 +1593,7 @@ def test_manager_with_aeid(): assert pp.algo == keeping.Algos.salty assert manager.decrypter.decrypt(ser=pp.salt).qb64 == salt assert pp.stem == '' - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(spre) assert ps.old.pubs == [] @@ -1673,7 +1676,7 @@ def test_manager_with_aeid(): assert pp.algo == keeping.Algos.salty assert manager.decrypter.decrypt(ser=pp.salt).qb64 == salt assert pp.stem == '' - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(spre) assert ps.old.pubs == ['DFRtyHAjSuJaRX6TDPva35GN11VHAruaOXMc79ZYDKsT'] @@ -1703,8 +1706,8 @@ def test_manager_with_aeid(): assert manager.algo == keeping.Algos.salty assert manager.salt == salt assert manager.pidx == 1 - assert manager.tier == coring.Tiers.low - saltCipher1 = coring.Cipher(qb64=manager.ks.gbls.get('salt')) + assert manager.tier == core.Tiers.low + saltCipher1 = core.Cipher(qb64=manager.ks.gbls.get('salt')) assert saltCipher1.decrypt(seed=seed1).qb64 == salt assert not saltCipher0.qb64 == saltCipher1.qb64 # old cipher different @@ -1802,7 +1805,7 @@ def test_manager_sign_dual_indices(): When ondices is not provided then all sigers .ondex is None. """ raw = b'0123456789abcdef' - salt = coring.Salter(raw=raw).qb64 + salt = core.Salter(raw=raw).qb64 # the particular serialization does not matter for test purposes ser = (b"See ya later Alligator. In a while Crocodile. " @@ -1814,7 +1817,7 @@ def test_manager_sign_dual_indices(): manager = keeping.Manager(ks=keeper, salt=salt) assert manager.ks.opened assert manager.pidx == 0 - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low assert manager.salt == salt assert manager.aeid == "" assert manager.seed == "" @@ -1842,7 +1845,7 @@ def test_manager_sign_dual_indices(): assert pp.algo == keeping.Algos.salty assert pp.salt == salt assert pp.stem == stem - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(pre) assert len(ps.old.pubs) == 0 @@ -1951,7 +1954,7 @@ def test_manager_sign_dual_indices(): assert pp.algo == keeping.Algos.salty assert pp.salt == salt assert pp.stem == stem - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(pre) assert len(ps.old.pubs) == ocount diff --git a/tests/app/test_signify.py b/tests/app/test_signify.py index 1d9a7d08f..205d44bf9 100644 --- a/tests/app/test_signify.py +++ b/tests/app/test_signify.py @@ -6,16 +6,20 @@ import pytest from keri import kering + +from keri import core +from keri.core import coring, eventing + from keri.app import habbing from keri.app.keeping import SaltyCreator -from keri.core import coring, eventing + def test_remote_salty_hab(): name = "test" - tier = coring.Tiers.low + tier = core.Tiers.low raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = coring.Salter(raw=raw, tier=tier) + salter = core.Salter(raw=raw, tier=tier) with habbing.openHby(name="remoteSalty") as remote, \ habbing.openHby(name="local", salt=salter.qb64, temp=True, tier=tier) as local: diff --git a/tests/app/test_signing.py b/tests/app/test_signing.py index 3561d2201..8d334543a 100644 --- a/tests/app/test_signing.py +++ b/tests/app/test_signing.py @@ -3,11 +3,13 @@ tests.app.signing module """ +from keri import core +from keri.core import coring, parsing, eventing +from keri.core.eventing import SealEvent from keri.app import habbing, configing, keeping from keri.app import signing -from keri.core import coring, parsing, eventing -from keri.core.eventing import SealEvent + from keri.db import basing from keri.vc import proving from keri.vdr import verifying, credentialing @@ -126,7 +128,7 @@ def test_sad_signature(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): # Test multisig identifier with configing.openCF(name="mel", base="mel", temp=True) as cf, \ - habbing.openHby(name="mel", temp=True, salt=coring.Salter(raw=b'0123456789abcdef').qb64b, + habbing.openHby(name="mel", temp=True, salt=core.Salter(raw=b'0123456789abcdef').qb64b, base="mel", cf=cf) as hby: hab = hby.makeHab(name="mel", icount=3, isith='3', ncount=3, nsith='3') seeder.seedSchema(hby.db) @@ -176,8 +178,8 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs seed1 = (b'\x83B~\x04\x94\xe3\xceUQy\x11f\x0c\x93]\x1e\xbf\xacQ\xb5\xd6Y^\xa2E\xfa\x015' b'\x98Y\xdd\xe8') - signer0 = coring.Signer(raw=seed, transferable=True) # original signing keypair non transferable - signer1 = coring.Signer(raw=seed1) # next signing keypair transferable is default + signer0 = core.Signer(raw=seed, transferable=True) # original signing keypair non transferable + signer1 = core.Signer(raw=seed1) # next signing keypair transferable is default keys0 = [signer0.verfer.qb64] nxt1 = [coring.Diger(ser=signer1.verfer.qb64b).qb64] # dfault sith is 1 serder = eventing.incept(keys=keys0, ndigs=nxt1, code=coring.MtrDex.Blake3_256, intive=True) @@ -313,7 +315,7 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs def test_signatory(): - salt = coring.Salter(raw=b'0123456789abcdef') # init sig Salter + salt = core.Salter(raw=b'0123456789abcdef') # init sig Salter with basing.openDB(name="sig") as db, keeping.openKS(name="sig") as ks, \ habbing.openHab(name="sig", salt=salt.raw) as (sigHby, sigHab): diff --git a/tests/comply/test_direct_mode.py b/tests/comply/test_direct_mode.py index 6ca1c4823..3bb8a69ef 100644 --- a/tests/comply/test_direct_mode.py +++ b/tests/comply/test_direct_mode.py @@ -2,11 +2,15 @@ import os -from keri.core.coring import Salter, MtrDex -from keri.core.coring import Seqner -from keri.app.keeping import Manager, openKS +from keri import core from keri.core import eventing, parsing +from keri.core import MtrDex +from keri.core.coring import Seqner + from keri.core.eventing import (incept, rotate, interact, messagize, SealEvent, receipt) + +from keri.app.keeping import Manager, openKS + from keri.db.dbing import dgKey, snKey from keri.db.basing import openDB @@ -24,10 +28,10 @@ def test_direct_mode_with_manager(): # but goes both ways once initiated. # set of secrets (seeds for private keys) - coeSalt = Salter(raw=b'0123456789abcdea').qb64 + coeSalt = core.Salter(raw=b'0123456789abcdea').qb64 # set of secrets (seeds for private keys) - valSalt = Salter(raw=b'1123456789abcdea').qb64 + valSalt = core.Salter(raw=b'1123456789abcdea').qb64 with openDB(name="controller") as coeLogger, openDB(name="validator") as valLogger, \ diff --git a/tests/core/test_bare.py b/tests/core/test_bare.py index 5794d22ce..c6b161e73 100644 --- a/tests/core/test_bare.py +++ b/tests/core/test_bare.py @@ -8,12 +8,16 @@ """ from keri import kering +from keri import help + +from keri import core from keri.core import eventing -from keri.core.coring import MtrDex, Salter, Diger + +from keri.core.coring import MtrDex, Diger from keri.core.eventing import (SealEvent, messagize) -from keri import help + logger = help.ogler.getLogger() @@ -45,7 +49,7 @@ def test_bare(): # use same salter for all but different path # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create transferable key pair for controller of service endpoint designation signerC = salter.signer(path="C", temp=True) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 552072c4b..f6ebf78b1 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -36,19 +36,18 @@ from keri.help.helping import (sceil, intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, B64_CHARS, Reb64, nabSextets) - +from keri import core from keri.core import coring -from keri.core import eventing from keri.core.coring import (Saids, Sadder, Tholder, Seqner, NumDex, Number, Dater, Bexter, Texter, TagDex, PadTagDex, Tagger, Ilker, Traitor, Verser, Versage, ) -from keri.core.coring import Serialage, Serials, Tiers +from keri.core.coring import Serialage, Serials from keri.core.coring import (Sizage, MtrDex, Matter, CtrDex, Counter) -from keri.core.coring import (Verfer, Cigar, Signer, Salter, Saider, DigDex, - Diger, Prefixer, Cipher, Encrypter, Decrypter) +from keri.core.coring import (Verfer, Cigar, Saider, DigDex, + Diger, Prefixer,) from keri.core.coring import versify, deversify, Rever, MAXVERFULLSPAN -from keri.core.coring import generateSigners + from keri.core.indexing import (Siger, Xizage, IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, Indexer) @@ -4703,687 +4702,6 @@ def test_cigar(): """ Done Test """ -def test_signer(): - """ - Test the support functionality for signer subclass of crymat - """ - signer = Signer() # defaults provide Ed25519 signer Ed25519 verfer - assert signer.code == MtrDex.Ed25519_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.Ed25519 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - # create something to sign and verify - ser = b'abcdefghijklmnopqrstuvwxyz0123456789' - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.Ed25519_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result == True - - index = 0 - siger = signer.sign(ser, index=index) - assert siger.code == IdrDex.Ed25519_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') - assert result == False - - assert cigar.raw == siger.raw - - with pytest.raises(ValueError): # use invalid code not SEED type code - signer = Signer(code=MtrDex.Ed25519N) - - # Non transferable defaults - signer = Signer(transferable=False) # Ed25519N verifier - assert signer.code == MtrDex.Ed25519_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.Ed25519N - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.Ed25519_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result == True - - siger = signer.sign(ser, index=0) - assert siger.code == IdrDex.Ed25519_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') - assert result == False - - - # non default seed - seed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - signer = Signer(raw=seed, code=MtrDex.Ed25519_Seed) - assert signer.code == MtrDex.Ed25519_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.verfer.code == MtrDex.Ed25519 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.Ed25519_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result == True - - index = 1 - siger = signer.sign(ser, index=index) - assert siger.code == IdrDex.Ed25519_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - assert cigar.raw == siger.raw - - # different both so Big - ondex = 3 - siger = signer.sign(ser, index=index, ondex=ondex) - assert siger.code == IdrDex.Ed25519_Big_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == ondex - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # same but Big - index = 67 - siger = signer.sign(ser, index=index) - assert siger.code == IdrDex.Ed25519_Big_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # different both so Big - ondex = 67 - siger = signer.sign(ser, index=index, ondex=ondex) - assert siger.code == IdrDex.Ed25519_Big_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == ondex - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # current only - index = 4 - siger = signer.sign(ser, index=index, only=True) - assert siger.code == IdrDex.Ed25519_Crt_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == None - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # ignores ondex if only - siger = signer.sign(ser, index=index, only=True, ondex=index+2) - assert siger.code == IdrDex.Ed25519_Crt_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == None - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # big current only - index = 65 - siger = signer.sign(ser, index=index, only=True) - assert siger.code == IdrDex.Ed25519_Big_Crt_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == None - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # ignores ondex if only - siger = signer.sign(ser, index=index, only=True, ondex=index+2) - assert siger.code == IdrDex.Ed25519_Big_Crt_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == None - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - with pytest.raises(ValueError): # use invalid code not SEED type code - signer = Signer(raw=seed, code=MtrDex.Ed25519N) - - # Test Secp256r1, default seed - signer = Signer(code=MtrDex.ECDSA_256r1_Seed) - assert signer.code == MtrDex.ECDSA_256r1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.ECDSA_256r1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.ECDSA_256r1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result is True - - # Test non-default seed - seed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - signer = Signer(raw=seed, code=MtrDex.ECDSA_256r1_Seed) - assert signer.code == MtrDex.ECDSA_256r1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.verfer.code == MtrDex.ECDSA_256r1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - # Test hardcoded seed - seed = (b'\x9f{\xa8\xa7\xa8C9\x96&\xfa\xb1\x99\xeb\xaa \xc4\x1bG\x11\xc4\xaeSAR\xc9\xbd\x04\x9d\x85)~\x93') - signer = Signer(raw=seed, code=MtrDex.ECDSA_256r1_Seed) - assert signer.code == MtrDex.ECDSA_256r1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.verfer.code == MtrDex.ECDSA_256r1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.qb64 == "QJ97qKeoQzmWJvqxmeuqIMQbRxHErlNBUsm9BJ2FKX6T" - assert signer.verfer.qb64 == "1AAJA3cK_P2CDlh-_EMFPvyqTPI1POkw-dr14DANx5JEXDCZ" - - # Test vectors from CERSide - seed = (b'\x35\x86\xc9\xa0\x4d\x33\x67\x85\xd5\xe4\x6a\xda\x62\xf0\x54\xc5\xa5\xf4\x32\x3f\x46\xcb\x92\x23\x07' - b'\xe0\xe2\x79\xb7\xe5\xf5\x0a') - verkey = (b"\x03\x16\x99\xbc\xa0\x51\x8f\xa6\x6c\xb3\x5d\x6b\x0a\x92\xf6\x84\x96\x28\x7b\xb6\x64\xe8\xe8\x57\x69" - b"\x15\xb8\xea\x9a\x02\x06\x2a\xff") - sig = (b'\x8c\xfa\xb4\x40\x01\xd2\xab\x4a\xbc\xc5\x96\x8b\xa2\x65\x76\xcd\x51\x9d\x3b\x40\xc3\x35\x21\x73\x9a\x1b' - b'\xe8\x2f\xe1\x30\x28\xe1\x07\x90\x08\xa6\x42\xd7\x3f\x36\x8c\x96\x32\xff\x01\x64\x03\x18\x08\x85\xb8\xa4' - b'\x97\x76\xbe\x9c\xe4\xd7\xc5\xe7\x05\xda\x51\x23') - - signerqb64 = "QDWGyaBNM2eF1eRq2mLwVMWl9DI_RsuSIwfg4nm35fUK" - verferqb64 = "1AAJAxaZvKBRj6Zss11rCpL2hJYoe7Zk6OhXaRW46poCBir_" - cigarqb64 = "0ICM-rRAAdKrSrzFlouiZXbNUZ07QMM1IXOaG-gv4TAo4QeQCKZC1z82jJYy_wFkAxgIhbikl3a-nOTXxecF2lEj" - - ser = b'abc' - signer = Signer(raw=seed, code=MtrDex.ECDSA_256r1_Seed) - cigar = signer.sign(ser) - assert signer.code == MtrDex.ECDSA_256r1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.qb64 == signerqb64 - - assert signer.verfer.code == MtrDex.ECDSA_256r1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.verfer.raw == verkey - assert signer.verfer.qb64 == verferqb64 - - assert cigar.code == MtrDex.ECDSA_256r1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - assert signer.verfer.verify(cigar.raw, ser) - assert signer.verfer.verify(sig, ser) - - cigar = Cigar(raw=sig, code=MtrDex.ECDSA_256r1_Sig) - assert cigar.qb64 == cigarqb64 - - - # Test Secp256k1, default seed - signer = Signer(code=MtrDex.ECDSA_256k1_Seed) - assert signer.code == MtrDex.ECDSA_256k1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.ECDSA_256k1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - # create something to sign and verify - ser = b'abcdefghijklmnopqrstuvwxyz0123456789' - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.ECDSA_256k1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result is True - - index = 0 - siger = signer.sign(ser, index=index) - assert siger.code == IdrDex.ECDSA_256k1_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') - assert result == False - - # Non transferable - signer = Signer(code=MtrDex.ECDSA_256k1_Seed, transferable=False) # ECDSA_256k1N verifier - assert signer.code == MtrDex.ECDSA_256k1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.ECDSA_256k1N - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.ECDSA_256k1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result == True - - siger = signer.sign(ser, index=0) - assert siger.code == IdrDex.ECDSA_256k1_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') - assert result == False - - # Test non-default seed - seed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - signer = Signer(raw=seed, code=MtrDex.ECDSA_256k1_Seed) - assert signer.code == MtrDex.ECDSA_256k1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.verfer.code == MtrDex.ECDSA_256k1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.ECDSA_256k1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result == True - - index = 1 - siger = signer.sign(ser, index=index) - assert siger.code == IdrDex.ECDSA_256k1_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') - assert result == False - - # different both so Big - ondex = 3 - siger = signer.sign(ser, index=index, ondex=ondex) - assert siger.code == IdrDex.ECDSA_256k1_Big_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == ondex - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # Test hardcoded seed from CERSide - seed = (b'\x9f{\xa8\xa7\xa8C9\x96&\xfa\xb1\x99\xeb\xaa \xc4\x1bG\x11\xc4\xaeSAR\xc9\xbd\x04\x9d\x85)~\x93') - signer = Signer(raw=seed, code=MtrDex.ECDSA_256k1_Seed) - assert signer.code == MtrDex.ECDSA_256k1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.verfer.code == MtrDex.ECDSA_256k1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.qb64 == "JJ97qKeoQzmWJvqxmeuqIMQbRxHErlNBUsm9BJ2FKX6T" - assert signer.verfer.qb64 == "1AABAg299p5IMvuw71HW_TlbzGq5cVOQ7bRbeDuhheF-DPYk" - - # Test vectors from CERSide - seed = (b'\x7f\x98\x0a\x3b\xe4\x45\xd7\x8c\xc9\x79\xa1\xee\x26\x20\x9c\x17\x71\x16\xab\xa6\xd6\xf1\x6a\x01\xe7\xb3\xce\xfe\xe2\x6c\x06\x08') - verkey = (b"\x02\xdb\x98\x33\x85\xa8\x0e\xbb\x7c\x15\x5d\xdd\xc6\x47\x6a\x24\x07\x9a\x7c\x96\x5f\x05\x0f\x62\xde\x2d\x47\x56\x9b\x54\x29\x16\x79") - sig = (b'\x5f\x80\xc0\x5a\xe4\x71\x32\x5d\xf7\xcb\xdb\x1b\xc2\xf4\x11\xc3\x05\xaf\xf4\xbe\x3b\x7e\xac\x3e\x8c\x15' - b'\x3a\x9f\xa5\x0a\x3d\x69\x75\x45\x93\x34\xc8\x96\x2b\xfe\x79\x8d\xd1\x4e\x9c\x1f\x6c\xa7\xc8\x12\xd6' - b'\x7a\x6c\xc5\x74\x9f\xef\x8d\xa7\x25\xa2\x95\x47\xcc') - - signerqb64 = "JH-YCjvkRdeMyXmh7iYgnBdxFqum1vFqAeezzv7ibAYI" - verferqb64 = "1AABAtuYM4WoDrt8FV3dxkdqJAeafJZfBQ9i3i1HVptUKRZ5" - cigarqb64 = "0CBfgMBa5HEyXffL2xvC9BHDBa_0vjt-rD6MFTqfpQo9aXVFkzTIliv-eY3RTpwfbKfIEtZ6bMV0n--NpyWilUfM" - - ser = b'abc' - signer = Signer(raw=seed, code=MtrDex.ECDSA_256k1_Seed) - cigar = signer.sign(ser) - assert signer.code == MtrDex.ECDSA_256k1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.qb64 == signerqb64 - - assert signer.verfer.code == MtrDex.ECDSA_256k1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.verfer.raw == verkey - assert signer.verfer.qb64 == verferqb64 - - assert cigar.code == MtrDex.ECDSA_256k1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - assert signer.verfer.verify(cigar.raw, ser) - assert signer.verfer.verify(sig, ser) - - cigar = Cigar(raw=sig, code=MtrDex.ECDSA_256k1_Sig) - assert cigar.qb64 == cigarqb64 - - - # test with only and ondex parameters - - """ Done Test """ - -def test_cipher(): - """ - Test Cipher subclass of Matter - """ - # conclusion never use box_seed_keypair always use sign_seed_keypair and - # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey - # is always the same. - - assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 - - # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' - b'\xde\x06\xc0+') - seedqb64b = Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b - assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' - #b'AGDswxA8qdkb646JFZWUflm_OKUeF41iG2gTw3N4GwCs' - - # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - salt = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' - saltqb64b = Matter(raw=salt, code=MtrDex.Salt_128).qb64b - assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' - - # seed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) - cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' - verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) - pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) - prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) - - with pytest.raises(EmptyMaterialError): - cipher = Cipher() - - raw = pysodium.crypto_box_seal(seedqb64b, pubkey) # uses nonce so different everytime - cipher = Cipher(raw=raw) - assert cipher.code == MtrDex.X25519_Cipher_Seed - uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) - assert uncb == seedqb64b - - # test .decrypt method needs qb64 - prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b - assert cipher.decrypt(prikey=prikeyqb64).qb64b == seedqb64b - - cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b - assert cipher.decrypt(seed=cryptseedqb64).qb64b == seedqb64b - - raw = pysodium.crypto_box_seal(saltqb64b, pubkey) # uses nonce so different everytime - cipher = Cipher(raw=raw) - assert cipher.code == MtrDex.X25519_Cipher_Salt - uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) - assert uncb == saltqb64b - - # test .decrypt method needs qb64 - prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b - assert cipher.decrypt(prikey=prikeyqb64).qb64b == saltqb64b - - cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b - assert cipher.decrypt(seed=cryptseedqb64).qb64b == saltqb64b - - with pytest.raises(ValueError): # bad code - cipher = Cipher(raw=raw, code=MtrDex.Ed25519N) - """ Done Test """ - - -def test_encrypter(): - """ - Test Encrypter subclass of Matter - """ - # conclusion never use box_seed_keypair always use sign_seed_keypair and - # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey - # is always the same. - - assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 - - # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' - b'\xde\x06\xc0+') - seedqb64b = Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b - assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' - - # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - salt = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' - saltqb64b = Matter(raw=salt, code=MtrDex.Salt_128).qb64b - assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' - - # seed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) - cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' - cryptsigner = Signer(raw=cryptseed, code=MtrDex.Ed25519_Seed, transferable=True) - verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) # raw - pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) - prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) - - with pytest.raises(EmptyMaterialError): - encrypter = Encrypter() - - encrypter = Encrypter(raw=pubkey) - assert encrypter.code == MtrDex.X25519 - assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' - assert encrypter.raw == pubkey - assert encrypter.verifySeed(seed=cryptsigner.qb64) - - cipher = encrypter.encrypt(ser=seedqb64b) - assert cipher.code == MtrDex.X25519_Cipher_Seed - uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) - assert uncb == seedqb64b - - cipher = encrypter.encrypt(ser=saltqb64b) - assert cipher.code == MtrDex.X25519_Cipher_Salt - uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) - assert uncb == saltqb64b - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) - - encrypter = Encrypter(verkey=verfer.qb64) - assert encrypter.code == MtrDex.X25519 - assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' - assert encrypter.raw == pubkey - - encrypter = Encrypter(verkey=verfer.qb64b) - assert encrypter.code == MtrDex.X25519 - assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' - assert encrypter.raw == pubkey - - # user Prefixer to generate original verkey - prefixer = Prefixer(qb64=verfer.qb64) - encrypter = Encrypter(verkey=prefixer.qb64b) - assert encrypter.code == MtrDex.X25519 - assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' - assert encrypter.raw == pubkey - """ Done Test """ - - -def test_decrypter(): - """ - Test Decrypter subclass of Matter - """ - # conclusion never use box_seed_keypair always use sign_seed_keypair and - # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey - # is always the same. - - assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 - - # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' - b'\xde\x06\xc0+') - signer = Signer(raw=seed, code=MtrDex.Ed25519_Seed) - assert signer.verfer.code == MtrDex.Ed25519 - assert signer.verfer.transferable # default - seedqb64b = signer.qb64b - assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' - # also works for Matter - assert seedqb64b == Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b - - # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - raw = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' - salter = Salter(raw=raw, code=MtrDex.Salt_128) - assert salter.code == MtrDex.Salt_128 - saltqb64b = salter.qb64b - assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' - # also works for Matter - assert saltqb64b == Matter(raw=raw, code=MtrDex.Salt_128).qb64b # - - # cryptseed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) - cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' - cryptsigner = Signer(raw=cryptseed, code=MtrDex.Ed25519_Seed, transferable=True) - verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) # raw - pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) - prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) - - with pytest.raises(EmptyMaterialError): - decrypter = Decrypter() - - # create encrypter - encrypter = Encrypter(raw=pubkey) - assert encrypter.code == MtrDex.X25519 - assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' - assert encrypter.raw == pubkey - - # create cipher of seed - seedcipher = encrypter.encrypt(ser=seedqb64b) - assert seedcipher.code == MtrDex.X25519_Cipher_Seed - # each encryption uses a nonce so not a stable representation for testing - - # create decrypter from prikey - decrypter = Decrypter(raw=prikey) - assert decrypter.code == MtrDex.X25519_Private - assert decrypter.qb64 == 'OLCFxqMz1z1UUS0TEJnvZP_zXHcuYdQsSGBWdOZeY5VQ' - assert decrypter.raw == prikey - - # decrypt seed cipher using ser - designer = decrypter.decrypt(ser=seedcipher.qb64b, transferable=signer.verfer.transferable) - assert designer.qb64b == seedqb64b - assert designer.code == MtrDex.Ed25519_Seed - assert designer.verfer.code == MtrDex.Ed25519 - assert signer.verfer.transferable - - # decrypt seed cipher using cipher - designer = decrypter.decrypt(cipher=seedcipher, transferable=signer.verfer.transferable) - assert designer.qb64b == seedqb64b - assert designer.code == MtrDex.Ed25519_Seed - assert designer.verfer.code == MtrDex.Ed25519 - assert signer.verfer.transferable - - # create cipher of salt - saltcipher = encrypter.encrypt(ser=saltqb64b) - assert saltcipher.code == MtrDex.X25519_Cipher_Salt - # each encryption uses a nonce so not a stable representation for testing - - # decrypt salt cipher using ser - desalter = decrypter.decrypt(ser=saltcipher.qb64b) - assert desalter.qb64b == saltqb64b - assert desalter.code == MtrDex.Salt_128 - - # decrypt salt cipher using cipher - desalter = decrypter.decrypt(cipher=saltcipher) - assert desalter.qb64b == saltqb64b - assert desalter.code == MtrDex.Salt_128 - - # use previously stored fully qualified seed cipher with different nonce - # get from seedcipher above - cipherseed = ('PM9jOGWNYfjM_oLXJNaQ8UlFSAV5ACjsUY7J16xfzrlpc9Ve3A5WYrZ4o_' - 'NHtP5lhp78Usspl9fyFdnCdItNd5JyqZ6dt8SXOt6TOqOCs-gy0obrwFkPPqBvVkEw') - designer = decrypter.decrypt(ser=cipherseed, transferable=signer.verfer.transferable) - assert designer.qb64b == seedqb64b - assert designer.code == MtrDex.Ed25519_Seed - assert designer.verfer.code == MtrDex.Ed25519 - - # use previously stored fully qualified salt cipher with different nonce - # get from saltcipher above - ciphersalt = ('1AAHjlR2QR9J5Et67Wy-ZaVdTryN6T6ohg44r73GLRPnHw-5S3ABFkhWy' - 'IwLOI6TXUB_5CT13S8JvknxLxBaF8ANPK9FSOPD8tYu') - desalter = decrypter.decrypt(ser=ciphersalt) - assert desalter.qb64b == saltqb64b - assert desalter.code == MtrDex.Salt_128 - - # Create new decrypter but use seed parameter to init prikey - decrypter = Decrypter(seed=cryptsigner.qb64b) - assert decrypter.code == MtrDex.X25519_Private - assert decrypter.qb64 == 'OLCFxqMz1z1UUS0TEJnvZP_zXHcuYdQsSGBWdOZeY5VQ' - assert decrypter.raw == prikey - - # decrypt ciphersalt - desalter = decrypter.decrypt(ser=saltcipher.qb64b) - assert desalter.qb64b == saltqb64b - assert desalter.code == MtrDex.Salt_128 - - """ Done Test """ - - -def test_salter(): - """ - Test the support functionality for salter subclass of crymat - """ - salter = Salter() # defaults to CryTwoDex.Salt_128 - assert salter.code == MtrDex.Salt_128 - assert len(salter.raw) == Matter._rawSize(salter.code) == 16 - - raw = b'0123456789abcdef' - salter = Salter(raw=raw) - assert salter.raw == raw - assert salter.qb64 == '0AAwMTIzNDU2Nzg5YWJjZGVm' #'0ACDEyMzQ1Njc4OWFiY2RlZg' - - signer = salter.signer(path="01", temp=True) # defaults to Ed25519 - assert signer.code == MtrDex.Ed25519_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.Ed25519 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.qb64 == 'AMPsqBZxWdtYpBhrWnKYitwFa77s902Q-nX3sPTzqs0R' - #'Aw-yoFnFZ21ikGGtacpiK3AVrvuz3TZD6dfew9POqzRE' - assert signer.verfer.qb64 == 'DFYFwZJOMNy3FknECL8tUaQZRBUyQ9xCv6F8ckG-UCrC' # - # 'DVgXBkk4w3LcWScQIvy1RpBlEFTJD3EK_oXxyQb5QKsI' - - signer = salter.signer(path="01") # defaults to Ed25519 temp = False level="low" - assert signer.code == MtrDex.Ed25519_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.Ed25519 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.qb64 == 'AEkqQiNTexWB9fTLpgJp_lXW63tFlT-Y0_mgQww4o-dC' - # 'ASSpCI1N7FYH19MumAmn-Vdbre0WVP5jT-aBDDDij50I' - assert signer.verfer.qb64 == 'DPJGyH9H1M_SUSf18RzX8OqdyhxEyZJpKm5Em0PnpsWd' - #'D8kbIf0fUz9JRJ_XxHNfw6p3KHETJkmkqbkSbQ-emxZ0' - - salter = Salter(qb64='0AAwMTIzNDU2Nzg5YWJjZGVm') - assert salter.raw == raw - assert salter.qb64 == '0AAwMTIzNDU2Nzg5YWJjZGVm' - - with pytest.raises(ShortageError): - salter = Salter(qb64='') - - salter = Salter(raw=raw) - assert salter.stretch(temp=True) == b'\xd4@\xeb\xa6x\x86\xdf\x93\xd6C\xdc\xb8\xa6\x9b\x02\xafh\xc1m(L\xd6\xf6\x86YU>$[\xf9\xef\xc0' - assert salter.stretch(tier=Tiers.low) == b'\xf8e\x80\xbaX\x08\xb9\xba\xc6\x1e\x84\r\x1d\xac\xa7\\\x82Wc@`\x13\xfd\x024t\x8ct\xd3\x01\x19\xe9' - assert salter.stretch(tier=Tiers.med) == b',\xf3\x8c\xbb\xe9)\nSQ\xec\xad\x8c9?\xaf\xb8\xb0\xb3\xcdB\xda\xd8\xb6\xf7\r\xf6D}Z\xb9Y\x16' - assert salter.stretch(tier=Tiers.high) == b'(\xcd\xc4\xb85\xcd\xe8:\xfc\x00\x8b\xfd\xa6\tj.y\x98\x0b\x04\x1c\xe3hBc!I\xe49K\x16-' - - """ Done Test """ - - -def test_generatesigners(): - """ - Test the support function genSigners - - """ - signers = generateSigners(count=2, transferable=False) - assert len(signers) == 2 - for signer in signers: - assert signer.verfer.code == MtrDex.Ed25519N - - # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) # raw salt - raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - assert len(raw) == 16 - signers = generateSigners(raw=raw, count=4) # default is transferable - assert len(signers) == 4 - for signer in signers: - assert signer.code == MtrDex.Ed25519_Seed - assert signer.verfer.code == MtrDex.Ed25519 - - sigkeys = [signer.qb64 for signer in signers] - assert sigkeys == ['AK8F6AAiYDpXlWdj2O5F5-6wNCCNJh2A4XOlqwR_HwwH', - 'AOs8-zNPPh0EhavdrCfCiTk9nGeO8e6VxUCzwdKXJAd0', - 'AHMBU5PsIJN2U9m7j0SGyvs8YD8fkym2noELzxIrzfdG', - 'AJZ7ZLd7unQ4IkMUwE69NXcvDO9rrmmRH_Xk3TPu9BpP'] - - """ End Test """ - def test_diger(): """ @@ -5763,7 +5081,7 @@ def test_prefixer(): 'AC6vsNVCpHa6acGcxk7c-D1mBHlptPrAx8zr-bKvesSW'] # create signers from secrets - signers = [Signer(qb64=secret) for secret in secrets] # faster + signers = [core.Signer(qb64=secret) for secret in secrets] # faster assert [siger.qb64 for siger in signers] == secrets # each signer has verfer for keys @@ -6754,7 +6072,6 @@ def test_tholder(): #test_number() #test_seqner() #test_siger() - #test_signer() #test_nexter() #test_tholder() #test_labels() diff --git a/tests/core/test_delegating.py b/tests/core/test_delegating.py index b6685331d..5d659f5ce 100644 --- a/tests/core/test_delegating.py +++ b/tests/core/test_delegating.py @@ -6,8 +6,12 @@ import os from keri import help -from keri.app import keeping, habbing + +from keri import core from keri.core import coring, eventing, parsing + +from keri.app import keeping, habbing + from keri.db import dbing, basing logger = help.ogler.getLogger() @@ -20,8 +24,8 @@ def test_delegation(): """ # bob is the delegator del is bob's delegate - bobSalt = coring.Salter(raw=b'0123456789abcdef').qb64 - delSalt = coring.Salter(raw=b'abcdef0123456789').qb64 + bobSalt = core.Salter(raw=b'0123456789abcdef').qb64 + delSalt = core.Salter(raw=b'abcdef0123456789').qb64 with (basing.openDB(name="bob") as bobDB, \ keeping.openKS(name="bob") as bobKS, \ @@ -313,12 +317,12 @@ def test_load_event(mockHelpingNowUTC): """ - topSalt = coring.Salter(raw=b'0123456789abcdef').qb64 - wopSalt = coring.Salter(raw=b'0123456789abcdef').qb64 - midSalt = coring.Salter(raw=b'abcdef0123456789').qb64 - widSalt = coring.Salter(raw=b'abcdef0123456789').qb64 - botSalt = coring.Salter(raw=b'zyxwvutsrponmlkj').qb64 - wotSalt = coring.Salter(raw=b'zyxwvutsrponmlkj').qb64 + topSalt = core.Salter(raw=b'0123456789abcdef').qb64 + wopSalt = core.Salter(raw=b'0123456789abcdef').qb64 + midSalt = core.Salter(raw=b'abcdef0123456789').qb64 + widSalt = core.Salter(raw=b'abcdef0123456789').qb64 + botSalt = core.Salter(raw=b'zyxwvutsrponmlkj').qb64 + wotSalt = core.Salter(raw=b'zyxwvutsrponmlkj').qb64 with (habbing.openHby(name="top", base="test", salt=topSalt) as topHby, habbing.openHby(name="wop", base="test", salt=wopSalt) as wopHby, diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index 07ac3d6c0..a100cf03a 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -9,9 +9,13 @@ from keri import help from keri.help import helping + +from keri import core +from keri.core import coring, eventing, parsing + from keri.db import dbing, basing from keri.app import keeping -from keri.core import coring, eventing, parsing + logger = help.ogler.getLogger() @@ -21,7 +25,7 @@ def test_partial_signed_escrow(): Test partially signed escrow """ - salt = coring.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter psr = parsing.Parser() # init event DB and keep DB @@ -348,8 +352,8 @@ def test_missing_delegator_escrow(): """ # bob is the delegator del is bob's delegate - bobSalt = coring.Salter(raw=b'0123456789abcdef').qb64 - delSalt = coring.Salter(raw=b'abcdef0123456789').qb64 + bobSalt = core.Salter(raw=b'0123456789abcdef').qb64 + delSalt = core.Salter(raw=b'abcdef0123456789').qb64 psr = parsing.Parser() @@ -592,7 +596,7 @@ def test_misfit_escrow(): Test misfit escrow """ - salt = coring.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter # stub for now @@ -605,7 +609,7 @@ def test_out_of_order_escrow(): Test out of order escrow """ - salt = coring.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter psr = parsing.Parser() # init event DB and keep DB @@ -801,7 +805,7 @@ def test_unverified_receipt_escrow(): Test unverified receipt escrow """ - salt = coring.Salter(raw=b'0123456789abcdef').qb64 # init Salter + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter psr = parsing.Parser() # init event DB and keep DB @@ -1086,7 +1090,7 @@ def test_unverified_trans_receipt_escrow(): Test unverified transferable receipt escrow """ - salt = coring.Salter(raw=b'0123456789abcdef').qb64 # init Salter + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter psr = parsing.Parser() # init event DB and keep DB diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index d6c50c361..d2a330850 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -12,11 +12,12 @@ from keri import kering from keri.app import habbing, keeping from keri.app.keeping import openKS, Manager +from keri import core +from keri.core import Signer from keri.core import coring, eventing, parsing, serdering from keri.core.coring import (Diger, MtrDex, Matter, - CtrDex, Counter, Salter, Cigar, - Seqner, Verfer, Signer, Prefixer, - generateSigners, DigDex) + CtrDex, Counter, Cigar, + Seqner, Verfer, Prefixer, DigDex) from keri.core.indexing import (IdrDex, IdxSigDex, Indexer, Siger) from keri.core.eventing import Kever, Kevery from keri.core.eventing import (SealDigest, SealRoot, SealBacker, @@ -1521,7 +1522,7 @@ def test_state(mockHelpingNowUTC): # use same salter for all but different path # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # State NonDelegated (key state notification) # create transferable key pair for controller of KEL @@ -1802,7 +1803,7 @@ def test_messagize(): """ Test messagize utility function """ - salter = Salter(raw=b'0123456789abcdef') + salter = core.Salter(raw=b'0123456789abcdef') with openDB(name="edy") as db, openKS(name="edy") as ks: # Init key pair manager mgr = Manager(ks=ks, salt=salter.qb64) @@ -2063,7 +2064,7 @@ def test_kever(mockHelpingNowUTC): with openDB() as db: # Transferable case # Setup inception key event dict raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create current key sith = 1 # one signer # original signing keypair transferable default @@ -2146,7 +2147,7 @@ def test_kever(mockHelpingNowUTC): # test exposeds raw = b"raw salt to test" # create signers with verfers - signers = coring.Salter(raw=raw).signers(count=3, path="next", temp=True) + signers = core.Salter(raw=raw).signers(count=3, path="next", temp=True) # create something to sign ser = b'abcdefghijklmnopqrstuvwxyz0123456789' @@ -2452,7 +2453,7 @@ def test_keyeventsequence_0(): """ # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(raw=salt, count=8, transferable=True) + signers = core.generateSigners(raw=salt, count=8, transferable=True) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2722,7 +2723,7 @@ def test_keyeventsequence_1(): # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(raw=salt, count=8, transferable=True) + signers = core.generateSigners(raw=salt, count=8, transferable=True) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2819,7 +2820,7 @@ def test_multisig_digprefix(): # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(raw=salt, count=8, transferable=True) + signers = core.generateSigners(raw=salt, count=8, transferable=True) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2973,7 +2974,7 @@ def test_recovery(): """ # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(raw=salt, count=8, transferable=True) + signers = core.generateSigners(raw=salt, count=8, transferable=True) with openDB(name="controller") as conlgr, openDB(name="validator") as vallgr: event_digs = [] # list of event digs in sequence to verify against database @@ -3204,7 +3205,7 @@ def test_receipt(): """ raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create coe's signers coeSigners = salter.signers(count=8, path='coe', temp=True) @@ -3501,7 +3502,7 @@ def test_direct_mode(): # but goes both ways once initiated. raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create coe's signers coeSigners = salter.signers(count=8, path='coe', temp=True) @@ -3967,7 +3968,7 @@ def test_direct_mode_cbor_mgpk(): # but goes both ways once initiated. raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create coe's signers coeSigners = salter.signers(count=8, path='coe', temp=True) @@ -4731,7 +4732,7 @@ def test_reload_kever(mockHelpingNowUTC): Test reload Kever from keystate state message """ - with habbing.openHby(name="nat", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as natHby: + with habbing.openHby(name="nat", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as natHby: # setup Nat's habitat using default salt multisig already incepts natHab = natHby.makeHab(name="nat", isith='2', icount=3) assert natHab.name == 'nat' @@ -4800,10 +4801,10 @@ def test_reload_kever(mockHelpingNowUTC): def test_load_event(mockHelpingNowUTC): - with habbing.openHby(name="tor", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as torHby, \ - habbing.openHby(name="wil", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as wilHby, \ - habbing.openHby(name="wan", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as wanHby, \ - habbing.openHby(name="tee", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as teeHby: + with habbing.openHby(name="tor", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as torHby, \ + habbing.openHby(name="wil", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as wilHby, \ + habbing.openHby(name="wan", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as wanHby, \ + habbing.openHby(name="tee", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as teeHby: wanKvy = Kevery(db=wanHby.db, lax=False, local=False) torKvy = Kevery(db=torHby.db, lax=False, local=False) diff --git a/tests/core/test_indexing.py b/tests/core/test_indexing.py index fbdc7bffd..705e7743c 100644 --- a/tests/core/test_indexing.py +++ b/tests/core/test_indexing.py @@ -1,6 +1,6 @@ # -*- encoding: utf-8 -*- """ -tests.core.test_coring module +tests.core.test_indexing module """ from dataclasses import dataclass, astuple, asdict diff --git a/tests/core/test_kevery.py b/tests/core/test_kevery.py index e34a8281c..183b073a2 100644 --- a/tests/core/test_kevery.py +++ b/tests/core/test_kevery.py @@ -2,16 +2,22 @@ import pytest +from keri.kering import (ValidationError) + from keri import help -from keri.app import habbing + +from keri import core from keri.core import parsing, eventing, coring, serdering from keri.core.coring import CtrDex, Counter -from keri.core.coring import Salter + from keri.core.eventing import Kever, Kevery from keri.core.eventing import (incept, rotate, interact) + +from keri.app import habbing + from keri.db import dbing from keri.db.basing import openDB -from keri.kering import (ValidationError) + logger = help.ogler.getLogger() @@ -25,7 +31,7 @@ def test_kevery(): # create signers raw = b"ABCDEFGH01234567" - signers = Salter(raw=raw).signers(count=8, path='kev', temp=True) + signers = core.Salter(raw=raw).signers(count=8, path='kev', temp=True) with openDB(name="controller") as conlgr, openDB(name="validator") as vallgr: event_digs = [] # list of event digs in sequence @@ -246,7 +252,7 @@ def test_witness_state(): """ # with basing.openDB(name="controller") as bobDB, keeping.openKS(name="controller") as bobKS: - with habbing.openHby(name="controller", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(name="controller", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: wits = [ "BAMUu4hpUYY4FKd4LtsvpMN6claZKF2AUmXIgXiAI9ZQ", @@ -336,11 +342,11 @@ def test_stale_event_receipts(): Bam is verifying the key events with receipts from Bob """ # openHby default temp=True - with (habbing.openHby(name="bob", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as bobHby, - habbing.openHby(name="bam", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as bamHby, - habbing.openHby(name="wes", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as wesHby, - habbing.openHby(name="wan", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as wanHby, - habbing.openHby(name="wil", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as wilHby): + with (habbing.openHby(name="bob", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as bobHby, + habbing.openHby(name="bam", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as bamHby, + habbing.openHby(name="wes", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as wesHby, + habbing.openHby(name="wan", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as wanHby, + habbing.openHby(name="wil", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as wilHby): # setup Wes's habitat nontrans wesHab = wesHby.makeHab(name="wes", isith='1', icount=1, transferable=False,) diff --git a/tests/core/test_keystate.py b/tests/core/test_keystate.py index a9160a10f..64a790446 100644 --- a/tests/core/test_keystate.py +++ b/tests/core/test_keystate.py @@ -6,8 +6,12 @@ routes: /ksn """ + +from keri import core +from keri.core import eventing, parsing, routing, serdering + from keri.app import habbing -from keri.core import coring, eventing, parsing, routing, serdering + def test_keystate(mockHelpingNowUTC): @@ -53,11 +57,11 @@ def test_keystate(mockHelpingNowUTC): """ raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = coring.Salter(raw=raw) + salter = core.Salter(raw=raw) salt = salter.qb64 assert salt == '0AAFqo8tU5rp-lWcApybCEh1' - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 # Bob is the controller # Wes is his witness diff --git a/tests/core/test_parsing.py b/tests/core/test_parsing.py index ad647506a..95ebc7e9e 100644 --- a/tests/core/test_parsing.py +++ b/tests/core/test_parsing.py @@ -8,14 +8,18 @@ import pytest from hio.help import decking -from keri.app import habbing + from keri.kering import ValidationError + +from keri import help + +from keri import core from keri.core import parsing, coring -from keri.core.coring import (CtrDex, Counter, Signer, Salter) +from keri.core.coring import (CtrDex, Counter,) from keri.core.eventing import (Kever, Kevery, incept, rotate, interact) -from keri.db.basing import openDB -from keri import help +from keri.db.basing import openDB +from keri.app import habbing from keri.peer import exchanging logger = help.ogler.getLogger() @@ -29,7 +33,7 @@ def test_parser(): # create signers raw = b"ABCDEFGH01234567" - signers = Salter(raw=raw).signers(count=8, path='psr', temp=True) + signers = core.Salter(raw=raw).signers(count=8, path='psr', temp=True) with openDB(name="controller") as conDB, openDB(name="validator") as valDB: event_digs = [] # list of event digs in sequence diff --git a/tests/core/test_parsing_pathed.py b/tests/core/test_parsing_pathed.py index e429ee622..399991445 100644 --- a/tests/core/test_parsing_pathed.py +++ b/tests/core/test_parsing_pathed.py @@ -7,9 +7,12 @@ from hio.help import decking from keri import help -from keri.app import habbing -from keri.core import parsing, coring, serdering + +from keri import core +from keri.core import parsing, coring from keri.peer import exchanging +from keri.app import habbing + logger = help.ogler.getLogger() @@ -27,8 +30,8 @@ def handle(self, serder, attachments=None): self.msgs.append(serder) self.atcs.append(attachments) - with (habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby, - habbing.openHby(name="deb", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as debHby): + with (habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby, + habbing.openHby(name="deb", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as debHby): sith = ["1/2", "1/2", "1/2"] # weighted signing threshold palHab = hby.makeHab(name="pal") debHab = debHby.makeHab(name="deb", isith=sith, icount=3) diff --git a/tests/core/test_partial_rotation.py b/tests/core/test_partial_rotation.py index b19c1b55c..698308e2e 100644 --- a/tests/core/test_partial_rotation.py +++ b/tests/core/test_partial_rotation.py @@ -6,8 +6,10 @@ import pytest from keri import kering + +from keri import core from keri.core import coring, eventing -from keri.core.coring import Salter + from keri.db.basing import openDB @@ -16,7 +18,7 @@ def test_partial_rotation(): # create signers raw = b"ABCDEFGH01234567" - signers = Salter(raw=raw).signers(count=18, path='rot', temp=True) + signers = core.Salter(raw=raw).signers(count=18, path='rot', temp=True) # partial rotation with numeric thresholds with openDB(name="controller") as db: diff --git a/tests/core/test_replay.py b/tests/core/test_replay.py index 5aebcf30f..29547a88c 100644 --- a/tests/core/test_replay.py +++ b/tests/core/test_replay.py @@ -9,9 +9,11 @@ from keri import help from keri.help import helping -from keri.app import habbing +from keri import core from keri.core import coring, eventing, parsing, serdering, indexing +from keri.app import habbing + logger = help.ogler.getLogger() @@ -26,8 +28,8 @@ def test_replay(): Deb replays Deb's events with both Cam's and Bev's receipts to Cam Compare replay of Deb's events with receipts by both Deb and Cam to confirm identical """ - artSalt = coring.Salter(raw=b'abcdef0123456789').qb64 - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + artSalt = core.Salter(raw=b'abcdef0123456789').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 with (habbing.openHby(name="deb", base="test", salt=default_salt) as debHby, habbing.openHby(name="cam", base="test", salt=default_salt) as camHby, @@ -495,8 +497,8 @@ def test_replay_all(): Replay all the events in database. """ - artSalt = coring.Salter(raw=b'abcdef0123456789').qb64 - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + artSalt = core.Salter(raw=b'abcdef0123456789').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 with (habbing.openHby(name="deb", base="test", salt=default_salt) as debHby, habbing.openHby(name="cam", base="test", salt=default_salt) as camHby, diff --git a/tests/core/test_reply.py b/tests/core/test_reply.py index 153bcd25c..4705065eb 100644 --- a/tests/core/test_reply.py +++ b/tests/core/test_reply.py @@ -14,13 +14,16 @@ from keri import kering +from keri import help + +from keri import core from keri.core import eventing, parsing, routing -from keri.core.coring import MtrDex, Salter +from keri.core.coring import MtrDex from keri.db import basing from keri.app import habbing, keeping -from keri import help + logger = help.ogler.getLogger() @@ -63,7 +66,7 @@ def test_reply(mockHelpingNowUTC): # use same salter for all but different path # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salt = Salter(raw=raw).qb64 + salt = core.Salter(raw=raw).qb64 assert salt == '0AAFqo8tU5rp-lWcApybCEh1' # makHab uses stem=name to make different names have differnt AID pre with (habbing.openHby(name="wes", base="test", salt=salt) as wesHby, diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 8151f3944..2d9bb346f 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -18,7 +18,7 @@ VERRAWSIZE, VERFMT, MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN,) -from keri.core import coring +from keri import core from keri.core.serdering import (FieldDom, FieldDom, Serdery, Serder, SerderKERI, SerderACDC, ) @@ -451,7 +451,7 @@ def test_serder(): # Test with non-digestive code for 'i' saidive field no sad serder = Serder(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519}, + saids = {'i': core.PreDex.Ed25519}, verify=False) assert serder.sad == {'v': 'KERI10JSON0000a3_', @@ -892,7 +892,7 @@ def test_serderkeri_icp(): # Test with non-digestive code for 'i' saidive field no sad serder = SerderKERI(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519}, + saids = {'i': core.PreDex.Ed25519}, verify=False) assert serder.sad == {'v': 'KERI10JSON0000a3_', @@ -1391,7 +1391,7 @@ def test_serderkeri_dip(): # Test with non-digestive code for 'i' saidive field no sad serder = SerderKERI(makify=True, ilk=kering.Ilks.dip, - saids = {'i': coring.PreDex.Ed25519}, + saids = {'i': core.PreDex.Ed25519}, verify=False) assert serder.sad == {'v': 'KERI10JSON0000ab_', @@ -1449,7 +1449,7 @@ def test_serderkeri_dip(): serder = SerderKERI(sad=sad, makify=True, - saids = {'i': coring.PreDex.Blake3_256}) + saids = {'i': core.PreDex.Blake3_256}) assert serder.sad == {'v': 'KERI10JSON000103_', 't': 'dip', @@ -2647,10 +2647,10 @@ def test_cesr_native_dumps(): # use same salter for all but different path # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = coring.Salter(raw=raw) + salter = core.Salter(raw=raw) - csigners = coring.generateSigners(raw=salter.raw, count=3) - wsigners = coring.generateSigners(raw=salter.raw, count=3, transferable=False) + csigners = core.generateSigners(raw=salter.raw, count=3) + wsigners = core.generateSigners(raw=salter.raw, count=3, transferable=False) keys = ["EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf87Bc70J"] diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py new file mode 100644 index 000000000..4465c58d0 --- /dev/null +++ b/tests/core/test_signing.py @@ -0,0 +1,717 @@ +# -*- encoding: utf-8 -*- +""" +tests.core.test_indexing module + +""" + + +import pysodium + +import pytest + +from keri import kering + +from keri.help import helping + +from keri import core +from keri.core import (Tiers, ) +from keri.core import (Matter, MtrDex, Cigar, Verfer, Prefixer) +from keri.core import (Indexer, IdrDex, ) +from keri.core import (Signer, generateSigners, Salter, + Cipher, Encrypter, Decrypter, ) + + + +def test_signer(): + """ + Test Signer instance + """ + signer = Signer() # defaults provide Ed25519 signer Ed25519 verfer + assert signer.code == MtrDex.Ed25519_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.verfer.code == MtrDex.Ed25519 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + + # create something to sign and verify + ser = b'abcdefghijklmnopqrstuvwxyz0123456789' + + cigar = signer.sign(ser) + assert cigar.code == MtrDex.Ed25519_Sig + assert len(cigar.raw) == Matter._rawSize(cigar.code) + result = signer.verfer.verify(cigar.raw, ser) + assert result == True + + index = 0 + siger = signer.sign(ser, index=index) + assert siger.code == IdrDex.Ed25519_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == index + result = signer.verfer.verify(siger.raw, ser) + assert result == True + result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') + assert result == False + + assert cigar.raw == siger.raw + + with pytest.raises(ValueError): # use invalid code not SEED type code + signer = Signer(code=MtrDex.Ed25519N) + + # Non transferable defaults + signer = Signer(transferable=False) # Ed25519N verifier + assert signer.code == MtrDex.Ed25519_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.verfer.code == MtrDex.Ed25519N + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + + cigar = signer.sign(ser) + assert cigar.code == MtrDex.Ed25519_Sig + assert len(cigar.raw) == Matter._rawSize(cigar.code) + result = signer.verfer.verify(cigar.raw, ser) + assert result == True + + siger = signer.sign(ser, index=0) + assert siger.code == IdrDex.Ed25519_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == index + result = signer.verfer.verify(siger.raw, ser) + assert result == True + result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') + assert result == False + + + # non default seed + seed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + signer = Signer(raw=seed, code=MtrDex.Ed25519_Seed) + assert signer.code == MtrDex.Ed25519_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.raw == seed + assert signer.verfer.code == MtrDex.Ed25519 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + + cigar = signer.sign(ser) + assert cigar.code == MtrDex.Ed25519_Sig + assert len(cigar.raw) == Matter._rawSize(cigar.code) + result = signer.verfer.verify(cigar.raw, ser) + assert result == True + + index = 1 + siger = signer.sign(ser, index=index) + assert siger.code == IdrDex.Ed25519_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == index + result = signer.verfer.verify(siger.raw, ser) + assert result == True + + assert cigar.raw == siger.raw + + # different both so Big + ondex = 3 + siger = signer.sign(ser, index=index, ondex=ondex) + assert siger.code == IdrDex.Ed25519_Big_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == ondex + result = signer.verfer.verify(siger.raw, ser) + assert result == True + + # same but Big + index = 67 + siger = signer.sign(ser, index=index) + assert siger.code == IdrDex.Ed25519_Big_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == index + result = signer.verfer.verify(siger.raw, ser) + assert result == True + + # different both so Big + ondex = 67 + siger = signer.sign(ser, index=index, ondex=ondex) + assert siger.code == IdrDex.Ed25519_Big_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == ondex + result = signer.verfer.verify(siger.raw, ser) + assert result == True + + # current only + index = 4 + siger = signer.sign(ser, index=index, only=True) + assert siger.code == IdrDex.Ed25519_Crt_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == None + result = signer.verfer.verify(siger.raw, ser) + assert result == True + + # ignores ondex if only + siger = signer.sign(ser, index=index, only=True, ondex=index+2) + assert siger.code == IdrDex.Ed25519_Crt_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == None + result = signer.verfer.verify(siger.raw, ser) + assert result == True + + # big current only + index = 65 + siger = signer.sign(ser, index=index, only=True) + assert siger.code == IdrDex.Ed25519_Big_Crt_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == None + result = signer.verfer.verify(siger.raw, ser) + assert result == True + + # ignores ondex if only + siger = signer.sign(ser, index=index, only=True, ondex=index+2) + assert siger.code == IdrDex.Ed25519_Big_Crt_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == None + result = signer.verfer.verify(siger.raw, ser) + assert result == True + + with pytest.raises(ValueError): # use invalid code not SEED type code + signer = Signer(raw=seed, code=MtrDex.Ed25519N) + + # Test Secp256r1, default seed + signer = Signer(code=MtrDex.ECDSA_256r1_Seed) + assert signer.code == MtrDex.ECDSA_256r1_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.verfer.code == MtrDex.ECDSA_256r1 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + + cigar = signer.sign(ser) + assert cigar.code == MtrDex.ECDSA_256r1_Sig + assert len(cigar.raw) == Matter._rawSize(cigar.code) + result = signer.verfer.verify(cigar.raw, ser) + assert result is True + + # Test non-default seed + seed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + signer = Signer(raw=seed, code=MtrDex.ECDSA_256r1_Seed) + assert signer.code == MtrDex.ECDSA_256r1_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.raw == seed + assert signer.verfer.code == MtrDex.ECDSA_256r1 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + + # Test hardcoded seed + seed = (b'\x9f{\xa8\xa7\xa8C9\x96&\xfa\xb1\x99\xeb\xaa \xc4\x1bG\x11\xc4\xaeSAR\xc9\xbd\x04\x9d\x85)~\x93') + signer = Signer(raw=seed, code=MtrDex.ECDSA_256r1_Seed) + assert signer.code == MtrDex.ECDSA_256r1_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.raw == seed + assert signer.verfer.code == MtrDex.ECDSA_256r1 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + assert signer.qb64 == "QJ97qKeoQzmWJvqxmeuqIMQbRxHErlNBUsm9BJ2FKX6T" + assert signer.verfer.qb64 == "1AAJA3cK_P2CDlh-_EMFPvyqTPI1POkw-dr14DANx5JEXDCZ" + + # Test vectors from CERSide + seed = (b'\x35\x86\xc9\xa0\x4d\x33\x67\x85\xd5\xe4\x6a\xda\x62\xf0\x54\xc5\xa5\xf4\x32\x3f\x46\xcb\x92\x23\x07' + b'\xe0\xe2\x79\xb7\xe5\xf5\x0a') + verkey = (b"\x03\x16\x99\xbc\xa0\x51\x8f\xa6\x6c\xb3\x5d\x6b\x0a\x92\xf6\x84\x96\x28\x7b\xb6\x64\xe8\xe8\x57\x69" + b"\x15\xb8\xea\x9a\x02\x06\x2a\xff") + sig = (b'\x8c\xfa\xb4\x40\x01\xd2\xab\x4a\xbc\xc5\x96\x8b\xa2\x65\x76\xcd\x51\x9d\x3b\x40\xc3\x35\x21\x73\x9a\x1b' + b'\xe8\x2f\xe1\x30\x28\xe1\x07\x90\x08\xa6\x42\xd7\x3f\x36\x8c\x96\x32\xff\x01\x64\x03\x18\x08\x85\xb8\xa4' + b'\x97\x76\xbe\x9c\xe4\xd7\xc5\xe7\x05\xda\x51\x23') + + signerqb64 = "QDWGyaBNM2eF1eRq2mLwVMWl9DI_RsuSIwfg4nm35fUK" + verferqb64 = "1AAJAxaZvKBRj6Zss11rCpL2hJYoe7Zk6OhXaRW46poCBir_" + cigarqb64 = "0ICM-rRAAdKrSrzFlouiZXbNUZ07QMM1IXOaG-gv4TAo4QeQCKZC1z82jJYy_wFkAxgIhbikl3a-nOTXxecF2lEj" + + ser = b'abc' + signer = Signer(raw=seed, code=MtrDex.ECDSA_256r1_Seed) + cigar = signer.sign(ser) + assert signer.code == MtrDex.ECDSA_256r1_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.raw == seed + assert signer.qb64 == signerqb64 + + assert signer.verfer.code == MtrDex.ECDSA_256r1 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + assert signer.verfer.raw == verkey + assert signer.verfer.qb64 == verferqb64 + + assert cigar.code == MtrDex.ECDSA_256r1_Sig + assert len(cigar.raw) == Matter._rawSize(cigar.code) + assert signer.verfer.verify(cigar.raw, ser) + assert signer.verfer.verify(sig, ser) + + cigar = Cigar(raw=sig, code=MtrDex.ECDSA_256r1_Sig) + assert cigar.qb64 == cigarqb64 + + + # Test Secp256k1, default seed + signer = Signer(code=MtrDex.ECDSA_256k1_Seed) + assert signer.code == MtrDex.ECDSA_256k1_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.verfer.code == MtrDex.ECDSA_256k1 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + + # create something to sign and verify + ser = b'abcdefghijklmnopqrstuvwxyz0123456789' + + cigar = signer.sign(ser) + assert cigar.code == MtrDex.ECDSA_256k1_Sig + assert len(cigar.raw) == Matter._rawSize(cigar.code) + result = signer.verfer.verify(cigar.raw, ser) + assert result is True + + index = 0 + siger = signer.sign(ser, index=index) + assert siger.code == IdrDex.ECDSA_256k1_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == index + result = signer.verfer.verify(siger.raw, ser) + assert result == True + result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') + assert result == False + + # Non transferable + signer = Signer(code=MtrDex.ECDSA_256k1_Seed, transferable=False) # ECDSA_256k1N verifier + assert signer.code == MtrDex.ECDSA_256k1_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.verfer.code == MtrDex.ECDSA_256k1N + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + + cigar = signer.sign(ser) + assert cigar.code == MtrDex.ECDSA_256k1_Sig + assert len(cigar.raw) == Matter._rawSize(cigar.code) + result = signer.verfer.verify(cigar.raw, ser) + assert result == True + + siger = signer.sign(ser, index=0) + assert siger.code == IdrDex.ECDSA_256k1_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == index + result = signer.verfer.verify(siger.raw, ser) + assert result == True + result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') + assert result == False + + # Test non-default seed + seed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + signer = Signer(raw=seed, code=MtrDex.ECDSA_256k1_Seed) + assert signer.code == MtrDex.ECDSA_256k1_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.raw == seed + assert signer.verfer.code == MtrDex.ECDSA_256k1 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + + cigar = signer.sign(ser) + assert cigar.code == MtrDex.ECDSA_256k1_Sig + assert len(cigar.raw) == Matter._rawSize(cigar.code) + result = signer.verfer.verify(cigar.raw, ser) + assert result == True + + index = 1 + siger = signer.sign(ser, index=index) + assert siger.code == IdrDex.ECDSA_256k1_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == index + result = signer.verfer.verify(siger.raw, ser) + assert result == True + result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') + assert result == False + + # different both so Big + ondex = 3 + siger = signer.sign(ser, index=index, ondex=ondex) + assert siger.code == IdrDex.ECDSA_256k1_Big_Sig + assert len(siger.raw) == Indexer._rawSize(siger.code) + assert siger.index == index + assert siger.ondex == ondex + result = signer.verfer.verify(siger.raw, ser) + assert result == True + + # Test hardcoded seed from CERSide + seed = (b'\x9f{\xa8\xa7\xa8C9\x96&\xfa\xb1\x99\xeb\xaa \xc4\x1bG\x11\xc4\xaeSAR\xc9\xbd\x04\x9d\x85)~\x93') + signer = Signer(raw=seed, code=MtrDex.ECDSA_256k1_Seed) + assert signer.code == MtrDex.ECDSA_256k1_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.raw == seed + assert signer.verfer.code == MtrDex.ECDSA_256k1 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + assert signer.qb64 == "JJ97qKeoQzmWJvqxmeuqIMQbRxHErlNBUsm9BJ2FKX6T" + assert signer.verfer.qb64 == "1AABAg299p5IMvuw71HW_TlbzGq5cVOQ7bRbeDuhheF-DPYk" + + # Test vectors from CERSide + seed = (b'\x7f\x98\x0a\x3b\xe4\x45\xd7\x8c\xc9\x79\xa1\xee\x26\x20\x9c\x17\x71\x16\xab\xa6\xd6\xf1\x6a\x01\xe7\xb3\xce\xfe\xe2\x6c\x06\x08') + verkey = (b"\x02\xdb\x98\x33\x85\xa8\x0e\xbb\x7c\x15\x5d\xdd\xc6\x47\x6a\x24\x07\x9a\x7c\x96\x5f\x05\x0f\x62\xde\x2d\x47\x56\x9b\x54\x29\x16\x79") + sig = (b'\x5f\x80\xc0\x5a\xe4\x71\x32\x5d\xf7\xcb\xdb\x1b\xc2\xf4\x11\xc3\x05\xaf\xf4\xbe\x3b\x7e\xac\x3e\x8c\x15' + b'\x3a\x9f\xa5\x0a\x3d\x69\x75\x45\x93\x34\xc8\x96\x2b\xfe\x79\x8d\xd1\x4e\x9c\x1f\x6c\xa7\xc8\x12\xd6' + b'\x7a\x6c\xc5\x74\x9f\xef\x8d\xa7\x25\xa2\x95\x47\xcc') + + signerqb64 = "JH-YCjvkRdeMyXmh7iYgnBdxFqum1vFqAeezzv7ibAYI" + verferqb64 = "1AABAtuYM4WoDrt8FV3dxkdqJAeafJZfBQ9i3i1HVptUKRZ5" + cigarqb64 = "0CBfgMBa5HEyXffL2xvC9BHDBa_0vjt-rD6MFTqfpQo9aXVFkzTIliv-eY3RTpwfbKfIEtZ6bMV0n--NpyWilUfM" + + ser = b'abc' + signer = Signer(raw=seed, code=MtrDex.ECDSA_256k1_Seed) + cigar = signer.sign(ser) + assert signer.code == MtrDex.ECDSA_256k1_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.raw == seed + assert signer.qb64 == signerqb64 + + assert signer.verfer.code == MtrDex.ECDSA_256k1 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + assert signer.verfer.raw == verkey + assert signer.verfer.qb64 == verferqb64 + + assert cigar.code == MtrDex.ECDSA_256k1_Sig + assert len(cigar.raw) == Matter._rawSize(cigar.code) + assert signer.verfer.verify(cigar.raw, ser) + assert signer.verfer.verify(sig, ser) + + cigar = Cigar(raw=sig, code=MtrDex.ECDSA_256k1_Sig) + assert cigar.qb64 == cigarqb64 + + + # test with only and ondex parameters + + """ Done Test """ + + +# deprecated uses Salter.signers() instead +def test_generatesigners(): + """ + Test the support function genSigners + + """ + signers = generateSigners(count=2, transferable=False) + assert len(signers) == 2 + for signer in signers: + assert signer.verfer.code == MtrDex.Ed25519N + + # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) # raw salt + raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' + assert len(raw) == 16 + signers = generateSigners(raw=raw, count=4) # default is transferable + assert len(signers) == 4 + for signer in signers: + assert signer.code == MtrDex.Ed25519_Seed + assert signer.verfer.code == MtrDex.Ed25519 + + sigkeys = [signer.qb64 for signer in signers] + assert sigkeys == ['AK8F6AAiYDpXlWdj2O5F5-6wNCCNJh2A4XOlqwR_HwwH', + 'AOs8-zNPPh0EhavdrCfCiTk9nGeO8e6VxUCzwdKXJAd0', + 'AHMBU5PsIJN2U9m7j0SGyvs8YD8fkym2noELzxIrzfdG', + 'AJZ7ZLd7unQ4IkMUwE69NXcvDO9rrmmRH_Xk3TPu9BpP'] + + """ End Test """ + + +def test_salter(): + """ + Test the support functionality for salter subclass of crymat + """ + salter = Salter() # defaults to CryTwoDex.Salt_128 + assert salter.code == MtrDex.Salt_128 + assert len(salter.raw) == Matter._rawSize(salter.code) == 16 + + raw = b'0123456789abcdef' + salter = Salter(raw=raw) + assert salter.raw == raw + assert salter.qb64 == '0AAwMTIzNDU2Nzg5YWJjZGVm' #'0ACDEyMzQ1Njc4OWFiY2RlZg' + + signer = salter.signer(path="01", temp=True) # defaults to Ed25519 + assert signer.code == MtrDex.Ed25519_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.verfer.code == MtrDex.Ed25519 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + assert signer.qb64 == 'AMPsqBZxWdtYpBhrWnKYitwFa77s902Q-nX3sPTzqs0R' + #'Aw-yoFnFZ21ikGGtacpiK3AVrvuz3TZD6dfew9POqzRE' + assert signer.verfer.qb64 == 'DFYFwZJOMNy3FknECL8tUaQZRBUyQ9xCv6F8ckG-UCrC' # + # 'DVgXBkk4w3LcWScQIvy1RpBlEFTJD3EK_oXxyQb5QKsI' + + signer = salter.signer(path="01") # defaults to Ed25519 temp = False level="low" + assert signer.code == MtrDex.Ed25519_Seed + assert len(signer.raw) == Matter._rawSize(signer.code) + assert signer.verfer.code == MtrDex.Ed25519 + assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) + assert signer.qb64 == 'AEkqQiNTexWB9fTLpgJp_lXW63tFlT-Y0_mgQww4o-dC' + # 'ASSpCI1N7FYH19MumAmn-Vdbre0WVP5jT-aBDDDij50I' + assert signer.verfer.qb64 == 'DPJGyH9H1M_SUSf18RzX8OqdyhxEyZJpKm5Em0PnpsWd' + #'D8kbIf0fUz9JRJ_XxHNfw6p3KHETJkmkqbkSbQ-emxZ0' + + salter = Salter(qb64='0AAwMTIzNDU2Nzg5YWJjZGVm') + assert salter.raw == raw + assert salter.qb64 == '0AAwMTIzNDU2Nzg5YWJjZGVm' + + with pytest.raises(kering.ShortageError): + salter = Salter(qb64='') + + salter = Salter(raw=raw) + assert salter.stretch(temp=True) == b'\xd4@\xeb\xa6x\x86\xdf\x93\xd6C\xdc\xb8\xa6\x9b\x02\xafh\xc1m(L\xd6\xf6\x86YU>$[\xf9\xef\xc0' + assert salter.stretch(tier=Tiers.low) == b'\xf8e\x80\xbaX\x08\xb9\xba\xc6\x1e\x84\r\x1d\xac\xa7\\\x82Wc@`\x13\xfd\x024t\x8ct\xd3\x01\x19\xe9' + assert salter.stretch(tier=Tiers.med) == b',\xf3\x8c\xbb\xe9)\nSQ\xec\xad\x8c9?\xaf\xb8\xb0\xb3\xcdB\xda\xd8\xb6\xf7\r\xf6D}Z\xb9Y\x16' + assert salter.stretch(tier=Tiers.high) == b'(\xcd\xc4\xb85\xcd\xe8:\xfc\x00\x8b\xfd\xa6\tj.y\x98\x0b\x04\x1c\xe3hBc!I\xe49K\x16-' + + """ Done Test """ + + +def test_cipher(): + """ + Test Cipher subclass of Matter + """ + # conclusion never use box_seed_keypair always use sign_seed_keypair and + # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey + # is always the same. + + assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 + + # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' + b'\xde\x06\xc0+') + seedqb64b = Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b + assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' + #b'AGDswxA8qdkb646JFZWUflm_OKUeF41iG2gTw3N4GwCs' + + # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + salt = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' + saltqb64b = Matter(raw=salt, code=MtrDex.Salt_128).qb64b + assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' + + # seed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) + cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' + verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) + pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) + prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) + + with pytest.raises(kering.EmptyMaterialError): + cipher = Cipher() + + raw = pysodium.crypto_box_seal(seedqb64b, pubkey) # uses nonce so different everytime + cipher = Cipher(raw=raw) + assert cipher.code == MtrDex.X25519_Cipher_Seed + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == seedqb64b + + # test .decrypt method needs qb64 + prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b + assert cipher.decrypt(prikey=prikeyqb64).qb64b == seedqb64b + + cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b + assert cipher.decrypt(seed=cryptseedqb64).qb64b == seedqb64b + + raw = pysodium.crypto_box_seal(saltqb64b, pubkey) # uses nonce so different everytime + cipher = Cipher(raw=raw) + assert cipher.code == MtrDex.X25519_Cipher_Salt + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == saltqb64b + + # test .decrypt method needs qb64 + prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b + assert cipher.decrypt(prikey=prikeyqb64).qb64b == saltqb64b + + cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b + assert cipher.decrypt(seed=cryptseedqb64).qb64b == saltqb64b + + with pytest.raises(ValueError): # bad code + cipher = Cipher(raw=raw, code=MtrDex.Ed25519N) + """ Done Test """ + + +def test_encrypter(): + """ + Test Encrypter subclass of Matter + """ + # conclusion never use box_seed_keypair always use sign_seed_keypair and + # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey + # is always the same. + + assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 + + # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' + b'\xde\x06\xc0+') + seedqb64b = Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b + assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' + + # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + salt = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' + saltqb64b = Matter(raw=salt, code=MtrDex.Salt_128).qb64b + assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' + + # seed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) + cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' + cryptsigner = Signer(raw=cryptseed, code=MtrDex.Ed25519_Seed, transferable=True) + verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) # raw + pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) + prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) + + with pytest.raises(kering.EmptyMaterialError): + encrypter = Encrypter() + + encrypter = Encrypter(raw=pubkey) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + assert encrypter.verifySeed(seed=cryptsigner.qb64) + + cipher = encrypter.encrypt(ser=seedqb64b) + assert cipher.code == MtrDex.X25519_Cipher_Seed + uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) + assert uncb == seedqb64b + + cipher = encrypter.encrypt(ser=saltqb64b) + assert cipher.code == MtrDex.X25519_Cipher_Salt + uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) + assert uncb == saltqb64b + + verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) + + encrypter = Encrypter(verkey=verfer.qb64) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + + encrypter = Encrypter(verkey=verfer.qb64b) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + + # user Prefixer to generate original verkey + prefixer = Prefixer(qb64=verfer.qb64) + encrypter = Encrypter(verkey=prefixer.qb64b) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + """ Done Test """ + + +def test_decrypter(): + """ + Test Decrypter subclass of Matter + """ + # conclusion never use box_seed_keypair always use sign_seed_keypair and + # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey + # is always the same. + + assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 + + # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' + b'\xde\x06\xc0+') + signer = Signer(raw=seed, code=MtrDex.Ed25519_Seed) + assert signer.verfer.code == MtrDex.Ed25519 + assert signer.verfer.transferable # default + seedqb64b = signer.qb64b + assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' + # also works for Matter + assert seedqb64b == Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b + + # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + raw = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' + salter = Salter(raw=raw, code=MtrDex.Salt_128) + assert salter.code == MtrDex.Salt_128 + saltqb64b = salter.qb64b + assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' + # also works for Matter + assert saltqb64b == Matter(raw=raw, code=MtrDex.Salt_128).qb64b # + + # cryptseed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) + cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' + cryptsigner = Signer(raw=cryptseed, code=MtrDex.Ed25519_Seed, transferable=True) + verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) # raw + pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) + prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) + + with pytest.raises(kering.EmptyMaterialError): + decrypter = Decrypter() + + # create encrypter + encrypter = Encrypter(raw=pubkey) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + + # create cipher of seed + seedcipher = encrypter.encrypt(ser=seedqb64b) + assert seedcipher.code == MtrDex.X25519_Cipher_Seed + # each encryption uses a nonce so not a stable representation for testing + + # create decrypter from prikey + decrypter = Decrypter(raw=prikey) + assert decrypter.code == MtrDex.X25519_Private + assert decrypter.qb64 == 'OLCFxqMz1z1UUS0TEJnvZP_zXHcuYdQsSGBWdOZeY5VQ' + assert decrypter.raw == prikey + + # decrypt seed cipher using ser + designer = decrypter.decrypt(ser=seedcipher.qb64b, transferable=signer.verfer.transferable) + assert designer.qb64b == seedqb64b + assert designer.code == MtrDex.Ed25519_Seed + assert designer.verfer.code == MtrDex.Ed25519 + assert signer.verfer.transferable + + # decrypt seed cipher using cipher + designer = decrypter.decrypt(cipher=seedcipher, transferable=signer.verfer.transferable) + assert designer.qb64b == seedqb64b + assert designer.code == MtrDex.Ed25519_Seed + assert designer.verfer.code == MtrDex.Ed25519 + assert signer.verfer.transferable + + # create cipher of salt + saltcipher = encrypter.encrypt(ser=saltqb64b) + assert saltcipher.code == MtrDex.X25519_Cipher_Salt + # each encryption uses a nonce so not a stable representation for testing + + # decrypt salt cipher using ser + desalter = decrypter.decrypt(ser=saltcipher.qb64b) + assert desalter.qb64b == saltqb64b + assert desalter.code == MtrDex.Salt_128 + + # decrypt salt cipher using cipher + desalter = decrypter.decrypt(cipher=saltcipher) + assert desalter.qb64b == saltqb64b + assert desalter.code == MtrDex.Salt_128 + + # use previously stored fully qualified seed cipher with different nonce + # get from seedcipher above + cipherseed = ('PM9jOGWNYfjM_oLXJNaQ8UlFSAV5ACjsUY7J16xfzrlpc9Ve3A5WYrZ4o_' + 'NHtP5lhp78Usspl9fyFdnCdItNd5JyqZ6dt8SXOt6TOqOCs-gy0obrwFkPPqBvVkEw') + designer = decrypter.decrypt(ser=cipherseed, transferable=signer.verfer.transferable) + assert designer.qb64b == seedqb64b + assert designer.code == MtrDex.Ed25519_Seed + assert designer.verfer.code == MtrDex.Ed25519 + + # use previously stored fully qualified salt cipher with different nonce + # get from saltcipher above + ciphersalt = ('1AAHjlR2QR9J5Et67Wy-ZaVdTryN6T6ohg44r73GLRPnHw-5S3ABFkhWy' + 'IwLOI6TXUB_5CT13S8JvknxLxBaF8ANPK9FSOPD8tYu') + desalter = decrypter.decrypt(ser=ciphersalt) + assert desalter.qb64b == saltqb64b + assert desalter.code == MtrDex.Salt_128 + + # Create new decrypter but use seed parameter to init prikey + decrypter = Decrypter(seed=cryptsigner.qb64b) + assert decrypter.code == MtrDex.X25519_Private + assert decrypter.qb64 == 'OLCFxqMz1z1UUS0TEJnvZP_zXHcuYdQsSGBWdOZeY5VQ' + assert decrypter.raw == prikey + + # decrypt ciphersalt + desalter = decrypter.decrypt(ser=saltcipher.qb64b) + assert desalter.qb64b == saltqb64b + assert desalter.code == MtrDex.Salt_128 + + """ Done Test """ + + + +if __name__ == "__main__": + test_signer() + test_generatesigners() + test_salter() + test_cipher() + test_encrypter() + test_decrypter() + diff --git a/tests/core/test_weighted_threshold.py b/tests/core/test_weighted_threshold.py index aac8c1995..3f65bc88d 100644 --- a/tests/core/test_weighted_threshold.py +++ b/tests/core/test_weighted_threshold.py @@ -8,10 +8,14 @@ import pytest from keri import help -from keri.db import dbing, basing -from keri.app import keeping + +from keri import core from keri.core import coring, eventing, parsing +from keri.db import basing +from keri.app import keeping + + logger = help.ogler.getLogger() @@ -20,7 +24,7 @@ def test_weighted(): Test multisig with weighted threshold """ - wesSalt = coring.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + wesSalt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter # init event DB and keep DB with basing.openDB(name="wes") as wesDB, keeping.openKS(name="wes") as wesKS: diff --git a/tests/core/test_witness.py b/tests/core/test_witness.py index 161c92704..052e6c37b 100644 --- a/tests/core/test_witness.py +++ b/tests/core/test_witness.py @@ -6,8 +6,12 @@ import os from keri import help + +from keri import core +from keri.core import eventing, parsing, serdering, indexing + from keri.app import habbing -from keri.core import coring, eventing, parsing, serdering, indexing + from keri.db import dbing logger = help.ogler.getLogger() @@ -24,7 +28,7 @@ def test_indexed_witness_replay(): wam is a witness """ - salt = coring.Salter(raw=b'abcdef0123456789').qb64 + salt = core.Salter(raw=b'abcdef0123456789').qb64 with habbing.openHby(name="cam", base="test", salt=salt) as camHby, \ habbing.openHby(name="van", base="test", salt=salt) as vanHby, \ @@ -291,7 +295,7 @@ def test_nonindexed_witness_receipts(): wam is a witness """ - salt = coring.Salter(raw=b'abcdef0123456789').qb64 + salt = core.Salter(raw=b'abcdef0123456789').qb64 with habbing.openHby(name="cam", base="test", salt=salt) as camHby, \ habbing.openHby(name="van", base="test", salt=salt) as vanHby, \ @@ -576,7 +580,7 @@ def test_out_of_order_witnessed_events(): # Wes is his witness # Bam is verifying the key state for Bob from Wes - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 with habbing.openHby(name="wes", base="test", salt=default_salt) as wesHby, \ habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, \ habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby: diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 575342f21..b44ba1ad9 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -11,20 +11,24 @@ import pytest from hio.base import doing -from tests.app import openMultiSig -from keri.kering import Versionage -from keri.app import habbing +from keri.help.helping import datify, dictify + +from keri import core from keri.core import coring, eventing, serdering -from keri.core.coring import MtrDex + from keri.core.coring import Serials, versify -from keri.core.coring import Salter + from keri.core.eventing import incept, rotate, interact, Kever + +from keri.app import habbing + from keri.db import basing from keri.db import dbing from keri.db.basing import openDB, Baser, KeyStateRecord from keri.db.dbing import (dgKey, onKey, snKey) from keri.db.dbing import openLMDB -from keri.help.helping import datify, dictify + +from tests.app import openMultiSig def test_baser(): @@ -1707,7 +1711,7 @@ def test_clean_baser(): """ name = "nat" # with basing.openDB(name="nat") as natDB, keeping.openKS(name="nat") as natKS: - with habbing.openHby(name=name, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True + with habbing.openHby(name=name, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True natHab = hby.makeHab(name=name, isith='2', icount=3) # default Hab # setup Nat's habitat using default salt multisig already incepts #natHab = habbing.Habitat(name='nat', ks=natKS, db=natDB, @@ -1937,7 +1941,7 @@ def test_usebaser(): Test using Baser """ raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create coe's signers signers = salter.signers(count=8, path='db', temp=True) @@ -1949,7 +1953,7 @@ def test_usebaser(): count = len(keys) nxtkeys = [signers[3].verfer.qb64b, signers[4].verfer.qb64b, signers[5].verfer.qb64b] sith = "2" - code = MtrDex.Blake3_256 # Blake3 digest of incepting data + code = core.MtrDex.Blake3_256 # Blake3 digest of incepting data serder = incept(keys=keys, code=code, isith=sith, diff --git a/tests/db/test_escrowing.py b/tests/db/test_escrowing.py index 9ca309126..87d12d9bc 100644 --- a/tests/db/test_escrowing.py +++ b/tests/db/test_escrowing.py @@ -4,8 +4,12 @@ """ from keri import kering -from keri.app import habbing + +from keri import core from keri.core import coring, eventing, serdering + +from keri.app import habbing + from keri.core.eventing import SealEvent from keri.db import escrowing, dbing, subing from keri.help import helping @@ -27,7 +31,7 @@ def test_broker(): def test_broker_nontrans(): raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = coring.Salter(raw=raw) + salter = core.Salter(raw=raw) salt = salter.qb64 assert salt == '0AAFqo8tU5rp-lWcApybCEh1' diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 3a550ddb4..4cee7586e 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -9,8 +9,11 @@ import pysodium -from keri.help import helping +from keri import help + +from keri import core from keri.core import coring, eventing, serdering, indexing + from keri.db import dbing, subing from keri.app import keeping @@ -1320,13 +1323,13 @@ def test_signer_suber(): sdb = subing.SignerSuber(db=db, subkey='bags.') # default klas is Signer assert isinstance(sdb, subing.SignerSuber) - assert issubclass(sdb.klas, coring.Signer) + assert issubclass(sdb.klas, core.Signer) assert not sdb.sdb.flags()["dupsort"] # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) seed0 = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' b'\xde\x06\xc0+') - signer0 = coring.Signer(raw=seed0, code=coring.MtrDex.Ed25519_Seed) + signer0 = core.Signer(raw=seed0, code=coring.MtrDex.Ed25519_Seed) assert signer0.verfer.code == coring.MtrDex.Ed25519 assert signer0.verfer.transferable # default assert signer0.qb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' @@ -1336,7 +1339,7 @@ def test_signer_suber(): seed1 = (b'`\x05\x93\xb9\x9b6\x1e\xe0\xd7\x98^\x94\xc8Et\xf2\xc4\xcd\x94\x18' b'\xc6\xae\xb9\xb6m\x12\xc4\x80\x03\x07\xfc\xf7') - signer1 = coring.Signer(raw=seed1, code=coring.MtrDex.Ed25519_Seed) + signer1 = core.Signer(raw=seed1, code=coring.MtrDex.Ed25519_Seed) assert signer1.verfer.code == coring.MtrDex.Ed25519 assert signer1.verfer.transferable # default assert signer1.qb64b == b'AGAFk7mbNh7g15helMhFdPLEzZQYxq65tm0SxIADB_z3' @@ -1345,7 +1348,7 @@ def test_signer_suber(): keys = (signer0.verfer.qb64, ) # must be verfer as key to get transferable sdb.put(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1355,14 +1358,14 @@ def test_signer_suber(): sdb.put(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 # try put different val when already put result = sdb.put(keys=keys, val=signer1) assert not result - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1371,7 +1374,7 @@ def test_signer_suber(): result = sdb.pin(keys=keys, val=signer1) assert result actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer1.qb64 assert actual.verfer.qb64 == signer1.verfer.qb64 @@ -1380,7 +1383,7 @@ def test_signer_suber(): sdb.pin(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1423,7 +1426,7 @@ def test_crypt_signer_suber(): # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) seed0 = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' b'\xde\x06\xc0+') - signer0 = coring.Signer(raw=seed0, code=coring.MtrDex.Ed25519_Seed) + signer0 = core.Signer(raw=seed0, code=coring.MtrDex.Ed25519_Seed) assert signer0.verfer.code == coring.MtrDex.Ed25519 assert signer0.verfer.transferable # default assert signer0.qb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' @@ -1433,7 +1436,7 @@ def test_crypt_signer_suber(): seed1 = (b'`\x05\x93\xb9\x9b6\x1e\xe0\xd7\x98^\x94\xc8Et\xf2\xc4\xcd\x94\x18' b'\xc6\xae\xb9\xb6m\x12\xc4\x80\x03\x07\xfc\xf7') - signer1 = coring.Signer(raw=seed1, code=coring.MtrDex.Ed25519_Seed) + signer1 = core.Signer(raw=seed1, code=coring.MtrDex.Ed25519_Seed) assert signer1.verfer.code == coring.MtrDex.Ed25519 assert signer1.verfer.transferable # default assert signer1.qb64b == b'AGAFk7mbNh7g15helMhFdPLEzZQYxq65tm0SxIADB_z3' @@ -1442,27 +1445,27 @@ def test_crypt_signer_suber(): # rawsalt =pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) rawsalt = b'0123456789abcdef' - salter = coring.Salter(raw=rawsalt) + salter = core.Salter(raw=rawsalt) salt = salter.qb64 assert salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' stem = "blue" # cryptseed0 = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) cryptseed0 = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' - cryptsigner0 = coring.Signer(raw=cryptseed0, code=coring.MtrDex.Ed25519_Seed, + cryptsigner0 = core.Signer(raw=cryptseed0, code=coring.MtrDex.Ed25519_Seed, transferable=False) seed0 = cryptsigner0.qb64 aeid0 = cryptsigner0.verfer.qb64 assert aeid0 == 'BCa7mK96FwxkU0TdF54Yqg3qBDXUWpOhQ_Mtr7E77yZB' - decrypter = coring.Decrypter(seed=seed0) - encrypter = coring.Encrypter(verkey=aeid0) + decrypter = core.Decrypter(seed=seed0) + encrypter = core.Encrypter(verkey=aeid0) assert encrypter.verifySeed(seed=seed0) # cryptseed1 = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) cryptseed1 = (b"\x89\xfe{\xd9'\xa7\xb3\x89#\x19\xbec\xee\xed\xc0\xf9\x97\xd0\x8f9\x1dyNI" b'I\x98\xbd\xa4\xf6\xfe\xbb\x03') - cryptsigner1 = coring.Signer(raw=cryptseed1, code=coring.MtrDex.Ed25519_Seed, + cryptsigner1 = core.Signer(raw=cryptseed1, code=coring.MtrDex.Ed25519_Seed, transferable=False) @@ -1473,14 +1476,14 @@ def test_crypt_signer_suber(): sdb = subing.CryptSignerSuber(db=db, subkey='bags.') # default klas is Signer assert isinstance(sdb, subing.CryptSignerSuber) - assert issubclass(sdb.klas, coring.Signer) + assert issubclass(sdb.klas, core.Signer) assert not sdb.sdb.flags()["dupsort"] # Test without encrypter or decrypter keys = (signer0.verfer.qb64, ) # must be verfer as key to get transferable sdb.put(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1490,14 +1493,14 @@ def test_crypt_signer_suber(): sdb.put(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 # try put different val when already put result = sdb.put(keys=keys, val=signer1) assert not result - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1506,7 +1509,7 @@ def test_crypt_signer_suber(): result = sdb.pin(keys=keys, val=signer1) assert result actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer1.qb64 assert actual.verfer.qb64 == signer1.verfer.qb64 @@ -1515,7 +1518,7 @@ def test_crypt_signer_suber(): sdb.pin(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1538,8 +1541,8 @@ def test_crypt_signer_suber(): # now test with encrypter and decrypter - encrypter0 = coring.Encrypter(verkey=cryptsigner0.verfer.qb64) - decrypter0 = coring.Decrypter(seed=cryptsigner0.qb64b) + encrypter0 = core.Encrypter(verkey=cryptsigner0.verfer.qb64) + decrypter0 = core.Decrypter(seed=cryptsigner0.qb64b) # first pin with encrypter assert sdb.pin(keys=signer0.verfer.qb64b, val=signer0, encrypter=encrypter0) @@ -1547,12 +1550,12 @@ def test_crypt_signer_suber(): # now get actual0 = sdb.get(keys=signer0.verfer.qb64b, decrypter=decrypter0) - assert isinstance(actual0, coring.Signer) + assert isinstance(actual0, core.Signer) assert actual0.qb64 == signer0.qb64 assert actual0.verfer.qb64 == signer0.verfer.qb64 actual1 = sdb.get(keys=signer1.verfer.qb64b, decrypter=decrypter0) - assert isinstance(actual1, coring.Signer) + assert isinstance(actual1, core.Signer) assert actual1.qb64 == signer1.qb64 assert actual1.verfer.qb64 == signer1.verfer.qb64 @@ -1589,8 +1592,8 @@ def test_crypt_signer_suber(): # test re-encrypt - encrypter1 = coring.Encrypter(verkey=cryptsigner1.verfer.qb64) - decrypter1 = coring.Decrypter(seed=cryptsigner1.qb64b) + encrypter1 = core.Encrypter(verkey=cryptsigner1.verfer.qb64) + decrypter1 = core.Decrypter(seed=cryptsigner1.qb64b) for keys, sgnr in sdb.getItemIter(decrypter=decrypter0): sdb.pin(keys, sgnr, encrypter=encrypter1) @@ -1618,15 +1621,15 @@ def test_crypt_signer_suber(): assert manager.algo == keeping.Algos.salty assert manager.salt == salt # encrypted on disk but property decrypts if seed assert manager.pidx == 0 - assert manager.tier == coring.Tiers.low - saltCipher0 = coring.Cipher(qb64=manager.ks.gbls.get('salt')) + assert manager.tier == core.Tiers.low + saltCipher0 = core.Cipher(qb64=manager.ks.gbls.get('salt')) assert saltCipher0.decrypt(seed=seed0).qb64 == salt manager.updateAeid(aeid=cryptsigner1.verfer.qb64, seed=cryptsigner1.qb64) assert manager.aeid == cryptsigner1.verfer.qb64 == 'BEcOrMrG_7r_NWaLl6h8UJapwIfQWIkjrIPXkCZm2fFM' assert manager.salt == salt - saltCipher1 = coring.Cipher(qb64=manager.ks.gbls.get('salt')) + saltCipher1 = core.Cipher(qb64=manager.ks.gbls.get('salt')) assert not saltCipher0.qb64 == saltCipher1.qb64 # old cipher different """End Test""" diff --git a/tests/demo/test_demo.py b/tests/demo/test_demo.py index e3c0c26e9..f0c8e4e7c 100644 --- a/tests/demo/test_demo.py +++ b/tests/demo/test_demo.py @@ -11,8 +11,12 @@ from hio.core.tcp import clienting, serving from keri import help # logger support -from keri.app import habbing, directing + +from keri import core from keri.core import eventing, coring + +from keri.app import habbing, directing + from keri.demo import demoing @@ -25,7 +29,7 @@ def test_direct_mode_bob_eve_demo(): raw = b"raw salt to test" # create bob signers and secrecies - bobSigners = coring.Salter(raw=raw).signers(count=8, path="bob", temp=True) + bobSigners = core.Salter(raw=raw).signers(count=8, path="bob", temp=True) bobSecrecies = [[signer.qb64] for signer in bobSigners] # bob inception transferable (nxt digest not empty) @@ -37,7 +41,7 @@ def test_direct_mode_bob_eve_demo(): assert bob == 'EFa1wAk_coghxxGCID6jEN79Kmvyj0Y1wWN_ndUv3LjW' # create eve signers and secrecies - eveSigners = coring.Salter(raw=raw).signers(count=8, path="eve", temp=True) + eveSigners = core.Salter(raw=raw).signers(count=8, path="eve", temp=True) eveSecrecies = [[signer.qb64] for signer in eveSigners] # eve inception transferable (nxt digest not empty) @@ -156,7 +160,7 @@ def test_direct_mode_sam_eve_demo(): raw = b"raw salt to test" # create sam signers and secrecies - samSigners = coring.Salter(raw=raw).signers(count=8, path="sam", temp=True) + samSigners = core.Salter(raw=raw).signers(count=8, path="sam", temp=True) samSecrecies = [[signer.qb64] for signer in samSigners] # sam inception transferable (nxt digest not empty) @@ -168,7 +172,7 @@ def test_direct_mode_sam_eve_demo(): assert sam == 'EDkU2U_TPKca14VElEItpj7twohQL60GIaUPvSHAghga' # create eve signers and secrecies - eveSigners = coring.Salter(raw=raw).signers(count=8, path="eve", temp=True) + eveSigners = core.Salter(raw=raw).signers(count=8, path="eve", temp=True) eveSecrecies = [[signer.qb64] for signer in eveSigners] # eve inception transferable (nxt digest not empty) @@ -309,7 +313,7 @@ def test_run_bob_eve_demo(): # create bob secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path="bob", temp=True)] @@ -325,7 +329,7 @@ def test_run_bob_eve_demo(): # create eve secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path="eve", temp=True)] @@ -363,7 +367,7 @@ def test_run_sam_eve_demo(): # create sam secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path="sam", temp=True)] @@ -380,7 +384,7 @@ def test_run_sam_eve_demo(): # create eve secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path="eve", temp=True)] @@ -410,16 +414,16 @@ def test_indirect_mode_sam_cam_wit_demo(): raw = b"raw salt to test" # create sam signers and secrecies - samSigners = coring.Salter(raw=raw).signers(count=8, path="sam", temp=True) + samSigners = core.Salter(raw=raw).signers(count=8, path="sam", temp=True) samSecrecies = [[signer.qb64] for signer in samSigners] # create cam signers and secrecies - camSigners = coring.Salter(raw=raw).signers(count=8, path="cam", temp=True) + camSigners = core.Salter(raw=raw).signers(count=8, path="cam", temp=True) camSecrecies = [[signer.qb64] for signer in camSigners] - with (habbing.openHby(name="cam", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as camHby, - habbing.openHby(name="sam", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as samHby, - habbing.openHby(name="wit", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as witHby): + with (habbing.openHby(name="cam", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as camHby, + habbing.openHby(name="sam", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as samHby, + habbing.openHby(name="wit", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as witHby): samPort = 5620 # sam's TCP listening port for server witPort = 5621 # wit' TCP listening port for server diff --git a/tests/end/test_ending.py b/tests/end/test_ending.py index 18782a025..60b3d0811 100644 --- a/tests/end/test_ending.py +++ b/tests/end/test_ending.py @@ -13,8 +13,12 @@ from hio.help import Hict from keri import help, kering -from keri.app import habbing + +from keri import core from keri.core import coring, serdering + +from keri.app import habbing + from keri.end import ending logger = help.ogler.getLogger() @@ -59,7 +63,7 @@ def test_signature_designature(): # db = basing.Baser(name=name, temp=temp, reopen=reopen) # Setup Habery and Hab - with habbing.openHby(name=name, base=base, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(name=name, base=base, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: # hby = habbing.Habery(name=name, base=base, temp=temp, free=True) hab = hby.makeHab(name=name, icount=3) print() @@ -313,7 +317,7 @@ def test_seid_api(): # Setup Habery and Hab name = 'zoe' base = 'test' - with habbing.openHby(name=name, base=base, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(name=name, base=base, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name=name) # hab = setupTestHab(name='zoe') # must do it here to inject into Falcon endpoint resource instances @@ -382,7 +386,7 @@ def test_get_admin(): # Setup Habery and Hab name = 'zoe' base = 'test' - with habbing.openHby(name=name, base=base, salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(name=name, base=base, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name=name) # hab = setupTestHab(name='zoe') @@ -407,7 +411,7 @@ def test_get_oobi(): # Setup Habery and Hab name = 'oobi' base = 'test' - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = core.Salter(raw=b'0123456789abcdef').qb64 with habbing.openHby(name=name, base=base, salt=salt) as hby: hab = hby.makeHab(name=name) msgs = bytearray() diff --git a/tests/peer/test_exchanging.py b/tests/peer/test_exchanging.py index b6d5e187e..6cb20d739 100644 --- a/tests/peer/test_exchanging.py +++ b/tests/peer/test_exchanging.py @@ -3,8 +3,11 @@ tests.peer.test_exchanging module """ -from keri.app import habbing, forwarding, storing, signing +from keri import core from keri.core import coring, serdering + +from keri.app import habbing, forwarding, storing, signing + from keri.peer import exchanging from keri.vdr.eventing import incept @@ -76,7 +79,7 @@ def test_exchanger(): def test_hab_exchange(mockHelpingNowUTC): - with habbing.openHby(salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name="test") assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" diff --git a/tests/vc/test_protocoling.py b/tests/vc/test_protocoling.py index 2fe2bdd72..356a3f329 100644 --- a/tests/vc/test_protocoling.py +++ b/tests/vc/test_protocoling.py @@ -4,24 +4,26 @@ """ -from keri.app import habbing, notifying +from keri import core from keri.core import coring, scheming, parsing from keri.core.eventing import SealEvent + from keri.peer import exchanging from keri.vc import protocoling from keri.vc.proving import credential from keri.vdr import credentialing, verifying +from keri.app import habbing, notifying def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingNowUTC): """ Test IPEX exchange protocol """ - sidSalt = coring.Salter(raw=b'0123456789abcdef').qb64 + sidSalt = core.Salter(raw=b'0123456789abcdef').qb64 assert sidSalt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - wanSalt = coring.Salter(raw=b'wann-the-witness').qb64 + wanSalt = core.Salter(raw=b'wann-the-witness').qb64 assert wanSalt == '0AB3YW5uLXRoZS13aXRuZXNz' - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 with (habbing.openHby(name="red", base="test", salt=default_salt) as redHby, habbing.openHby(name="sid", base="test", salt=sidSalt) as sidHby): diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index 8675fa1ca..9860ca072 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -6,12 +6,15 @@ import pytest from keri import kering -from keri.app import habbing -from keri.core import coring, scheming, parsing, serdering, indexing +from keri.kering import Versionage + +from keri import core +from keri.core import coring, scheming, parsing, serdering from keri.core.coring import Serials, Counter, CtrDex, Prefixer, Seqner, Diger from keri.core.indexing import Siger from keri.core.scheming import CacheResolver -from keri.kering import Versionage + +from keri.app import habbing from keri.vc.proving import credential from keri.vdr import verifying, credentialing @@ -19,7 +22,7 @@ def test_proving(mockHelpingNowIso8601): """Test credential proof with SerderACDC""" - sidSalt = coring.Salter(raw=b'0123456789abcdef').qb64 + sidSalt = core.Salter(raw=b'0123456789abcdef').qb64 with habbing.openHby(name="sid", base="test", salt=sidSalt) as sidHby: sidHab = sidHby.makeHab(name="test", ) @@ -249,7 +252,7 @@ def test_privacy_preserving_credential(mockHelpingNowIso8601): engagementContextRole="Project Manager", ) - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = core.Salter(raw=b'0123456789abcdef').qb64 cred = credential(schema="EZllThM1rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q", recipient="EM_S2MdMaKgP6P2Yyno6-flV6GqrwPencTIw8tCMR7iB", private=True, diff --git a/tests/vc/test_walleting.py b/tests/vc/test_walleting.py index 2f1c55f23..d514171c6 100644 --- a/tests/vc/test_walleting.py +++ b/tests/vc/test_walleting.py @@ -3,16 +3,18 @@ tests.vc.walleting module """ - -from keri.app import habbing, signing +from keri import core from keri.core import coring, parsing from keri.core.eventing import SealEvent + +from keri.app import habbing + from keri.vc.proving import credential from keri.vdr import verifying, credentialing def test_wallet(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): - sidSalt = coring.Salter(raw=b'0123456789abcdef').qb64 + sidSalt = core.Salter(raw=b'0123456789abcdef').qb64 with habbing.openHby(name="sid", base="test", salt=sidSalt) as sidHby: sidHab = sidHby.makeHab(name="test") diff --git a/tests/vdr/__init__.py b/tests/vdr/__init__.py index ffbf7bcb8..fc59a8734 100644 --- a/tests/vdr/__init__.py +++ b/tests/vdr/__init__.py @@ -2,7 +2,10 @@ """ Test utilities for vdr """ -from keri.core import coring + +from keri import core + +from keri import app from keri.app import habbing def buildHab(db, ks, name="test"): @@ -15,7 +18,7 @@ def buildHab(db, ks, name="test"): # create secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path="name", temp=True)] diff --git a/tests/vdr/test_eventing.py b/tests/vdr/test_eventing.py index 3da4f055c..5978edc90 100644 --- a/tests/vdr/test_eventing.py +++ b/tests/vdr/test_eventing.py @@ -6,9 +6,10 @@ import pytest from keri.app import habbing, keeping +from keri.core import Signer from keri.core import coring, serdering from keri.core import eventing as keventing -from keri.core.coring import versify, Serials, Ilks, MtrDex, Prefixer, Signer, Seqner, Saider +from keri.core.coring import versify, Serials, Ilks, MtrDex, Prefixer, Seqner, Saider from keri.db import basing from keri.db.dbing import snKey, dgKey from keri.kering import Version, EmptyMaterialError, DerivationError, MissingAnchorError, ValidationError, \ diff --git a/tests/vdr/test_txn_state.py b/tests/vdr/test_txn_state.py index f1aa1896d..c9ae51a32 100644 --- a/tests/vdr/test_txn_state.py +++ b/tests/vdr/test_txn_state.py @@ -1,9 +1,12 @@ from dataclasses import asdict -from keri.app import habbing +from keri import core from keri.core import routing, parsing, coring, serdering + from keri.core.eventing import Kevery, SealEvent +from keri.app import habbing + from keri.vc import proving from keri.vdr import viring, credentialing, eventing @@ -12,7 +15,7 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): # Bob is the controller # Bam is verifying the key state for Bob with a stale key state in the way - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): @@ -92,7 +95,7 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): # Bob is the controller # Bam is verifying the key state for Bob with a stale key state in the way - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): @@ -185,7 +188,7 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): # Wes is his witness # Bam is verifying the key state for Bob from Wes # Habery.makeHab uses name as stem path for salt so different pre - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 with (habbing.openHby(name="wes", base="test", salt=default_salt) as wesHby, habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): @@ -305,11 +308,11 @@ def test_tsn_from_no_one(mockHelpingNowUTC, mockCoringRandomNonce): # Bam is verifying the key state for Bob from Wes # Wes is no one #raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - #salter = coring.Salter(raw=raw) + #salter = core.Salter(raw=raw) #salt = salter.qb64 #assert salt == '0AAFqo8tU5rp-lWcApybCEh1' # Habery.makeHab uses name as stem path for salt so different pre - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 with (habbing.openHby(name="wes", base="test", salt=default_salt) as wesHby, habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): @@ -402,7 +405,7 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe # Bob is the controller # Bam is verifying the key state for Bob with a stale key state in the way - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): @@ -530,7 +533,7 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe def test_tever_reload(mockHelpingNowUTC, mockCoringRandomNonce, mockHelpingNowIso8601): - with habbing.openHby(name="bob", base="test", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as hby: + with habbing.openHby(name="bob", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: bobHab = hby.makeHab(name="bob", isith='1', icount=1,) assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' From 90a167194a52f3bb12b13e316fc9fca8f06624bd Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 10 Apr 2024 08:49:28 -0600 Subject: [PATCH 134/418] started tests support for Seals in CESR native keri event messages --- src/keri/core/serdering.py | 25 ++++++++++++++++++------- tests/core/test_serdering.py | 1 + 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index f010cbca1..5b211b404 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -27,18 +27,22 @@ MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN) from ..kering import SMELLSIZE, Smellage, smell -from ..kering import Protocols, Serials, Rever, versify, deversify, Ilks -from ..core import coring +from ..kering import Protocols, Serials, versify, deversify, Ilks + +from .. import help +from ..help import helping + + +from . import coring from .coring import MtrDex, DigDex, PreDex, Saids, Digestage from .coring import (Matter, Saider, Verfer, Diger, Number, Tholder, Tagger, Ilker, Traitor, Verser, ) -from ..core import counting -from ..core.counting import GenDex, AllTags, Counter +from .counting import GenDex, AllTags, Counter + +from .structing import Sealer + -from .. import help -from ..help import helping -from ..help.helping import nonStringSequence logger = help.ogler.getLogger() @@ -1280,6 +1284,13 @@ def _dumps(self, sad): case "a": # list of seals or field map of attributes frame = bytearray() for e in v: # list of seal dicts + try: + sealer = Sealer(crew=e) + frame.extend(sealer.qb64b) + except kering.InvalidValueError: + pass + #unknown seal type so serialize as field map + pass #if tuple(v) == eventing.SealEvent._fields: #eseal = eventing.SealEvent(**v) # convert to namedtuple diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 2d9bb346f..31583a84f 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -2649,6 +2649,7 @@ def test_cesr_native_dumps(): raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' salter = core.Salter(raw=raw) + # replace with Salter.signers() csigners = core.generateSigners(raw=salter.raw, count=3) wsigners = core.generateSigners(raw=salter.raw, count=3, transferable=False) From afda4ed4b44d44a631bebf6e04fb21a13100098b Mon Sep 17 00:00:00 2001 From: Rodolfo Date: Thu, 11 Apr 2024 10:14:59 -0300 Subject: [PATCH 135/418] fix: EndRole escrow idempotent check (#746) --- src/keri/core/eventing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 772fd03ab..157fdee62 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -4198,6 +4198,8 @@ def processReplyEndRole(self, *, serder, saider, route, aid = cid # authorizing attribution id keys = (aid, role, eid) osaider = self.db.eans.get(keys=keys) # get old said if any + if osaider is not None and osaider.qb64b == saider.qb64b: # check idempotent + osaider = None # BADA Logic accepted = self.rvy.acceptReply(serder=serder, saider=saider, route=route, aid=aid, osaider=osaider, cigars=cigars, From 79f94adcdcde336d1a2c417b0d3be601a768204e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lenksj=C3=B6?= <5889538+lenkan@users.noreply.github.com> Date: Thu, 11 Apr 2024 16:06:47 +0200 Subject: [PATCH 136/418] update comments and help texts referring to 22 character passcode (#745) --- src/keri/app/cli/commands/challenge/respond.py | 2 +- src/keri/app/cli/commands/challenge/verify.py | 2 +- src/keri/app/cli/commands/clean.py | 2 +- src/keri/app/cli/commands/contacts/list.py | 2 +- src/keri/app/cli/commands/delegate/confirm.py | 2 +- src/keri/app/cli/commands/delegate/request.py | 2 +- src/keri/app/cli/commands/did/generate.py | 2 +- src/keri/app/cli/commands/ends/add.py | 2 +- src/keri/app/cli/commands/ends/export.py | 2 +- src/keri/app/cli/commands/ends/list.py | 2 +- src/keri/app/cli/commands/escrow.py | 2 +- src/keri/app/cli/commands/export.py | 2 +- src/keri/app/cli/commands/incept.py | 2 +- src/keri/app/cli/commands/init.py | 4 ++-- src/keri/app/cli/commands/interact.py | 2 +- src/keri/app/cli/commands/ipex/admit.py | 2 +- src/keri/app/cli/commands/ipex/grant.py | 2 +- src/keri/app/cli/commands/ipex/join.py | 2 +- src/keri/app/cli/commands/ipex/list.py | 2 +- src/keri/app/cli/commands/ipex/spurn.py | 2 +- src/keri/app/cli/commands/kevers.py | 2 +- src/keri/app/cli/commands/list.py | 2 +- src/keri/app/cli/commands/local/watch.py | 2 +- src/keri/app/cli/commands/mailbox/debug.py | 2 +- src/keri/app/cli/commands/mailbox/update.py | 2 +- src/keri/app/cli/commands/migrate.py | 2 +- src/keri/app/cli/commands/migrate/list.py | 2 +- src/keri/app/cli/commands/migrate/run.py | 2 +- src/keri/app/cli/commands/migrate/show.py | 2 +- src/keri/app/cli/commands/multisig/continue.py | 2 +- src/keri/app/cli/commands/multisig/incept.py | 2 +- src/keri/app/cli/commands/multisig/interact.py | 2 +- src/keri/app/cli/commands/multisig/join.py | 2 +- src/keri/app/cli/commands/multisig/notice.py | 2 +- src/keri/app/cli/commands/multisig/rotate.py | 2 +- src/keri/app/cli/commands/multisig/shell.py | 2 +- src/keri/app/cli/commands/multisig/update.py | 2 +- src/keri/app/cli/commands/oobi/clean.py | 2 +- src/keri/app/cli/commands/oobi/generate.py | 2 +- src/keri/app/cli/commands/oobi/resolve.py | 2 +- src/keri/app/cli/commands/passcode/remove.py | 2 +- src/keri/app/cli/commands/passcode/set.py | 6 +++--- src/keri/app/cli/commands/query.py | 2 +- src/keri/app/cli/commands/rename.py | 2 +- src/keri/app/cli/commands/rollback.py | 2 +- src/keri/app/cli/commands/rotate.py | 2 +- src/keri/app/cli/commands/sign.py | 2 +- src/keri/app/cli/commands/ssh/export.py | 2 +- src/keri/app/cli/commands/status.py | 2 +- src/keri/app/cli/commands/vc/create.py | 2 +- src/keri/app/cli/commands/vc/export.py | 2 +- src/keri/app/cli/commands/vc/list.py | 2 +- src/keri/app/cli/commands/vc/registry/incept.py | 4 ++-- src/keri/app/cli/commands/vc/registry/list.py | 2 +- src/keri/app/cli/commands/vc/registry/status.py | 2 +- src/keri/app/cli/commands/vc/revoke.py | 2 +- src/keri/app/cli/commands/verify.py | 2 +- src/keri/app/cli/commands/version.py | 2 +- src/keri/app/cli/commands/witness/start.py | 2 +- src/keri/app/cli/commands/witness/submit.py | 2 +- 60 files changed, 64 insertions(+), 64 deletions(-) diff --git a/src/keri/app/cli/commands/challenge/respond.py b/src/keri/app/cli/commands/challenge/respond.py index 75c31094a..1bca6554a 100644 --- a/src/keri/app/cli/commands/challenge/respond.py +++ b/src/keri/app/cli/commands/challenge/respond.py @@ -19,7 +19,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--words', '-d', help='JSON formatted array of words to sign, \'@\' allowed to load from a file', action="store", required=True) diff --git a/src/keri/app/cli/commands/challenge/verify.py b/src/keri/app/cli/commands/challenge/verify.py index db27501ee..4811e6098 100644 --- a/src/keri/app/cli/commands/challenge/verify.py +++ b/src/keri/app/cli/commands/challenge/verify.py @@ -28,7 +28,7 @@ default=None) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--words', '-d', help='JSON formatted array of words to verfiy, \'@\' allowed to load from a file', diff --git a/src/keri/app/cli/commands/clean.py b/src/keri/app/cli/commands/clean.py index 7afc752bd..63c3b4907 100644 --- a/src/keri/app/cli/commands/clean.py +++ b/src/keri/app/cli/commands/clean.py @@ -36,7 +36,7 @@ def handler(args): # Parameters for Manager creation # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) diff --git a/src/keri/app/cli/commands/contacts/list.py b/src/keri/app/cli/commands/contacts/list.py index 1ba574483..d4c626d37 100644 --- a/src/keri/app/cli/commands/contacts/list.py +++ b/src/keri/app/cli/commands/contacts/list.py @@ -23,7 +23,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index ded7ed9c9..816132b52 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -26,7 +26,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--interact", "-i", help="anchor the delegation approval in an interaction event. " "Default is to use a rotation event.", action="store_true") diff --git a/src/keri/app/cli/commands/delegate/request.py b/src/keri/app/cli/commands/delegate/request.py index 8cd07f0ca..272856d83 100644 --- a/src/keri/app/cli/commands/delegate/request.py +++ b/src/keri/app/cli/commands/delegate/request.py @@ -25,7 +25,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran def request(args): diff --git a/src/keri/app/cli/commands/did/generate.py b/src/keri/app/cli/commands/did/generate.py index 9e0cb28d1..0200d40ba 100644 --- a/src/keri/app/cli/commands/did/generate.py +++ b/src/keri/app/cli/commands/did/generate.py @@ -30,7 +30,7 @@ # Parameters for Manager access # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) parser.add_argument('--url', '-u', help="generate a DID URL instead of a DID", action="store_true") diff --git a/src/keri/app/cli/commands/ends/add.py b/src/keri/app/cli/commands/ends/add.py index e672abfda..44f4fd639 100644 --- a/src/keri/app/cli/commands/ends/add.py +++ b/src/keri/app/cli/commands/ends/add.py @@ -26,7 +26,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--role", "-r", help="KERI enpoint authorization role.", required=True) diff --git a/src/keri/app/cli/commands/ends/export.py b/src/keri/app/cli/commands/ends/export.py index e1d54a2f3..6eed26a16 100644 --- a/src/keri/app/cli/commands/ends/export.py +++ b/src/keri/app/cli/commands/ends/export.py @@ -21,7 +21,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--aid", "-a", help="qualified base64 of AID to export rpy messages for all endpoints.", diff --git a/src/keri/app/cli/commands/ends/list.py b/src/keri/app/cli/commands/ends/list.py index 533e39d95..28e9cc3d9 100644 --- a/src/keri/app/cli/commands/ends/list.py +++ b/src/keri/app/cli/commands/ends/list.py @@ -24,7 +24,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--aid", help="qualified base64 of AID to export rpy messages for all endpoints.", required=True) diff --git a/src/keri/app/cli/commands/escrow.py b/src/keri/app/cli/commands/escrow.py index 24d78f335..62546fa16 100644 --- a/src/keri/app/cli/commands/escrow.py +++ b/src/keri/app/cli/commands/escrow.py @@ -24,7 +24,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--escrow", "-e", help="show values for one specific escrow", default=None) diff --git a/src/keri/app/cli/commands/export.py b/src/keri/app/cli/commands/export.py index 7fe7dfff9..e6ada36a7 100644 --- a/src/keri/app/cli/commands/export.py +++ b/src/keri/app/cli/commands/export.py @@ -23,7 +23,7 @@ required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--files", help="export artifacts to individual files keyed off of AIDs or SAIDS, default is " "stdout", action="store_true") diff --git a/src/keri/app/cli/commands/incept.py b/src/keri/app/cli/commands/incept.py index 084d6315c..68e771c45 100644 --- a/src/keri/app/cli/commands/incept.py +++ b/src/keri/app/cli/commands/incept.py @@ -29,7 +29,7 @@ parser.add_argument('--file', '-f', help='Filename to use to create the identifier', default="", required=False) # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) diff --git a/src/keri/app/cli/commands/init.py b/src/keri/app/cli/commands/init.py index a4eedbefd..52fc51398 100644 --- a/src/keri/app/cli/commands/init.py +++ b/src/keri/app/cli/commands/init.py @@ -52,7 +52,7 @@ def handler(args): # Parameters for Manager creation # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) parser.add_argument('--nopasscode', help='create an unencrypted keystore', action='store_true') parser.add_argument('--aeid', '-a', help='qualified base64 of non-transferable identifier prefix for authentication ' @@ -86,7 +86,7 @@ def initialize(self, tymth, tock=0.0): configDir = args.configDir if not args.nopasscode and not bran: - print("Creating encrypted keystore, please enter your 22 character passcode:") + print("Creating encrypted keystore, please enter your 21 character passcode:") while True: bran = getpass.getpass("Passcode: ") retry = getpass.getpass("Re-enter passcode: ") diff --git a/src/keri/app/cli/commands/interact.py b/src/keri/app/cli/commands/interact.py index e768d2994..2e7a4bc1d 100644 --- a/src/keri/app/cli/commands/interact.py +++ b/src/keri/app/cli/commands/interact.py @@ -19,7 +19,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--data', '-d', help='Anchor data, \'@\' allowed', default=None, action="store", required=False) diff --git a/src/keri/app/cli/commands/ipex/admit.py b/src/keri/app/cli/commands/ipex/admit.py index 842533805..b7ab7be99 100644 --- a/src/keri/app/cli/commands/ipex/admit.py +++ b/src/keri/app/cli/commands/ipex/admit.py @@ -25,7 +25,7 @@ required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--said", "-s", help="SAID of the exn grant message to admit", required=True) diff --git a/src/keri/app/cli/commands/ipex/grant.py b/src/keri/app/cli/commands/ipex/grant.py index 38fc0d33b..dad3a79ab 100644 --- a/src/keri/app/cli/commands/ipex/grant.py +++ b/src/keri/app/cli/commands/ipex/grant.py @@ -23,7 +23,7 @@ required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--recipient", "-r", help="alias or qb64 identifier prefix of the self.recp of " diff --git a/src/keri/app/cli/commands/ipex/join.py b/src/keri/app/cli/commands/ipex/join.py index 0f8401514..897d9287d 100644 --- a/src/keri/app/cli/commands/ipex/join.py +++ b/src/keri/app/cli/commands/ipex/join.py @@ -23,7 +23,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--auto", "-Y", help="auto approve any delegation request non-interactively", action="store_true") diff --git a/src/keri/app/cli/commands/ipex/list.py b/src/keri/app/cli/commands/ipex/list.py index 4f59d9816..70569a52e 100644 --- a/src/keri/app/cli/commands/ipex/list.py +++ b/src/keri/app/cli/commands/ipex/list.py @@ -33,7 +33,7 @@ default=None) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print full JSON of all credentials", action="store_true") diff --git a/src/keri/app/cli/commands/ipex/spurn.py b/src/keri/app/cli/commands/ipex/spurn.py index 3d4ecebe8..7af44eacb 100644 --- a/src/keri/app/cli/commands/ipex/spurn.py +++ b/src/keri/app/cli/commands/ipex/spurn.py @@ -26,7 +26,7 @@ required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--said", "-s", help="SAID of the exn IPEX message to spurn", required=True) diff --git a/src/keri/app/cli/commands/kevers.py b/src/keri/app/cli/commands/kevers.py index 3eaf42aca..4509c476f 100644 --- a/src/keri/app/cli/commands/kevers.py +++ b/src/keri/app/cli/commands/kevers.py @@ -25,7 +25,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--prefix', help='qb64 identifier prefix to display', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--poll", "-P", help="Poll mailboxes for any events", action="store_true") diff --git a/src/keri/app/cli/commands/list.py b/src/keri/app/cli/commands/list.py index d02f162ca..8e15b3b1b 100644 --- a/src/keri/app/cli/commands/list.py +++ b/src/keri/app/cli/commands/list.py @@ -20,7 +20,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") diff --git a/src/keri/app/cli/commands/local/watch.py b/src/keri/app/cli/commands/local/watch.py index 6529f80e9..9e17841a7 100644 --- a/src/keri/app/cli/commands/local/watch.py +++ b/src/keri/app/cli/commands/local/watch.py @@ -26,7 +26,7 @@ required=False, default="") # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) diff --git a/src/keri/app/cli/commands/mailbox/debug.py b/src/keri/app/cli/commands/mailbox/debug.py index de7f425ba..d305a1ad1 100644 --- a/src/keri/app/cli/commands/mailbox/debug.py +++ b/src/keri/app/cli/commands/mailbox/debug.py @@ -25,7 +25,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--witness', '-w', help='The qualified b64 AID of the witness to poll', required=True) parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") diff --git a/src/keri/app/cli/commands/mailbox/update.py b/src/keri/app/cli/commands/mailbox/update.py index 0c191bb94..d648702b6 100644 --- a/src/keri/app/cli/commands/mailbox/update.py +++ b/src/keri/app/cli/commands/mailbox/update.py @@ -27,7 +27,7 @@ parser.add_argument("--index", "-i", help="new index for topic on witness", required=True) # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) diff --git a/src/keri/app/cli/commands/migrate.py b/src/keri/app/cli/commands/migrate.py index 2116ee267..36d3b3c0c 100644 --- a/src/keri/app/cli/commands/migrate.py +++ b/src/keri/app/cli/commands/migrate.py @@ -25,7 +25,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--force', action="store_true", required=False, help='True means perform migration without prompting the user') diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py index 198533fd9..104fa8baf 100644 --- a/src/keri/app/cli/commands/migrate/list.py +++ b/src/keri/app/cli/commands/migrate/list.py @@ -37,7 +37,7 @@ def handler(args): # Parameters for Manager creation # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py index b45f4e9aa..9a9e4c818 100644 --- a/src/keri/app/cli/commands/migrate/run.py +++ b/src/keri/app/cli/commands/migrate/run.py @@ -39,7 +39,7 @@ def handler(args): # Parameters for Manager creation # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) diff --git a/src/keri/app/cli/commands/migrate/show.py b/src/keri/app/cli/commands/migrate/show.py index b8b9c6060..29c140889 100644 --- a/src/keri/app/cli/commands/migrate/show.py +++ b/src/keri/app/cli/commands/migrate/show.py @@ -38,7 +38,7 @@ def handler(args): # Parameters for Manager creation # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) diff --git a/src/keri/app/cli/commands/multisig/continue.py b/src/keri/app/cli/commands/multisig/continue.py index e38746711..a93caa190 100644 --- a/src/keri/app/cli/commands/multisig/continue.py +++ b/src/keri/app/cli/commands/multisig/continue.py @@ -23,7 +23,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the local identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran diff --git a/src/keri/app/cli/commands/multisig/incept.py b/src/keri/app/cli/commands/multisig/incept.py index 30face97f..3e24f8f58 100644 --- a/src/keri/app/cli/commands/multisig/incept.py +++ b/src/keri/app/cli/commands/multisig/incept.py @@ -28,7 +28,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the local identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--wait", "-w", help="number of seconds to wait for other multisig events, defaults to 10", default=10) diff --git a/src/keri/app/cli/commands/multisig/interact.py b/src/keri/app/cli/commands/multisig/interact.py index 77b98ff73..5b45a11ac 100644 --- a/src/keri/app/cli/commands/multisig/interact.py +++ b/src/keri/app/cli/commands/multisig/interact.py @@ -26,7 +26,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the local identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--data', '-d', help='Anchor data, \'@\' allowed', default=[], action="store", required=True) parser.add_argument("--aids", "-g", help="List of other participant qb64 identifiers to include in interaction event", diff --git a/src/keri/app/cli/commands/multisig/join.py b/src/keri/app/cli/commands/multisig/join.py index 0e868e8ee..b1e9d9f60 100644 --- a/src/keri/app/cli/commands/multisig/join.py +++ b/src/keri/app/cli/commands/multisig/join.py @@ -25,7 +25,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--auto", "-Y", help="auto approve any delegation request non-interactively", action="store_true") diff --git a/src/keri/app/cli/commands/multisig/notice.py b/src/keri/app/cli/commands/multisig/notice.py index a442eab0c..3ba464797 100644 --- a/src/keri/app/cli/commands/multisig/notice.py +++ b/src/keri/app/cli/commands/multisig/notice.py @@ -26,7 +26,7 @@ parser.add_argument("--config", "-c", help="directory override for configuration data") # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) diff --git a/src/keri/app/cli/commands/multisig/rotate.py b/src/keri/app/cli/commands/multisig/rotate.py index af301b813..fa73ea724 100644 --- a/src/keri/app/cli/commands/multisig/rotate.py +++ b/src/keri/app/cli/commands/multisig/rotate.py @@ -27,7 +27,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the local identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--smids", "-s", help="List of other participant qb64 identifiers with signing authority in " "rotation event", diff --git a/src/keri/app/cli/commands/multisig/shell.py b/src/keri/app/cli/commands/multisig/shell.py index 6d0da88c2..ec4c3d093 100644 --- a/src/keri/app/cli/commands/multisig/shell.py +++ b/src/keri/app/cli/commands/multisig/shell.py @@ -15,7 +15,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--static-threshold', '-t', help='Specify this to switch to using one static number as the signing' diff --git a/src/keri/app/cli/commands/multisig/update.py b/src/keri/app/cli/commands/multisig/update.py index 4ef7d19fc..37f4d6d84 100644 --- a/src/keri/app/cli/commands/multisig/update.py +++ b/src/keri/app/cli/commands/multisig/update.py @@ -27,7 +27,7 @@ parser.add_argument('--witness', '-w', help='QB64 identifier of witness to query', default="", required=True) # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) diff --git a/src/keri/app/cli/commands/oobi/clean.py b/src/keri/app/cli/commands/oobi/clean.py index cd2dc1d46..0af65e04f 100644 --- a/src/keri/app/cli/commands/oobi/clean.py +++ b/src/keri/app/cli/commands/oobi/clean.py @@ -19,7 +19,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran def list_oobis(args): diff --git a/src/keri/app/cli/commands/oobi/generate.py b/src/keri/app/cli/commands/oobi/generate.py index f2ac4ba68..f487f3546 100644 --- a/src/keri/app/cli/commands/oobi/generate.py +++ b/src/keri/app/cli/commands/oobi/generate.py @@ -28,7 +28,7 @@ # Parameters for Manager access # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) diff --git a/src/keri/app/cli/commands/oobi/resolve.py b/src/keri/app/cli/commands/oobi/resolve.py index 9b835b30f..dc6e62dc4 100644 --- a/src/keri/app/cli/commands/oobi/resolve.py +++ b/src/keri/app/cli/commands/oobi/resolve.py @@ -34,7 +34,7 @@ # Parameters for Manager access # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) diff --git a/src/keri/app/cli/commands/passcode/remove.py b/src/keri/app/cli/commands/passcode/remove.py index 82b6b04f7..61b3dcf79 100644 --- a/src/keri/app/cli/commands/passcode/remove.py +++ b/src/keri/app/cli/commands/passcode/remove.py @@ -19,7 +19,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='existing 22 character encryption passcode for keystore', +parser.add_argument('--passcode', '-p', help='existing 21 character encryption passcode for keystore', dest="bran", default=None) # passcode => bran diff --git a/src/keri/app/cli/commands/passcode/set.py b/src/keri/app/cli/commands/passcode/set.py index a31f978bb..353e29140 100644 --- a/src/keri/app/cli/commands/passcode/set.py +++ b/src/keri/app/cli/commands/passcode/set.py @@ -18,11 +18,11 @@ parser = argparse.ArgumentParser(description='Initialize a prefix') parser.set_defaults(handler=lambda args: set_passcode(args), transferable=True) -parser.add_argument("--new", help="new 22 character encryption passcode for keystore", required=False, default=None) +parser.add_argument("--new", help="new 21 character encryption passcode for keystore", required=False, default=None) parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='existing 22 character encryption passcode for keystore', +parser.add_argument('--passcode', '-p', help='existing 21 character encryption passcode for keystore', dest="bran", default=None) # passcode => bran @@ -50,7 +50,7 @@ def set_passcode(tymth, tock=0.0, **opts): with existing.existingHby(name=name, base=base, bran=bran) as hby: if newpasscode is None: - print("Re-encrypting keystore, please enter the new 22 character passcode:") + print("Re-encrypting keystore, please enter the new 21 character passcode:") while True: newpasscode = getpass.getpass("New passcode: ") retry = getpass.getpass("Re-enter new passcode: ") diff --git a/src/keri/app/cli/commands/query.py b/src/keri/app/cli/commands/query.py index dc9a1b220..f3b9414f5 100644 --- a/src/keri/app/cli/commands/query.py +++ b/src/keri/app/cli/commands/query.py @@ -28,7 +28,7 @@ parser.add_argument('--prefix', help='QB64 identifier to query', default="", required=True) # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) diff --git a/src/keri/app/cli/commands/rename.py b/src/keri/app/cli/commands/rename.py index af0fb0cf6..10bcc54d8 100644 --- a/src/keri/app/cli/commands/rename.py +++ b/src/keri/app/cli/commands/rename.py @@ -22,7 +22,7 @@ required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the identifier prefix', required=True) parser.add_argument('new', help='new human readable alias for the identifier') -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran diff --git a/src/keri/app/cli/commands/rollback.py b/src/keri/app/cli/commands/rollback.py index f7d573f45..386f631f1 100644 --- a/src/keri/app/cli/commands/rollback.py +++ b/src/keri/app/cli/commands/rollback.py @@ -25,7 +25,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index 56092c617..d8316edcb 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -19,7 +19,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--file', '-f', help='file path of config options (JSON) for rotation', default="", required=False) parser.add_argument('--next-count', '-C', help='Count of pre-rotated keys (signing keys after next rotation).', diff --git a/src/keri/app/cli/commands/sign.py b/src/keri/app/cli/commands/sign.py index fa2c44d8b..b6324d6e4 100644 --- a/src/keri/app/cli/commands/sign.py +++ b/src/keri/app/cli/commands/sign.py @@ -18,7 +18,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--text', '-t', help='Text or file (starts with "@") to sign', required=True) diff --git a/src/keri/app/cli/commands/ssh/export.py b/src/keri/app/cli/commands/ssh/export.py index 0f0eff8da..542aead0e 100644 --- a/src/keri/app/cli/commands/ssh/export.py +++ b/src/keri/app/cli/commands/ssh/export.py @@ -26,7 +26,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--private", help="export private key instead of public key", action="store_true") diff --git a/src/keri/app/cli/commands/status.py b/src/keri/app/cli/commands/status.py index 787bae9ad..662a6b625 100644 --- a/src/keri/app/cli/commands/status.py +++ b/src/keri/app/cli/commands/status.py @@ -22,7 +22,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") diff --git a/src/keri/app/cli/commands/vc/create.py b/src/keri/app/cli/commands/vc/create.py index e30b632dc..b57ea18a6 100644 --- a/src/keri/app/cli/commands/vc/create.py +++ b/src/keri/app/cli/commands/vc/create.py @@ -36,7 +36,7 @@ parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) parser.add_argument("--private", help="flag to indicate if this credential needs privacy preserving features", action="store_true") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--time", help="timestamp for the credential creation", required=False, default=None) diff --git a/src/keri/app/cli/commands/vc/export.py b/src/keri/app/cli/commands/vc/export.py index f79e2b693..71ce933dc 100644 --- a/src/keri/app/cli/commands/vc/export.py +++ b/src/keri/app/cli/commands/vc/export.py @@ -25,7 +25,7 @@ required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--said", "-s", help="SAID of the credential to export.", required=False, default=None) diff --git a/src/keri/app/cli/commands/vc/list.py b/src/keri/app/cli/commands/vc/list.py index 45717c015..a7d5d8706 100644 --- a/src/keri/app/cli/commands/vc/list.py +++ b/src/keri/app/cli/commands/vc/list.py @@ -29,7 +29,7 @@ default=None) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print full JSON of all credentials", action="store_true") diff --git a/src/keri/app/cli/commands/vc/registry/incept.py b/src/keri/app/cli/commands/vc/registry/incept.py index ac0a01ba8..83dd6b320 100644 --- a/src/keri/app/cli/commands/vc/registry/incept.py +++ b/src/keri/app/cli/commands/vc/registry/incept.py @@ -32,7 +32,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--usage', '-u', help='For multisig issuers, a message to other participants about how this' ' registry is to be used', @@ -74,7 +74,7 @@ def __init__(self, name, base, alias, bran, registryName, usage, **kwa): name (str): name of habery and shared db and file path base (str): optional if "" path component of shared db and files. alias(str): human readable name of identifier to use for registry inception - bran (str): Base64 22 char string that is used as base material for + bran (str): Base64 21 char string that is used as base material for seed. bran allows alphanumeric passcodes generated by key managers like 1password to be key store for seed. registryName(str): human readable name for newly created registry diff --git a/src/keri/app/cli/commands/vc/registry/list.py b/src/keri/app/cli/commands/vc/registry/list.py index 3c299895a..0087d6530 100644 --- a/src/keri/app/cli/commands/vc/registry/list.py +++ b/src/keri/app/cli/commands/vc/registry/list.py @@ -21,7 +21,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") diff --git a/src/keri/app/cli/commands/vc/registry/status.py b/src/keri/app/cli/commands/vc/registry/status.py index f24f57a50..ad7efdbaf 100644 --- a/src/keri/app/cli/commands/vc/registry/status.py +++ b/src/keri/app/cli/commands/vc/registry/status.py @@ -18,7 +18,7 @@ default=None, required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") diff --git a/src/keri/app/cli/commands/vc/revoke.py b/src/keri/app/cli/commands/vc/revoke.py index a37059255..072b86389 100644 --- a/src/keri/app/cli/commands/vc/revoke.py +++ b/src/keri/app/cli/commands/vc/revoke.py @@ -24,7 +24,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--said', help='is SAID vc content qb64') parser.add_argument('--send', help='alias of contact to send the revocation events to (can be repeated)', diff --git a/src/keri/app/cli/commands/verify.py b/src/keri/app/cli/commands/verify.py index 1de4fa25b..17033bb20 100644 --- a/src/keri/app/cli/commands/verify.py +++ b/src/keri/app/cli/commands/verify.py @@ -17,7 +17,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--prefix", help="Identifier prefix of the signer", required=True) diff --git a/src/keri/app/cli/commands/version.py b/src/keri/app/cli/commands/version.py index 98cc81c5a..d2a9a0385 100644 --- a/src/keri/app/cli/commands/version.py +++ b/src/keri/app/cli/commands/version.py @@ -16,7 +16,7 @@ default=None) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran diff --git a/src/keri/app/cli/commands/witness/start.py b/src/keri/app/cli/commands/witness/start.py index 3ea02fe6e..400b75468 100644 --- a/src/keri/app/cli/commands/witness/start.py +++ b/src/keri/app/cli/commands/witness/start.py @@ -36,7 +36,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--config-dir", "-c", dest="configDir", help="directory override for configuration data") parser.add_argument('--config-file', diff --git a/src/keri/app/cli/commands/witness/submit.py b/src/keri/app/cli/commands/witness/submit.py index 1aaae2808..f2f317ac0 100644 --- a/src/keri/app/cli/commands/witness/submit.py +++ b/src/keri/app/cli/commands/witness/submit.py @@ -23,7 +23,7 @@ parser.add_argument("--config", "-c", help="directory override for configuration data") # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) From de7886b0bc0b296d35da36167ebaaad9f2179d52 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 11 Apr 2024 11:38:02 -0600 Subject: [PATCH 137/418] some refactoring change name of MapCodex to MapDom to be consistent for uses that are not Codexes fixed missing code entry for SealLast and updated references --- src/keri/core/coring.py | 6 +-- src/keri/core/counting.py | 39 ++++++++++++++-- src/keri/core/eventing.py | 12 ++--- src/keri/core/serdering.py | 29 +++++++++--- src/keri/core/signing.py | 21 +++++++++ src/keri/core/structing.py | 89 +++++++++++++++++++----------------- tests/core/test_coring.py | 6 +-- tests/core/test_counting.py | 28 +++++++++++- tests/core/test_serdering.py | 6 +-- tests/core/test_structing.py | 60 ++++++++++++++++++------ 10 files changed, 217 insertions(+), 79 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 599b948d7..f4cda654e 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -210,8 +210,8 @@ def randomNonce(): @dataclass -class MapDom: - """Base class for dataclasses that support map syntax +class MapHood: + """Base class for mutable dataclasses that support map syntax Adds support for dunder methods for map syntax dc[name]. Converts exceptions from attribute syntax to raise map syntax when using map syntax. @@ -241,7 +241,7 @@ def __delitem__(self, name): @dataclass(frozen=True) -class MapCodex: +class MapDom: """Base class for frozen dataclasses (codexes) that support map syntax Adds support for dunder methods for map syntax dc[name]. Converts exceptions from attribute syntax to raise map syntax when using diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 62ea757ca..a79cf6a28 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -16,7 +16,7 @@ from .. import kering from ..kering import (Versionage, Vrsn_1_0, Vrsn_2_0) -from ..core.coring import Sizage, MapCodex +from ..core.coring import Sizage, MapDom @@ -45,7 +45,7 @@ def __iter__(self): @dataclass(frozen=True) -class CounterCodex_1_0(MapCodex): +class CounterCodex_1_0(MapDom): """ CounterCodex is codex hard (stable) part of all counter derivation codes. Only provide defined codes. @@ -77,7 +77,7 @@ def __iter__(self): CtrDex_1_0 = CounterCodex_1_0() @dataclass(frozen=True) -class CounterCodex_2_0(MapCodex): +class CounterCodex_2_0(MapDom): """ CounterCodex is codex hard (stable) part of all counter derivation codes. Only provide defined codes. @@ -134,6 +134,8 @@ class CounterCodex_2_0(MapCodex): BigMerkleRootSealSingles: str = '-0W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. BackerRegistrarSealCouples: str = '-X' # Backer Registrar Seal Couple(s), brid+dig of sealed data. BigBackerRegistrarSealCouples: str = '-0X' # Big Backer Registrar Seal Couple(s), brid+dig of sealed data. + SealSourceLastSingles: str = '-Y' # Seal Source Couple(s), pre of last source sealing or sealed event. + BigSealSourceLastSingles: str = '-0Y' # Big Seal Source Couple(s), pre of last source sealing or sealed event. ESSRPayloadGroup: str = '-Z' # ESSR Payload Group. BigESSRPayloadGroup: str = '-0Z' # Big ESSR Payload Group. KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Stack CESR Protocol Genus Version (Universal) @@ -157,6 +159,35 @@ def __iter__(self): AllTags = AllTagage() # uses defaults +@dataclass(frozen=True) +class SealCodex_2_0(MapDom): + """ + SealCodex_2_0 is codex of seal counter derivation codes. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get codes with item syntax using tag variables. + Example: codex[tag] + """ + SealSourceCouples: str = '-Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. + BigSealSourceCouples: str = '-0Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. + SealSourceTriples: str = '-R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. + BigSealSourceTriples: str = '-0R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. + DigestSealSingles: str = '-V' # Digest Seal Single(s), dig of sealed data. + BigDigestSealSingles: str = '-0V' # Big Digest Seal Single(s), dig of sealed data. + MerkleRootSealSingles: str = '-W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. + BigMerkleRootSealSingles: str = '-0W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. + BackerRegistrarSealCouples: str = '-X' # Backer Registrar Seal Couple(s), brid+dig of sealed data. + BigBackerRegistrarSealCouples: str = '-0X' # Big Backer Registrar Seal Couple(s), brid+dig of sealed data. + SealSourceLastSingles: str = '-Y' # Seal Source Couple(s), pre of last source sealing event. + BigSealSourceLastSingles: str = '-0Y' # Big Seal Source Couple(s), pre of last source sealing event. + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +SealDex_2_0 = SealCodex_2_0() + + class Counter: """ Counter is fully qualified cryptographic material primitive base class for @@ -373,6 +404,8 @@ class Counter: '-0X': Sizage(hs=3, ss=5, fs=8, ls=0), '-Y': Sizage(hs=2, ss=2, fs=4, ls=0), '-0Y': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0Z': Sizage(hs=3, ss=5, fs=8, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), }, }, diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 772fd03ab..0a4bd9535 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -76,12 +76,6 @@ # d = digest qb64 of backer metadata anchored to event usually SAID of data SealBacker = namedtuple("SealBacker", 'bi d') -# Event Seal: triple (i, s, d) -# i = pre is qb64 of identifier prefix of KEL for event, -# s = sn of event as lowercase hex string no leading zeros, -# d = SAID digest qb64 of event -SealEvent = namedtuple("SealEvent", 'i s d') - # Last Estalishment Event Seal: uniple (i,) # i = pre is qb64 of identifier prefix of KEL from which to get last est, event # used to indicate to get the latest keys available from KEL for 'i' @@ -95,6 +89,12 @@ # use SealSourceCouples count code for attachment SealTrans = namedtuple("SealTrans", 's d') +# Event Seal: triple (i, s, d) +# i = pre is qb64 of identifier prefix of KEL for event, +# s = sn of event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of event +SealEvent = namedtuple("SealEvent", 'i s d') + # Following are not seals only used in database # State Establishment Event (latest current) : quadruple (s, d, br, ba) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 5b211b404..07b6e5c72 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -38,9 +38,9 @@ from .coring import (Matter, Saider, Verfer, Diger, Number, Tholder, Tagger, Ilker, Traitor, Verser, ) -from .counting import GenDex, AllTags, Counter +from .counting import GenDex, AllTags, Counter, SealDex_2_0 -from .structing import Sealer +from .structing import Sealer, ClanDom @@ -347,8 +347,10 @@ class Serder: # Spans dict keyed by version (Versionage instance) of version string span (size) Spans = {Vrsn_1_0: VER1FULLSPAN, Vrsn_2_0: VER2FULLSPAN} - # should be same set of codes as in coring.DigestCodex coring.DigDex so - # .digestive property works. Use unit tests to ensure codex sets match + # Maps digest codes to Digestages of algorithms for computing digest. + # Should be based on the same set of codes as in coring.DigestCodex + # coring.DigDex so .digestive property works. + # Use unit tests to ensure codex elements sets match Digests = { DigDex.Blake3_256: Digestage(klas=blake3.blake3, size=None, length=None), DigDex.Blake2b_256: Digestage(klas=hashlib.blake2b, size=32, length=None), @@ -361,6 +363,18 @@ class Serder: DigDex.SHA2_512: Digestage(klas=hashlib.sha512, size=None, length=None), } + # map seal clan names to seal counter code for grouping seals in anchor list + ClanSeals = dict() + ClanSeals[ClanDom.SealDigest.__name__] = SealDex_2_0.DigestSealSingles + ClanSeals[ClanDom.SealRoot.__name__] = SealDex_2_0.MerkleRootSealSingles + ClanSeals[ClanDom.SealBacker.__name__] = SealDex_2_0.BackerRegistrarSealCouples + ClanSeals[ClanDom.SealLast.__name__] = SealDex_2_0.SealSourceLastSingles + ClanSeals[ClanDom.SealTrans.__name__] = SealDex_2_0.SealSourceCouples + ClanSeals[ClanDom.SealEvent.__name__] = SealDex_2_0.SealSourceTriples + + # map seal counter code to seal clan name for parsing seal groups in anchor list + SealClans = {} + #override in subclass to enforce specific protocol Protocol = None # class based message protocol, None means any in Protocols is ok Proto = Protocols.keri # default message protocol type for makify on base Serder @@ -1284,14 +1298,17 @@ def _dumps(self, sad): case "a": # list of seals or field map of attributes frame = bytearray() for e in v: # list of seal dicts + # need support for grouping consecutive seals of same type with same counter + try: sealer = Sealer(crew=e) frame.extend(sealer.qb64b) + # lookup counter type by sealer.clan.name except kering.InvalidValueError: pass #unknown seal type so serialize as field map - pass + #if tuple(v) == eventing.SealEvent._fields: #eseal = eventing.SealEvent(**v) # convert to namedtuple #SealSourceCouples: str = '-Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. @@ -1305,7 +1322,7 @@ def _dumps(self, sad): #tuple(d) #('a', 'b') - #frame.extend(Anchor(seal=e).qb64b) + #frame.extend(Sealer(seal=e).qb64b) # else: generic seal no count type (v, Mapping): #for l, e in v.items(): #pass diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index f3571016f..f956a4d02 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -364,6 +364,27 @@ class Salter(Matter): Salter is Matter subclass to maintain random salt for secrets (private keys) Its .raw is random salt, .code as cipher suite for salt + To initialize with deterministic salt pass in 16 bytes for raw: + salter = Salter(raw=b'0123456789abcdef') + + To create a deterministic secret, seed, or private key from salt + call .signer: + signer = salter.signer(code=MtrDex.Ed25519_Seed, + transferable=True, + path="", + tier=None, + temp=False) + + To create a deterministic set of secrets or seeds or private keys from salt + call signers: + signers = salter.signers(count=1, + start=0, + path="", + code=MtrDex.Ed25519_Seed, + transferable=True, + tier=None, + temp=False) + Attributes: .level is str security level code. Provides default level diff --git a/src/keri/core/structing.py b/src/keri/core/structing.py index 5c1a77176..cb7e1fea8 100644 --- a/src/keri/core/structing.py +++ b/src/keri/core/structing.py @@ -17,7 +17,7 @@ from ..help import nonStringSequence from . import coring -from .coring import (MapCodex, Matter, Diger, Prefixer, Number) +from .coring import (MapDom, Matter, Diger, Prefixer, Number) # ToDo Change seal namedtuple definitions to NamedTuple subclasses so can @@ -48,12 +48,6 @@ # d = digest qb64 of backer metadata anchored to event usually SAID of data SealBacker = namedtuple("SealBacker", 'bi d') -# Event Seal: triple (i, s, d) -# i = pre is qb64 of identifier prefix of KEL for event, -# s = sn of event as lowercase hex string no leading zeros, -# d = SAID digest qb64 of event -SealEvent = namedtuple("SealEvent", 'i s d') - # Last Establishment Event Seal: uniple (i,) # i = pre is qb64 of identifier prefix of KEL from which to get last est, event # used to indicate to get the latest keys available from KEL for 'i' @@ -67,6 +61,12 @@ # use SealSourceCouples count code for attachment SealTrans = namedtuple("SealTrans", 's d') +# Event Seal: triple (i, s, d) +# i = pre is qb64 of identifier prefix of KEL for event, +# s = sn of event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of event +SealEvent = namedtuple("SealEvent", 'i s d') + # Following are not seals only used in database @@ -86,7 +86,7 @@ @dataclass(frozen=True) -class EmptyClanCodex(MapCodex): +class EmptyClanCodex(MapDom): """ SealClanDom is dataclass of namedtuple seal class references (clans) each indexed by its class name. @@ -107,7 +107,7 @@ def __iter__(self): @dataclass(frozen=True) -class EmptyCastCodex(MapCodex): +class EmptyCastCodex(MapDom): """ SealCastCodex is dataclass of namedtuple instances (seal casts) whose values are named primitive class references @@ -130,9 +130,9 @@ def __iter__(self): @dataclass(frozen=True) -class SealClanCodex(MapCodex): +class SealClanDom(MapDom): """ - SealClanCodex is dataclass of namedtuple seal class references (clans) each + SealClanDom is dataclass of namedtuple seal class references (clans) each indexed by its class name. Only provide defined classes. @@ -141,25 +141,25 @@ class SealClanCodex(MapCodex): As subclass of MapCodex can get class reference with item syntax using name variables. - Example: SealClanDex[name] + Example: ClanDom[name] """ SealDigest: type[NamedTuple] = SealDigest # SealDigest class reference SealRoot: type[NamedTuple] = SealRoot # SealRoot class reference SealBacker: type[NamedTuple] = SealBacker # SealBacker class reference - SealEvent: type[NamedTuple] = SealEvent # SealEvent class reference - SealLast: type[NamedTuple] = SealLast # SealLast class reference - SealTrans: type[NamedTuple] = SealTrans # SealTrans class reference + SealLast: type[NamedTuple] = SealLast # SealLast class reference single + SealTrans: type[NamedTuple] = SealTrans # SealTrans class reference couple + SealEvent: type[NamedTuple] = SealEvent # SealEvent class reference triple def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" -SealClanDex = SealClanCodex() # create instance +ClanDom = SealClanDom() # create instance @dataclass(frozen=True) -class SealCastCodex(MapCodex): +class SealCastDom(MapDom): """ - SealCastCodex is dataclass of namedtuple instances (seal casts) whose values + SealCastDom is dataclass of namedtuple instances (seal casts) whose values are named primitive class references indexed by its namedtuple class name. @@ -170,19 +170,19 @@ class SealCastCodex(MapCodex): As subclass of MapCodex can get namedtuple instance with item syntax using name variables. - Example: SealCastDom[name] + Example: CastDom[name] """ SealDigest: NamedTuple = SealDigest(d=Diger) # SealDigest class reference SealRoot: NamedTuple = SealRoot(rd=Diger) # SealRoot class reference SealBacker: NamedTuple = SealBacker(bi=Prefixer, d=Diger) # SealBacker class reference - SealEvent: NamedTuple = SealEvent(i=Prefixer, s=Number, d=Diger) # SealEvent class reference - SealLast: NamedTuple = SealLast(i=Prefixer) # SealLast class reference - SealTrans: NamedTuple = SealTrans(s=Number, d=Diger) # SealTrans class reference + SealLast: NamedTuple = SealLast(i=Prefixer) # SealLast class reference single + SealTrans: NamedTuple = SealTrans(s=Number, d=Diger) # SealTrans class reference couple + SealEvent: NamedTuple = SealEvent(i=Prefixer, s=Number, d=Diger) # SealEvent class reference triple def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" -SealCastDex = SealCastCodex() # create instance +CastDom = SealCastDom() # create instance @@ -254,9 +254,10 @@ class Structor: """ Clans = EmptyClanDex # known namedtuple clans. Override in subclass with non-empty Casts = EmptyCastDex # known namedtuple casts. Override in subclass with non-empty - # Create .Names dict that maps clan/cast fields names to its namedtuple - # class type name so can look up a know clan or cast given a matching set - # of either field names from a namedtuple or keys from a dict. + # Create .Names dict that maps tuple of clan/cast fields names to its namedtuple + # class type name so can look up a know clan or cast given a matching tuple + # of either field names from a namedtuple or keys from a dict. The tuple of + # field names is a mark of the structor type. Maps mark to class name. Names = {tuple(clan._fields): clan.__name__ for clan in Clans} @@ -308,22 +309,22 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, clan = crew.__class__ if not clan and isinstance(cast, Mapping): # get clan from cast - names = tuple(cast) # create custom clan based on cast - if (cname := self.Names.get(names)): # get known else None + mark = tuple(cast) # create custom clan based on cast mark + if (cname := self.Names.get(mark)): # get known else None clan = self.Clans[cname] cast = self.Casts[cname] else: # create custom clan from cast - clan = namedtuple("_".join(names), names) # custom clan + clan = namedtuple("_".join(mark), mark) # custom clan if not clan and isinstance(crew, Mapping): # get clan from crew - names = tuple(crew) # create custom clan based on cast - if (cname := self.Names.get(names)): # get known else None + mark = tuple(crew) # create custom clan based on crew mark + if (cname := self.Names.get(mark)): # get known else None clan = self.Clans[cname] cast = self.Casts[cname] else: # create custom clan from crew - clan = namedtuple("_".join(names), names) # custom clan from cast keys + clan = namedtuple("_".join(mark), mark) # custom clan from cast keys if clan: if not (issubclass(clan, tuple) and hasattr(clan, "_fields")): @@ -335,17 +336,15 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, if cast: if not isinstance(cast, clan): if isinstance(cast, tuple) and hasattr(cast, "_fields"): - if cast._fields != clan._fields: + if cast._fields != clan._fields: # fields is mark raise InvalidValueError(f"Mismatching fields clan=" f"{clan._fields} and cast=" f"{cast._fields}.") cast = clan(**cast._asdict()) # convert to clan - - elif isinstance(cast, Mapping): - if tuple(cast) != clan._fields: + if tuple(cast) != clan._fields: # fields is mark raise InvalidValueError(f"Mismatching fields clan=" f"{clan._fields} and keys cast=" f"{tuple(cast)}.") @@ -359,7 +358,7 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, raise InvalidValueError(f"Invalid {cast=}.") else: # get cast from known .Casts if possible - if (cname := self.Names.get(clan._fields)): + if (cname := self.Names.get(clan._fields)): # fields is mark cast = self.Casts[cname] # get known cast else: # cast missing or unobtainable raise InvalidValueError(f"Missing or unobtainable cast.") @@ -374,7 +373,7 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, if crew: if not isinstance(crew, clan): if isinstance(crew, tuple) and hasattr(crew, "_fields"): - if crew._fields != clan._fields: + if crew._fields != clan._fields: # fields is mark raise InvalidValueError(f"Mismatching fields clan=" f"{clan._fields} and crew=" f"{crew._fields}.") @@ -382,7 +381,7 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, crew = clan(**crew._asdict()) # convert to clan elif isinstance(crew, Mapping): - if tuple(crew) != clan._fields: + if tuple(crew) != clan._fields: # fields is mark raise InvalidValueError(f"Mismatching fields clan=" f"{clan._fields} and keys crew=" f"{tuple(crew)}.") @@ -458,6 +457,14 @@ def clan(self): """ return self.data.__class__ + @property + def name(self): + """Returns: + name (str): name of class of .data + + """ + return self.data.__class__.__name__ + @property def cast(self): """Return: @@ -556,8 +563,8 @@ class Sealer(Structor): """ - Clans = SealClanDex # known namedtuple clans. Override in subclass with non-empty - Casts = SealCastDex # known namedtuple casts. Override in subclass with non-empty + Clans = ClanDom # known namedtuple clans. Override in subclass with non-empty + Casts = CastDom # known namedtuple casts. Override in subclass with non-empty # Create .Names dict that maps clan/cast fields names to its namedtuple # class type name so can look up a know clan or cast given a matching set # of either field names from a namedtuple or keys from a dict. diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index f6ebf78b1..e2c49c800 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -52,7 +52,7 @@ IdxCrtSigDex, IdxBthSigDex, Indexer) -from keri.core.coring import MapDom, MapCodex +from keri.core.coring import MapHood, MapDom @@ -60,7 +60,7 @@ def test_mapdom(): """Test MapDom base dataclass""" @dataclass - class TestMapDom(MapDom): + class TestMapDom(MapHood): """ """ @@ -151,7 +151,7 @@ def test_mapcodex(): @dataclass(frozen=True) - class TestMapCodex(MapCodex): + class TestMapCodex(MapDom): """ """ diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index da1c21aeb..39534f68d 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -21,7 +21,7 @@ from keri.core import counting -from keri.core.counting import GenDex, Sizage, MapCodex, Counter +from keri.core.counting import GenDex, Sizage, MapDom, Counter from keri.core.counting import Versionage, Vrsn_1_0, Vrsn_2_0, AllTags @@ -130,6 +130,8 @@ def test_codexes_tags(): 'BigMerkleRootSealSingles': '-0W', 'BackerRegistrarSealCouples': '-X', 'BigBackerRegistrarSealCouples': '-0X', + 'SealSourceLastSingles': '-Y', + 'BigSealSourceLastSingles': '-0Y', 'ESSRPayloadGroup': '-Z', 'BigESSRPayloadGroup': '-0Z', 'KERIACDCGenusVersion': '--AAA' @@ -206,6 +208,8 @@ def test_codexes_tags(): 'BigMerkleRootSealSingles': 'BigMerkleRootSealSingles', 'BackerRegistrarSealCouples': 'BackerRegistrarSealCouples', 'BigBackerRegistrarSealCouples': 'BigBackerRegistrarSealCouples', + 'SealSourceLastSingles': 'SealSourceLastSingles', + 'BigSealSourceLastSingles': 'BigSealSourceLastSingles', 'ESSRPayloadGroup': 'ESSRPayloadGroup', 'BigESSRPayloadGroup': 'BigESSRPayloadGroup', 'KERIACDCGenusVersion': 'KERIACDCGenusVersion' @@ -263,6 +267,8 @@ def test_codexes_tags(): 'BigMerkleRootSealSingles': 'BigMerkleRootSealSingles', 'BackerRegistrarSealCouples': 'BackerRegistrarSealCouples', 'BigBackerRegistrarSealCouples': 'BigBackerRegistrarSealCouples', + 'SealSourceLastSingles': 'SealSourceLastSingles', + 'BigSealSourceLastSingles': 'BigSealSourceLastSingles', 'ESSRPayloadGroup': 'ESSRPayloadGroup', 'BigESSRPayloadGroup': 'BigESSRPayloadGroup', 'KERIACDCGenusVersion': 'KERIACDCGenusVersion' @@ -270,6 +276,24 @@ def test_codexes_tags(): assert counting.AllTags.ControllerIdxSigs == 'ControllerIdxSigs' + assert asdict(counting.SealDex_2_0) == \ + { + 'SealSourceCouples': '-Q', + 'BigSealSourceCouples': '-0Q', + 'SealSourceTriples': '-R', + 'BigSealSourceTriples': '-0R', + 'DigestSealSingles': '-V', + 'BigDigestSealSingles': '-0V', + 'MerkleRootSealSingles': '-W', + 'BigMerkleRootSealSingles': '-0W', + 'BackerRegistrarSealCouples': '-X', + 'BigBackerRegistrarSealCouples': '-0X', + 'SealSourceLastSingles': '-Y', + 'BigSealSourceLastSingles': '-0Y', + } + + + """End Test""" @@ -388,6 +412,8 @@ def test_counter_class(): '-0X': Sizage(hs=3, ss=5, fs=8, ls=0), '-Y': Sizage(hs=2, ss=2, fs=4, ls=0), '-0Y': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0Z': Sizage(hs=3, ss=5, fs=8, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) }, }, diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 31583a84f..472eee39b 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -2649,11 +2649,11 @@ def test_cesr_native_dumps(): raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' salter = core.Salter(raw=raw) - # replace with Salter.signers() - csigners = core.generateSigners(raw=salter.raw, count=3) - wsigners = core.generateSigners(raw=salter.raw, count=3, transferable=False) + csigners = salter.signers(count=3, transferable=True, temp=True) + wsigners = salter.signers(count=3, transferable=False, temp=True) + keys = [csigners[0].qb64] keys = ["EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf87Bc70J"] serder = incept(keys, version=Vrsn_2_0) diff --git a/tests/core/test_structing.py b/tests/core/test_structing.py index a2ade17b8..b34cd6d0d 100644 --- a/tests/core/test_structing.py +++ b/tests/core/test_structing.py @@ -23,7 +23,7 @@ from keri.core.structing import (SealDigest, SealRoot, SealBacker, SealEvent, SealLast, SealTrans) from keri.core.structing import (Structor, EmptyClanDex, EmptyCastDex, - Sealer, SealClanDex, SealCastDex, ) + Sealer, ClanDom, CastDom, ) def test_structor_class(): @@ -59,6 +59,7 @@ def test_structor(): clan = SealDigest cast = SealDigest(d=Diger) crew = SealDigest(d=dig) + name = SealDigest.__name__ dcast = cast._asdict() dcrew = crew._asdict() @@ -76,6 +77,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.asdict == data._asdict() assert structor.asdict == {'d': diger} assert structor.qb64 == qb64 @@ -87,6 +89,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -95,6 +98,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -103,6 +107,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -111,6 +116,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -120,6 +126,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -129,6 +136,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -137,6 +145,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -146,6 +155,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -156,6 +166,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -164,6 +175,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -172,6 +184,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -180,6 +193,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -189,6 +203,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -198,6 +213,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -206,6 +222,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -214,7 +231,7 @@ def test_structor(): structor = Structor(cast=dcast, crew=dcrew) assert structor.data.__class__.__name__ == "d" assert structor.clan != clan - assert structor.clan.__name__ == "d" + assert structor.name == "d" assert structor.cast == cast # tuple compare is by field value not type assert structor.cast.__class__.__name__ == "d" assert structor.crew == crew @@ -229,6 +246,7 @@ def test_structor(): clan = SealEvent cast = SealEvent(i=Prefixer, s=Number, d=Diger) crew = SealEvent(i=aid, s=snq, d=dig) + name = SealEvent.__name__ dcast = cast._asdict() dcrew = crew._asdict() @@ -246,6 +264,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.asdict == data._asdict() assert structor.asdict == \ { @@ -262,6 +281,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -270,6 +290,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -278,6 +299,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -286,6 +308,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -295,6 +318,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -304,6 +328,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -312,6 +337,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -321,6 +347,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -331,6 +358,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -339,6 +367,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -347,6 +376,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -355,6 +385,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -365,6 +396,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -375,6 +407,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -383,6 +416,7 @@ def test_structor(): assert structor.clan == clan assert structor.cast == cast assert structor.crew == crew + assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -391,7 +425,7 @@ def test_structor(): structor = Structor(cast=dcast, crew=dcrew) assert structor.data.__class__.__name__ == "i_s_d" assert structor.clan != clan - assert structor.clan.__name__ == "i_s_d" + assert structor.name == "i_s_d" assert structor.cast == cast # tuple compare is by field value not type assert structor.cast.__class__.__name__ == "i_s_d" assert structor.crew == crew @@ -413,40 +447,40 @@ def test_seal_dexes(): test Seal Codexes """ - assert asdict(SealClanDex) == \ + assert asdict(ClanDom) == \ { 'SealDigest': SealDigest, 'SealRoot': SealRoot, 'SealBacker': SealBacker, - 'SealEvent': SealEvent, 'SealLast': SealLast, - 'SealTrans': SealTrans + 'SealTrans': SealTrans, + 'SealEvent': SealEvent, } - assert asdict(SealCastDex) == \ + assert asdict(CastDom) == \ { 'SealDigest': SealDigest(d=Diger), 'SealRoot': SealRoot(rd=Diger), 'SealBacker': SealBacker(bi=Prefixer, d=Diger), - 'SealEvent': SealEvent(i=Prefixer, s=Number, d=Diger), 'SealLast': SealLast(i=Prefixer), - 'SealTrans': SealTrans(s=Number, d=Diger) + 'SealTrans': SealTrans(s=Number, d=Diger), + 'SealEvent': SealEvent(i=Prefixer, s=Number, d=Diger), } def test_sealer_class(): """ test sealer class variables etc """ - assert Sealer.Clans == SealClanDex - assert Sealer.Casts == SealCastDex + assert Sealer.Clans == ClanDom + assert Sealer.Casts == CastDom assert Sealer.Names == \ { ('d',): 'SealDigest', ('rd',): 'SealRoot', ('bi', 'd'): 'SealBacker', - ('i', 's', 'd'): 'SealEvent', ('i',): 'SealLast', - ('s', 'd'): 'SealTrans' + ('s', 'd'): 'SealTrans', + ('i', 's', 'd'): 'SealEvent', } """End Test""" From 1968bbe95fbeb2ba470d8be862b05fd877f3ee22 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 11 Apr 2024 11:54:16 -0600 Subject: [PATCH 138/418] added Serder.ClanCodes and .CodeClans as lookup for Seals added unit test --- src/keri/core/serdering.py | 20 ++++++++++-------- tests/core/test_serdering.py | 40 +++++++++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 07b6e5c72..babfc1ca6 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -364,16 +364,16 @@ class Serder: } # map seal clan names to seal counter code for grouping seals in anchor list - ClanSeals = dict() - ClanSeals[ClanDom.SealDigest.__name__] = SealDex_2_0.DigestSealSingles - ClanSeals[ClanDom.SealRoot.__name__] = SealDex_2_0.MerkleRootSealSingles - ClanSeals[ClanDom.SealBacker.__name__] = SealDex_2_0.BackerRegistrarSealCouples - ClanSeals[ClanDom.SealLast.__name__] = SealDex_2_0.SealSourceLastSingles - ClanSeals[ClanDom.SealTrans.__name__] = SealDex_2_0.SealSourceCouples - ClanSeals[ClanDom.SealEvent.__name__] = SealDex_2_0.SealSourceTriples + ClanCodes = dict() + ClanCodes[ClanDom.SealDigest.__name__] = SealDex_2_0.DigestSealSingles + ClanCodes[ClanDom.SealRoot.__name__] = SealDex_2_0.MerkleRootSealSingles + ClanCodes[ClanDom.SealBacker.__name__] = SealDex_2_0.BackerRegistrarSealCouples + ClanCodes[ClanDom.SealLast.__name__] = SealDex_2_0.SealSourceLastSingles + ClanCodes[ClanDom.SealTrans.__name__] = SealDex_2_0.SealSourceCouples + ClanCodes[ClanDom.SealEvent.__name__] = SealDex_2_0.SealSourceTriples # map seal counter code to seal clan name for parsing seal groups in anchor list - SealClans = {} + CodeClans = { val: key for key, val in ClanCodes.items()} # invert dict #override in subclass to enforce specific protocol Protocol = None # class based message protocol, None means any in Protocols is ok @@ -1296,7 +1296,9 @@ def _dumps(self, sad): val.extend(frame) case "a": # list of seals or field map of attributes - frame = bytearray() + frame = bytearray() # whole list + gctr = None # counter for consecutive same type seals + gframe = bytearray() # consecutive same type seals for e in v: # list of seal dicts # need support for grouping consecutive seals of same type with same counter diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 472eee39b..9134ee0b2 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -60,12 +60,31 @@ def test_spans(): """End Test""" -def test_serder(): - """ - Test Serder - """ +def test_serder_class(): + """Test Serder class""" + + assert Serder.ClanCodes + assert Serder.ClanCodes == \ + { + 'SealDigest': '-V', + 'SealRoot': '-W', + 'SealBacker': '-X', + 'SealLast': '-Y', + 'SealTrans': '-Q', + 'SealEvent': '-R' + } + + assert Serder.CodeClans + assert Serder.CodeClans == \ + { + '-V': 'SealDigest', + '-W': 'SealRoot', + '-X': 'SealBacker', + '-Y': 'SealLast', + '-Q': 'SealTrans', + '-R': 'SealEvent' + } - # Test Serder assert Serder.Fields @@ -100,6 +119,16 @@ def test_serder(): assert set(fields.saids) <= set(fields.alls) + """End Test""" + + +def test_serder(): + """Test Serder instances""" + + # Test Serder + + + with pytest.raises(kering.InvalidValueError): serder = Serder() @@ -2736,6 +2765,7 @@ def test_cesr_native_dumps(): if __name__ == "__main__": test_fielddom() test_spans() + test_serder_class() test_serder() test_serderkeri() test_serderkeri_icp() From 140eeaad6949d3113f9d47f0e598481a18367a55 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Thu, 11 Apr 2024 16:23:25 -0700 Subject: [PATCH 139/418] Secure Witness Support (#750) * Changes to support secure witness transactions. Signed-off-by: pfeairheller * Enhanced interact and rotate kli commands to allow for --authenticate parameter to ask user for a 2-factor auth code to send to witnesses for secure witness interactions. Signed-off-by: pfeairheller * Updated setup to include QR code library. Signed-off-by: pfeairheller * Update misfits to be a suber instead of a CESR suber. Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller --- scripts/demo/basic/essr.sh | 19 --- setup.py | 3 +- src/keri/app/agenting.py | 16 ++- src/keri/app/cli/commands/decrypt.py | 65 +++++++++ src/keri/app/cli/commands/export.py | 2 +- src/keri/app/cli/commands/interact.py | 45 ++++-- src/keri/app/cli/commands/rotate.py | 17 ++- .../app/cli/commands/witness/authenticate.py | 129 ++++++++++++++++++ src/keri/app/httping.py | 13 +- src/keri/app/kiwiing.py | 0 src/keri/core/eventing.py | 6 +- src/keri/db/basing.py | 2 +- tests/app/test_httping.py | 5 + 13 files changed, 272 insertions(+), 50 deletions(-) delete mode 100755 scripts/demo/basic/essr.sh create mode 100644 src/keri/app/cli/commands/decrypt.py create mode 100644 src/keri/app/cli/commands/witness/authenticate.py delete mode 100644 src/keri/app/kiwiing.py diff --git a/scripts/demo/basic/essr.sh b/scripts/demo/basic/essr.sh deleted file mode 100755 index e3a5cb957..000000000 --- a/scripts/demo/basic/essr.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -#kli init --name sender --salt 0ACDEyMzQ1Njc4OWxtbm9aBc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis -#kli incept --name sender --alias sender --file ${KERI_DEMO_SCRIPT_DIR}/data/gleif-sample.json -# -#kli init --name recipient --salt 0ACDEyMzQ1Njc4OWxtbm9qWc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis -#kli incept --name recipient --alias recipient --file ${KERI_DEMO_SCRIPT_DIR}/data/gleif-sample.json -# -#kli oobi resolve --name sender --oobi-alias recipient --oobi http://127.0.0.1:5642/oobi/EFXJsTFSo10FAZGR-8_Uw1DlhU8nuRFOAN9Z8ajJ56ci/witness -#kli oobi resolve --name recipient --oobi-alias sender --oobi http://127.0.0.1:5642/oobi/EJf7MfzNmehwY5310MUWXPSxhAA_3ifPW2bdsjwqnvae/witness -# -#kli vc registry incept --name sender --alias sender --registry-name vLEI -# -#kli vc create --name sender --alias sender --registry-name vLEI --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao --recipient ELjSFdrTdCebJlmvbFNX9-TLhR2PO0_60al1kQp5_e6k --data @${KERI_DEMO_SCRIPT_DIR}/data/credential-data.json -#SAID=$(kli vc list --name sender --alias sender --issued --said --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao) -# -#kli ipex grant --name sender --alias sender --said "${SAID}" --recipient ELjSFdrTdCebJlmvbFNX9-TLhR2PO0_60al1kQp5_e6k - -kli essr send --name sender --alias sender --recipient recipient \ No newline at end of file diff --git a/setup.py b/setup.py index c280298d3..4af2c48bf 100644 --- a/setup.py +++ b/setup.py @@ -94,7 +94,8 @@ 'PrettyTable>=3.10.0', 'http_sfv>=0.9.9', 'cryptography>=42.0.5', - 'semver>=3.0.2' + 'semver>=3.0.2', + 'qrcode>=7.4.2' ], extras_require={ }, diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index e51ddc7f3..b0b866726 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -39,21 +39,23 @@ def __init__(self, hby, msgs=None, gets=None, cues=None): super(Receiptor, self).__init__(doers=doers) - def receipt(self, pre, sn=None): + def receipt(self, pre, sn=None, auths=None): """ Returns a generator for witness receipting The returns a generator that will submit the designated event to witnesses for receipts using - the synchronous witness API, the propogate the receipts to each of the other witnesses. + the synchronous witness API, then propogate the receipts to each of the other witnesses. Parameters: pre (str): qualified base64 identifier to gather receipts for sn: (Optiona[int]): sequence number of event to gather receipts for, latest is used if not provided + auths: (Options[dict]): map of witness AIDs to (time,auth) tuples for providing TOTP auth for witnessing Returns: list: identifiers of witnesses that returned receipts. """ + auths = auths if auths is not None else dict() if pre not in self.hby.prefixes: raise kering.MissingEntryError(f"{pre} not a valid AID") @@ -86,7 +88,11 @@ def receipt(self, pre, sn=None): rcts = dict() for wit, client in clients.items(): - httping.streamCESRRequests(client=client, dest=wit, ims=bytearray(msg), path="/receipts") + headers = dict() + if wit in auths: + headers["Authorization"] = auths[wit] + + httping.streamCESRRequests(client=client, dest=wit, ims=bytearray(msg), path="receipts", headers=headers) while not client.responses: yield self.tock @@ -101,7 +107,7 @@ def receipt(self, pre, sn=None): coring.Counter(qb64b=rct, strip=True) rcts[wit] = rct else: - logger.error(f"invalid response {rep.status} from witnesses {wit}") + print(f"invalid response {rep.status} from witnesses {wit}") for wit in rcts: ewits = [w for w in rcts if w != wit] @@ -1057,7 +1063,7 @@ def httpClient(hab, wit): url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] up = urlparse(url) - client = http.clienting.Client(scheme=up.scheme, hostname=up.hostname, port=up.port) + client = http.clienting.Client(scheme=up.scheme, hostname=up.hostname, port=up.port, path=up.path) clientDoer = http.clienting.ClientDoer(client=client) return client, clientDoer diff --git a/src/keri/app/cli/commands/decrypt.py b/src/keri/app/cli/commands/decrypt.py new file mode 100644 index 000000000..de460ded8 --- /dev/null +++ b/src/keri/app/cli/commands/decrypt.py @@ -0,0 +1,65 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio.base import doing + +from keri import kering +from keri.app.cli.common import existing +from keri.core import indexing, coring, MtrDex + +parser = argparse.ArgumentParser(description='Decrypt arbitrary data for AIDs with Ed25519 public keys only') +parser.set_defaults(handler=lambda args: handler(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--data', '-d', help='Encrypted data or file (starts with "@")', required=True) + + +def handler(args): + """ + Verify signatures on arbitrary data + + Args: + args(Namespace): arguments object from command line + """ + kwa = dict(args=args) + return [doing.doify(decrypt, **kwa)] + + +def decrypt(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + + name = args.name + alias = args.alias + base = args.base + bran = args.bran + + try: + with existing.existingHab(name=name, alias=alias, base=base, bran=bran) as (_, hab): + + data = args.data + if data.startswith("@"): + f = open(data[1:], "r") + data = f.read() + else: + data = data + + m = coring.Matter(qb64=data) + d = coring.Matter(qb64=hab.decrypt(m.raw)) + print(d.raw) + + except kering.ConfigurationError: + print(f"prefix for {name} does not exist, incept must be run first", ) + except FileNotFoundError: + print("unable to open file", args.text[1:]) diff --git a/src/keri/app/cli/commands/export.py b/src/keri/app/cli/commands/export.py index e6ada36a7..5f6b2e418 100644 --- a/src/keri/app/cli/commands/export.py +++ b/src/keri/app/cli/commands/export.py @@ -15,7 +15,7 @@ logger = help.ogler.getLogger() -parser = argparse.ArgumentParser(description='List credentials and check mailboxes for any newly issued credentials') +parser = argparse.ArgumentParser(description='Export key events in CESR stream format') parser.set_defaults(handler=lambda args: export(args), transferable=True) parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) diff --git a/src/keri/app/cli/commands/interact.py b/src/keri/app/cli/commands/interact.py index 2e7a4bc1d..eedea2d0f 100644 --- a/src/keri/app/cli/commands/interact.py +++ b/src/keri/app/cli/commands/interact.py @@ -5,11 +5,11 @@ """ import argparse import json -from ordered_set import OrderedSet as oset from hio.base import doing from keri import kering +from keri.help import helping from ..common import existing from ... import habbing, agenting, indirecting @@ -22,6 +22,10 @@ parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--data', '-d', help='Anchor data, \'@\' allowed', default=None, action="store", required=False) +parser.add_argument("--receipt-endpoint", help="Attempt to connect to witness receipt endpoint for witness receipts.", + dest="endpoint", action='store_true') +parser.add_argument("--authenticate", '-z', help="Prompt the controller for authentication codes for each witness", + action='store_true') def interact(args): @@ -52,7 +56,8 @@ def interact(args): else: data = None - ixnDoer = InteractDoer(name=name, base=base, alias=alias, bran=bran, data=data) + ixnDoer = InteractDoer(name=name, base=base, alias=alias, bran=bran, data=data, authenticate=args.authenticate, + endpoint=args.endpoint) return [ixnDoer] @@ -63,7 +68,7 @@ class InteractDoer(doing.DoDoer): to all appropriate witnesses """ - def __init__(self, name, base, bran, alias, data: list = None): + def __init__(self, name, base, bran, alias, data: list = None, endpoint=False, authenticate=False): """ Returns DoDoer with all registered Doers needed to perform interaction event. @@ -75,6 +80,8 @@ def __init__(self, name, base, bran, alias, data: list = None): self.alias = alias self.data = data + self.endpoint = endpoint + self.authenticate = authenticate self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer @@ -96,20 +103,36 @@ def interactDo(self, tymth, tock=0.0, **opts): hab = self.hby.habByName(name=self.alias) hab.interact(data=self.data) - witDoer = agenting.WitnessReceiptor(hby=self.hby) - self.extend(doers=[witDoer]) + if self.endpoint or self.authenticate: + receiptor = agenting.Receiptor(hby=self.hby) + self.extend([receiptor]) - if hab.kever.wits: - witDoer.msgs.append(dict(pre=hab.pre)) - while not witDoer.cues: - _ = yield self.tock + auths = {} + if self.authenticate: + for wit in hab.kever.wits: + code = input(f"Entire code for {wit}: ") + auths[wit] = f"{code}#{helping.nowIso8601()}" + yield from receiptor.receipt(hab.pre, sn=hab.kever.sn, auths=auths) + self.remove([receiptor]) + + else: + + witDoer = agenting.WitnessReceiptor(hby=self.hby) + self.extend(doers=[witDoer]) + + if hab.kever.wits: + witDoer.msgs.append(dict(pre=hab.pre)) + while not witDoer.cues: + _ = yield self.tock + + self.remove([witDoer]) print(f'Prefix {hab.pre}') print(f'New Sequence No. {hab.kever.sn}') for idx, verfer in enumerate(hab.kever.verfers): - print(f'\tPublic key {idx+1}: {verfer.qb64}') + print(f'\tPublic key {idx + 1}: {verfer.qb64}') - toRemove = [self.hbyDoer, witDoer, self.mbx] + toRemove = [self.hbyDoer, self.mbx] self.remove(toRemove) return diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index d8316edcb..cb9b34c9e 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -11,6 +11,7 @@ from keri import kering from keri.app.cli.common import rotating, existing, config from keri.core import coring +from keri.help import helping from ... import habbing, agenting, indirecting, delegating, forwarding parser = argparse.ArgumentParser(description='Rotate keys') @@ -26,6 +27,8 @@ default=None, type=int, required=False) parser.add_argument("--receipt-endpoint", help="Attempt to connect to witness receipt endpoint for witness receipts.", dest="endpoint", action='store_true') +parser.add_argument("--authenticate", '-z', help="Prompt the controller for authentication codes for each witness", + action='store_true') parser.add_argument("--proxy", help="alias for delegation communication proxy", default="") rotating.addRotationArgs(parser) @@ -60,7 +63,7 @@ def rotate(args): cuts=opts.witsCut, adds=opts.witsAdd, isith=opts.isith, nsith=opts.nsith, count=opts.ncount, toad=opts.toad, - data=opts.data, proxy=args.proxy) + data=opts.data, proxy=args.proxy, authenticate=args.authenticate) doers = [rotDoer] @@ -115,7 +118,7 @@ class RotateDoer(doing.DoDoer): """ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=None, count=None, - toad=None, wits=None, cuts=None, adds=None, data: list = None, proxy=None): + toad=None, wits=None, cuts=None, adds=None, data: list = None, proxy=None, authenticate=False): """ Returns DoDoer with all registered Doers needed to perform rotation. @@ -140,6 +143,7 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.data = data self.endpoint = endpoint self.proxy = proxy + self.authenticate = authenticate self.wits = wits if wits is not None else [] self.cuts = cuts if cuts is not None else [] @@ -194,8 +198,13 @@ def rotateDo(self, tymth, tock=0.0): yield self.tock elif hab.kever.wits: - if self.endpoint: - yield from receiptor.receipt(hab.pre, sn=hab.kever.sn) + if self.endpoint or self.authenticate: + auths = {} + if self.authenticate: + for wit in hab.kever.wits: + code = input(f"Entire code for {wit}: ") + auths[wit] = f"{code}#{helping.nowIso8601()}" + yield from receiptor.receipt(hab.pre, sn=hab.kever.sn, auths=auths) else: for wit in self.adds: self.mbx.addPoller(hab, witness=wit) diff --git a/src/keri/app/cli/commands/witness/authenticate.py b/src/keri/app/cli/commands/witness/authenticate.py new file mode 100644 index 000000000..bea4c8a6d --- /dev/null +++ b/src/keri/app/cli/commands/witness/authenticate.py @@ -0,0 +1,129 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse +import json +import sys + +import qrcode +from hio import help +from hio.base import doing +from hio.help import Hict + +from keri.app import httping, connecting +from keri.app.agenting import httpClient +from keri.app.cli.common import existing +from keri.app.httping import CESR_CONTENT_TYPE +from keri.core import coring + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Perform authentication against an witness to get a OTP code') +parser.set_defaults(handler=lambda args: auth(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--witness", '-w', help="the witness AID or alias to authenticate against", required=True) + + +def auth(args): + """ Command line list credential registries handler + + """ + + ed = AuthDoer(name=args.name, + alias=args.alias, + base=args.base, + bran=args.bran, + witness=args.witness) + return [ed] + + +class AuthDoer(doing.DoDoer): + + def __init__(self, name, alias, base, bran, witness): + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hab = self.hby.habByName(alias) + self.org = connecting.Organizer(hby=self.hby) + + if witness in self.hby.kevers: + wit = witness + else: + wit = self.org.find("alias", witness) + if len(wit) != 1: + raise ValueError(f"invalid recipient {witness}") + wit = wit[0]['id'] + + if not wit: + raise ValueError(f"unknown witness {witness}") + + self.witness = witness + self.clienter = httping.Clienter() + doers = [doing.doify(self.authDo), self.clienter] + + super(AuthDoer, self).__init__(doers=doers) + + def authDo(self, tymth, tock=0.0): + """ Export credential from store and any related material + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + body = bytearray() + for msg in self.hab.db.clonePreIter(pre=self.hab.pre): + body.extend(msg) + + headers = (Hict([ + ("Content-Type", "application/cesr"), + ("Content-Length", len(body)), + ])) + + client, clientDoer = httpClient(self.hab, self.witness) + self.extend([clientDoer]) + + client.request( + method="POST", + path=f"{client.requester.path}/aids", + headers=headers, + body=bytes(body) + ) + while not client.responses: + yield self.tock + + rep = client.respond() + if rep.status == 200: + data = json.loads(rep.body) + + totp = data["totp"] + m = coring.Matter(qb64=totp) + d = coring.Matter(qb64=self.hab.decrypt(m.raw)) + otpurl = f"otpauth://totp/KERIpy:{self.witness}?secret={d.raw.decode('utf-8')}&issuer=KERIpy" + + qr = qrcode.QRCode() + qr.add_data(otpurl) + + qr.print_ascii() + print(otpurl) + + else: + raise ValueError(rep.body) + + self.remove([clientDoer, self.clienter]) diff --git a/src/keri/app/httping.py b/src/keri/app/httping.py index eb1ce6c48..d3ac5e8ac 100644 --- a/src/keri/app/httping.py +++ b/src/keri/app/httping.py @@ -7,7 +7,7 @@ import json from dataclasses import dataclass from urllib import parse -from urllib.parse import urlparse +from pathlib import Path import falcon from hio.base import doing @@ -152,7 +152,7 @@ def createCESRRequest(msg, client, dest, path=None): ) -def streamCESRRequests(client, ims, dest, path=None): +def streamCESRRequests(client, ims, dest, path=None, headers=None): """ Turns a stream of KERI messages into CESR http requests against the provided hio http Client @@ -167,6 +167,7 @@ def streamCESRRequests(client, ims, dest, path=None): """ path = path if path is not None else "/" + path = str(Path(client.requester.path) / path) cold = kering.sniff(ims) # check for spurious counters at front of stream if cold in (parsing.Colds.txt, parsing.Colds.bny): # not message error out to flush stream @@ -191,17 +192,19 @@ def streamCESRRequests(client, ims, dest, path=None): body = serder.raw - headers = Hict([ + headers = headers if headers is not None else Hict() + heads = (Hict([ ("Content-Type", CESR_CONTENT_TYPE), ("Content-Length", len(body)), (CESR_ATTACHMENT_HEADER, attachment), (CESR_DESTINATION_HEADER, dest) - ]) + ])) + heads.update(headers) client.request( method="POST", path=path, - headers=headers, + headers=heads, body=body ) cnt += 1 diff --git a/src/keri/app/kiwiing.py b/src/keri/app/kiwiing.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 157fdee62..52920056b 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2259,7 +2259,6 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, if (not local and (self.locallyOwned() or self.locallyWitnessed(wits=wits))): - self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) raise MisfitEventSourceError(f"Nonlocal source for locally owned" @@ -2944,7 +2943,8 @@ def escrowMFEvent(self, serder, sigers, wigers=None, if seqner and saider: couple = seqner.qb64b + saider.qb64b self.db.putPde(dgkey, couple) # idempotent - self.db.misfits.add(snKey(serder.preb, serder.sn), serder.saidb) + + res = self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed logger.info("Kever state: escrowed misfit event=\n%s\n", json.dumps(serder.ked, indent=1)) @@ -4659,7 +4659,7 @@ def escrowMFEvent(self, serder, sigers, wigers=None, if seqner and saider: couple = seqner.qb64b + saider.qb64b self.db.putPde(dgkey, couple) # idempotent - self.db.misfits.add(snKey(serder.preb, serder.sn), serder.saidb) + self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed logger.info("Kevery process: escrowed misfit event=\n%s\n", json.dumps(serder.ked, indent=1)) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 395119bfb..f71402305 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -946,7 +946,7 @@ def reopen(self, **kwa): subkey='esrs.') # misfit escrows whose processing may change the .esrs event source record - self.misfits = subing.CesrIoSetSuber(db=self, subkey='mfes.', klas=coring.Diger) + self.misfits = subing.IoSetSuber(db=self, subkey='mfes.') # delegable events escrows. events with local delegator that need approval self.delegables = subing.CesrIoSetSuber(db=self, subkey='dees.', klas=coring.Diger) diff --git a/tests/app/test_httping.py b/tests/app/test_httping.py index 896ba75c2..231a1c065 100644 --- a/tests/app/test_httping.py +++ b/tests/app/test_httping.py @@ -56,10 +56,15 @@ def test_parse_cesr_request(): assert cr.attachments == "-H000000000" +class MockRequester: + path = '/' + + class MockClient: def __init__(self): self.args = [] + self.requester = MockRequester() def request(self, **kwargs): self.args.append(kwargs) From 18cf9180eaffc7a5520173d7556d85d8c50e91bc Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 11 Apr 2024 18:50:05 -0600 Subject: [PATCH 140/418] preliminary basic support for cesr native serialization of keri event messages. With tests for inception --- src/keri/core/coring.py | 2 +- src/keri/core/counting.py | 41 +++++----- src/keri/core/eventing.py | 3 +- src/keri/core/serdering.py | 86 ++++++++++---------- tests/core/test_serdering.py | 151 ++++++++++++++++++++++++++++++++--- 5 files changed, 211 insertions(+), 72 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index f4cda654e..d90d43f43 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -4285,7 +4285,7 @@ class Tholder: when unweighted is size of int thold since don't have anyway to know size of keys list in this case - .limen is qualified b64 signing threshold suitable for CESR serialization. + .limen is qualified b64b signing threshold suitable for CESR serialization. either Number.qb64b or Bexter.qb64b. The b64 portion of limen with code stripped (Bexter.bext) of [["1/2", "1/2", "1/4", "1/4", "1/4"], ["1", "1"]] diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index a79cf6a28..590a5ed16 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -212,25 +212,25 @@ class Counter: .version (Versionage): current CESR code table protocol genus version .codes (CounterCodex_1_0 | CounterCodex_1_0): version specific codex .sizes (dict): version specific sizes table - .code (str) derivation code to indicate cypher suite - .raw is bytes crypto material only without code - .pad is int number of pad chars given raw - .count is int count of quadlets/triplets of following framed material - (not including code) - .qb64 is str in Base64 fully qualified with derivation code + crypto mat - .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat - .qb2 is bytes in binary with derivation code + crypto material + .code (str): hard part of derivation code to indicate cypher suite + .raw (bytes): crypto material only without code + .pad (int): number of pad chars given raw + .count (int): count of quadlets/triplets of following framed material + (not including code) + .qb64 (str | bytes | bytearray): in Base64 fully qualified with + derivation code + crypto mat + .qb64b (bytes | bytearray): in Base64 fully qualified with + derivation code + crypto mat + .qb2 (bytes | bytearray): in binary with derivation code + + crypto material Hidden: ._version (Versionage): value for .version property ._codes (CounterCodex_1_0 | CounterCodex_1_0): version specific codex ._sizes (dict): version specific sizes table - ._code is str value for .code property - ._raw is bytes value for .raw property - ._pad is method to compute .pad property - ._count is int value for .count property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 + ._code (str): value for .code property + ._raw (bytes): value for .raw property + ._count (int): value for .count property Versioning: @@ -423,13 +423,12 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, code (str | None): stable (hard) part of derivation code if tag provided lookup code from tag else if tag is None and code provided use code - count (int | None): count of framed material for composition - Count does not include code. - Count represents quadlets/triplets - When both count and countB64 are None then count defaults to 1 - countB64 (str | None): count of framed material for composition - as Base64 - countB64 represents quadlets/triplets + count (int | None): count of framed material in quadlets/triplets + for composition. Count does not include code. + When both count and countB64 are None then count + defaults to 1 + countB64 (str | None): count of framed material in quadlets/triplets + for composition as Base64 representation of int. qb64b (bytes | bytearray | None): fully qualified crypto material text domain if code nor tag is provided qb64 (str | None) fully qualified crypto material text domain diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 0a4bd9535..8780cefc9 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -643,7 +643,8 @@ def state(pre, ) return ksr # return KeyStateRecord use asdict(ksr) to get dict version - +# should remove intive as its not standard KERI so confusing and leads to errors +# this is an old feature that is now deprecated. def incept(keys, *, diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index babfc1ca6..e4010b7b6 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1199,14 +1199,14 @@ def dumps(self, sad, kind=Serials.json): return raw - def _dumps(self, sad): + def _dumps(self, sad=None): """CESR native serialization of sad Returns: raw (bytes): CESR native serialization of sad dict Parameters: - sad (dict | list)): serializable dict or list to serialize + sad (dict | None)): serializable dict to serialize Versioning: CESR native serialization includes in its fixed version field @@ -1220,6 +1220,8 @@ def _dumps(self, sad): tables are backwards compatible across major versions. """ + sad = sad if sad else self.sad + if (self.gvrsn.major < Vrsn_2_0.major or self.vrsn.major < Vrsn_2_0.major): raise SerializeError(f"Invalid major genus version={self.gvrsn}" @@ -1234,9 +1236,10 @@ def _dumps(self, sad): f"genus={self.genus}.") - fixed = True # True = use fixed field, False= use field map - raw = bytearray() + + raw = bytearray() # message as qb64 + bdy = bytearray() # message body as qb64 ilks = self.Fields[self.proto][self.vrsn] # get fields keyed by ilk ilk = sad.get('t') # returns None if missing message type (ilk) @@ -1249,6 +1252,8 @@ def _dumps(self, sad): if fields.opts or not fields.strict: # optional or extra fields allowed fixed = False # so must use field map + else: + fixed = True #fixed field # assumes that sad's field ordering and field inclusion is correct @@ -1256,11 +1261,10 @@ def _dumps(self, sad): # need to fix ._verify and .makify to account for CESR native serialization if self.proto == Protocols.keri: - for l, v in sad.items(): # assumes valid field order & presence - if not fixed: # prepend label - pass + if not fixed: # prepend label + pass # raise error - # should dispatch or use match instead of big if else + for l, v in sad.items(): # assumes valid field order & presence match l: # label case "v": # protocol+version do not use version string itself val = Verser(proto=self.proto, vrsn=self.vrsn).qb64b @@ -1283,67 +1287,69 @@ def _dumps(self, sad): frame.extend(e.encode("utf-8")) val = bytearray(Counter(tag=AllTags.GenericListGroup, - count=len(frame) % 4).qb64b) + count=len(frame) // 4).qb64b) val.extend(frame) case "c": # list of config traits strings frame = bytearray() for e in v: # list - frame.extend(Traitor(trait=e).qb64n) + frame.extend(Traitor(trait=e).qb64b) val = bytearray(Counter(tag=AllTags.GenericListGroup, - count=len(frame) % 4).qb64b) + count=len(frame) // 4).qb64b) val.extend(frame) case "a": # list of seals or field map of attributes frame = bytearray() # whole list - gctr = None # counter for consecutive same type seals + gcode = None # code for counter for consecutive same type seals gframe = bytearray() # consecutive same type seals for e in v: # list of seal dicts # need support for grouping consecutive seals of same type with same counter try: sealer = Sealer(crew=e) - frame.extend(sealer.qb64b) - # lookup counter type by sealer.clan.name - except kering.InvalidValueError: - pass - #unknown seal type so serialize as field map + code = self.ClanCodes[sealer.name] + if gcode and gcode == code: + gframe.extend(sealer.qb64b) + else: + if gframe: # not same so close off and rotate group + counter = Counter(code=gcode, count=len(gframe) // 4) + frame.extend(counter.qb64b + gframe) + gframe = bytearray() # new group + gcode = code # new group or keep same group + gframe.extend(sealer.qb64b) # extend in new group + except kering.InvalidValueError: + if gframe: + counter = Counter(code=gcode, count=len(gframe) // 4) + frame.extend(counter.qb64b + gframe) + gframe = bytearray() + gcode = None - #if tuple(v) == eventing.SealEvent._fields: - #eseal = eventing.SealEvent(**v) # convert to namedtuple - #SealSourceCouples: str = '-Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. - #SealSourceTriples: str = '-R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. - #DigestSealSingles: str = '-V' # Digest Seal Single(s), dig of sealed data. - #MerkleRootSealSingles: str = '-W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. - #BackerRegistrarSealCouples: str = '-X' # Backer Registrar Seal Couple(s), brid+dig of sealed data. - - # SealMark == tuple of seal dict field names tuple(dict) - #d = dict(a=1, b=2) - #tuple(d) - #('a', 'b') - - #frame.extend(Sealer(seal=e).qb64b) - # else: generic seal no count type (v, Mapping): + #unknown seal type so serialize as field map + #generic seal no count type (v, Mapping): #for l, e in v.items(): #pass #val = bytearray(Counter(tag=AllTags.GenericMapGroup, - # count=len(frame) % 4).qb64b) + # count=len(frame) // 4).qb64b) #val.extend(mapframe) + if gframe: # close off last group if any + counter = Counter(code=gcode, count=len(gframe) // 4) + frame.extend(counter.qb64b + gframe) + gframe = bytearray() + gcode = None + val = bytearray(Counter(tag=AllTags.GenericListGroup, - count=len(frame) % 4).qb64b) + count=len(frame) // 4).qb64b) val.extend(frame) - case _: # if extra fields this is where logic would be raise SerializeError(f"Unsupported protocol field label" f"='{l}' for protocol={self.proto}" f" version={self.vrsn}.") - - raw.extend(val) + bdy.extend(val) elif self.protocol == Protocols.acdc: @@ -1360,9 +1366,9 @@ def _dumps(self, sad): # prepend count code for message if fixed: - val = bytearray(Counter(tag=AllTags.FixedMessageBodyGroup, - count=len(raw) % 4).qb64b) - val.extend(raw) + raw = bytearray(Counter(tag=AllTags.FixedMessageBodyGroup, + count=len(bdy) // 4).qb64b) + raw.extend(bdy) else: pass diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 9134ee0b2..ba1ac0e9e 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -6,6 +6,8 @@ import dataclasses import json from collections import namedtuple +from base64 import urlsafe_b64encode as encodeB64 +from base64 import urlsafe_b64decode as decodeB64 import cbor2 as cbor import msgpack @@ -20,6 +22,8 @@ from keri import core +from keri.core.structing import Sealer, SealEvent, SealTrans + from keri.core.serdering import (FieldDom, FieldDom, Serdery, Serder, SerderKERI, SerderACDC, ) @@ -2675,26 +2679,28 @@ def test_cesr_native_dumps(): # use same salter for all but different path # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = core.Salter(raw=raw) + rawsalt = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' + salter = core.Salter(raw=rawsalt) csigners = salter.signers(count=3, transferable=True, temp=True) wsigners = salter.signers(count=3, transferable=False, temp=True) - keys = [csigners[0].qb64] - keys = ["EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf87Bc70J"] + # simple event + + keys = [csigners[0].verfer.qb64] + assert keys == ['DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ'] serder = incept(keys, version=Vrsn_2_0) assert serder.sad == \ { 'v': 'KERICAAJSONAAD8.', 't': 'icp', - 'd': 'EF_SoHnCdQ0N9Kivxl54u3l1-sKwDL0gs729_REO6koi', - 'i': 'EF_SoHnCdQ0N9Kivxl54u3l1-sKwDL0gs729_REO6koi', + 'd': 'EP9O8aDcloRpvTmk8pnfq3KE2eH_-_wDYWqwOsSgpPws', + 'i': 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', 's': '0', 'kt': '1', - 'k': ['EDGnGYIa5obfFUhxcAuUmM4fJyeRYj2ti3KGf87Bc70J'], + 'k': ['DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ'], 'nt': '0', 'n': [], 'bt': '0', @@ -2703,6 +2709,134 @@ def test_cesr_native_dumps(): 'a': [] } + rawqb64 = serder._dumps() # default is it dumps self.sad + assert rawqb64 == (b'-FAtYKERICAAXicpEP9O8aDcloRpvTmk8pnfq3KE2eH_-_wDYWqwOsSgpPwsDG9X' + b'hvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAAMAAB-LALDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAA-LAAMAAA-LAA-LAA-LAA') + assert len(rawqb64) == 184 + + rawqb2 = decodeB64(rawqb64) + assert len(rawqb2) == 138 + assert rawqb64 == encodeB64(rawqb2) # round trips + + rawjson = serder.dumps(serder.sad) + assert len(rawjson) == 252 + + rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + assert len(rawcbor) == 202 + + rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + assert len(rawmgpk) == 202 + + raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] + ratios = [ round(len(raw) / len(rawqb2), 2) for raw in raws] + + assert ratios == [1.0, 1.33, 1.46, 1.46, 1.83] + + # more complex event + + keys = [signer.verfer.qb64 for signer in csigners] + ndigs = [core.Diger(ser=key.encode()).qb64 for key in keys] + wits = [signer.verfer.qb64 for signer in wsigners] + data = [dict(i=keys[0], s=core.Number(num=0).qb64, d=ndigs[0]), + dict(i=keys[1], s=core.Number(num=1).qb64, d=ndigs[1]), + dict(s=core.Number(num=15).qb64, d=ndigs[2])] + + + serder = incept(keys, + ndigs=ndigs, + wits=wits, + cnfg=['DND'], + data=data, + code=core.MtrDex.Blake3_256, + version=Vrsn_2_0) + + assert serder.sad == \ + { + 'v': 'KERICAAJSONAAOc.', + 't': 'icp', + 'd': 'EO1m0X8audpEosDBc65cpXzSPDCxZ1HnYXQRwM-AivK8', + 'i': 'EO1m0X8audpEosDBc65cpXzSPDCxZ1HnYXQRwM-AivK8', + 's': '0', + 'kt': '2', + 'k': [ + 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', + 'DK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', + 'DMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0' + ], + 'nt': '2', + 'n': [ + 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_', + 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG', + 'EEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZpZEyP' + ], + 'bt': '3', + 'b': [ + 'BG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', + 'BK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', + 'BMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0' + ], + 'c': ['DND'], + 'a': [ + { + 'i': 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', + 's': 'MAAA', + 'd': 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_' + }, + { + 'i': 'DK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', + 's': 'MAAB', + 'd': 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG' + }, + { + 's': 'MAAP', + 'd': 'EEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZpZEyP' + } + ] + } + + rawqb64 = serder._dumps() # default is it dumps self.sad + assert rawqb64 == (b'-FDCYKERICAAXicpEO1m0X8audpEosDBc65cpXzSPDCxZ1HnYXQRwM-AivK8EO1m' + b'0X8audpEosDBc65cpXzSPDCxZ1HnYXQRwM-AivK8MAAAMAAC-LAhDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' + b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-LAh' + b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUGEEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZp' + b'ZEyPMAAD-LAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' + b'o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnBMOmBoddcrRHShSajb4d60S6RK34' + b'gXZ2WYbr3AiPY1M0-LABXDND-LA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'4fBJre3NGwTQMAAAEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' + b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAABEMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' + b'ZJGJqQZpZEyP') + + assert len(rawqb64) == 780 + + rawqb2 = decodeB64(rawqb64) + assert len(rawqb2) == 585 + assert rawqb64 == encodeB64(rawqb2) # round trips + + rawjson = serder.dumps(serder.sad) + assert len(rawjson) == 924 + + rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + assert len(rawcbor) == 838 + + rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + assert len(rawmgpk) == 838 + + raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] + ratios = [ round(len(raw) / len(rawqb2), 2) for raw in raws] + + assert ratios == [1.0, 1.33, 1.43, 1.43, 1.58] + + """End Test""" + +def test_cesr_native_dumps_hby(): + """Test Serder._dumps with habery""" + + rawsalt = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' + salter = core.Salter(raw=rawsalt) salt = salter.qb64 assert salt == '0AAFqo8tU5rp-lWcApybCEh1' @@ -2760,8 +2894,6 @@ def test_cesr_native_dumps(): """End Test""" - - if __name__ == "__main__": test_fielddom() test_spans() @@ -2784,4 +2916,5 @@ def test_cesr_native_dumps(): test_serder_v2() test_serdery() test_cesr_native_dumps() + test_cesr_native_dumps_hby() From 6f298c3de00e2b64153558dbc672d904f4a012be Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 12 Apr 2024 10:08:49 -0600 Subject: [PATCH 141/418] Change Number init so can override dynamic code sizing. Change default on Serder.dumps to use self.sad Now Number can replace Seqner. --- src/keri/core/coring.py | 47 +++++++++++++++++++++++--------------- src/keri/core/serdering.py | 10 ++++---- tests/core/test_coring.py | 22 +++++++++++++++--- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index d90d43f43..4231ee72e 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -1696,11 +1696,13 @@ class Number(Matter): """ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, - code=NumDex.Short, num=None, numh=None, **kwa): + code=None, num=None, numh=None, **kwa): """ Inherited Parameters: (see Matter) raw (bytes): unqualified crypto material usable for crypto operations - code (str): stable (hard) part of derivation code + code (str | None): stable (hard) part of derivation code. + None means pick code based on value of num or numh + otherwise raise error rize (int): raw size in bytes when variable sized material else None qb64b (bytes): fully qualified crypto material Base64 qb64 (str, bytes): fully qualified crypto material Base64 @@ -1733,33 +1735,40 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, except ValueError as ex: raise InvalidValueError(f"Not whole number={num} .") from ex - if not isinstance(num, int) or num < 0: - raise InvalidValueError(f"Not whole number={num}.") + if code is None: # dynamically size code + if not isinstance(num, int) or num < 0: + raise InvalidValueError(f"Not whole number={num}.") - if num <= (256 ** 2 - 1): # make short version of code - code = NumDex.Short + if num <= (256 ** 2 - 1): # make short version of code + code = NumDex.Short - elif num <= (256 ** 5 - 1): # make tall version of code - code = code = NumDex.Tall + elif num <= (256 ** 5 - 1): # make tall version of code + code = code = NumDex.Tall - elif num <= (256 ** 8 - 1): # make big version of code - code = code = NumDex.Big + elif num <= (256 ** 8 - 1): # make big version of code + code = code = NumDex.Big - elif num <= (256 ** 11 - 1): # make large version of code - code = code = NumDex.Large + elif num <= (256 ** 11 - 1): # make large version of code + code = code = NumDex.Large - elif num <= (256 ** 14 - 1): # make great version of code - code = code = NumDex.Great + elif num <= (256 ** 14 - 1): # make great version of code + code = code = NumDex.Great - elif num <= (256 ** 17 - 1): # make vast version of code - code = code = NumDex.Vast + elif num <= (256 ** 17 - 1): # make vast version of code + code = code = NumDex.Vast - else: - raise InvalidValueError(f"Invalid num = {num}, too large to encode.") + else: + raise InvalidValueError(f"Invalid num = {num}, too large to encode.") # default to_bytes parameter signed is False. If negative raises # OverflowError: can't convert negative int to unsigned - raw = num.to_bytes(Matter._rawSize(code), 'big') # big endian unsigned + try: + raw = num.to_bytes(Matter._rawSize(code), 'big') # big endian unsigned + except Exception as ex: + raise InvalidValueError(f"Not convertable to bytes {num=}.") from ex + + if len(raw) > Matter._rawSize(code): + raise InvalidValueError(f"To big {num=} for {code=}.") super(Number, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, code=code, **kwa) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index e4010b7b6..5136cf779 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1164,7 +1164,7 @@ def _exhale(self, sad): self._size = size - def dumps(self, sad, kind=Serials.json): + def dumps(self, sad=None, kind=Serials.json): """Method to handle serialization by kind Assumes sad fields are properly filled out for serialization kind. @@ -1172,7 +1172,7 @@ def dumps(self, sad, kind=Serials.json): raw (bytes): serialization of sad dict using serialization kind Parameters: - sad (dict | list)): serializable dict or list to serialize + sad (dict | list | None)): serializable dict or list to serialize kind (str): value of Serials (Serialage) serialization kind "JSON", "MGPK", "CBOR", "CSER" @@ -1180,6 +1180,8 @@ def dumps(self, sad, kind=Serials.json): dumps of json uses str whereas dumps of cbor and msgpack use bytes crypto opts want bytes not bytearray """ + sad = sad if sad is not None else self.sad + if kind == Serials.json: raw = json.dumps(sad, separators=(",", ":"), ensure_ascii=False).encode("utf-8") @@ -1190,7 +1192,7 @@ def dumps(self, sad, kind=Serials.json): elif kind == Serials.cbor: raw = cbor.dumps(sad) - elif kind == Serials.cser: + elif kind == Serials.cser: # does not support list only dict raw = self._dumps(sad) else: @@ -1220,7 +1222,7 @@ def _dumps(self, sad=None): tables are backwards compatible across major versions. """ - sad = sad if sad else self.sad + sad = sad if sad is not None else self.sad if (self.gvrsn.major < Vrsn_2_0.major or self.vrsn.major < Vrsn_2_0.major): diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index e2c49c800..70937b90e 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -2615,14 +2615,21 @@ def test_number(): 'Vast': 'U' } + with pytest.raises(EmptyMaterialError): + number = Number(raw=b'') # missing code with pytest.raises(RawMaterialError): - number = Number(raw=b'') + number = Number(raw=b'', code=MtrDex.Short) # empty raw + + with pytest.raises(InvalidValueError): + number = Number(num=-1) # negative + # when code provided does not dynamically size code with pytest.raises(InvalidValueError): - number = Number(num=-1) + number = Number(num=256 ** 2, code=MtrDex.Short) # wrong code for num - number = Number() # test None defaults to zero + + number = Number() # test defaults, num is None forces to zero, code dynamic assert number.code == NumDex.Short assert number.raw == b'\x00\x00' assert number.qb64 == 'MAAA' @@ -2664,6 +2671,15 @@ def test_number(): with pytest.raises(InvalidValueError): number = Number(num=" :") + # force bigger code for smaller number like for lexicographic namespace + # which must be fixed length no matter the numeric value such as sequence + # numbers in namespaces for lmdb + number = Number(num=1, code=NumDex.Huge) + assert number.qb64 == '0AAAAAAAAAAAAAAAAAAAAAAB' + assert len(number.raw) == 16 + assert NumDex.Huge == MtrDex.Salt_128 + + num = (256 ** 18 - 1) # too big to represent assert num == 22300745198530623141535718272648361505980415 numh = f"{num:x}" From ed910e91e19c7369d395a9e75a955dc06693778c Mon Sep 17 00:00:00 2001 From: Daniel Comisar Date: Fri, 12 Apr 2024 17:37:07 -0400 Subject: [PATCH 142/418] Fix SyntaxWarning: invalid escape sequence Use a raw byte string for the regex so that \Z is properly interpreted as an anchor representing the end of the input, not an escape sequence for the character \Z (which doesn't exist). --- src/keri/help/helping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index 7e3b8717a..838f69bf4 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -264,7 +264,7 @@ def fromIso8601(dts): B64_CHARS = tuple(B64ChrByIdx.values()) # tuple of characters in Base64 -B64REX = b'^[0-9A-Za-z_-]*\Z' # [A-Za-z0-9\-\_] +B64REX = rb'^[0-9A-Za-z_-]*\Z' # [A-Za-z0-9\-\_] Reb64 = re.compile(B64REX) # compile is faster # to use if Reb64.match(bext): From f909152170f2d425c2958a837ee21075ed82245d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 12 Apr 2024 15:38:30 -0600 Subject: [PATCH 143/418] removed .seqner property from Number. Added .huge property. started arduous process of converting Seqner instances to appropriate Number instances but not finished. Some point need to identify exactly which Seqner need to be lexicogrphic full size NumDex.Huge and only manually do those so all rest can be simply replaced with refactorying tool --- src/keri/app/cli/commands/delegate/confirm.py | 15 ++-- src/keri/app/cli/commands/vc/create.py | 13 +-- src/keri/core/coring.py | 34 +++++--- src/keri/db/basing.py | 12 +-- src/keri/db/subing.py | 2 +- tests/core/test_coring.py | 83 ++++--------------- tests/db/test_basing.py | 6 +- 7 files changed, 66 insertions(+), 99 deletions(-) diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index 816132b52..0161a2eec 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -13,6 +13,7 @@ from keri.app import habbing, indirecting, agenting, grouping, forwarding, delegating, notifying from keri.app.cli.common import existing from keri.app.habbing import GroupHab +from keri import core from keri.core import coring, serdering from keri.db import dbing from keri.peer import exchanging @@ -130,8 +131,8 @@ def confirmDo(self, tymth, tock=0.0): if isinstance(hab, GroupHab): aids = hab.smids - seqner = coring.Seqner(sn=eserder.sn) - anchor = dict(i=eserder.ked["i"], s=seqner.snh, d=eserder.said) + #seqner = coring.Seqner(sn=eserder.sn) + anchor = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) if self.interact: msg = hab.interact(data=[anchor]) else: @@ -148,12 +149,12 @@ def confirmDo(self, tymth, tock=0.0): attachment=atc) prefixer = coring.Prefixer(qb64=hab.pre) - seqner = coring.Seqner(sn=serder.sn) + sner = core.Number(num=serder.sn, code=core.NumDex.Huge) # maybe serder.sner instead so not Huge saider = coring.Saider(qb64b=serder.saidb) - self.counselor.start(ghab=hab, prefixer=prefixer, seqner=seqner, saider=saider) + self.counselor.start(ghab=hab, prefixer=prefixer, seqner=sner, saider=saider) while True: - saider = self.hby.db.cgms.get(keys=(prefixer.qb64, seqner.qb64)) + saider = self.hby.db.cgms.get(keys=(prefixer.qb64, sner.qb64)) if saider is not None: break @@ -166,8 +167,8 @@ def confirmDo(self, tymth, tock=0.0): else: cur = hab.kever.sner.num - seqner = coring.Seqner(sn=eserder.sn) - anchor = dict(i=eserder.ked["i"], s=seqner.snh, d=eserder.said) + #seqner = coring.Seqner(sn=eserder.sn) + anchor = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) if self.interact: hab.interact(data=[anchor]) else: diff --git a/src/keri/app/cli/commands/vc/create.py b/src/keri/app/cli/commands/vc/create.py index b57ea18a6..c8a36e049 100644 --- a/src/keri/app/cli/commands/vc/create.py +++ b/src/keri/app/cli/commands/vc/create.py @@ -5,9 +5,11 @@ from hio.base import doing from keri import kering +from keri import core +from keri.core import coring, eventing, serdering + from keri.app import indirecting, habbing, grouping, connecting, forwarding, signing, notifying from keri.app.cli.common import existing -from keri.core import coring, eventing, serdering from keri.help import helping from keri.peer import exchanging from keri.vdr import credentialing, verifying @@ -207,9 +209,9 @@ def createDo(self, tymth, tock=0.0): dt = self.creder.attrib["dt"] if "dt" in self.creder.attrib else helping.nowIso8601() iserder = registry.issue(said=self.creder.said, dt=dt) - vcid = iserder.ked["i"] - rseq = coring.Seqner(snh=iserder.ked["s"]) - rseal = eventing.SealEvent(vcid, rseq.snh, iserder.said) + #vcid = iserder.ked["i"] + #rseq = coring.Seqner(snh=iserder.ked["s"]) + rseal = eventing.SealEvent(iserder.pre, iserder.snh, iserder.said) rseal = dict(i=rseal.i, s=rseal.s, d=rseal.d) if registry.estOnly: @@ -222,7 +224,8 @@ def createDo(self, tymth, tock=0.0): self.credentialer.issue(self.creder, iserder) self.registrar.issue(self.creder, iserder, aserder) - acdc = signing.serialize(self.creder, coring.Prefixer(qb64=iserder.pre), coring.Seqner(sn=iserder.sn), + acdc = signing.serialize(self.creder, coring.Prefixer(qb64=iserder.pre), + core.Number(num=iserder.sn, code=core.NumDex.Huge), coring.Saider(qb64=iserder.said)) if isinstance(self.hab, habbing.GroupHab): diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 4231ee72e..63d8a43a9 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -1676,8 +1676,14 @@ class Number(Matter): Properties: num (int): int representation of number humh (str): hex string representation of number with no leading zeros + sn (int): alias for num + snh (str): alias for numh + huge (str): qb64 of num but with code NumDex.Huge so 24 char compatible + with fixed size seq num for lexicographic lmdb key space positive (bool): True if .num > 0, False otherwise. Because .num must be - non-negative, .positive == False means .num == 0 + non-negative, .positive == False means .num == 0 + inceptive (bool): True means .num == 0 False otherwise. + Hidden: _code (str): value for .code property @@ -1850,6 +1856,21 @@ def snh(self): return self.numh + @property + def huge(self): + """Provides number value as qb64 but with code NumDex.huge. This is the + same as Seqner.qb64. Raises error if too big. + + Returns: + huge (str): qb64 of num coded as NumDex.Huge + """ + num = self.num + if num > MaxON: # too big for ordinal 256 ** 16 - 1 + raise InvalidValueError(f"Non-ordinal {num} exceeds {MaxON}.") + + return Number(num=num, code=NumDex.Huge).qb64 + + @property def positive(self): """ @@ -1864,20 +1885,11 @@ def positive(self): def inceptive(self): """ Returns True if .num == 0 False otherwise. - Valid number .num must be non-negative, + """ return True if self.num == 0 else False - @property - def seqner(self): - """Seqner getter. - - Returns: - seqner (Seqner): instance made from number - """ - return Seqner(sn=self.sn) - class Dater(Matter): """ diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index f71402305..8a28698cf 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -38,7 +38,7 @@ import keri from . import dbing, koming, subing from .. import kering - +from .. import core from ..core import coring, eventing, parsing, serdering, indexing from .. import help @@ -952,7 +952,7 @@ def reopen(self, **kwa): self.delegables = subing.CesrIoSetSuber(db=self, subkey='dees.', klas=coring.Diger) # events as ordered by first seen ordinals - self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=coring.Seqner) + self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=core.Number) # Kever state made of KeyStateRecord key states # TODO: clean self.states = koming.Komer(db=self, @@ -1024,15 +1024,15 @@ def reopen(self, **kwa): # group partial signature escrow self.gpse = subing.CatCesrIoSetSuber(db=self, subkey='gpse.', - klas=(coring.Seqner, coring.Saider)) + klas=(core.Number, coring.Saider)) # group delegate escrow self.gdee = subing.CatCesrIoSetSuber(db=self, subkey='gdee.', - klas=(coring.Seqner, coring.Saider)) + klas=(core.Number, coring.Saider)) # group partial witness escrow self.gpwe = subing.CatCesrIoSetSuber(db=self, subkey='gdwe.', - klas=(coring.Seqner, coring.Saider)) + klas=(core.Number, coring.Saider)) # completed group multisig # TODO: clean @@ -1522,7 +1522,7 @@ def cloneEvtMsg(self, pre, fn, dig): raise kering.MissingEntryError("Missing datetime for dig={}.".format(dig)) atc.extend(coring.Counter(code=coring.CtrDex.FirstSeenReplayCouples, count=1).qb64b) - atc.extend(coring.Seqner(sn=fn).qb64b) + atc.extend(core.Number(num=fn, code=core.NumDex.Huge).qb64b) # may not need to be Huge atc.extend(coring.Dater(dts=bytes(dts)).qb64b) # prepend pipelining counter to attachments diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 4fbfc8ff2..f007a28aa 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -317,7 +317,7 @@ def _des(self, val: Union[str, memoryview, bytes]): """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes - return self.klas(qb64b=val) # converts to bytes + return self.klas(qb64b=val) class CesrSuber(CesrSuberBase, Suber): diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 70937b90e..28548a043 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -2639,6 +2639,8 @@ def test_number(): assert number.numh == '0' assert number.sn == 0 assert number.snh == '0' + assert number.huge == '0AAAAAAAAAAAAAAAAAAAAAAA' + assert len(number.huge) == 24 assert not number.positive assert number.inceptive assert hex(int.from_bytes(number.qb2, 'big')) == '0x300000' @@ -2714,9 +2716,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn - number = Number(num=numh) # num can be hext str too assert number.code == code @@ -2729,8 +2728,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(numh=numh) @@ -2744,8 +2741,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(qb64=nqb64) assert number.code == code @@ -2758,8 +2753,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(qb2=nqb2) assert number.code == code @@ -2772,8 +2765,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(raw=raw, code=code) assert number.code == code @@ -2786,8 +2777,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn num = (256 ** 5 - 1) assert num == 1099511627775 @@ -2809,8 +2798,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(numh=numh) assert number.code == code @@ -2823,8 +2810,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(qb64=nqb64) assert number.code == code @@ -2837,8 +2822,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(qb2=nqb2) assert number.code == code @@ -2849,8 +2832,6 @@ def test_number(): assert number.num == num assert number.numh == numh assert number.positive - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(raw=raw, code=code) assert number.code == code @@ -2863,8 +2844,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn num = (256 ** 8 - 1) assert num == 18446744073709551615 @@ -2886,8 +2865,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(numh=numh) assert number.code == code @@ -2900,8 +2877,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(qb64=nqb64) assert number.code == code @@ -2914,8 +2889,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(qb2=nqb2) assert number.code == code @@ -2928,8 +2901,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(raw=raw, code=code) assert number.code == code @@ -2942,8 +2913,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn num = (256 ** 11 - 1) assert num == 309485009821345068724781055 @@ -2965,8 +2934,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(numh=numh) assert number.code == code @@ -2979,8 +2946,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(qb64=nqb64) assert number.code == code @@ -2993,8 +2958,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(qb2=nqb2) assert number.code == code @@ -3007,8 +2970,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(raw=raw, code=code) assert number.code == code @@ -3021,8 +2982,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn num = (256 ** 14 - 1) assert num == 5192296858534827628530496329220095 @@ -3045,8 +3004,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(numh=numh) assert number.code == code @@ -3059,8 +3016,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(qb64=nqb64) assert number.code == code @@ -3073,8 +3028,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn number = Number(qb2=nqb2) assert number.code == code @@ -3087,9 +3040,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn - number = Number(raw=raw, code=code) assert number.code == code @@ -3102,8 +3052,6 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - assert isinstance(number.seqner, Seqner) - assert number.seqner.sn == number.sn num = (256 ** 17 - 1) assert num == 87112285931760246646623899502532662132735 @@ -3126,9 +3074,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - with pytest.raises(ValidationError): # too big to be ordinal - number.seqner - + with pytest.raises(InvalidValueError): + number.huge # too big for huge number = Number(numh=numh) assert number.code == code @@ -3141,8 +3088,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - with pytest.raises(ValidationError): # too big to be ordinal - number.seqner + with pytest.raises(InvalidValueError): + number.huge # too big for huge number = Number(qb64=nqb64) assert number.code == code @@ -3155,8 +3102,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - with pytest.raises(ValidationError): # too big to be ordinal - number.seqner + with pytest.raises(InvalidValueError): + number.huge # too big for huge number = Number(qb2=nqb2) assert number.code == code @@ -3169,8 +3116,9 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - with pytest.raises(ValidationError): # too big to be ordinal - number.seqner + with pytest.raises(InvalidValueError): + number.huge # too big for huge + number = Number(raw=raw, code=code) @@ -3184,9 +3132,8 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - with pytest.raises(ValidationError): # too big to be ordinal - number.seqner - + with pytest.raises(InvalidValueError): + number.huge # too big for huge # tests with wrong size raw for code short num = (256 ** 2 - 1) @@ -3368,7 +3315,7 @@ def test_number(): number = Number(raw=raw2bad, code=code) - # tests with wrong size raw for code huge + # tests with wrong size raw for code Vast num = (256 ** 17 - 1) assert num == 87112285931760246646623899502532662132735 numh = f"{num:x}" @@ -6085,7 +6032,7 @@ def test_tholder(): #test_counter() #test_prodex() #test_indexer() - #test_number() + test_number() #test_seqner() #test_siger() #test_nexter() diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index b44ba1ad9..43d575007 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -28,7 +28,11 @@ from keri.db.dbing import (dgKey, onKey, snKey) from keri.db.dbing import openLMDB -from tests.app import openMultiSig +# this breaks when running as __main__ better to do a custom import call to +# walk the directory tree and import explicity rather than depend on it +# being a known package. Works with pytest because pytest contructs a path +# its test runner and imports the tests explicity +from tests.app import openMultiSig # this breaks when running as __main__ def test_baser(): From 8160ded52dcd38e1586aed890f0e4fdc7c74803b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 12 Apr 2024 15:52:51 -0600 Subject: [PATCH 144/418] refactor names of ClanDom and CastDoms Add Castage becasue need to specify keyword paramter when casting --- src/keri/core/serdering.py | 14 +++++++------- src/keri/core/structing.py | 28 ++++++++++++++++++---------- tests/core/test_structing.py | 16 ++++++++-------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 5136cf779..db8c0cd6c 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -40,7 +40,7 @@ from .counting import GenDex, AllTags, Counter, SealDex_2_0 -from .structing import Sealer, ClanDom +from .structing import Sealer, SClanDom @@ -365,12 +365,12 @@ class Serder: # map seal clan names to seal counter code for grouping seals in anchor list ClanCodes = dict() - ClanCodes[ClanDom.SealDigest.__name__] = SealDex_2_0.DigestSealSingles - ClanCodes[ClanDom.SealRoot.__name__] = SealDex_2_0.MerkleRootSealSingles - ClanCodes[ClanDom.SealBacker.__name__] = SealDex_2_0.BackerRegistrarSealCouples - ClanCodes[ClanDom.SealLast.__name__] = SealDex_2_0.SealSourceLastSingles - ClanCodes[ClanDom.SealTrans.__name__] = SealDex_2_0.SealSourceCouples - ClanCodes[ClanDom.SealEvent.__name__] = SealDex_2_0.SealSourceTriples + ClanCodes[SClanDom.SealDigest.__name__] = SealDex_2_0.DigestSealSingles + ClanCodes[SClanDom.SealRoot.__name__] = SealDex_2_0.MerkleRootSealSingles + ClanCodes[SClanDom.SealBacker.__name__] = SealDex_2_0.BackerRegistrarSealCouples + ClanCodes[SClanDom.SealLast.__name__] = SealDex_2_0.SealSourceLastSingles + ClanCodes[SClanDom.SealTrans.__name__] = SealDex_2_0.SealSourceCouples + ClanCodes[SClanDom.SealEvent.__name__] = SealDex_2_0.SealSourceTriples # map seal counter code to seal clan name for parsing seal groups in anchor list CodeClans = { val: key for key, val in ClanCodes.items()} # invert dict diff --git a/src/keri/core/structing.py b/src/keri/core/structing.py index cb7e1fea8..ae065ee3a 100644 --- a/src/keri/core/structing.py +++ b/src/keri/core/structing.py @@ -85,8 +85,14 @@ StateEvent = namedtuple("StateEvent", 's t d') +# Cast conversion: duple (c, k) +# s = cast as appropriate namedtuple with values as primitive classes +# k = keyword parameter name to use when casting +Castage = namedtuple('Castage', "c k") + + @dataclass(frozen=True) -class EmptyClanCodex(MapDom): +class EmptyClanDom(MapDom): """ SealClanDom is dataclass of namedtuple seal class references (clans) each indexed by its class name. @@ -103,11 +109,11 @@ class EmptyClanCodex(MapDom): def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" -EmptyClanDex = EmptyClanCodex() # create instance +EClanDom = EmptyClanDom() # create instance @dataclass(frozen=True) -class EmptyCastCodex(MapDom): +class EmptyCastDom(MapDom): """ SealCastCodex is dataclass of namedtuple instances (seal casts) whose values are named primitive class references @@ -126,7 +132,7 @@ class EmptyCastCodex(MapDom): def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" -EmptyCastDex = EmptyCastCodex() # create instance +ECastDom = EmptyCastDom() # create instance @dataclass(frozen=True) @@ -153,7 +159,9 @@ class SealClanDom(MapDom): def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" -ClanDom = SealClanDom() # create instance +SClanDom = SealClanDom() # create instance + + @dataclass(frozen=True) @@ -182,7 +190,7 @@ class SealCastDom(MapDom): def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" -CastDom = SealCastDom() # create instance +SCastDom = SealCastDom() # create instance @@ -252,8 +260,8 @@ class Structor: """ - Clans = EmptyClanDex # known namedtuple clans. Override in subclass with non-empty - Casts = EmptyCastDex # known namedtuple casts. Override in subclass with non-empty + Clans = EClanDom # known namedtuple clans. Override in subclass with non-empty + Casts = ECastDom # known namedtuple casts. Override in subclass with non-empty # Create .Names dict that maps tuple of clan/cast fields names to its namedtuple # class type name so can look up a know clan or cast given a matching tuple # of either field names from a namedtuple or keys from a dict. The tuple of @@ -563,8 +571,8 @@ class Sealer(Structor): """ - Clans = ClanDom # known namedtuple clans. Override in subclass with non-empty - Casts = CastDom # known namedtuple casts. Override in subclass with non-empty + Clans = SClanDom # known namedtuple clans. Override in subclass with non-empty + Casts = SCastDom # known namedtuple casts. Override in subclass with non-empty # Create .Names dict that maps clan/cast fields names to its namedtuple # class type name so can look up a know clan or cast given a matching set # of either field names from a namedtuple or keys from a dict. diff --git a/tests/core/test_structing.py b/tests/core/test_structing.py index b34cd6d0d..94f0905d6 100644 --- a/tests/core/test_structing.py +++ b/tests/core/test_structing.py @@ -22,16 +22,16 @@ from keri.core import structing from keri.core.structing import (SealDigest, SealRoot, SealBacker, SealEvent, SealLast, SealTrans) -from keri.core.structing import (Structor, EmptyClanDex, EmptyCastDex, - Sealer, ClanDom, CastDom, ) +from keri.core.structing import (Structor, EClanDom, ECastDom, + Sealer, SClanDom, SCastDom, ) def test_structor_class(): """ test Structor class variables etc """ - assert Structor.Clans == EmptyClanDex - assert Structor.Casts == EmptyCastDex + assert Structor.Clans == EClanDom + assert Structor.Casts == ECastDom assert Structor.Names == {} """End Test""" @@ -447,7 +447,7 @@ def test_seal_dexes(): test Seal Codexes """ - assert asdict(ClanDom) == \ + assert asdict(SClanDom) == \ { 'SealDigest': SealDigest, 'SealRoot': SealRoot, @@ -457,7 +457,7 @@ def test_seal_dexes(): 'SealEvent': SealEvent, } - assert asdict(CastDom) == \ + assert asdict(SCastDom) == \ { 'SealDigest': SealDigest(d=Diger), 'SealRoot': SealRoot(rd=Diger), @@ -471,8 +471,8 @@ def test_sealer_class(): """ test sealer class variables etc """ - assert Sealer.Clans == ClanDom - assert Sealer.Casts == CastDom + assert Sealer.Clans == SClanDom + assert Sealer.Casts == SCastDom assert Sealer.Names == \ { ('d',): 'SealDigest', From 1583f56c21380c647744d3fe188f3dd028f420b7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 13 Apr 2024 13:23:47 -0600 Subject: [PATCH 145/418] Update Structor Sealer to support Casts Seals where the value can be something other than qb64 to support legacy seals formats where numbers are hex str not qb64. This means supporting an option parameter name to apply when casting. --- src/keri/core/structing.py | 80 ++++++++++------ tests/core/test_serdering.py | 108 ++++++++++++--------- tests/core/test_structing.py | 180 ++++++++++++++++++++++++++++------- 3 files changed, 263 insertions(+), 105 deletions(-) diff --git a/src/keri/core/structing.py b/src/keri/core/structing.py index ae065ee3a..b40b30f65 100644 --- a/src/keri/core/structing.py +++ b/src/keri/core/structing.py @@ -86,9 +86,11 @@ # Cast conversion: duple (c, k) -# s = cast as appropriate namedtuple with values as primitive classes -# k = keyword parameter name to use when casting -Castage = namedtuple('Castage', "c k") +# kls = primitive class reference in order to cast as appropriate +# namedtuple with values as primitive classes +# prm = primitive __init__ keyword parameter name to use when casting +# default None. When default then use qb64 or qb64b as appropriate. +Castage = namedtuple('Castage', "kls prm", defaults=(None, )) @dataclass(frozen=True) @@ -180,12 +182,16 @@ class SealCastDom(MapDom): Example: CastDom[name] """ - SealDigest: NamedTuple = SealDigest(d=Diger) # SealDigest class reference - SealRoot: NamedTuple = SealRoot(rd=Diger) # SealRoot class reference - SealBacker: NamedTuple = SealBacker(bi=Prefixer, d=Diger) # SealBacker class reference - SealLast: NamedTuple = SealLast(i=Prefixer) # SealLast class reference single - SealTrans: NamedTuple = SealTrans(s=Number, d=Diger) # SealTrans class reference couple - SealEvent: NamedTuple = SealEvent(i=Prefixer, s=Number, d=Diger) # SealEvent class reference triple + SealDigest: NamedTuple = SealDigest(d=Castage(Diger)) # SealDigest class reference + SealRoot: NamedTuple = SealRoot(rd=Castage(Diger)) # SealRoot class reference + SealBacker: NamedTuple = SealBacker(bi=Castage(Prefixer), + d=Castage(Diger)) # SealBacker class reference + SealLast: NamedTuple = SealLast(i=Castage(Prefixer)) # SealLast class reference single + SealTrans: NamedTuple = SealTrans(s=Castage(Number, 'numh'), + d=Castage(Diger)) # SealTrans class reference couple + SealEvent: NamedTuple = SealEvent(i=Castage(Prefixer), + s=Castage(Number, 'numh'), + d=Castage(Diger)) # SealEvent class reference triple def __iter__(self): return iter(astuple(self)) # enables value not key inclusion test with "in" @@ -244,8 +250,9 @@ class Structor: Properties: data (NamedTuple): fields are named instances of CESR primitives clan (type[NamedTuple]): class reference of .data's class - cast (NamedTuple): CESR primitive class references of .data's primitive - instances + cast (NamedTuple | None): values are Castage instances that each provide + CESR primitive class references and primitive init parameter + used to .data's primitive instances. crew (NamedTuple): named qb64 values of .data's primitive instances qb64 (str): concatenated data values as qb64 str of data's primitives qb64b (bytes): concatenated data values as qb64b of data's primitives @@ -274,21 +281,26 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, """Initialize instance Parameters: - data (NamedTuple): fields are named primitive instances for .data + data (NamedTuple | None): fields are named primitive instances for .data Given data can derive clan, cast, crew, qb64, and qb2 clan (type[NamedTuple]): provides class reference for generated .data when data missing. - cast (NamedTuple | dict | Iterable): each value provides CESR + cast (NamedTuple | dict | Iterable | None): values are Castage + instances that each provide CESR primitive class references + and primitive init parameter used to .data's primitive + instances. None means .data provided directly not generated + from cast + each value provides CESR primitive subclass reference used to create primitive instances for generating .data. Can be used to infer namedtuple type of .data when data and clan missing. Takes precendence over crew. - crew (NamedTuple | dict | Iterable): each value provides qb64 value + crew (NamedTuple | dict | Iterable | None): each value provides qb64 value of primitive for generating .data with .cast when data missing. Can be used to infer namedtuple type of .data when data and clan missing. - qb64 (str | bytes | bytearray): concatenation of qb64 data values to + qb64 (str | bytes | bytearray | None): concatenation of qb64 data values to generate .data with data and crew missing. - qb2 (bytes | bytearray): concatenation of qb2 data values to generate + qb2 (bytes | bytearray | None): concatenation of qb2 data values to generate .data when data and crew and qb64 missing. strip (bool): False means do not strip each value from qb64 or qb2. Default is False. @@ -306,6 +318,8 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, if not (hasattr(pi, "qb64") and hasattr(pi, "qb2")): raise InvalidValueError(f"Non-primitive data member={pi}.") + cast = None # ensure cast is None since not used to generate data + else: if not clan: # attempt to get from cast and/or crew @@ -372,9 +386,9 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, raise InvalidValueError(f"Missing or unobtainable cast.") # have cast now - for klas in cast: - if not (hasattr(klas, "qb64") and hasattr(klas, "qb2")): - raise InvalidValueError(f"Cast member {klas=} not CESR" + for cstg in cast: + if not (hasattr(cstg.kls, "qb64") and hasattr(cstg.kls, "qb2")): + raise InvalidValueError(f"Cast member {cstg.kls=} not CESR" " Primitive.") # have clan and cast but may not have crew @@ -402,7 +416,9 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, else: raise InvalidValueError(f"Invalid {crew=}.") - data = clan(*(klas(qb64=val) for klas, val in zip(cast, crew))) + data = clan(*(cstg.kls(**{cstg.prm if cstg.prm is not None else 'qb64': val}) + for cstg, val in zip(cast, crew))) + # data = clan(*(klas(qb64=val) for klas, val in zip(cast, crew))) elif qb64: if hasattr(qb64, "encode"): @@ -412,14 +428,14 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, if not isinstance(qb64, bytearray): qb64 = bytearray(qb64) - data = clan(*(klas(qb64b=qb64, strip=strip) for klas in cast)) + data = clan(*(cstg.kls(qb64b=qb64, strip=strip) for cstg in cast)) else: o = 0 # offset into memoryview of qb64 pis = [] # primitive instances mv = memoryview(qb64) - for klas in cast: - pi = klas(qb64b=mv[o:]) + for cstg in cast: # Castage + pi = cstg.kls(qb64b=mv[o:]) pis.append(pi) o += len(pi.qb64b) data = clan(*pis) @@ -429,14 +445,14 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, if not isinstance(qb2, bytearray): qb2 = bytearray(qb2) - data = clan(*(klas(qb2=qb2, strip=strip) for klas in cast)) + data = clan(*(cstg.kls(qb2=qb2, strip=strip) for cstg in cast)) else: o = 0 # offset into memoryview of qb2 pis = [] # primitive instances mv = memoryview(qb2) - for klas in cast: - pi = klas(qb2=mv[o:]) + for cstg in cast: # Castage + pi = cstg.kls(qb2=mv[o:]) pis.append(pi) o += len(pi.qb2) data = clan(*pis) @@ -444,7 +460,12 @@ def __init__(self, data=None, *, clan=None, cast=None, crew=None, else: raise EmptyMaterialError("Need crew or qb64 or qb2.") + self._data = data + self._cast = (cast if cast is not None else + self.clan(*(Castage(val.__class__) for val in self.data))) + + @property @@ -478,8 +499,9 @@ def cast(self): """Return: cast (NamedTuple): named primitive classes in .data + Getter for ._cast makes it read only when not None """ - return self.clan(*(val.__class__ for val in self.data)) + return self._cast @property def crew(self): @@ -487,7 +509,9 @@ def crew(self): crew (NamedTuple): named qb64 field values from .data """ - return self.clan(*(val.qb64 for val in self.data)) + return (self.clan(*(getattr(val, cstg.prm if cstg.prm is not None else "qb64") + for cstg, val in zip(self.cast, self.data)))) + # return self.clan(*(val.qb64 for val in self.data)) @property diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index ba1ac0e9e..f89aa6dce 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -2738,9 +2738,27 @@ def test_cesr_native_dumps(): keys = [signer.verfer.qb64 for signer in csigners] ndigs = [core.Diger(ser=key.encode()).qb64 for key in keys] wits = [signer.verfer.qb64 for signer in wsigners] - data = [dict(i=keys[0], s=core.Number(num=0).qb64, d=ndigs[0]), - dict(i=keys[1], s=core.Number(num=1).qb64, d=ndigs[1]), - dict(s=core.Number(num=15).qb64, d=ndigs[2])] + data = [dict(i=keys[0], s=core.Number(num=0).numh, d=ndigs[0]), + dict(i=keys[1], s=core.Number(num=1).numh, d=ndigs[1]), + dict(s=core.Number(num=15).numh, d=ndigs[2])] + + assert data == \ + [ + { + 'i': 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', + 's': '0', + 'd': 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_' + }, + { + 'i': 'DK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', + 's': '1', + 'd': 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG' + }, + { + 's': 'f', + 'd': 'EEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZpZEyP' + } + ] serder = incept(keys, @@ -2753,51 +2771,54 @@ def test_cesr_native_dumps(): assert serder.sad == \ { - 'v': 'KERICAAJSONAAOc.', + 'v': 'KERICAAJSONAAOT.', 't': 'icp', - 'd': 'EO1m0X8audpEosDBc65cpXzSPDCxZ1HnYXQRwM-AivK8', - 'i': 'EO1m0X8audpEosDBc65cpXzSPDCxZ1HnYXQRwM-AivK8', + 'd': 'ECdA32v8SkQ2ZFpliiK0RBcfNERaVtxk4pmaulKsakXY', + 'i': 'ECdA32v8SkQ2ZFpliiK0RBcfNERaVtxk4pmaulKsakXY', 's': '0', 'kt': '2', - 'k': [ - 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', - 'DK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', - 'DMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0' - ], + 'k': + [ + 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', + 'DK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', + 'DMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0' + ], 'nt': '2', - 'n': [ - 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_', - 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG', - 'EEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZpZEyP' - ], + 'n': + [ + 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_', + 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG', + 'EEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZpZEyP' + ], 'bt': '3', - 'b': [ - 'BG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', - 'BK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', - 'BMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0' - ], + 'b': + [ + 'BG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', + 'BK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', + 'BMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0' + ], 'c': ['DND'], - 'a': [ - { - 'i': 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', - 's': 'MAAA', - 'd': 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_' - }, - { - 'i': 'DK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', - 's': 'MAAB', - 'd': 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG' - }, - { - 's': 'MAAP', - 'd': 'EEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZpZEyP' - } - ] + 'a': + [ + { + 'i': 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', + 's': '0', + 'd': 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_' + }, + { + 'i': 'DK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', + 's': '1', + 'd': 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG'}, + { + 's': 'f', + 'd': 'EEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZpZEyP' + } + ] } rawqb64 = serder._dumps() # default is it dumps self.sad - assert rawqb64 == (b'-FDCYKERICAAXicpEO1m0X8audpEosDBc65cpXzSPDCxZ1HnYXQRwM-AivK8EO1m' - b'0X8audpEosDBc65cpXzSPDCxZ1HnYXQRwM-AivK8MAAAMAAC-LAhDG9XhvcVryHj' + assert rawqb64 == (b'-FDCYKERICAAXicpECdA32v8SkQ2ZFpliiK0RBcfNERaVtxk4pmaulKsakXYECdA' + b'32v8SkQ2ZFpliiK0RBcfNERaVtxk4pmaulKsakXYMAAAMAAC-LAhDG9XhvcVryHj' b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-LAh' b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' @@ -2810,6 +2831,7 @@ def test_cesr_native_dumps(): b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' b'ZJGJqQZpZEyP') + assert len(rawqb64) == 780 rawqb2 = decodeB64(rawqb64) @@ -2817,18 +2839,18 @@ def test_cesr_native_dumps(): assert rawqb64 == encodeB64(rawqb2) # round trips rawjson = serder.dumps(serder.sad) - assert len(rawjson) == 924 + assert len(rawjson) == 915 rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) - assert len(rawcbor) == 838 + assert len(rawcbor) == 829 rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) - assert len(rawmgpk) == 838 + assert len(rawmgpk) == 829 raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] ratios = [ round(len(raw) / len(rawqb2), 2) for raw in raws] - assert ratios == [1.0, 1.33, 1.43, 1.43, 1.58] + assert ratios == [1.0, 1.33, 1.42, 1.42, 1.56] """End Test""" diff --git a/tests/core/test_structing.py b/tests/core/test_structing.py index 94f0905d6..47f2066dc 100644 --- a/tests/core/test_structing.py +++ b/tests/core/test_structing.py @@ -22,10 +22,52 @@ from keri.core import structing from keri.core.structing import (SealDigest, SealRoot, SealBacker, SealEvent, SealLast, SealTrans) -from keri.core.structing import (Structor, EClanDom, ECastDom, +from keri.core.structing import (Castage, + Structor, EClanDom, ECastDom, Sealer, SClanDom, SCastDom, ) +def test_structor_doms(): + """ + test doms in structure + """ + assert EClanDom == structing.EmptyClanDom() + assert ECastDom == structing.EmptyCastDom() + assert SClanDom == structing.SealClanDom() + assert SCastDom == structing.SealCastDom() + + assert asdict(EClanDom) == {} + assert asdict(ECastDom) == {} + + assert asdict(SClanDom) == \ + { + 'SealDigest': SealDigest, + 'SealRoot': SealRoot, + 'SealBacker': SealBacker, + 'SealLast': SealLast, + 'SealTrans': SealTrans, + 'SealEvent': SealEvent, + } + + assert asdict(SCastDom) == \ + { + 'SealDigest': SealDigest(d=Castage(kls=Diger, prm=None)), + 'SealRoot': SealRoot(rd=Castage(kls=Diger, prm=None)), + 'SealBacker': SealBacker(bi=Castage(Prefixer, prm=None), + d=Castage(kls=Diger, prm=None)), + 'SealLast': SealLast(i=Castage(kls=Prefixer, prm=None)), + 'SealTrans': SealTrans(s=Castage(kls=Number, prm='numh'), + d=Castage(kls=Diger, prm=None)), + 'SealEvent': SealEvent(i=Castage(kls=Prefixer, prm=None), + s=Castage(kls=Number, prm='numh'), + d=Castage(kls=Diger, prm=None)) + } + + + """End Test""" + + + def test_structor_class(): """ test Structor class variables etc @@ -50,6 +92,7 @@ def test_structor(): num = 14 number = Number(num=num) snq = number.qb64 + snh = number.numh dig = 'ELC5L3iBVD77d_MYbYGGCUQgqQBju1o4x1Ud-z2sL-ux' diger = Diger(qb64=dig) @@ -57,7 +100,7 @@ def test_structor(): data = SealDigest(d=diger) clan = SealDigest - cast = SealDigest(d=Diger) + cast = SealDigest(d=Castage(Diger)) crew = SealDigest(d=dig) name = SealDigest.__name__ @@ -75,9 +118,9 @@ def test_structor(): structor = Structor(data=data) assert structor.data == data assert structor.clan == clan + assert structor.name == name assert structor.cast == cast assert structor.crew == crew - assert structor.name == name assert structor.asdict == data._asdict() assert structor.asdict == {'d': diger} assert structor.qb64 == qb64 @@ -86,19 +129,20 @@ def test_structor(): # Test cast structor = Structor(cast=cast, crew=crew) + #assert structor.data == data different instances so not == assert structor.clan == clan + assert structor.name == name assert structor.cast == cast assert structor.crew == crew - assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 structor = Structor(cast=cast, qb64=qb64) assert structor.clan == clan + assert structor.name == name assert structor.cast == cast assert structor.crew == crew - assert structor.name == name assert structor.qb64 == qb64 assert structor.qb64b == qb64.encode() assert structor.qb2 == qb2 @@ -242,10 +286,21 @@ def test_structor(): # Test with multiple field namedtuple for data + data = SealEvent(i=prefixer, s=number, d=diger) clan = SealEvent - cast = SealEvent(i=Prefixer, s=Number, d=Diger) - crew = SealEvent(i=aid, s=snq, d=dig) + cast = SealEvent(i=Castage(Prefixer), + s=Castage(Number, 'numh'), + d=Castage(Diger)) + + #naive cast doesn't know about prm for Number + ncast = SealEvent(i=Castage(Prefixer), + s=Castage(Number), + d=Castage(Diger)) + crew = SealEvent(i=aid, s=snh, d=dig) + + # naive crew does't know about prm for Number + ncrew = SealEvent(i=aid, s=snq, d=dig) name = SealEvent.__name__ dcast = cast._asdict() @@ -262,8 +317,8 @@ def test_structor(): structor = Structor(data=data) assert structor.data == data assert structor.clan == clan - assert structor.cast == cast - assert structor.crew == crew + assert structor.cast == ncast + assert structor.crew == ncrew assert structor.name == name assert structor.asdict == data._asdict() assert structor.asdict == \ @@ -442,30 +497,7 @@ def test_structor(): """End Test""" -def test_seal_dexes(): - """ - test Seal Codexes - """ - assert asdict(SClanDom) == \ - { - 'SealDigest': SealDigest, - 'SealRoot': SealRoot, - 'SealBacker': SealBacker, - 'SealLast': SealLast, - 'SealTrans': SealTrans, - 'SealEvent': SealEvent, - } - - assert asdict(SCastDom) == \ - { - 'SealDigest': SealDigest(d=Diger), - 'SealRoot': SealRoot(rd=Diger), - 'SealBacker': SealBacker(bi=Prefixer, d=Diger), - 'SealLast': SealLast(i=Prefixer), - 'SealTrans': SealTrans(s=Number, d=Diger), - 'SealEvent': SealEvent(i=Prefixer, s=Number, d=Diger), - } def test_sealer_class(): """ @@ -500,6 +532,7 @@ def test_sealer(): num = 14 number = Number(num=num) snq = number.qb64 + snh = number.snh dig = 'ELC5L3iBVD77d_MYbYGGCUQgqQBju1o4x1Ud-z2sL-ux' diger = Diger(qb64=dig) @@ -507,8 +540,9 @@ def test_sealer(): data = SealDigest(d=diger) clan = SealDigest - cast = SealDigest(d=Diger) + cast = SealDigest(d=Castage(Diger)) crew = SealDigest(d=dig) + name = SealDigest.__name__ dcast = cast._asdict() dcrew = crew._asdict() @@ -524,6 +558,7 @@ def test_sealer(): sealer = Sealer(data=data) assert sealer.data == data assert sealer.clan == clan + assert sealer.name == name assert sealer.cast == cast assert sealer.crew == crew assert sealer.asdict == data._asdict() @@ -535,6 +570,79 @@ def test_sealer(): # Test no clan but with one or the other of cast and crew as dict or namedtuple sealer = Sealer(crew=crew) # uses known cast assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == cast + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + sealer = Sealer(crew=dcrew) # uses known cast + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == cast + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + # uses known class + sealer = Sealer(cast=dcast, crew=dcrew) + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == cast # tuple compare is by field value not type + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + # Test with multiple field namedtuple for data + + data = SealEvent(i=prefixer, s=number, d=diger) + clan = SealEvent + cast = SealEvent(i=Castage(Prefixer), + s=Castage(Number, "numh"), + d=Castage(Diger)) + # naive cast since data does not provide prm for number + ncast = SealEvent(i=Castage(Prefixer), + s=Castage(Number), + d=Castage(Diger)) + + crew = SealEvent(i=aid, s=snh, d=dig) + # naive crew since data does not provide prm for number + ncrew = SealEvent(i=aid, s=snq, d=dig) + + name = SealEvent.__name__ + + dcast = cast._asdict() + dcrew = crew._asdict() + + assert data._fields == SealEvent._fields + klas = data.__class__ + assert klas == clan + + qb64 = prefixer.qb64 + number.qb64 + diger.qb64 # ''.join(crew) + qb2 = prefixer.qb2 + number.qb2 + diger.qb2 + + # Test data + sealer = Sealer(data=data) + assert sealer.data == data + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == ncast + assert sealer.crew == ncrew + assert sealer.asdict == data._asdict() + assert sealer.asdict == {'i': prefixer, + 's': number, + 'd': diger} + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + # Test no clan but with one or the other of cast and crew as dict or namedtuple + sealer = Sealer(crew=crew) # uses known cast + assert sealer.clan == clan + assert sealer.name == name assert sealer.cast == cast assert sealer.crew == crew assert sealer.qb64 == qb64 @@ -543,6 +651,7 @@ def test_sealer(): sealer = Sealer(crew=dcrew) # uses known cast assert sealer.clan == clan + assert sealer.name == name assert sealer.cast == cast assert sealer.crew == crew assert sealer.qb64 == qb64 @@ -552,16 +661,19 @@ def test_sealer(): # uses known class sealer = Sealer(cast=dcast, crew=dcrew) assert sealer.clan == clan + assert sealer.name == name assert sealer.cast == cast # tuple compare is by field value not type assert sealer.crew == crew assert sealer.qb64 == qb64 assert sealer.qb64b == qb64.encode() assert sealer.qb2 == qb2 + + if __name__ == "__main__": + test_structor_doms() test_structor_class() test_structor() - test_seal_dexes() test_sealer_class() test_sealer() From 928a747a5361483c9ea73dc98a24ae8bae960cfb Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 13 Apr 2024 14:40:44 -0600 Subject: [PATCH 146/418] Added support for Serder.makify and .verify for CESR native keri basic --- src/keri/core/serdering.py | 56 +++++++++++++++----------- tests/core/test_serdering.py | 77 +++++++++++++++++++++++++----------- 2 files changed, 86 insertions(+), 47 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index db8c0cd6c..312bc4b5b 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1001,7 +1001,7 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, # compute saidive digestive field values using raw from sized dummied sad - raw = self.dumps(sad, kind=kind) # serialize sized dummied sad + raw = self.dumps(sad, kind=kind, proto=proto, vrsn=vrsn) # serialize sized dummied sad for label, code in _saids.items(): if code in DigDex: # subclass override if non digestive allowed klas, dsize, dlen = self.Digests[code] # digest algo size & length @@ -1014,7 +1014,7 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, dig = Matter(raw=klas(raw, **ikwa).digest(**dkwa), code=code).qb64 sad[label] = dig - raw = self.dumps(sad, kind=kind) # compute final raw + raw = self.dumps(sad, kind=kind, proto=proto, vrsn=vrsn) # compute final raw if kind == Serials.cesr:# cesr kind version string does not set size size = len(raw) # size of whole message @@ -1164,17 +1164,22 @@ def _exhale(self, sad): self._size = size - def dumps(self, sad=None, kind=Serials.json): + def dumps(self, sad=None, kind=Serials.json, proto=None, vrsn=None): """Method to handle serialization by kind Assumes sad fields are properly filled out for serialization kind. Returns: - raw (bytes): serialization of sad dict using serialization kind + raw (bytes): serialization of sad dict using serialization kind Parameters: - sad (dict | list | None)): serializable dict or list to serialize - kind (str): value of Serials (Serialage) serialization kind + sad (dict | list | None)): serializable dict or list to serialize + kind (str): value of Serials (Serialage) serialization kind "JSON", "MGPK", "CBOR", "CSER" + proto (str | None): desired protocol type str value of Protocols + If None then eventually use self.proto + vrsn (Versionage | None): instance desired protocol version + If None then eventually self.vrsn + Notes: dumps of json uses str whereas dumps of cbor and msgpack use bytes @@ -1192,8 +1197,8 @@ def dumps(self, sad=None, kind=Serials.json): elif kind == Serials.cbor: raw = cbor.dumps(sad) - elif kind == Serials.cser: # does not support list only dict - raw = self._dumps(sad) + elif kind == Serials.cesr: # does not support list only dict + raw = self._dumps(sad, proto=proto, vrsn=vrsn) else: raise SerializeError(f"Invalid serialization kind = {kind}") @@ -1201,7 +1206,7 @@ def dumps(self, sad=None, kind=Serials.json): return raw - def _dumps(self, sad=None): + def _dumps(self, sad=None, proto=None, vrsn=None): """CESR native serialization of sad Returns: @@ -1209,6 +1214,10 @@ def _dumps(self, sad=None): Parameters: sad (dict | None)): serializable dict to serialize + proto (str | None): desired protocol type str value of Protocols + If None then self.proto + vrsn (Versionage | None): instance desired protocol version + If None then self.vrsn Versioning: CESR native serialization includes in its fixed version field @@ -1223,18 +1232,19 @@ def _dumps(self, sad=None): """ sad = sad if sad is not None else self.sad + proto = proto if proto is not None else self.proto + vrsn = vrsn if vrsn is not None else self.vrsn - if (self.gvrsn.major < Vrsn_2_0.major or - self.vrsn.major < Vrsn_2_0.major): - raise SerializeError(f"Invalid major genus version={self.gvrsn}" - f"or Invalid major protocol version={self.vrsn}" + if (self.gvrsn.major < Vrsn_2_0.major or vrsn.major < Vrsn_2_0.major): + raise SerializeError(f"Invalid major genus version={self.gvrsn}" + f"or Invalid major protocol version={vrsn}" f" for native CESR serialization.") if self.genus not in GenDex: # ensures self.genus != None raise SerializeError(f"Invalid genus={self.genus}.") - if getattr(GenDex, self.proto, None) != self.genus: - raise SerializeError(f"Incompatible protocol={self.proto} with " + if getattr(GenDex, proto, None) != self.genus: + raise SerializeError(f"Incompatible protocol={proto} with " f"genus={self.genus}.") @@ -1242,13 +1252,13 @@ def _dumps(self, sad=None): raw = bytearray() # message as qb64 bdy = bytearray() # message body as qb64 - ilks = self.Fields[self.proto][self.vrsn] # get fields keyed by ilk + ilks = self.Fields[proto][vrsn] # get fields keyed by ilk ilk = sad.get('t') # returns None if missing message type (ilk) if ilk not in ilks: # raise SerializeError(f"Missing message type field " - f"'t' for protocol={self.proto} " - f"version={self.vrsn} with {sad=}.") + f"'t' for protocol={proto} " + f"version={vrsn} with {sad=}.") fields = ilks[ilk] # FieldDom for given protocol and ilk @@ -1262,14 +1272,14 @@ def _dumps(self, sad=None): # so can serialize in order to compute saidive fields # need to fix ._verify and .makify to account for CESR native serialization - if self.proto == Protocols.keri: + if proto == Protocols.keri: if not fixed: # prepend label pass # raise error for l, v in sad.items(): # assumes valid field order & presence match l: # label case "v": # protocol+version do not use version string itself - val = Verser(proto=self.proto, vrsn=self.vrsn).qb64b + val = Verser(proto=proto, vrsn=vrsn).qb64b case "t": # message type (ilk), already got ilk val = Ilker(ilk=v).qb64b # assumes same @@ -1348,13 +1358,13 @@ def _dumps(self, sad=None): case _: # if extra fields this is where logic would be raise SerializeError(f"Unsupported protocol field label" - f"='{l}' for protocol={self.proto}" - f" version={self.vrsn}.") + f"='{l}' for protocol={proto}" + f" version={vrsn}.") bdy.extend(val) - elif self.protocol == Protocols.acdc: + elif proto == Protocols.acdc: for l, val in sad.items(): # assumes valid field order & presence if not fixed: pass # prepend label diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index f89aa6dce..02f2d436a 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -20,6 +20,8 @@ VERRAWSIZE, VERFMT, MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN,) +from keri.help import helping + from keri import core from keri.core.structing import Sealer, SealEvent, SealTrans @@ -2690,13 +2692,13 @@ def test_cesr_native_dumps(): keys = [csigners[0].verfer.qb64] assert keys == ['DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ'] - serder = incept(keys, version=Vrsn_2_0) + serder = incept(keys, version=Vrsn_2_0, kind=kering.Serials.cesr) assert serder.sad == \ { - 'v': 'KERICAAJSONAAD8.', + 'v': 'KERICAACESRAAAA.', 't': 'icp', - 'd': 'EP9O8aDcloRpvTmk8pnfq3KE2eH_-_wDYWqwOsSgpPws', + 'd': 'EO6lMLcTbUhdpbQVXCh78MShuT_69th6tiZhEbAfPCj4', 'i': 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', 's': '0', 'kt': '1', @@ -2709,10 +2711,16 @@ def test_cesr_native_dumps(): 'a': [] } + assert serder.raw == (b'-FAtYKERICAAXicpEO6lMLcTbUhdpbQVXCh78MShuT_69th6tiZhEbAfPCj4DG9X' + b'hvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAAMAAB-LALDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAA-LAAMAAA-LAA-LAA-LAA') + assert len(serder.raw) == serder.size == 184 + sizeh = serder.raw[2:4] + assert sizeh == b"At" + assert helping.b64ToInt(sizeh) * 4 + 4 == serder.size == 184 + rawqb64 = serder._dumps() # default is it dumps self.sad - assert rawqb64 == (b'-FAtYKERICAAXicpEP9O8aDcloRpvTmk8pnfq3KE2eH_-_wDYWqwOsSgpPwsDG9X' - b'hvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAAMAAB-LALDG9XhvcVryHj' - b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAA-LAAMAAA-LAA-LAA-LAA') + assert rawqb64 == serder.raw assert len(rawqb64) == 184 rawqb2 = decodeB64(rawqb64) @@ -2767,14 +2775,15 @@ def test_cesr_native_dumps(): cnfg=['DND'], data=data, code=core.MtrDex.Blake3_256, - version=Vrsn_2_0) + version=Vrsn_2_0, + kind=kering.Serials.cesr) assert serder.sad == \ { - 'v': 'KERICAAJSONAAOT.', + 'v': 'KERICAACESRAAAA.', 't': 'icp', - 'd': 'ECdA32v8SkQ2ZFpliiK0RBcfNERaVtxk4pmaulKsakXY', - 'i': 'ECdA32v8SkQ2ZFpliiK0RBcfNERaVtxk4pmaulKsakXY', + 'd': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6', + 'i': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6', 's': '0', 'kt': '2', 'k': @@ -2808,7 +2817,8 @@ def test_cesr_native_dumps(): { 'i': 'DK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', 's': '1', - 'd': 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG'}, + 'd': 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG' + }, { 's': 'f', 'd': 'EEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZpZEyP' @@ -2816,20 +2826,39 @@ def test_cesr_native_dumps(): ] } + assert serder.raw == (b'-FDCYKERICAAXicpEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6EMEv' + b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAAAMAAC-LAhDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' + b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-LAh' + b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUGEEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZp' + b'ZEyPMAAD-LAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' + b'o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnBMOmBoddcrRHShSajb4d60S6RK34' + b'gXZ2WYbr3AiPY1M0-LABXDND-LA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'4fBJre3NGwTQMAAAEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' + b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAABEMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' + b'ZJGJqQZpZEyP') + + assert len(serder.raw) == serder.size == 780 + sizeh = serder.raw[2:4] + assert sizeh == b"DC" + assert helping.b64ToInt(sizeh) * 4 + 4 == serder.size == 780 + rawqb64 = serder._dumps() # default is it dumps self.sad - assert rawqb64 == (b'-FDCYKERICAAXicpECdA32v8SkQ2ZFpliiK0RBcfNERaVtxk4pmaulKsakXYECdA' - b'32v8SkQ2ZFpliiK0RBcfNERaVtxk4pmaulKsakXYMAAAMAAC-LAhDG9XhvcVryHj' - b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' - b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-LAh' - b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' - b'TPUtc7uSAvrchIPN3I8S6vUGEEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZp' - b'ZEyPMAAD-LAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' - b'o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnBMOmBoddcrRHShSajb4d60S6RK34' - b'gXZ2WYbr3AiPY1M0-LABXDND-LA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' - b'4fBJre3NGwTQMAAAEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' - b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAABEMrowWRk6u1imR32ZNHn' - b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' - b'ZJGJqQZpZEyP') + assert rawqb64 == (b'-FDCYKERICAAXicpEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6EMEv' + b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAAAMAAC-LAhDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' + b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-LAh' + b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUGEEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZp' + b'ZEyPMAAD-LAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' + b'o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnBMOmBoddcrRHShSajb4d60S6RK34' + b'gXZ2WYbr3AiPY1M0-LABXDND-LA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'4fBJre3NGwTQMAAAEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' + b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAABEMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' + b'ZJGJqQZpZEyP') assert len(rawqb64) == 780 From d343dd7faa9cbd04531210ba510caff333d0300d Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Sat, 13 Apr 2024 15:17:49 -0700 Subject: [PATCH 147/418] Logging updates (#755) * Add support to stream CESR over HTTP path for authentication headers. Signed-off-by: pfeairheller * Updates to logging. - Removed logging at any level bug debug for messages in the escrow loops. - Removed line feeds from all log levels but debug - Ensured stack traces are only logged in debug - Removed full event logging from info, moved to debug level. Replaced with SAID in info. - Replaced print statements in grouping and delegating with logger.info statements. Signed-off-by: pfeairheller * Makes things more pythonic Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller --- src/keri/app/agenting.py | 25 +- src/keri/app/cli/commands/interact.py | 15 +- src/keri/app/cli/commands/rotate.py | 15 +- src/keri/app/cli/commands/witness/start.py | 2 +- src/keri/app/cli/commands/witness/submit.py | 55 ++-- src/keri/app/delegating.py | 6 +- src/keri/app/grouping.py | 18 +- src/keri/app/habbing.py | 5 +- src/keri/app/indirecting.py | 6 +- src/keri/core/eventing.py | 287 +++++++++++--------- src/keri/core/parsing.py | 61 ++--- src/keri/core/routing.py | 48 ++-- src/keri/db/escrowing.py | 19 +- src/keri/peer/exchanging.py | 20 +- src/keri/vdr/credentialing.py | 3 +- src/keri/vdr/eventing.py | 51 ++-- src/keri/vdr/verifying.py | 13 +- 17 files changed, 359 insertions(+), 290 deletions(-) diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index b0b866726..2fd13cf49 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -273,7 +273,7 @@ class WitnessReceiptor(doing.DoDoer): """ - def __init__(self, hby, msgs=None, cues=None, force=False, **kwa): + def __init__(self, hby, msgs=None, cues=None, force=False, auths=None, **kwa): """ For the current event, gather the current set of witnesses, send the event, gather all receipts and send them to all other witnesses @@ -289,6 +289,7 @@ def __init__(self, hby, msgs=None, cues=None, force=False, **kwa): self.force = force self.msgs = msgs if msgs is not None else decking.Deck() self.cues = cues if cues is not None else decking.Deck() + self.auths = auths if auths is not None else dict() super(WitnessReceiptor, self).__init__(doers=[doing.doify(self.receiptDo)], **kwa) @@ -332,7 +333,8 @@ def receiptDo(self, tymth=None, tock=0.0): witers = [] for wit in wits: - witer = messenger(hab, wit) + auth = self.auths[wit] if wit in self.auths else None + witer = messenger(hab, wit, auth=auth) witers.append(witer) self.extend([witer]) @@ -836,7 +838,7 @@ class HTTPMessenger(doing.DoDoer): """ - def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): + def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, auth=None, **kwa): """ For the current event, gather the current set of witnesses, send the event, gather all receipts and send them to all other witnesses @@ -851,6 +853,7 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): self.msgs = msgs if msgs is not None else decking.Deck() self.sent = sent if sent is not None else decking.Deck() self.parser = None + self.auth = auth doers = doers if doers is not None else [] doers.extend([doing.doify(self.msgDo), doing.doify(self.responseDo)]) @@ -881,7 +884,11 @@ def msgDo(self, tymth=None, tock=0.0): yield self.tock msg = self.msgs.popleft() - self.posted += httping.streamCESRRequests(client=self.client, dest=self.wit, ims=msg) + headers = dict() + if self.auth is not None: + headers["Authorization"] = self.auth + + self.posted += httping.streamCESRRequests(client=self.client, dest=self.wit, ims=msg, headers=headers) while self.client.requests: yield self.tock @@ -983,34 +990,36 @@ def mailbox(hab, cid): return mbx -def messenger(hab, pre): +def messenger(hab, pre, auth=None): """ Create a Messenger (tcp or http) based on available endpoints Parameters: hab (Habitat): Environment to use to look up witness URLs pre (str): qb64 identifier prefix of recipient to create a messanger for + auth (str): optional auth code to send with any request for messenger Returns: Optional(TcpWitnesser, HTTPMessenger): witnesser for ensuring full reciepts """ urls = hab.fetchUrls(eid=pre) - return messengerFrom(hab, pre, urls) + return messengerFrom(hab, pre, urls, auth) -def messengerFrom(hab, pre, urls): +def messengerFrom(hab, pre, urls, auth=None): """ Create a Witnesser (tcp or http) based on provided endpoints Parameters: hab (Habitat): Environment to use to look up witness URLs pre (str): qb64 identifier prefix of recipient to create a messanger for urls (dict): map of schemes to urls of available endpoints + auth (str): optional auth code to send with any request for messenger Returns: Optional(TcpWitnesser, HTTPMessenger): witnesser for ensuring full reciepts """ if kering.Schemes.http in urls or kering.Schemes.https in urls: url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - witer = HTTPMessenger(hab=hab, wit=pre, url=url) + witer = HTTPMessenger(hab=hab, wit=pre, url=url, auth=auth) elif kering.Schemes.tcp in urls: url = urls[kering.Schemes.tcp] witer = TCPMessenger(hab=hab, wit=pre, url=url) diff --git a/src/keri/app/cli/commands/interact.py b/src/keri/app/cli/commands/interact.py index eedea2d0f..53cabce75 100644 --- a/src/keri/app/cli/commands/interact.py +++ b/src/keri/app/cli/commands/interact.py @@ -103,21 +103,22 @@ def interactDo(self, tymth, tock=0.0, **opts): hab = self.hby.habByName(name=self.alias) hab.interact(data=self.data) - if self.endpoint or self.authenticate: + auths = {} + if self.authenticate: + for wit in hab.kever.wits: + code = input(f"Entire code for {wit}: ") + auths[wit] = f"{code}#{helping.nowIso8601()}" + + if self.endpoint: receiptor = agenting.Receiptor(hby=self.hby) self.extend([receiptor]) - auths = {} - if self.authenticate: - for wit in hab.kever.wits: - code = input(f"Entire code for {wit}: ") - auths[wit] = f"{code}#{helping.nowIso8601()}" yield from receiptor.receipt(hab.pre, sn=hab.kever.sn, auths=auths) self.remove([receiptor]) else: - witDoer = agenting.WitnessReceiptor(hby=self.hby) + witDoer = agenting.WitnessReceiptor(hby=self.hby, auths=auths) self.extend(doers=[witDoer]) if hab.kever.wits: diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index cb9b34c9e..c5eb3b586 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -191,6 +191,12 @@ def rotateDo(self, tymth, tock=0.0): cuts=list(self.cuts), adds=list(self.adds), data=self.data) + auths = {} + if self.authenticate: + for wit in hab.kever.wits: + code = input(f"Entire code for {wit}: ") + auths[wit] = f"{code}#{helping.nowIso8601()}" + if hab.kever.delpre: self.swain.delegation(pre=hab.pre, sn=hab.kever.sn, proxy=self.hby.habByName(self.proxy)) print("Waiting for delegation approval...") @@ -198,19 +204,14 @@ def rotateDo(self, tymth, tock=0.0): yield self.tock elif hab.kever.wits: - if self.endpoint or self.authenticate: - auths = {} - if self.authenticate: - for wit in hab.kever.wits: - code = input(f"Entire code for {wit}: ") - auths[wit] = f"{code}#{helping.nowIso8601()}" + if self.endpoint: yield from receiptor.receipt(hab.pre, sn=hab.kever.sn, auths=auths) else: for wit in self.adds: self.mbx.addPoller(hab, witness=wit) print("Waiting for witness receipts...") - witDoer = agenting.WitnessReceiptor(hby=self.hby) + witDoer = agenting.WitnessReceiptor(hby=self.hby, auths=auths) self.extend(doers=[witDoer]) yield self.tock diff --git a/src/keri/app/cli/commands/witness/start.py b/src/keri/app/cli/commands/witness/start.py index 400b75468..8e47df39a 100644 --- a/src/keri/app/cli/commands/witness/start.py +++ b/src/keri/app/cli/commands/witness/start.py @@ -55,7 +55,7 @@ def launch(args): help.ogler.level = logging.getLevelName(args.loglevel) - if(args.logfile != None): + if args.logfile is not None: help.ogler.headDirPath = args.logfile help.ogler.reopen(name=args.name, temp=False, clear=True) logger = help.ogler.getLogger() diff --git a/src/keri/app/cli/commands/witness/submit.py b/src/keri/app/cli/commands/witness/submit.py index f2f317ac0..9f46a1a52 100644 --- a/src/keri/app/cli/commands/witness/submit.py +++ b/src/keri/app/cli/commands/witness/submit.py @@ -10,6 +10,7 @@ from keri.app import habbing, agenting, indirecting from keri.app.cli.common import existing, displaying +from keri.help import helping logger = help.ogler.getLogger() @@ -30,6 +31,10 @@ parser.add_argument('--force', action="store_true", required=False, help='True means to send witnesses all receipts even if we have a full compliment of receipts for ' 'the current event') +parser.add_argument("--receipt-endpoint", help="Attempt to connect to witness receipt endpoint for witness receipts.", + dest="endpoint", action='store_true') +parser.add_argument("--authenticate", '-z', help="Prompt the controller for authentication codes for each witness", + action='store_true') def handler(args): @@ -46,17 +51,18 @@ def handler(args): alias = args.alias force = args.force - icpDoer = InceptDoer(name=name, base=base, alias=alias, bran=bran, force=force) + subDoer = SubmitDoer(name=name, base=base, alias=alias, bran=bran, force=force, authenticate=args.authenticate, + endpoint=args.endpoint) - doers = [icpDoer] + doers = [subDoer] return doers -class InceptDoer(doing.DoDoer): +class SubmitDoer(doing.DoDoer): """ DoDoer for creating a new identifier prefix and Hab with an alias. """ - def __init__(self, name, base, alias, bran, force): + def __init__(self, name, base, alias, bran, force, endpoint=False, authenticate=False): hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer @@ -64,13 +70,14 @@ def __init__(self, name, base, alias, bran, force): self.alias = alias self.hby = hby self.force = force + self.endpoint = endpoint + self.authenticate = authenticate - self.witDoer = None - doers = [self.hbyDoer, self.mbx, doing.doify(self.inceptDo)] + doers = [self.hbyDoer, self.mbx, doing.doify(self.submitDo)] - super(InceptDoer, self).__init__(doers=doers) + super(SubmitDoer, self).__init__(doers=doers) - def inceptDo(self, tymth, tock=0.0): + def submitDo(self, tymth, tock=0.0): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -85,18 +92,34 @@ def inceptDo(self, tymth, tock=0.0): _ = (yield self.tock) hab = self.hby.habByName(name=self.alias) - self.witDoer = agenting.WitnessReceiptor(hby=self.hby, force=self.force) - self.extend([self.witDoer]) + auths = {} + if self.authenticate: + for wit in hab.kever.wits: + code = input(f"Entire code for {wit}: ") + auths[wit] = f"{code}#{helping.nowIso8601()}" - if hab.kever.wits: - print("Waiting for witness receipts...") - self.witDoer.msgs.append(dict(pre=hab.pre)) - while not self.witDoer.cues: - _ = yield self.tock + if self.endpoint: + receiptor = agenting.Receiptor(hby=self.hby) + self.extend([receiptor]) + + yield from receiptor.receipt(hab.pre, sn=hab.kever.sn, auths=auths) + self.remove([receiptor]) + + else: + witDoer = agenting.WitnessReceiptor(hby=self.hby, force=self.force, auths=auths) + self.extend([witDoer]) + + if hab.kever.wits: + print("Waiting for witness receipts...") + witDoer.msgs.append(dict(pre=hab.pre)) + while not witDoer.cues: + _ = yield self.tock + + self.remove([witDoer]) displaying.printIdentifier(self.hby, hab.pre) - toRemove = [self.hbyDoer, self.witDoer, self.mbx] + toRemove = [self.hbyDoer, self.mbx] self.remove(toRemove) return diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 0823af1b1..2ffa80ce6 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -12,7 +12,7 @@ from . import agenting, forwarding from .habbing import GroupHab from .. import kering -from ..core import coring, eventing, serdering +from ..core import coring, serdering from ..db import dbing from ..peer import exchanging @@ -162,7 +162,7 @@ def processUnanchoredEscrow(self): self.witDoer.msgs.append(dict(pre=pre, sn=serder.sn)) # Move to escrow waiting for witness receipts - print(f"Waiting for fully signed witness receipts for {serder.sn}") + logger.info(f"Waiting for fully signed witness receipts for {serder.sn}") self.hby.db.dpwe.pin(keys=(pre, said), val=serder) self.hby.db.dune.rem(keys=(pre, said)) @@ -188,7 +188,7 @@ def processPartialWitnessEscrow(self): witnessed = True if not witnessed: continue - print(f"Witness receipts complete, {pre} confirmed.") + logger.info(f"Witness receipts complete, {pre} confirmed.") self.hby.db.dpwe.rem(keys=(pre, said)) self.hby.db.cdel.put(keys=(pre, seqner.qb64), val=coring.Saider(qb64=serder.said)) diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 2f4bd3e82..bbefab2da 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -51,7 +51,7 @@ def start(self, ghab, prefixer, seqner, saider): serder = serdering.SerderKERI(raw=evt) del evt[:serder.size] - print(f"Waiting for other signatures for {serder.pre}:{seqner.sn}...") + logger.info(f"Waiting for other signatures for {serder.pre}:{seqner.sn}...") return self.hby.db.gpse.add(keys=(prefixer.qb64,), val=(seqner, saider)) def complete(self, prefixer, seqner, saider=None): @@ -133,7 +133,7 @@ def processPartialSignedEscrow(self): if kever.delegated and kever.ilk in (coring.Ilks.dip, coring.Ilks.drt): # We are a delegated identifier, must wait for delegator approval for dip and drt if witered: # We are elected to perform delegation and witnessing messaging - print(f"We are the witnesser, sending {pre} to delegator") + logger.info(f"We are the witnesser, sending {pre} to delegator") self.swain.delegation(pre=pre, sn=seqner.sn) else: anchor = dict(i=pre, s=seqner.snh, d=saider.qb64) @@ -142,15 +142,15 @@ def processPartialSignedEscrow(self): else: self.witq.query(src=ghab.mhab.pre, pre=kever.delpre, anchor=anchor) - print("Waiting for delegation approval...") + logger.info("Waiting for delegation approval...") self.hby.db.gdee.add(keys=(pre,), val=(seqner, saider)) else: # Non-delegation, move on to witnessing if witered: # We are elected witnesser, send off event to witnesses - print(f"We are the fully signed witnesser {seqner.sn}, sending to witnesses") + logger.info(f"We are the fully signed witnesser {seqner.sn}, sending to witnesses") self.witDoer.msgs.append(dict(pre=pre, sn=seqner.sn)) # Move to escrow waiting for witness receipts - print(f"Waiting for fully signed witness receipts for {seqner.sn}") + logger.info(f"Waiting for fully signed witness receipts for {seqner.sn}") self.hby.db.gpwe.add(keys=(pre,), val=(seqner, saider)) def processDelegateEscrow(self): @@ -170,7 +170,7 @@ def processDelegateEscrow(self): if witer: # We are elected witnesser, We've already done out part in Boatswain, we are done. if self.swain.complete(prefixer=kever.prefixer, seqner=coring.Seqner(sn=kever.sn)): self.hby.db.gdee.rem(keys=(pre,)) - print(f"Delegation approval for {pre} received.") + logger.info(f"Delegation approval for {pre} received.") self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) @@ -181,10 +181,10 @@ def processDelegateEscrow(self): dgkey = dbing.dgKey(pre, saider.qb64b) self.hby.db.setAes(dgkey, couple) # authorizer event seal (delegator/issuer) self.hby.db.gdee.rem(keys=(pre,)) - print(f"Delegation approval for {pre} received.") + logger.info(f"Delegation approval for {pre} received.") # Move to escrow waiting for witness receipts - print(f"Waiting for witness receipts for {pre}") + logger.info(f"Waiting for witness receipts for {pre}") self.hby.db.gdee.rem(keys=(pre,)) self.hby.db.gpwe.add(keys=(pre,), val=(seqner, saider)) @@ -212,7 +212,7 @@ def processPartialWitnessEscrow(self): witnessed = True if not witnessed: continue - print(f"Witness receipts complete, {pre} confirmed.") + logger.info(f"Witness receipts complete, {pre} confirmed.") self.hby.db.gpwe.rem(keys=(pre,)) self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) elif not witer: diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 95cbc8d51..0bf3f2c24 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -2084,8 +2084,9 @@ def processCuesIter(self, cues): cuedSerder = cue["serder"] # Serder of received event for other pre cuedKed = cuedSerder.ked cuedPrefixer = coring.Prefixer(qb64=cuedKed["i"]) - logger.info("%s got cue: kin=%s\n%s\n\n", self.pre, cueKin, - json.dumps(cuedKed, indent=1)) + logger.info("%s got cue: kin=%s%s", self.pre, cueKin, + cuedSerder.said) + logger.debug(f"event=\n{cuedSerder.pretty()}\n") if cuedKed["t"] == coring.Ilks.icp: dgkey = dbing.dgKey(self.pre, self.iserder.said) diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index c2aef5a14..031c9cf53 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -192,7 +192,7 @@ def msgDo(self, tymth=None, tock=0.0): _ = (yield self.tock) if self.parser.ims: - logger.info("Client %s received:\n%s\n...\n", self.kvy, self.parser.ims[:1024]) + logger.debug("Client %s received:\n%s\n...\n", self.kvy, self.parser.ims[:1024]) done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close @@ -381,7 +381,7 @@ def msgDo(self, tymth=None, tock=0.0): _ = (yield self.tock) if self.parser.ims: - logger.info("Client %s received:\n%s\n...\n", self.hab.pre, self.parser.ims[:1024]) + logger.debug("Client %s received:\n%s\n...\n", self.hab.pre, self.parser.ims[:1024]) done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close @@ -444,7 +444,7 @@ def sendMessage(self, msg, label=""): Sends message msg and loggers label if any """ self.client.tx(msg) # send to remote - logger.info("%s sent %s:\n%s\n\n", self.hab.pre, label, bytes(msg)) + logger.debug("%s sent %s:\n%s\n\n", self.hab.pre, label, bytes(msg)) class MailboxDirector(doing.DoDoer): diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index be5ed47d8..d256b5448 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -7,7 +7,6 @@ import json import logging from collections import namedtuple -from dataclasses import dataclass, astuple, asdict, field from urllib.parse import urlsplit from math import ceil from ordered_set import OrderedSet as oset @@ -16,27 +15,26 @@ from .. import kering from ..kering import (MissingEntryError, - ValidationError, DerivationError, MissingSignatureError, + ValidationError, MissingSignatureError, MissingWitnessSignatureError, UnverifiedReplyError, MissingDelegationError, OutOfOrderError, LikelyDuplicitousError, UnverifiedWitnessReceiptError, UnverifiedReceiptError, UnverifiedTransferableReceiptError, QueryNotFoundError, MisfitEventSourceError, MissingDelegableApprovalError) -from ..kering import Version, Versionage, TraitCodex, TraitDex -from ..kering import Coldage, Colds, ColdDex +from ..kering import Version, Versionage, TraitDex from .. import help from ..help import helping from . import coring -from .coring import (versify, Serials, Ilks, MtrDex, PreDex, DigDex, +from .coring import (versify, Serials, Ilks, PreDex, DigDex, NonTransDex, CtrDex, Counter, Number, Seqner, Cigar, Dater, Verfer, Diger, Prefixer, Tholder, Saider) from . import indexing -from .indexing import (IdrDex, Indexer, Siger) +from .indexing import Siger from . import serdering @@ -390,7 +388,7 @@ def verifySigs(raw, sigers, verfers): # verfer to each siger for siger in usigers: if siger.index >= len(verfers): - logger.info("Skipped sig: Index=%s to large.\n", siger.index) + logger.info("Skipped sig: Index=%s to large.", siger.index) siger.verfer = verfers[siger.index] # assign verfer # create lists of unique verified signatures and indices @@ -667,9 +665,9 @@ def incept(keys, Parameters: keys (list): current signing keys qb64 - sith (int | str | list | None): current signing threshold input to Tholder + isith (int | str | list | None): current signing threshold input to Tholder ndigs (list | None): current signing key digests qb64 - nsith int | str | list | None): next signing threshold input to Tholder + nsith (int | str | list | None): next signing threshold input to Tholder toad (int | str | None): witness threshold number if str then hex str wits (list | None): witness identifier prefixes qb64 cnfg (list | None): configuration traits from TraitDex @@ -818,9 +816,9 @@ def rotate(pre, dig (str): SAID of previous event qb64 ilk (str): ilk of event. Must be in (Ilks.rot, Ilks.drt) sn (int | str): sequence number int or hex str - sith (int | str | list | None): current signing threshold input to Tholder + isith (int | str | list | None): current signing threshold input to Tholder ndigs (list | None): current signing key digests qb64 - nsith int | str | list | None): next signing threshold input to Tholder + nsith (int | str | list | None): next signing threshold input to Tholder toad (int | str | None): witness threshold number if str then hex str wits (list | None): prior witness identifier prefixes qb64 cuts (list | None): witness prefixes to cut qb64 @@ -2893,19 +2891,22 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, if firner and fn != firner.sn: # cloned replay but replay fn not match if self.cues is not None: # cue to notice BadCloneFN self.cues.push(dict(kin="noticeBadCloneFN", serder=serder, - fn=fn, firner=firner, dater=dater)) + fn=fn, firner=firner, dater=dater)) logger.info("Kever Mismatch Cloned Replay FN: %s First seen " - "ordinal fn %s and clone fn %s \nEvent=\n%s\n", - serder.preb, fn, firner.sn, serder.pretty()) + "ordinal fn %s and clone fn %s, said=%s", + serder.preb, fn, firner.sn, serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") if dater: # cloned replay use original's dts from dater dtsb = dater.dtsb self.db.setDts(dgkey, dtsb) # first seen so set dts to now self.db.fons.pin(keys=dgkey, val=Seqner(sn=fn)) - logger.info("Kever state: %s First seen ordinal %s at %s\nEvent=\n%s\n", - serder.preb, fn, dtsb.decode("utf-8"), serder.pretty()) + logger.info("Kever state: %s First seen ordinal %s at %s, said=%s", + serder.pre, fn, dtsb.decode("utf-8"), serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") self.db.addKe(snKey(serder.preb, serder.sn), serder.saidb) - logger.info("Kever state: %s Added to KEL valid event=\n%s\n", - serder.preb, serder.pretty()) + logger.info("Kever state: %s Added to KEL valid said=%s", + serder.pre, serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") return (fn, dtsb.decode("utf-8")) # (fn int, dts str) if first else (None, dts str) @@ -2947,7 +2948,7 @@ def escrowMFEvent(self, serder, sigers, wigers=None, res = self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed - logger.info("Kever state: escrowed misfit event=\n%s\n", + logger.debug("Kever state: escrowed misfit event=\n%s\n", json.dumps(serder.ked, indent=1)) @@ -2985,7 +2986,7 @@ def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) self.db.delegables.add(snKey(serder.preb, serder.sn), serder.saidb) # log escrowed - logger.info("Kever state: escrowed delegable event=\n%s\n", + logger.debug("Kever state: escrowed delegable event=\n%s\n", json.dumps(serder.ked, indent=1)) @@ -3023,7 +3024,7 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): snkey = snKey(serder.preb, serder.sn) self.db.addPse(snkey, serder.saidb) # b'EOWwyMU3XA7RtWdelFt-6waurOTH_aW_Z9VTaU-CshGk.00000000000000000000000000000001' - logger.info("Kever state: Escrowed partially signed or delegated " + logger.debug("Kever state: Escrowed partially signed or delegated " "event = %s\n", serder.ked) @@ -3051,7 +3052,7 @@ def escrowPACouple(self, serder, seqner, saider, local=True): dgkey = dgKey(serder.preb, serder.saidb) couple = seqner.qb64b + saider.qb64b self.db.putPde(dgkey, couple) # idempotent - logger.info("Kever state: Escrowed source couple for partially signed " + logger.debug("Kever state: Escrowed source couple for partially signed " "or delegated event = %s\n", serder.ked) @@ -3094,7 +3095,7 @@ def escrowPWEvent(self, serder, wigers, sigers=None, esr = basing.EventSourceRecord(local=local) self.db.esrs.put(keys=dgkey, val=esr) - logger.info("Kever state: Escrowed partially witnessed " + logger.debug("Kever state: Escrowed partially witnessed " "event = %s\n", serder.ked) return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) @@ -3708,11 +3709,14 @@ def processReceiptWitness(self, serder, wigers, local=None): if pre in self.prefixes: # skip own receiptor of own event # sign own events not receipt them logger.info("Kevery process: skipped own receipt attachment" - " on own event receipt=\n%s\n", serder.pretty()) + " on own event receipt=%s", serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") + continue # skip own receipt attachment on own event if not local: # so skip own receipt on other event when non-local source logger.info("Kevery process: skipped own receipt attachment" - " on nonlocal event receipt=\n%s\n", serder.pretty()) + " on nonlocal event receipt=%s", serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") continue # skip own receipt attachment on non-local event if wiger.verfer.verify(wiger.raw, lserder.raw): @@ -3780,11 +3784,14 @@ def processReceipt(self, serder, cigars, local=None): if pre in self.prefixes: # skip own receipter of own event # sign own events not receipt them logger.info("Kevery process: skipped own receipt attachment" - " on own event receipt=\n%s\n", serder.pretty()) + " on own event receipt=%s", serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") + continue # skip own receipt attachment on own event if not local: # skip own receipt on other event when not local logger.info("Kevery process: skipped own receipt attachment" - " on nonlocal event receipt=\n%s\n", serder.pretty()) + " on nonlocal event receipt=%s", serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") continue # skip own receipt attachment on non-local event if cigar.verfer.verify(cigar.raw, lserder.raw): @@ -3861,11 +3868,15 @@ def processAttachedReceiptCouples(self, serder, cigars, firner=None, local=None) if pre in self.prefixes: # skip own receipter on own event # sign own events not receipt them logger.info("Kevery process: skipped own receipt attachment" - " on own event receipt=\n%s\n", serder.pretty()) + " on own event receipt=%s", serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") + continue # skip own receipt attachment on own event if not local: # own receipt on other event when not local logger.info("Kevery process: skipped own receipt attachment" - " on nonlocal event receipt=\n%s\n", serder.pretty()) + " on nonlocal event receipt=%s", serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") + continue # skip own receipt attachment on non-local event if cigar.verfer.verify(cigar.raw, serder.raw): @@ -4071,7 +4082,7 @@ def processAttachedReceiptQuadruples(self, serder, trqs, firner=None, local=None siger.verfer = sverfers[siger.index] # assign verfer if not siger.verfer.verify(siger.raw, serder.raw): # verify sig logger.info("Kevery unescrow error: Bad trans receipt sig." - "pre=%s sn=%x receipter=%s\n", pre, sn, sprefixer.qb64) + "pre=%s sn=%x receipter=%s", pre, sn, sprefixer.qb64) raise ValidationError("Bad escrowed trans receipt sig at " "pre={} sn={:x} receipter={}." @@ -4662,7 +4673,7 @@ def escrowMFEvent(self, serder, sigers, wigers=None, self.db.putPde(dgkey, couple) # idempotent self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed - logger.info("Kevery process: escrowed misfit event=\n%s\n", + logger.debug("Kevery process: escrowed misfit event=\n%s", json.dumps(serder.ked, indent=1)) @@ -4702,8 +4713,8 @@ def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None, l self.db.putPde(dgkey, couple) # idempotent self.db.addOoe(snKey(serder.preb, serder.sn), serder.saidb) # log escrowed - logger.info("Kevery process: escrowed out of order event=\n%s\n", - json.dumps(serder.ked, indent=1)) + logger.debug("Kevery process: escrowed out of order event=\n%s", + json.dumps(serder.ked, indent=1)) def escrowQueryNotFoundEvent(self, prefixer, serder, sigers, cigars=None): """ @@ -4726,8 +4737,8 @@ def escrowQueryNotFoundEvent(self, prefixer, serder, sigers, cigars=None): self.db.addRct(key=dgkey, val=cigar.verfer.qb64b + cigar.qb64b) # log escrowed - logger.info("Kevery process: escrowed query not found event=\n%s\n", - json.dumps(serder.ked, indent=1)) + logger.debug("Kevery process: escrowed query not found event=\n%s", + json.dumps(serder.ked, indent=1)) def escrowLDEvent(self, serder, sigers, local=True): """ @@ -4757,8 +4768,8 @@ def escrowLDEvent(self, serder, sigers, local=True): self.db.putEvt(dgkey, serder.raw) self.db.addLde(snKey(serder.preb, serder.sn), serder.saidb) # log duplicitous - logger.info("Kevery process: escrowed likely duplicitous event=\n%s\n", - json.dumps(serder.ked, indent=1)) + logger.debug("Kevery process: escrowed likely duplicitous event=\n%s", + json.dumps(serder.ked, indent=1)) def escrowUWReceipt(self, serder, wigers, said): """ @@ -4793,8 +4804,8 @@ def escrowUWReceipt(self, serder, wigers, said): couple = said.encode("utf-8") + wiger.qb64b self.db.addUwe(key=snKey(serder.preb, serder.sn), val=couple) # log escrowed - logger.info("Kevery process: escrowed unverified witness indexed receipt" - " of pre= %s sn=%x dig=%s\n", serder.pre, serder.sn, said) + logger.debug("Kevery process: escrowed unverified witness indexed receipt" + " of pre= %s sn=%x dig=%s", serder.pre, serder.sn, said) def escrowUReceipt(self, serder, cigars, said): """ @@ -4824,8 +4835,8 @@ def escrowUReceipt(self, serder, cigars, said): triple = said.encode("utf-8") + cigar.verfer.qb64b + cigar.qb64b self.db.addUre(key=snKey(serder.preb, serder.sn), val=triple) # should be snKey # log escrowed - logger.info("Kevery process: escrowed unverified receipt of pre= %s " - " sn=%x dig=%s\n", serder.pre, serder.sn, said) + logger.debug("Kevery process: escrowed unverified receipt of pre= %s " + " sn=%x dig=%s", serder.pre, serder.sn, said) def escrowTRGroups(self, serder, tsgs): """ @@ -4868,9 +4879,9 @@ def escrowTRGroups(self, serder, tsgs): quintuple = prelet + siger.qb64b # quintuple self.db.addVre(key=snKey(serder.preb, serder.sn), val=quintuple) # log escrowed - logger.info("Kevery process: escrowed unverified transferable receipt " - "of pre=%s sn=%x dig=%s by pre=%s\n", serder.pre, - serder.sn, serder.ked["d"], prefixer.qb64) + logger.debug("Kevery process: escrowed unverified transferable receipt " + "of pre=%s sn=%x dig=%s by pre=%s", serder.pre, + serder.sn, serder.ked["d"], prefixer.qb64) def escrowTReceipts(self, serder, prefixer, seqner, saider, sigers): """ @@ -4910,8 +4921,8 @@ def escrowTReceipts(self, serder, prefixer, seqner, saider, sigers): quintuple = prelet + siger.qb64b # quintuple self.db.addVre(key=snKey(serder.preb, serder.sn), val=quintuple) # log escrowed - logger.info("Kevery process: escrowed unverified transferable receipt " - "of pre=%s sn=%x dig=%s by pre=%s\n", serder.pre, + logger.debug("Kevery process: escrowed unverified transferable receipt " + "of pre=%s sn=%x dig=%s by pre=%s", serder.pre, serder.sn, serder.ked["d"], prefixer.qb64) def escrowTRQuadruple(self, serder, sprefixer, sseqner, saider, siger): @@ -4943,9 +4954,9 @@ def escrowTRQuadruple(self, serder, sprefixer, sseqner, saider, siger): saider.qb64b + siger.qb64b) self.db.addVre(key=snKey(serder.preb, serder.sn), val=quintuple) # log escrowed - logger.info("Kevery process: escrowed unverified transferabe validator " - "receipt of pre= %s sn=%x dig=%s\n", serder.pre, serder.sn, - serder.said) + logger.debug("Kevery process: escrowed unverified transferabe validator " + "receipt of pre= %s sn=%x dig=%s", serder.pre, serder.sn, + serder.said) def processEscrows(self): """ @@ -4966,9 +4977,9 @@ def processEscrows(self): except Exception as ex: # log diagnostics errors etc if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery escrow process error: %s\n", ex.args[0]) + logger.exception("Kevery escrow process error: %s", ex.args[0]) else: - logger.error("Kevery escrow process error: %s\n", ex.args[0]) + logger.error("Kevery escrow process error: %s", ex.args[0]) raise ex def processEscrowOutOfOrders(self): @@ -5025,7 +5036,7 @@ def processEscrowOutOfOrders(self): if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s\n", bytes(edig)) + " at dig = %s", bytes(edig)) raise ValidationError("Missing escrowed event datetime " "at dig = {}.".format(bytes(edig))) @@ -5036,7 +5047,7 @@ def processEscrowOutOfOrders(self): if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutOOE): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s\n", bytes(edig)) + " at dig = %s", bytes(edig)) raise ValidationError("Stale event escrow " "at dig = {}.".format(bytes(edig))) @@ -5046,7 +5057,7 @@ def processEscrowOutOfOrders(self): if eraw is None: # no event so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event at." - "dig = %s\n", bytes(edig)) + "dig = %s", bytes(edig)) raise ValidationError("Missing escrowed evt at dig = {}." "".format(bytes(edig))) @@ -5058,7 +5069,7 @@ def processEscrowOutOfOrders(self): if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s\n", bytes(edig)) + "dig = %s", bytes(edig)) raise ValidationError("Missing escrowed evt sigs at " "dig = {}.".format(bytes(edig))) @@ -5089,17 +5100,17 @@ def processEscrowOutOfOrders(self): except OutOfOrderError as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery unescrow failed: %s", ex.args[0]) else: - logger.error("Kevery unescrow failed: %s\n", ex.args[0]) + logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delOoe(snKey(pre, sn), edig) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed: %s", ex.args[0]) else: - logger.error("Kevery unescrowed: %s\n", ex.args[0]) + logger.error("Kevery unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -5107,7 +5118,8 @@ def processEscrowOutOfOrders(self): # valid event escrow. self.db.delOoe(snKey(pre, sn), edig) # removes one escrow at key val logger.info("Kevery unescrow succeeded in valid event: " - "event=\n%s\n", json.dumps(eserder.ked, indent=1)) + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") if ekey == key: # still same so no escrows found on last while iteration break @@ -5165,7 +5177,7 @@ def processEscrowPartialSigs(self): if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s\n", bytes(edig)) + " at dig = %s", bytes(edig)) raise ValidationError("Missing escrowed event datetime " "at dig = {}.".format(bytes(edig))) @@ -5176,7 +5188,7 @@ def processEscrowPartialSigs(self): if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s\n", bytes(edig)) + " at dig = %s", bytes(edig)) raise ValidationError("Stale event escrow " "at dig = {}.".format(bytes(edig))) @@ -5186,7 +5198,7 @@ def processEscrowPartialSigs(self): if eraw is None: # no event so so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event at." - "dig = %s\n", bytes(edig)) + "dig = %s", bytes(edig)) raise ValidationError("Missing escrowed evt at dig = {}." "".format(bytes(edig))) @@ -5197,7 +5209,7 @@ def processEscrowPartialSigs(self): if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s\n", bytes(edig)) + "dig = %s", bytes(edig)) raise ValidationError("Missing escrowed evt sigs at " "dig = {}.".format(bytes(edig))) @@ -5244,9 +5256,9 @@ def processEscrowPartialSigs(self): except (MissingSignatureError, MissingDelegationError) as ex: # still waiting on missing sigs or missing seal to validate if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery unescrow failed: %s", ex.args[0]) else: - logger.error("Kevery unescrow failed: %s\n", ex.args[0]) + logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow @@ -5256,9 +5268,9 @@ def processEscrowPartialSigs(self): self.cues.push(dict(kin="psUnescrow", serder=eserder)) if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed: %s", ex.args[0]) else: - logger.error("Kevery unescrowed: %s\n", ex.args[0]) + logger.error("Kevery unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -5271,7 +5283,8 @@ def processEscrowPartialSigs(self): self.cues.push(dict(kin="psUnescrow", serder=eserder)) logger.info("Kevery unescrow succeeded in valid event: " - "event=\n%s\n", json.dumps(eserder.ked, indent=1)) + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") if ekey == key: # still same so no escrows found on last while iteration break @@ -5330,7 +5343,7 @@ def processEscrowPartialWigs(self): if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s\n", bytes(edig)) + " at dig = %s", bytes(edig)) raise ValidationError("Missing escrowed event datetime " "at dig = {}.".format(bytes(edig))) @@ -5341,7 +5354,7 @@ def processEscrowPartialWigs(self): if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s\n", bytes(edig)) + " at dig = %s", bytes(edig)) raise ValidationError("Stale event escrow " "at dig = {}.".format(bytes(edig))) @@ -5351,7 +5364,7 @@ def processEscrowPartialWigs(self): if eraw is None: # no event so so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event at." - "dig = %s\n", bytes(edig)) + "dig = %s", bytes(edig)) raise ValidationError("Missing escrowed evt at dig = {}." "".format(bytes(edig))) @@ -5363,7 +5376,7 @@ def processEscrowPartialWigs(self): if not sigs: # empty list # no sigs so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s\n", bytes(edig)) + "dig = %s", bytes(edig)) raise ValidationError("Missing escrowed evt sigs at " "dig = {}.".format(bytes(edig))) @@ -5376,8 +5389,8 @@ def processEscrowPartialWigs(self): # which may not arrive until some time after event is fully signed # so just log for debugging but do not unescrow by raising # ValidationError - logger.info("Kevery unescrow wigs: No event wigs yet at." - "dig = %s\n", bytes(edig)) + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) # raise ValidationError("Missing escrowed evt wigs at " # "dig = {}.".format(bytes(edig))) @@ -5415,17 +5428,17 @@ def processEscrowPartialWigs(self): except MissingWitnessSignatureError as ex: # still waiting on missing witness sigs if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery unescrow failed: %s", ex.args[0]) else: - logger.error("Kevery unescrow failed: %s\n", ex.args[0]) + logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed: %s", ex.args[0]) else: - logger.error("Kevery unescrowed: %s\n", ex.args[0]) + logger.error("Kevery unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -5433,7 +5446,8 @@ def processEscrowPartialWigs(self): # valid event escrow. self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val logger.info("Kevery unescrow succeeded in valid event: " - "event=\n%s\n", json.dumps(eserder.ked, indent=1)) + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") if ekey == key: # still same so no escrows found on last while iteration break @@ -5504,7 +5518,7 @@ def processEscrowUnverWitness(self): if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s\n", rdiger.qb64b) + " at dig = %s", rdiger.qb64b) raise ValidationError("Missing escrowed event datetime " "at dig = {}.".format(rdiger.qb64b)) @@ -5515,7 +5529,7 @@ def processEscrowUnverWitness(self): if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutUWE): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s\n", rdiger.qb64b) + " at dig = %s", rdiger.qb64b) raise ValidationError("Stale event escrow " "at dig = {}.".format(rdiger.qb64b)) @@ -5529,8 +5543,8 @@ def processEscrowUnverWitness(self): if not found: # no partial witness escrow of event found # so keep in escrow by raising UnverifiedWitnessReceiptError - logger.info("Kevery unescrow error: Missing witness " - "receipted evt at pre=%s sn=%x\n", (pre, sn)) + logger.debug("Kevery unescrow error: Missing witness " + "receipted evt at pre=%s sn=%x", (pre, sn)) raise UnverifiedWitnessReceiptError("Missing witness " "receipted evt at pre={} sn={:x}".format(pre, sn)) @@ -5539,17 +5553,17 @@ def processEscrowUnverWitness(self): # still waiting on missing prior event to validate # only happens if we process above if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery unescrow failed: %s", ex.args[0]) else: - logger.error("Kevery unescrow failed: %s\n", ex.args[0]) + logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrowed: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed: %s", ex.args[0]) else: - logger.error("Kevery unescrowed: %s\n", ex.args[0]) + logger.error("Kevery unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -5557,7 +5571,7 @@ def processEscrowUnverWitness(self): # valid event escrow. self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val logger.info("Kevery unescrow succeeded for event pre=%s " - "sn=%s\n", pre, sn) + "sn=%s", pre, sn) if ekey == key: # still same so no escrows found on last while iteration break @@ -5624,7 +5638,7 @@ def processEscrowUnverNonTrans(self): if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s\n", rsaider.qb64b) + " at dig = %s", rsaider.qb64b) raise ValidationError("Missing escrowed event datetime " "at dig = {}.".format(rsaider.qb64b)) @@ -5635,7 +5649,7 @@ def processEscrowUnverNonTrans(self): if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutURE): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s\n", rsaider.qb64b) + " at dig = %s", rsaider.qb64b) raise ValidationError("Stale event escrow " "at dig = {}.".format(rsaider.qb64b)) @@ -5655,8 +5669,8 @@ def processEscrowUnverNonTrans(self): dig = self.db.getKeLast(snKey(pre, sn)) if dig is None: # no receipted event so keep in escrow - logger.info("Kevery unescrow error: Missing receipted " - "event at pre=%s sn=%x\n", pre, sn) + logger.debug("Kevery unescrow error: Missing receipted " + "event at pre=%s sn=%x", pre, sn) raise UnverifiedReceiptError("Missing receipted evt " "at pre={} sn={:x}".format(pre, sn)) @@ -5665,7 +5679,7 @@ def processEscrowUnverNonTrans(self): raw = self.db.getEvt(dgKey(pre, dig)) if raw is None: # receipted event superseded so remove from escrow logger.info("Kevery unescrow error: Invalid receipted " - "event refereance at pre=%s sn=%x\n", pre, sn) + "event refereance at pre=%s sn=%x", pre, sn) raise ValidationError("Invalid receipted evt reference" " at pre={} sn={:x}".format(pre, sn)) @@ -5675,7 +5689,7 @@ def processEscrowUnverNonTrans(self): # compare digs if rsaider.qb64b != serder.saidb: logger.info("Kevery unescrow error: Bad receipt dig." - "pre=%s sn=%x receipter=%s\n", pre, sn, sprefixer.qb64) + "pre=%s sn=%x receipter=%s", pre, sn, sprefixer.qb64) raise ValidationError("Bad escrowed receipt dig at " "pre={} sn={:x} receipter={}." @@ -5685,7 +5699,7 @@ def processEscrowUnverNonTrans(self): if not cigar.verfer.verify(cigar.raw, serder.raw): # no sigs so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Bad receipt sig." - "pre=%s sn=%x receipter=%s\n", pre, sn, sprefixer.qb64) + "pre=%s sn=%x receipter=%s", pre, sn, sprefixer.qb64) raise ValidationError("Bad escrowed receipt sig at " "pre={} sn={:x} receipter={}." @@ -5712,17 +5726,17 @@ def processEscrowUnverNonTrans(self): # still waiting on missing prior event to validate # only happens if we process above if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery unescrow failed: %s", ex.args[0]) else: - logger.error("Kevery unescrow failed: %s\n", ex.args[0]) + logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delUre(snKey(pre, sn), etriplet) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrowed: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed: %s", ex.args[0]) else: - logger.error("Kevery unescrowed: %s\n", ex.args[0]) + logger.error("Kevery unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -5730,7 +5744,7 @@ def processEscrowUnverNonTrans(self): # valid event escrow. self.db.delUre(snKey(pre, sn), etriplet) # removes one escrow at key val logger.info("Kevery unescrow succeeded for event pre=%s " - "sn=%s\n", pre, sn) + "sn=%s", pre, sn) if ekey == key: # still same so no escrows found on last while iteration break @@ -5773,7 +5787,7 @@ def processQueryNotFound(self): if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s\n", bytes(edig)) + " at dig = %s", bytes(edig)) raise ValidationError("Missing escrowed event datetime " "at dig = {}.".format(bytes(edig))) @@ -5784,7 +5798,7 @@ def processQueryNotFound(self): if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutQNF): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale qry event escrow " - " at dig = %s\n", bytes(edig)) + " at dig = %s", bytes(edig)) raise ValidationError("Stale qry event escrow " "at dig = {}.".format(bytes(edig))) @@ -5794,7 +5808,7 @@ def processQueryNotFound(self): if eraw is None: # no event so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event at." - "dig = %s\n", bytes(edig)) + "dig = %s", bytes(edig)) raise ValidationError("Missing escrowed evt at dig = {}." "".format(bytes(edig))) @@ -5806,7 +5820,7 @@ def processQueryNotFound(self): if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s\n", bytes(edig)) + "dig = %s", bytes(edig)) raise ValidationError("Missing escrowed evt sigs at " "dig = {}.".format(bytes(edig))) @@ -5833,24 +5847,25 @@ def processQueryNotFound(self): except QueryNotFoundError as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery unescrow failed: %s", ex.args[0]) else: - logger.error("Kevery unescrow failed: %s\n", ex.args[0]) + logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delQnf(dgKey(pre, edig), edig) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed: %s", ex.args[0]) else: - logger.error("Kevery unescrowed: %s\n", ex.args[0]) + logger.error("Kevery unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delQnf(dgKey(pre, edig), edig) # removes one escrow at key val logger.info("Kevery unescrow succeeded in valid event: " - "event=\n%s\n", json.dumps(eserder.ked, indent=1)) + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") if ekey == key: # still same so no escrows found on last while iteration break @@ -5942,7 +5957,7 @@ def _processEscrowFindUnver(self, pre, sn, rsaider, wiger=None, cigar=None): if wiger.index >= len(wits): # bad index # raise ValidationError which removes from escrow by caller logger.info("Kevery unescrow error: Bad witness receipt" - " index=%i for pre=%s sn=%x\n", wiger.index, pre, sn) + " index=%i for pre=%s sn=%x", wiger.index, pre, sn) raise ValidationError("Bad escrowed witness receipt index={}" " at pre={} sn={:x}.".format(wiger.index, pre, sn)) @@ -5954,7 +5969,7 @@ def _processEscrowFindUnver(self, pre, sn, rsaider, wiger=None, cigar=None): if not wiger.verfer.verify(wiger.raw, serder.raw): # not verify # raise ValidationError which unescrows .Uwes or .Ures in caller logger.info("Kevery unescrow error: Bad witness receipt" - " wig. pre=%s sn=%x\n", pre, sn) + " wig. pre=%s sn=%x", pre, sn) raise ValidationError("Bad escrowed witness receipt wig" " at pre={} sn={:x}." @@ -6023,7 +6038,7 @@ def processEscrowUnverTrans(self): if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s\n", esaider.qb64b) + " at dig = %s", esaider.qb64b) raise ValidationError("Missing escrowed event datetime " "at dig = {}.".format(esaider.qb64b)) @@ -6034,7 +6049,7 @@ def processEscrowUnverTrans(self): if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutVRE): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s\n", esaider.qb64b) + " at dig = %s", esaider.qb64b) raise ValidationError("Stale event escrow " "at dig = {}.".format(esaider.qb64b)) @@ -6043,8 +6058,8 @@ def processEscrowUnverTrans(self): raw = self.db.getKeLast(snKey(pre, sn)) if raw is None: # no event so keep in escrow - logger.info("Kevery unescrow error: Missing receipted " - "event at pre=%s sn=%x\n", pre, sn) + logger.debug("Kevery unescrow error: Missing receipted " + "event at pre=%s sn=%x", pre, sn) raise UnverifiedTransferableReceiptError("Missing receipted evt at pre={} " " sn={:x}".format(pre, sn)) @@ -6054,7 +6069,7 @@ def processEscrowUnverTrans(self): raw = self.db.getEvt(dgKey(pre, dig)) if raw is None: # receipted event superseded so remove from escrow logger.info("Kevery unescrow error: Invalid receipted " - "event referenace at pre=%s sn=%x\n", pre, sn) + "event referenace at pre=%s sn=%x", pre, sn) raise ValidationError("Invalid receipted evt reference " "at pre={} sn={:x}".format(pre, sn)) @@ -6064,7 +6079,7 @@ def processEscrowUnverTrans(self): # compare digs if esaider.qb64b != serder.saidb: logger.info("Kevery unescrow error: Bad receipt dig." - "pre=%s sn=%x receipter=%s\n", (pre, sn, sprefixer.qb64)) + "pre=%s sn=%x receipter=%s", (pre, sn, sprefixer.qb64)) raise ValidationError("Bad escrowed receipt dig at " "pre={} sn={:x} receipter={}." @@ -6076,8 +6091,8 @@ def processEscrowUnverTrans(self): sn=sseqner.sn)) if sdig is None: # no event so keep in escrow - logger.info("Kevery unescrow error: Missing receipted " - "event at pre=%s sn=%x\n", pre, sn) + logger.debug("Kevery unescrow error: Missing receipted " + "event at pre=%s sn=%x", pre, sn) raise UnverifiedTransferableReceiptError("Missing receipted evt at pre={} " " sn={:x}".format(pre, sn)) @@ -6108,7 +6123,7 @@ def processEscrowUnverTrans(self): siger.verfer = verfers[siger.index] # assign verfer if not siger.verfer.verify(siger.raw, serder.raw): # verify sig logger.info("Kevery unescrow error: Bad trans receipt sig." - "pre=%s sn=%x receipter=%s\n", pre, sn, sprefixer.qb64) + "pre=%s sn=%x receipter=%s", pre, sn, sprefixer.qb64) raise ValidationError("Bad escrowed trans receipt sig at " "pre={} sn={:x} receipter={}." @@ -6123,24 +6138,25 @@ def processEscrowUnverTrans(self): # still waiting on missing prior event to validate # only happens if we process above if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery unescrow failed: %s", ex.args[0]) else: - logger.error("Kevery unescrow failed: %s\n", ex.args[0]) + logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delVre(snKey(pre, sn), equinlet) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrowed: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed: %s", ex.args[0]) else: - logger.error("Kevery unescrowed: %s\n", ex.args[0]) + logger.error("Kevery unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delVre(snKey(pre, sn), equinlet) # removes one escrow at key val - logger.info("Kevery unescrow succeeded for event = %s\n", serder.ked) + logger.info("Kevery unescrow succeeded for event = %s", serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") if ekey == key: # still same so no escrows found on last while iteration break @@ -6198,7 +6214,7 @@ def processEscrowDuplicitous(self): if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s\n", bytes(edig)) + " at dig = %s", bytes(edig)) raise ValidationError("Missing escrowed event datetime " "at dig = {}.".format(bytes(edig))) @@ -6209,7 +6225,7 @@ def processEscrowDuplicitous(self): if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutLDE): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s\n", bytes(edig)) + " at dig = %s", bytes(edig)) raise ValidationError("Stale event escrow " "at dig = {}.".format(bytes(edig))) @@ -6219,7 +6235,7 @@ def processEscrowDuplicitous(self): if eraw is None: # no event so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event at." - "dig = %s\n", bytes(edig)) + "dig = %s", bytes(edig)) raise ValidationError("Missing escrowed evt at dig = {}." "".format(bytes(edig))) @@ -6231,7 +6247,7 @@ def processEscrowDuplicitous(self): if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s\n", bytes(edig)) + "dig = %s", bytes(edig)) raise ValidationError("Missing escrowed evt sigs at " "dig = {}.".format(bytes(edig))) @@ -6256,17 +6272,17 @@ def processEscrowDuplicitous(self): except LikelyDuplicitousError as ex: # still can't determine if duplicitous if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery unescrow failed: %s", ex.args[0]) else: - logger.error("Kevery unescrow failed: %s\n", ex.args[0]) + logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than likely duplicitous so remove from escrow self.db.delLde(snKey(pre, sn), edig) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed: %s", ex.args[0]) else: - logger.error("Kevery unescrowed: %s\n", ex.args[0]) + logger.error("Kevery unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -6274,7 +6290,8 @@ def processEscrowDuplicitous(self): # valid event escrow. self.db.delLde(snKey(pre, sn), edig) # removes one escrow at key val logger.info("Kevery unescrow succeeded in valid event: " - "event=\n%s\n", json.dumps(eserder.ked, indent=1)) + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") if ekey == key: # still same so no escrows found on last while iteration break diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index 88d572e4d..e1275075c 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -6,17 +6,14 @@ """ import logging -import traceback -from collections import namedtuple -from dataclasses import dataclass, astuple from .coring import (Ilks, CtrDex, Counter, Seqner, Cigar, - Dater, Verfer, Prefixer, Saider, Pather, Protocols ) + Dater, Verfer, Prefixer, Saider, Pather) from .indexing import (Siger, ) from . import serdering from .. import help from .. import kering -from ..kering import ColdDex, Colds, sniff +from ..kering import Colds, sniff logger = help.ogler.getLogger() @@ -430,25 +427,25 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Parser msg extraction error: %s", ex.args[0]) else: - logger.error("Parser msg extraction error: %s\n", ex.args[0]) + logger.error("Parser msg extraction error: %s", ex.args[0]) except (kering.ColdStartError, kering.ExtractionError) as ex: # some extraction error - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Parser msg extraction error: %s", ex.args[0]) else: - logger.error("Parser msg extraction error: %s\n", ex.args[0]) + logger.error("Parser msg extraction error: %s", ex.args[0]) del ims[:] # delete rest of stream to force cold restart except (kering.ValidationError, Exception) as ex: # non Extraction Error # Non extraction errors happen after successfully extracted from stream # so we don't flush rest of stream just resume - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg non-extraction error: %s\n", ex) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Parser msg non-extraction error: %s", ex) else: - logger.error("Parser msg non-extraction error: %s\n", ex) + logger.error("Parser msg non-extraction error: %s", ex) yield return True @@ -515,25 +512,25 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream - if logger.isEnabledFor(logging.ERROR): - logger.exception("Kevery msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery msg extraction error: %s", ex.args[0]) else: - logger.error("Kevery msg extraction error: %s\n", ex.args[0]) + logger.error("Kevery msg extraction error: %s", ex.args[0]) except (kering.ColdStartError, kering.ExtractionError) as ex: # some extraction error - if logger.isEnabledFor(logging.ERROR): - logger.exception("Kevery msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery msg extraction error: %s", ex.args[0]) else: - logger.error("Kevery msg extraction error: %s\n", ex.args[0]) + logger.error("Kevery msg extraction error: %s", ex.args[0]) del ims[:] # delete rest of stream to force cold restart except (kering.ValidationError, Exception) as ex: # non Extraction Error # Non extraction errors happen after successfully extracted from stream # so we don't flush rest of stream just resume - if logger.isEnabledFor(logging.ERROR): - logger.exception("Kevery msg non-extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery msg non-extraction error: %s", ex.args[0]) else: - logger.error("Kevery msg non-extraction error: %s\n", ex.args[0]) + logger.error("Kevery msg non-extraction error: %s", ex.args[0]) finally: done = True @@ -605,25 +602,25 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Parser msg extraction error: %s", ex.args[0]) else: - logger.error("Parser msg extraction error: %s\n", ex.args[0]) + logger.error("Parser msg extraction error: %s", ex.args[0]) except (kering.ColdStartError, kering.ExtractionError) as ex: # some extraction error - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Parser msg extraction error: %s", ex.args[0]) else: - logger.error("Parser msg extraction error: %s\n", ex.args[0]) + logger.error("Parser msg extraction error: %s", ex.args[0]) del ims[:] # delete rest of stream to force cold restart except (kering.ValidationError, Exception) as ex: # non Extraction Error # Non extraction errors happen after successfully extracted from stream # so we don't flush rest of stream just resume - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg non-extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Parser msg non-extraction error: %s", ex.args[0]) else: - logger.error("Parser msg non-extraction error: %s\n", ex.args[0]) + logger.error("Parser msg non-extraction error: %s", ex.args[0]) yield return True # should never return diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 2b3856b41..8d119a27b 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -267,25 +267,30 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, if not self.lax and cigar.verfer.qb64 in self.prefixes: # own cig if not self.local: # own cig when not local so ignore logger.info("Kevery process: skipped own attachment" - " on nonlocal reply msg=\n%s\n", serder.pretty()) + " on nonlocal reply said=", serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") + continue # skip own cig attachment on non-local reply msg if aid != cigar.verfer.qb64: # cig not by aid logger.info("Kevery process: skipped cig not from aid=" - "%s on reply msg=\n%s\n", aid, serder.pretty()) + "%s on reply said=%s", aid, serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") continue # skip invalid cig's verfer is not aid if odater: # get old compare datetimes to see if later if dater.datetime <= odater.datetime: logger.info("Kevery process: skipped stale update from " - "%s of reply msg=\n%s\n", aid, serder.pretty()) + "%s of reply said=%s", aid, serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") continue # skip if not later # raise ValidationError(f"Stale update of {route} from {aid} " # f"via {Ilks.rpy}={serder.ked}.") if not cigar.verfer.verify(cigar.raw, serder.raw): # cig not verify logger.info("Kevery process: skipped nonverifying cig from " - "%s on reply msg=\n%s\n", cigar.verfer.qb64, serder.pretty()) + "%s on reply said=%s", cigar.verfer.qb64, serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") continue # skip if cig not verify # All constraints satisfied so update @@ -298,13 +303,15 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, if not self.lax and prefixer.qb64 in self.prefixes: # own sig if not self.local: # own sig when not local so ignore logger.info("Kevery process: skipped own attachment" - " on nonlocal reply msg=\n%s\n", serder.pretty()) + " on nonlocal reply said=%s", serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") continue # skip own sig attachment on non-local reply msg spre = prefixer.qb64 if aid != spre: # sig not by aid logger.info("Kevery process: skipped signature not from aid=" - "%s on reply msg=\n%s\n", aid, serder.pretty()) + "%s on reply said=%s", aid, serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") continue # skip invalid signature is not from aid if osaider: # check if later logic sn > or sn == and dt > @@ -313,16 +320,18 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, if seqner.sn < osqr.sn: # sn earlier logger.info("Kevery process: skipped stale key state sig" - "from %s sn=%s<%s on reply msg=\n%s\n", - aid, seqner.sn, osqr.sn, serder.pretty()) + "from %s sn=%s<%s on reply said=%s", + aid, seqner.sn, osqr.sn, serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") continue # skip if sn earlier if seqner.sn == osqr.sn: # sn same so check datetime if odater: if dater.datetime <= odater.datetime: logger.info("Kevery process: skipped stale key" - "state sig datetime from %s on reply msg=\n%s\n", - aid, serder.pretty()) + "state sig datetime from %s on reply said=%s", + aid, serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") continue # skip if not later # retrieve sdig of last event at sn of signer. @@ -475,7 +484,7 @@ def processEscrowReply(self): datetime.timedelta(seconds=self.TimeoutRPE)): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale reply escrow " - " at route = %s\n", route) + " at route = %s", route) raise kering.ValidationError(f"Stale reply escrow at route = {route}.") @@ -484,30 +493,31 @@ def processEscrowReply(self): except kering.UnverifiedReplyError as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow attempt failed: %s\n", ex.args[0]) + logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) else: - logger.error("Kevery unescrow attempt failed: %s\n", ex.args[0]) + logger.error("Kevery unescrow attempt failed: %s", ex.args[0]) except Exception as ex: # other error so remove from reply escrow self.db.rpes.rem(keys=(route, ), val=saider) # remove escrow only self.removeReply(saider) # remove escrow reply artifacts if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: - logger.error("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.error("Kevery unescrowed due to error: %s", ex.args[0]) else: # unescrow succeded self.db.rpes.rem(keys=(route, ), val=saider) # remove escrow only - logger.info("Kevery unescrow succeeded for reply=\n%s\n", - serder.pretty()) + logger.info("Kevery unescrow succeeded for reply said=%s", + serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") except Exception as ex: # log diagnostics errors etc self.db.rpes.rem(keys=(route,), val=saider) # remove escrow only self.removeReply(saider) # remove escrow reply artifacts if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: - logger.error("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.error("Kevery unescrowed due to error: %s", ex.args[0]) class Route: diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 71fe06248..e1818b993 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -104,7 +104,7 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): datetime.timedelta(seconds=self.timeout)): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale txn state escrow " - " at pre = %s\n", pre) + " at pre = %s", pre) raise kering.ValidationError(f"Stale txn state escrow at pre = {pre}.") @@ -114,29 +114,30 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): except extype as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow attempt failed: %s\n", ex.args[0]) + logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) else: - logger.error("Kevery unescrow attempt failed: %s\n", ex.args[0]) + logger.error("Kevery unescrow attempt failed: %s", ex.args[0]) except Exception as ex: # other error so remove from reply escrow self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: - logger.error("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.error("Kevery unescrowed due to error: %s", ex.args[0]) else: # unescrow succeded self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow only - logger.info("Kevery unescrow succeeded for txn state=\n%s\n", - serder.pretty()) + logger.info("Kevery unescrow succeeded for txn state=%s", + serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") except Exception as ex: # log diagnostics errors etc self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow self.removeState(saider) if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: - logger.error("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.error("Kevery unescrowed due to error: %s", ex.args[0]) def escrowStateNotice(self, *, typ, pre, aid, serder, saider, dater, cigars=None, tsgs=None): """ diff --git a/src/keri/peer/exchanging.py b/src/keri/peer/exchanging.py index 570e1492d..fcbefa8a8 100644 --- a/src/keri/peer/exchanging.py +++ b/src/keri/peer/exchanging.py @@ -121,11 +121,13 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): # Perform behavior specific verification, think IPEX chaining requirements try: if not behavior.verify(serder=serder, attachments=attachments): - logger.info(f"exn event for route {route} failed behavior verfication. exn={serder.ked}") + logger.info(f"exn event for route {route} failed behavior verfication. said={serder.said}") + logger.debug(f"event=\n{serder.pretty()}\n") return except AttributeError: - logger.info(f"Behavior for {route} missing or does not have verify for exn={serder.ked}") + logger.info(f"Behavior for {route} missing or does not have verify for said={serder.said}") + logger.debug(f"event=\n{serder.pretty()}\n") # Always persis events self.logEvent(serder, pathed, tsgs, cigars) @@ -135,7 +137,8 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): try: behavior.handle(serder=serder, attachments=attachments) except AttributeError: - logger.info(f"Behavior for {route} missing or does not have handle for exn={serder.ked}") + logger.info(f"Behavior for {route} missing or does not have handle for said={serder.said}") + logger.debug(f"event=\n{serder.pretty()}\n") def processEscrow(self): """ Process all escrows for `exn` messages @@ -190,21 +193,22 @@ def processEscrowPartialSigned(self): except MissingSignatureError as ex: if logger.isEnabledFor(logging.DEBUG): - logger.info("Exchange partially signed unescrow failed: %s\n", ex.args[0]) + logger.info("Exchange partially signed unescrow failed: %s", ex.args[0]) else: - logger.info("Exchange partially signed failed: %s\n", ex.args[0]) + logger.info("Exchange partially signed failed: %s", ex.args[0]) except Exception as ex: self.hby.db.epse.rem(dig) self.hby.db.esigs.rem(dig) if logger.isEnabledFor(logging.DEBUG): - logger.info("Exchange partially signed unescrowed: %s\n", ex.args[0]) + logger.info("Exchange partially signed unescrowed: %s", ex.args[0]) else: - logger.info("Exchange partially signed unescrowed: %s\n", ex.args[0]) + logger.info("Exchange partially signed unescrowed: %s", ex.args[0]) else: self.hby.db.epse.rem(dig) self.hby.db.esigs.rem(dig) logger.info("Exchanger unescrow succeeded in valid exchange: " - "creder=\n%s\n", serder.pretty()) + "creder=%s", serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") def logEvent(self, serder, pathed=None, tsgs=None, cigars=None): dig = serder.said diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index 8a85c8e54..7200f8192 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -232,7 +232,8 @@ def processEvent(self, serder): try: self.tvy.processEvent(serder=serder) except kering.MissingAnchorError: - logger.info("Credential registry missing anchor for inception = {}".format(serder.ked)) + logger.info("Credential registry missing anchor for inception = {}".format(serder.said)) + logger.debug(f"event=\n{serder.pretty()}\n") def anchorMsg(self, pre, regd, seqner, saider): """ Create key event with seal to serder anchored as data. diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 3bd5ab55f..278eb4bac 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -1265,8 +1265,9 @@ def logEvent(self, pre, sn, serder, seqner, saider, bigers=None, baks=None): self.reger.tets.pin(keys=(pre.decode("utf-8"), dig.decode("utf-8")), val=coring.Dater()) self.reger.putTvt(key, serder.raw) self.reger.putTel(snKey(pre, sn), dig) - logger.info("Tever state: %s Added to TEL valid event=\n%s\n", - pre, json.dumps(serder.ked, indent=1)) + logger.info("Tever state: %s Added to TEL valid said=%s", + pre, serder.said) + logger.debug(f"event=\n{serder.pretty()}\n") def valAnchorBigs(self, serder, seqner, saider, bigers, toad, baks): """ Validate anchor and backer signatures (bigers) when provided. @@ -1394,8 +1395,8 @@ def escrowPWEvent(self, serder, seqner, saider, bigers=None): self.reger.putTibs(dgkey, [biger.qb64b for biger in bigers]) self.reger.putTvt(dgkey, serder.raw) self.reger.putTwe(snKey(serder.preb, serder.sn), serder.saidb) - logger.info("Tever state: Escrowed partially witnessed " - "event = %s\n", serder.ked) + logger.debug("Tever state: Escrowed partially witnessed " + "event = %s", serder.ked) def escrowALEvent(self, serder, seqner, saider, bigers=None, baks=None): """ Update associated logs for escrow of anchorless event @@ -1421,8 +1422,8 @@ def escrowALEvent(self, serder, seqner, saider, bigers=None, baks=None): self.reger.delBaks(key) self.reger.putBaks(key, [bak.encode("utf-8") for bak in baks]) self.reger.putTvt(key, serder.raw) - logger.info("Tever state: Escrowed anchorless event " - "event = %s\n", serder.ked) + logger.debug("Tever state: Escrowed anchorless event " + "event = %s", serder.ked) return self.reger.putTae(snKey(serder.preb, serder.sn), serder.saidb) def getBackerState(self, ked): @@ -2005,8 +2006,8 @@ def escrowOOEvent(self, serder, seqner, saider): sealet = seqner.qb64b + saider.qb64b self.reger.putAnc(key, sealet) self.reger.putOot(snKey(serder.preb, serder.sn), serder.saidb) - logger.info("Tever state: Escrowed our of order TEL event " - "event = %s\n", serder.ked) + logger.debug("Tever state: Escrowed our of order TEL event " + "event = %s", serder.ked) def processEscrows(self): """ Loop through escrows and process and events that may now be finalized """ @@ -2027,9 +2028,9 @@ def processEscrows(self): except Exception as ex: # log diagnostics errors etc if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery escrow process error: %s\n", ex.args[0]) + logger.exception("Tevery escrow process error: %s", ex.args[0]) else: - logger.error("Tevery escrow process error: %s\n", ex.args[0]) + logger.error("Tevery escrow process error: %s", ex.args[0]) def processEscrowOutOfOrders(self): """ Loop through out of order escrow: @@ -2050,7 +2051,7 @@ def processEscrowOutOfOrders(self): if traw is None: # no event so raise ValidationError which unescrows below logger.info("Tevery unescrow error: Missing event at." - "dig = %s\n", bytes(digb)) + "dig = %s", bytes(digb)) raise ValidationError("Missing escrowed evt at dig = {}." "".format(bytes(digb))) @@ -2064,7 +2065,7 @@ def processEscrowOutOfOrders(self): couple = self.reger.getAnc(dgkey) if couple is None: logger.info("Tevery unescrow error: Missing anchor at." - "dig = %s\n", bytes(digb)) + "dig = %s", bytes(digb)) raise ValidationError("Missing escrowed anchor at dig = {}." "".format(bytes(digb))) @@ -2077,17 +2078,17 @@ def processEscrowOutOfOrders(self): except OutOfOrderError as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery unescrow failed: %s\n", ex.args[0]) + logger.exception("Tevery unescrow failed: %s", ex.args[0]) else: - logger.error("Tevery unescrow failed: %s\n", ex.args[0]) + logger.error("Tevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.reger.delOot(snKey(pre, sn)) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery unescrowed: %s\n", ex.args[0]) + logger.exception("Tevery unescrowed: %s", ex.args[0]) else: - logger.error("Tevery unescrowed: %s\n", ex.args[0]) + logger.error("Tevery unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -2095,7 +2096,8 @@ def processEscrowOutOfOrders(self): # valid event escrow. self.reger.delOot(snKey(pre, sn)) # removes from escrow logger.info("Tevery unescrow succeeded in valid event: " - "event=\n%s\n", json.dumps(tserder.ked, indent=1)) + "said=%s", tserder.said) + logger.debug(f"event=\n{tserder.pretty()}\n") def processEscrowAnchorless(self): """ Process escrow of TEL events received before the anchoring KEL event. @@ -2117,7 +2119,7 @@ def processEscrowAnchorless(self): if traw is None: # no event so raise ValidationError which unescrows below logger.info("Tevery unescrow error: Missing event at." - "dig = %s\n", bytes(digb)) + "dig = %s", bytes(digb)) raise ValidationError("Missing escrowed evt at dig = {}." "".format(bytes(digb))) @@ -2131,7 +2133,7 @@ def processEscrowAnchorless(self): couple = self.reger.getAnc(dgkey) if couple is None: logger.info("Tevery unescrow error: Missing anchor at." - "dig = %s\n", bytes(digb)) + "dig = %s", bytes(digb)) raise MissingAnchorError("Missing escrowed anchor at dig = {}." "".format(bytes(digb))) @@ -2144,17 +2146,17 @@ def processEscrowAnchorless(self): except MissingAnchorError as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery unescrow failed: %s\n", ex.args[0]) + logger.exception("Tevery unescrow failed: %s", ex.args[0]) else: - logger.error("Tevery unescrow failed: %s\n", ex.args[0]) + logger.error("Tevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.reger.delTae(snKey(pre, sn)) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery unescrowed: %s\n", ex.args[0]) + logger.exception("Tevery unescrowed: %s", ex.args[0]) else: - logger.error("Tevery unescrowed: %s\n", ex.args[0]) + logger.error("Tevery unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -2162,4 +2164,5 @@ def processEscrowAnchorless(self): # valid event escrow. self.reger.delTae(snKey(pre, sn)) # removes from escrow logger.info("Tevery unescrow succeeded in valid event: " - "event=\n%s\n", json.dumps(tserder.ked, indent=1)) + "said=%s", tserder.said) + logger.debug(f"event=\n{tserder.pretty()}\n") diff --git a/src/keri/vdr/verifying.py b/src/keri/vdr/verifying.py index 7f1c1a237..e9bdf5858 100644 --- a/src/keri/vdr/verifying.py +++ b/src/keri/vdr/verifying.py @@ -255,7 +255,7 @@ def _processEscrow(self, db, timeout, etype: Type[Exception]): if (dtnow - dte) > datetime.timedelta(seconds=timeout): # escrow stale so raise ValidationError which unescrows below logger.info("Verifier unescrow error: Stale event escrow " - " at said = %s\n", said) + " at said = %s", said) raise kering.ValidationError("Stale event escrow " "at said = {}.".format(said)) @@ -264,20 +264,21 @@ def _processEscrow(self, db, timeout, etype: Type[Exception]): except etype as ex: if logger.isEnabledFor(logging.DEBUG): - logger.exception("Verifiery unescrow failed: %s\n", ex.args[0]) + logger.exception("Verifiery unescrow failed: %s", ex.args[0]) else: - logger.error("Verifier unescrow failed: %s\n", ex.args[0]) + logger.error("Verifier unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than missing sigs so remove from PA escrow db.rem(said) if logger.isEnabledFor(logging.DEBUG): - logger.exception("Verifier unescrowed: %s\n", ex.args[0]) + logger.exception("Verifier unescrowed: %s", ex.args[0]) else: - logger.error("Verifier unescrowed: %s\n", ex.args[0]) + logger.error("Verifier unescrowed: %s", ex.args[0]) else: db.rem(said) logger.info("Verifier unescrow succeeded in valid group op: " - "creder=\n%s\n", creder.pretty()) + "creder=%s", creder.said) + logger.debug(f"event=\n{creder.pretty()}\n") def saveCredential(self, creder, prefixer, seqner, saider): """ Write the credential and associated indicies to the database From b4b9b36af003dba25f30101ebef55f4cbf87b1a9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 13 Apr 2024 16:46:28 -0600 Subject: [PATCH 148/418] added tests for rotate for CESR native --- tests/core/test_serdering.py | 229 +++++++++++++++++++++++++++++++++-- 1 file changed, 221 insertions(+), 8 deletions(-) diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 02f2d436a..82f4507dc 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -29,7 +29,7 @@ from keri.core.serdering import (FieldDom, FieldDom, Serdery, Serder, SerderKERI, SerderACDC, ) -from keri.core.eventing import (incept, ) +from keri.core.eventing import (incept, interact, rotate, delcept, deltate) from keri.app import habbing @@ -2684,11 +2684,11 @@ def test_cesr_native_dumps(): rawsalt = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' salter = core.Salter(raw=rawsalt) - csigners = salter.signers(count=3, transferable=True, temp=True) - wsigners = salter.signers(count=3, transferable=False, temp=True) + csigners = salter.signers(count=12, transferable=True, temp=True) + wsigners = salter.signers(count=12, transferable=False, temp=True) - # simple event + # simple inception event keys = [csigners[0].verfer.qb64] assert keys == ['DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ'] @@ -2741,11 +2741,11 @@ def test_cesr_native_dumps(): assert ratios == [1.0, 1.33, 1.46, 1.46, 1.83] - # more complex event + # more complex inception event - keys = [signer.verfer.qb64 for signer in csigners] - ndigs = [core.Diger(ser=key.encode()).qb64 for key in keys] - wits = [signer.verfer.qb64 for signer in wsigners] + keys = [signer.verfer.qb64 for signer in csigners][:3] + ndigs = [core.Diger(ser=key.encode()).qb64 for key in keys][:3] + wits = [signer.verfer.qb64 for signer in wsigners][:3] data = [dict(i=keys[0], s=core.Number(num=0).numh, d=ndigs[0]), dict(i=keys[1], s=core.Number(num=1).numh, d=ndigs[1]), dict(s=core.Number(num=15).numh, d=ndigs[2])] @@ -2778,6 +2778,11 @@ def test_cesr_native_dumps(): version=Vrsn_2_0, kind=kering.Serials.cesr) + pre = serder.pre + assert pre == 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6' + said = serder.said + assert said == pre + assert serder.sad == \ { 'v': 'KERICAACESRAAAA.', @@ -2846,6 +2851,7 @@ def test_cesr_native_dumps(): assert helping.b64ToInt(sizeh) * 4 + 4 == serder.size == 780 rawqb64 = serder._dumps() # default is it dumps self.sad + assert rawqb64 == serder.raw assert rawqb64 == (b'-FDCYKERICAAXicpEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6EMEv' b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAAAMAAC-LAhDG9XhvcVryHj' b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' @@ -2881,6 +2887,213 @@ def test_cesr_native_dumps(): assert ratios == [1.0, 1.33, 1.42, 1.42, 1.56] + # complex interaction event + + prior = said + + data = \ + [ + dict(i=keys[0], s=core.Number(num=2).numh, d=ndigs[0]), + dict(i=keys[1], s=core.Number(num=34).numh, d=ndigs[1]), + dict(s=core.Number(num=67).numh, d=ndigs[2]), + dict(i=keys[2], s=core.Number(num=128).numh, d=ndigs[0]) + ] + + assert data == \ + [ + { + 'i': 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', + 's': '2', + 'd': 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_' + }, + { + 'i': 'DK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', + 's': '22', + 'd': 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG' + }, + { + 's': '43', + 'd': 'EEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZpZEyP' + }, + { + 'i': 'DMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0', + 's': '80', + 'd': 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_' + }, + ] + + + serder = interact(pre=pre, + dig=prior, + sn=1, + data=data, + version=Vrsn_2_0, + kind=kering.Serials.cesr) + + said = serder.said + assert said == 'EHeLJVa4LLNRRYVkLQsXHIDvllcmhDaahe5a_oMvXKeP' + + assert serder.sad == \ + { + 'v': 'KERICAACESRAAAA.', + 't': 'ixn', + 'd': 'EHeLJVa4LLNRRYVkLQsXHIDvllcmhDaahe5a_oMvXKeP', + 'i': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6', + 's': '1', + 'p': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6', + 'a': + [ + { + 'i': 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', + 's': '2', + 'd': 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_' + }, + { + 'i': 'DK58m521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cn', + 's': '22', + 'd': 'EMrowWRk6u1imR32ZNHnTPUtc7uSAvrchIPN3I8S6vUG' + }, + { + 's': '43', + 'd': 'EEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZpZEyP' + }, + { + 'i': 'DMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0', + 's': '80', + 'd': 'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_' + } + ] + } + + assert serder.raw == (b'-FB6YKERICAAXixnEHeLJVa4LLNRRYVkLQsXHIDvllcmhDaahe5a_oMvXKePEMEv' + b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAABEMEvSn0o6Iv2-3gInTDM' + b'MDTV0qQEfooM-yTzkj6Kynn6-LBU-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'4fBJre3NGwTQMAACEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' + b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAAiEMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMABDEEbufBpvagqe9kijKISOoQPYFEOpy22C' + b'ZJGJqQZpZEyP-RAXDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MACA' + b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_') + + assert len(serder.raw) == serder.size == 492 + sizeh = serder.raw[2:4] + assert sizeh == b"B6" + assert helping.b64ToInt(sizeh) * 4 + 4 == serder.size == 492 + + rawqb64 = serder._dumps() # default is it dumps self.sad + assert rawqb64 == serder.raw + assert len(rawqb64) == 492 + + rawqb2 = decodeB64(rawqb64) + assert len(rawqb2) == 369 + assert rawqb64 == encodeB64(rawqb2) # round trips + + rawjson = serder.dumps(serder.sad) + assert len(rawjson) == 601 + + rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + assert len(rawcbor) == 536 + + rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + assert len(rawmgpk) == 536 + + raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] + ratios = [ round(len(raw) / len(rawqb2), 2) for raw in raws] + + assert ratios ==[1.0, 1.33, 1.45, 1.45, 1.63] + + # complex rotation event + + prior = said + + keys = [signer.verfer.qb64 for signer in csigners][3:6] + ndigs = [core.Diger(ser=key.encode()).qb64 for key in keys] + cuts = [wits[0]] + adds = [signer.verfer.qb64 for signer in wsigners][3:4] + data = {} # no anchors + + + serder = rotate(pre=pre, + keys=keys, + dig=prior, + sn=2, + ndigs=ndigs, + wits=wits, #prior + cuts=cuts, + adds=adds, + data=data, + version=Vrsn_2_0, + kind=kering.Serials.cesr) + + said = serder.said + assert said == 'EDtBwgOB0uGrSMBJhOmnkRoCupjg-4sJApvOx04ujhKs' + + assert serder.sad == \ + { + 'v': 'KERICAACESRAAAA.', + 't': 'rot', + 'd': 'EDtBwgOB0uGrSMBJhOmnkRoCupjg-4sJApvOx04ujhKs', + 'i': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6', + 's': '2', + 'p': 'EHeLJVa4LLNRRYVkLQsXHIDvllcmhDaahe5a_oMvXKeP', + 'kt': '2', + 'k': ['DH7p14xo09rob5cEupmo8jSDi35ZOGt1k4t2nm1C1A68', + 'DIAdqJzLWEwQbhXEMOFjvFVZ7oMCJP4XXDP_ILaTEBAQ', + 'DKhYdMBeP6FoH3ajGJTf_4fH229rm_lTZXfYkfwGTMER'], + 'nt': '2', + 'n': + [ + 'EBvDSpcj3y0y9W2-1GzYJ85KEkDIPxu4y_TxAK49k7ci', + 'EEb97lh2oOd_yM3meBaRX5xSs8mIeBoPdhOTgVkd31jb', + 'ECQTrhKHgrOXJS4kdvifvOqoJ7RjfJSsN3nshclYStga' + ], + 'bt': '3', + 'br': ['BG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ'], + 'ba': ['BH7p14xo09rob5cEupmo8jSDi35ZOGt1k4t2nm1C1A68'], + 'c': [], + 'a': {} + } + + + assert serder.raw == (b'-FCGYKERICAAXrotEDtBwgOB0uGrSMBJhOmnkRoCupjg-4sJApvOx04ujhKsEMEv' + b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAACEHeLJVa4LLNRRYVkLQsX' + b'HIDvllcmhDaahe5a_oMvXKePMAAC-LAhDH7p14xo09rob5cEupmo8jSDi35ZOGt1' + b'k4t2nm1C1A68DIAdqJzLWEwQbhXEMOFjvFVZ7oMCJP4XXDP_ILaTEBAQDKhYdMBe' + b'P6FoH3ajGJTf_4fH229rm_lTZXfYkfwGTMERMAAC-LAhEBvDSpcj3y0y9W2-1GzY' + b'J85KEkDIPxu4y_TxAK49k7ciEEb97lh2oOd_yM3meBaRX5xSs8mIeBoPdhOTgVkd' + b'31jbECQTrhKHgrOXJS4kdvifvOqoJ7RjfJSsN3nshclYStgaMAAD-LALBG9XhvcV' + b'ryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ-LALBH7p14xo09rob5cEupmo8jSD' + b'i35ZOGt1k4t2nm1C1A68-LAA-LAA') + + assert len(serder.raw) == serder.size == 540 + sizeh = serder.raw[2:4] + assert sizeh == b"CG" + assert helping.b64ToInt(sizeh) * 4 + 4 == serder.size == 540 + + rawqb64 = serder._dumps() # default is it dumps self.sad + assert rawqb64 == serder.raw + assert len(rawqb64) == 540 + + rawqb2 = decodeB64(rawqb64) + assert len(rawqb2) == 405 + assert rawqb64 == encodeB64(rawqb2) # round trips + + rawjson = serder.dumps(serder.sad) + assert len(rawjson) == 638 + + rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + assert len(rawcbor) == 577 + + rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + assert len(rawmgpk) == 577 + + raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] + ratios = [ round(len(raw) / len(rawqb2), 2) for raw in raws] + + assert ratios == [1.0, 1.33, 1.42, 1.42, 1.58] + + + + """End Test""" def test_cesr_native_dumps_hby(): From c81dc0fb40d4cd6b1c3a138123e9142dfd248989 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Sat, 13 Apr 2024 15:55:33 -0700 Subject: [PATCH 149/418] Version bump --- Makefile | 4 ++-- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e16e41364..a99d41d9b 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev0 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev0-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev1 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev1-arm64 . .PHONY: build-witness-demo build-witness-demo: diff --git a/setup.py b/setup.py index 4af2c48bf..9a0cda9a5 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev0', # also change in src/keri/__init__.py + version='1.2.0-dev1', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index dcacb989d..ea009cc64 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev0' # also change in setup.py +__version__ = '1.2.0-dev1' # also change in setup.py From 6d195370969a7106fcd11acb7966030efb8516f9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 14 Apr 2024 13:37:10 -0600 Subject: [PATCH 150/418] some clean up --- src/keri/help/helping.py | 2 +- tests/core/test_serdering.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index 7e3b8717a..838f69bf4 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -264,7 +264,7 @@ def fromIso8601(dts): B64_CHARS = tuple(B64ChrByIdx.values()) # tuple of characters in Base64 -B64REX = b'^[0-9A-Za-z_-]*\Z' # [A-Za-z0-9\-\_] +B64REX = rb'^[0-9A-Za-z_-]*\Z' # [A-Za-z0-9\-\_] Reb64 = re.compile(B64REX) # compile is faster # to use if Reb64.match(bext): diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 82f4507dc..d73395ce6 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -2999,7 +2999,7 @@ def test_cesr_native_dumps(): raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] ratios = [ round(len(raw) / len(rawqb2), 2) for raw in raws] - assert ratios ==[1.0, 1.33, 1.45, 1.45, 1.63] + assert ratios == [1.0, 1.33, 1.45, 1.45, 1.63] # complex rotation event From faf580377d1798400e82ece4272c258468948f9b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 14 Apr 2024 14:19:49 -0600 Subject: [PATCH 151/418] added tests for delcept and deltate for CESR native serialization --- src/keri/core/eventing.py | 4 +- tests/core/test_serdering.py | 227 +++++++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+), 2 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index d256b5448..f7c9da324 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -770,7 +770,7 @@ def delcept(keys, delpre, **kwa): Parameters: keys (list): current signing keys qb64 - sith (int | str | list | None): current signing threshold input to Tholder + isith (int | str | list | None): current signing threshold input to Tholder ndigs (list | None): current signing key digests qb64 nsith int | str | list | None): next signing threshold input to Tholder toad (int | str | None): witness threshold number if str then hex str @@ -948,7 +948,7 @@ def deltate(pre, dig (str): said of previous event qb64 ilk (str): ilk of event. Must be in (Ilks.rot, Ilks.drt) sn (int | str): sequence number int or hex str - sith (int | str | list): current signing threshold input to Tholder + isith (int | str | list): current signing threshold input to Tholder ndigs (list): current signing key digests qb64 nsith int | str | list): next signing threshold input to Tholder toad (int | str ): witness threshold number if str then hex str diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index d73395ce6..91edd1469 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -3092,6 +3092,233 @@ def test_cesr_native_dumps(): assert ratios == [1.0, 1.33, 1.42, 1.42, 1.58] + # Test delcept + delpre = pre + keys = [signer.verfer.qb64 for signer in csigners][6:9] + ndigs = [core.Diger(ser=key.encode()).qb64 for key in keys][:3] + wits = [signer.verfer.qb64 for signer in wsigners][6:9] + data = [dict(i=keys[0], s=core.Number(num=3).numh, d=ndigs[0]), + dict(i=keys[1], s=core.Number(num=4).numh, d=ndigs[1]), + dict(s=core.Number(num=21).numh, d=ndigs[2]), + dict(s=core.Number(num=15890).numh, d=ndigs[0])] + + assert data == \ + [ + { + 'i': 'DIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV', + 's': '3', + 'd': 'EKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ44DS' + }, + { + 'i': 'DN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUti', + 's': '4', + 'd': 'EC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZ' + }, + { + 's': '15', + 'd': 'EHgewy_ymPxtSFwuX2KaI_mPmoIUkxClviX3f-M38kCD' + }, + { + 's': '3e12', + 'd': 'EKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ44DS' + } + ] + + + + serder = delcept(keys, + isith=["1/2", "1/2", "1/2"], + ndigs=ndigs, + nsith=["1/2", "1/2", "1/2"], + wits=wits, + data=data, + delpre=delpre, + code=core.MtrDex.Blake3_256, + version=Vrsn_2_0, + kind=kering.Serials.cesr) + + pre = serder.pre + assert pre == 'ECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8T' + said = serder.said + assert said == pre + + assert serder.sad == \ + { + 'v': 'KERICAACESRAAAA.', + 't': 'dip', + 'd': 'ECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8T', + 'i': 'ECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8T', + 's': '0', + 'kt': ['1/2', '1/2', '1/2'], + 'k': + [ + 'DIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV', + 'DN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUti', + 'DOE5jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl5nfY' + ], + 'nt': ['1/2', '1/2', '1/2'], + 'n': + [ + 'EKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ44DS', + 'EC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZ', + 'EHgewy_ymPxtSFwuX2KaI_mPmoIUkxClviX3f-M38kCD' + ], + 'bt': '3', + 'b': + [ + 'BIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV', + 'BN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUti', + 'BOE5jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl5nfY' + ], + 'c': [], + 'a': + [ + { + 'i': 'DIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV', + 's': '3', + 'd': 'EKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ44DS' + }, + { + 'i': 'DN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUti', + 's': '4', + 'd': 'EC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZ' + }, + { + 's': '15', + 'd': 'EHgewy_ymPxtSFwuX2KaI_mPmoIUkxClviX3f-M38kCD' + }, + { + 's': '3e12', + 'd': 'EKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ44DS' + } + ], + 'di': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6' + } + + + assert serder.raw == (b'-FDeYKERICAAXdipECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TECQs' + b'0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TMAAA4AADA1s2c1s2c1s2-LAh' + b'DIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uVDN7WiKyjLLBTK92xayCu' + b'ddZsBuwPmD2BKrl83h1xEUtiDOE5jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl' + b'5nfY4AADA1s2c1s2c1s2-LAhEKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ' + b'44DSEC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZEHgewy_ymPxtSFwu' + b'X2KaI_mPmoIUkxClviX3f-M38kCDMAAD-LAhBIR8GACw4z2GC5_XoReU4DMKbqi6' + b'-EdbgDZUAobRb8uVBN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiBOE5' + b'jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl5nfY-LAA-LBI-RAuDIR8GACw4z2G' + b'C5_XoReU4DMKbqi6-EdbgDZUAobRb8uVMAADEKFoJ9Conb37zSn8zHLKP3YwHbeQ' + b'iD1D9Qx0MagJ44DSDN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiMAAE' + b'EC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZ-QAYMAAVEHgewy_ymPxt' + b'SFwuX2KaI_mPmoIUkxClviX3f-M38kCDMD4SEKFoJ9Conb37zSn8zHLKP3YwHbeQ' + b'iD1D9Qx0MagJ44DSEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6') + + assert len(serder.raw) == serder.size == 892 + sizeh = serder.raw[2:4] + assert sizeh == b"De" + assert helping.b64ToInt(sizeh) * 4 + 4 == serder.size == 892 + + rawqb64 = serder._dumps() # default is it dumps self.sad + assert rawqb64 == serder.raw + + assert len(rawqb64) == 892 + + rawqb2 = decodeB64(rawqb64) + assert len(rawqb2) == 669 + assert rawqb64 == encodeB64(rawqb2) # round trips + + rawjson = serder.dumps(serder.sad) + assert len(rawjson) == 1059 + + rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + assert len(rawcbor) == 953 + + rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + assert len(rawmgpk) == 953 + + raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] + ratios = [ round(len(raw) / len(rawqb2), 2) for raw in raws] + + assert ratios == [1.0, 1.33, 1.42, 1.42, 1.58] + + + # Test deltate + + prior = said + + keys = [signer.verfer.qb64 for signer in csigners][9:10] + ndigs = [core.Diger(ser=key.encode()).qb64 for key in keys] + cuts = [wits[0]] + adds = [signer.verfer.qb64 for signer in wsigners][9:10] + data = {} # no anchors + + + serder = deltate(pre=pre, + keys=keys, + dig=prior, + sn=1, + ndigs=ndigs, + wits=wits, #prior + cuts=cuts, + adds=adds, + data=data, + version=Vrsn_2_0, + kind=kering.Serials.cesr) + + said = serder.said + assert said == 'EKwDKG0L9pAMbzV2e31-I5ObiEfkptfs8VqXYiHGCL1v' + + assert serder.sad == \ + { + 'v': 'KERICAACESRAAAA.', + 't': 'drt', + 'd': 'EKwDKG0L9pAMbzV2e31-I5ObiEfkptfs8VqXYiHGCL1v', + 'i': 'ECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8T', + 's': '1', + 'p': 'ECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8T', + 'kt': '1', + 'k': ['DJ0pLe3f2zGus0Va1dqWAnukWdZHGNWlK9NciJop9N4f'], + 'nt': '1', + 'n': ['ENX_LTL97uOSOkA1PEzam9vtmCLPprnbcpi71wXpmhFF'], + 'bt': '3', + 'br': ['BIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV'], + 'ba': ['BJ0pLe3f2zGus0Va1dqWAnukWdZHGNWlK9NciJop9N4f'], + 'c': [], + 'a': {} + } + + + assert serder.raw == (b'-FBaYKERICAAXdrtEKwDKG0L9pAMbzV2e31-I5ObiEfkptfs8VqXYiHGCL1vECQs' + b'0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TMAABECQs0t3_GL7-B3q4kMU-' + b'qLeRCugTFjrxR15mxUwYWp8TMAAB-LALDJ0pLe3f2zGus0Va1dqWAnukWdZHGNWl' + b'K9NciJop9N4fMAAB-LALENX_LTL97uOSOkA1PEzam9vtmCLPprnbcpi71wXpmhFF' + b'MAAD-LALBIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV-LALBJ0pLe3f' + b'2zGus0Va1dqWAnukWdZHGNWlK9NciJop9N4f-LAA-LAA') + + assert len(serder.raw) == serder.size == 364 + sizeh = serder.raw[2:4] + assert sizeh == b"Ba" + assert helping.b64ToInt(sizeh) * 4 + 4 == serder.size == 364 + + rawqb64 = serder._dumps() # default is it dumps self.sad + assert rawqb64 == serder.raw + assert len(rawqb64) == 364 + + rawqb2 = decodeB64(rawqb64) + assert len(rawqb2) == 273 + assert rawqb64 == encodeB64(rawqb2) # round trips + + rawjson = serder.dumps(serder.sad) + assert len(rawjson) == 450 + + rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + assert len(rawcbor) == 393 + + rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + assert len(rawmgpk) == 393 + + raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] + ratios = [ round(len(raw) / len(rawqb2), 2) for raw in raws] + + assert ratios == [1.0, 1.33, 1.44, 1.44, 1.65] """End Test""" From 74c2cec0c5a963ca3e5a16acf2c0c76ba4b99015 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 14 Apr 2024 15:25:44 -0600 Subject: [PATCH 152/418] added streaming.py and test_streaming.py preliminary setup for annotation of CESR streams --- src/keri/core/coring.py | 67 ------------------------- src/keri/core/streaming.py | 97 ++++++++++++++++++++++++++++++++++++ tests/core/test_serdering.py | 2 + tests/core/test_streaming.py | 37 ++++++++++++++ 4 files changed, 136 insertions(+), 67 deletions(-) create mode 100644 src/keri/core/streaming.py create mode 100644 tests/core/test_streaming.py diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 63d8a43a9..a02d29fbb 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -4797,73 +4797,6 @@ def _satisfy_weighted(self, indices): -class Streamer: - """ - Streamer is CESR sniffable stream class - - - Has the following public properties: - - Properties: - - - Methods: - - - Hidden: - - - - """ - - def __init__(self, stream): - """Initialize instance - - - Parameters: - stream (bytes | bytearray): sniffable CESR stream - - - """ - self._stream = bytes(stream) - - - @property - def stream(self): - """stream property getter - """ - return self._stream - - @property - def text(self): - """expanded stream as qb64 text - Returns: - stream (bytes): expanded text qb64 version of stream - - """ - return self._stream - - @property - def binary(self): - """compacted stream as qb2 binary - Returns: - stream (bytes): compacted binary qb2 version of stream - - """ - return self._stream - - @property - def texter(self): - """expanded stream as Texter instance - Returns: - texter (Texter): Texter primitive of stream suitable wrapping - - """ - return self._stream - - - - class Sadder: """ Sadder is self addressed data (SAD) serializer-deserializer class diff --git a/src/keri/core/streaming.py b/src/keri/core/streaming.py new file mode 100644 index 000000000..b154572d8 --- /dev/null +++ b/src/keri/core/streaming.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +""" +keri.core.streaming module + +Provides support for Streamer and Annotater +""" + + +from typing import NamedTuple +from collections import namedtuple + +from .. import kering + +from .. import help + + +from . import coring + + +def annot(): + """Annotate CESR stream""" + + + +def denot(): + """De-annotate CESR stream""" + + + +class Streamer: + """ + Streamer is CESR sniffable stream class + + + Has the following public properties: + + Properties: + + + Methods: + + + Hidden: + + + + """ + + def __init__(self, stream): + """Initialize instance + + + Parameters: + stream (bytes | bytearray): sniffable CESR stream + + + """ + self._stream = bytes(stream) + + + @property + def stream(self): + """stream property getter + """ + return self._stream + + @property + def text(self): + """expanded stream as qb64 text + Returns: + stream (bytes): expanded text qb64 version of stream + + """ + return self._stream + + @property + def binary(self): + """compacted stream as qb2 binary + Returns: + stream (bytes): compacted binary qb2 version of stream + + """ + return self._stream + + @property + def texter(self): + """expanded stream as Texter instance + Returns: + texter (Texter): Texter primitive of stream suitable wrapping + + """ + return self._stream + + + + + diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 91edd1469..6cdddfc31 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -3323,6 +3323,8 @@ def test_cesr_native_dumps(): """End Test""" + + def test_cesr_native_dumps_hby(): """Test Serder._dumps with habery""" diff --git a/tests/core/test_streaming.py b/tests/core/test_streaming.py new file mode 100644 index 000000000..b32c43344 --- /dev/null +++ b/tests/core/test_streaming.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +tests.core.test_streaming module + +""" + + +import pytest + + +from keri import kering + +from keri.help import helping + +from keri.core import (Matter, ) + + +from keri.core import streaming +from keri.core.streaming import (annot, denot, Streamer) + + +def test_annot(): + """Test annot function Annotate""" + + """End Test""" + +def test_streamer(): + """Test streamer instance""" + + """End Test""" + +if __name__ == "__main__": + test_annot() + test_streamer() + + + From 441db3ebc4685fa52e3b5fd03a5b5d67355338b1 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 15 Apr 2024 15:13:43 -0600 Subject: [PATCH 153/418] Added .name property to Matter, Indexer, and Counter which provides the code name. This is to support annotation with the code name. --- src/keri/core/coring.py | 20 +++++- src/keri/core/counting.py | 39 +++++++++-- src/keri/core/indexing.py | 17 ++++- tests/core/test_coring.py | 125 +++++++++++++++++++++++++++++++++++- tests/core/test_counting.py | 77 ++++++++++++++++++++-- tests/core/test_indexing.py | 41 ++++++++++-- 6 files changed, 301 insertions(+), 18 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index a02d29fbb..7c755abd7 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -8,7 +8,7 @@ from typing import Union from collections import namedtuple, deque from collections.abc import Sequence, Mapping -from dataclasses import dataclass, astuple +from dataclasses import dataclass, astuple, asdict from base64 import urlsafe_b64encode as encodeB64 from base64 import urlsafe_b64decode as decodeB64 from fractions import Fraction @@ -906,6 +906,9 @@ class Matter: '9AAE': Sizage(hs=4, ss=4, fs=None, ls=2), } + Codes = asdict(MtrDex) # map code name to code + Names = {val : key for key, val in Codes.items()} # invert map code to code name + def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, @@ -1116,6 +1119,17 @@ def code(self): return self._code + @property + def name(self): + """ + Returns: + name (str): code name for self.code. Used for annotation for + primitives like Matter + + """ + return self.Names[self.code] + + @property def hard(self): """ @@ -1700,6 +1714,10 @@ class Number(Matter): Methods: """ + Codes = asdict(NumDex) # map code name to code + Names = {val : key for key, val in Codes.items()} # invert map code to code name + + def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, code=None, num=None, numh=None, **kwa): diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 590a5ed16..9dee9fe14 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -419,10 +419,10 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, Parameters: tag (str | None): label of stable (hard) part of derivation code to lookup in codex so it can depend on version. - takes precedence over tag + takes precedence over code. code (str | None): stable (hard) part of derivation code if tag provided lookup code from tag - else if tag is None and code provided use code + else if tag is None and code provided use code. count (int | None): count of framed material in quadlets/triplets for composition. Count does not include code. When both count and countB64 are None then count @@ -517,6 +517,9 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, "(code and count) or qb64b or " "qb64 or qb2.") + codenames = { val: key for key, val in asdict(self.codes).items()} # map codes to code names + self._tag = codenames[self.code] + @property def version(self): """ @@ -525,6 +528,14 @@ def version(self): """ return self._version + @property + def gvrsn(self): + """ + Returns .version alias for .version + + """ + return self.version + @property def codes(self): """ @@ -536,10 +547,10 @@ def codes(self): @property def tags(self): """ - Returns ._tags + Returns tags for current .version Makes .tags read only """ - return self._tags + return self.Tags[self.version] # use own version @property def sizes(self): @@ -560,6 +571,26 @@ def code(self): """ return self._code + @property + def tag(self): + """ + Returns: + tag (str): code name for self.code + + Getter for ._tag. Makes .tag read only + """ + return self._tag + + @property + def name(self): + """ + Returns: + name (str): code name for self.code alias of .tag. Match interface + for annotation for primitives like Matter + + """ + return self.tag + @property def hard(self): diff --git a/src/keri/core/indexing.py b/src/keri/core/indexing.py index 6cb192a35..7022fbc1a 100644 --- a/src/keri/core/indexing.py +++ b/src/keri/core/indexing.py @@ -192,7 +192,6 @@ class Indexer: ._bexfil is method to extract .code and .raw from fully qualified Base2 """ - Codex = IdrDex # Hards table maps from bytes Base64 first code char to int of hard size, hs, # (stable) of code. The soft size, ss, (unstable) is always > 0 for Indexer. Hards = ({chr(c): 1 for c in range(65, 65 + 26)}) @@ -228,6 +227,11 @@ class Indexer: # converted from first code char. Used for ._bexfil. Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) + Codes = asdict(IdrDex) # map code name to code + Names = {val : key for key, val in Codes.items()} # invert map code to code name + + + def __init__(self, raw=None, code=IdrDex.Ed25519_Sig, index=0, ondex=None, qb64b=None, qb64=None, qb2=None, strip=False): """ @@ -341,6 +345,17 @@ def code(self): """ return self._code + + @property + def name(self): + """ + Returns: + name (str): code name for self.code. Used for annotation for + primitives like Matter + + """ + return self.Names[self.code] + @property def raw(self): """ diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 28548a043..0933c0e89 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -205,7 +205,7 @@ def test_matter_class(): Test Matter class attributes """ - assert asdict(MtrDex) == \ + assert Matter.Codes == \ { 'Ed25519_Seed': 'A', 'Ed25519N': 'B', @@ -300,6 +300,101 @@ def test_matter_class(): 'X25519_Cipher_QB2_Big_L2': '9AAE' } + assert Matter.Names == \ + { + 'A': 'Ed25519_Seed', + 'B': 'Ed25519N', + 'C': 'X25519', + 'D': 'Ed25519', + 'E': 'Blake3_256', + 'F': 'Blake2b_256', + 'G': 'Blake2s_256', + 'H': 'SHA3_256', + 'I': 'SHA2_256', + 'J': 'ECDSA_256k1_Seed', + 'K': 'Ed448_Seed', + 'L': 'X448', + 'M': 'Short', + 'N': 'Big', + 'O': 'X25519_Private', + 'P': 'X25519_Cipher_Seed', + 'Q': 'ECDSA_256r1_Seed', + 'R': 'Tall', + 'S': 'Large', + 'T': 'Great', + 'U': 'Vast', + 'V': 'Label1', + 'W': 'Label2', + 'X': 'Tag3', + 'Y': 'Tag7', + 'Z': 'Blind', + '0A': 'Salt_128', + '0B': 'Ed25519_Sig', + '0C': 'ECDSA_256k1_Sig', + '0D': 'Blake3_512', + '0E': 'Blake2b_512', + '0F': 'SHA3_512', + '0G': 'SHA2_512', + '0H': 'Long', + '0I': 'ECDSA_256r1_Sig', + '0J': 'Tag1', + '0K': 'Tag2', + '0L': 'Tag5', + '0M': 'Tag6', + '0N': 'Tag9', + '0O': 'Tag10', + '1AAA': 'ECDSA_256k1N', + '1AAB': 'ECDSA_256k1', + '1AAC': 'Ed448N', + '1AAD': 'Ed448', + '1AAE': 'Ed448_Sig', + '1AAF': 'Tag4', + '1AAG': 'DateTime', + '1AAH': 'X25519_Cipher_Salt', + '1AAI': 'ECDSA_256r1N', + '1AAJ': 'ECDSA_256r1', + '1AAK': 'Null', + '1AAL': 'No', + '1AAM': 'Yes', + '1AAN': 'Tag8', + '1__-': 'TBD0S', + '1___': 'TBD0', + '2__-': 'TBD1S', + '2___': 'TBD1', + '3__-': 'TBD2S', + '3___': 'TBD2', + '4A': 'StrB64_L0', + '5A': 'StrB64_L1', + '6A': 'StrB64_L2', + '7AAA': 'StrB64_Big_L0', + '8AAA': 'StrB64_Big_L1', + '9AAA': 'StrB64_Big_L2', + '4B': 'Bytes_L0', + '5B': 'Bytes_L1', + '6B': 'Bytes_L2', + '7AAB': 'Bytes_Big_L0', + '8AAB': 'Bytes_Big_L1', + '9AAB': 'Bytes_Big_L2', + '4C': 'X25519_Cipher_L0', + '5C': 'X25519_Cipher_L1', + '6C': 'X25519_Cipher_L2', + '7AAC': 'X25519_Cipher_Big_L0', + '8AAC': 'X25519_Cipher_Big_L1', + '9AAC': 'X25519_Cipher_Big_L2', + '4D': 'X25519_Cipher_QB64_L0', + '5D': 'X25519_Cipher_QB64_L1', + '6D': 'X25519_Cipher_QB64_L2', + '7AAD': 'X25519_Cipher_QB64_Big_L0', + '8AAD': 'X25519_Cipher_QB64_Big_L1', + '9AAD': 'X25519_Cipher_QB64_Big_L2', + '4E': 'X25519_Cipher_QB2_L0', + '5E': 'X25519_Cipher_QB2_L1', + '6E': 'X25519_Cipher_QB2_L2', + '7AAE': 'X25519_Cipher_QB2_Big_L0', + '8AAE': 'X25519_Cipher_QB2_Big_L1', + '9AAE': 'X25519_Cipher_QB2_Big_L2' + } + # first character of code with hard size of code assert Matter.Hards == { @@ -536,6 +631,7 @@ def test_matter(): matter = Matter(raw=verkey) # default code is MtrDex.Ed25519N assert matter.raw == verkey assert matter.code == MtrDex.Ed25519N == matter.hard + assert matter.name == 'Ed25519N' assert matter.soft == "" assert matter.both == MtrDex.Ed25519N assert matter.size == None @@ -850,6 +946,7 @@ def test_matter(): matter = Matter(raw=raw, code=code) assert matter.raw == raw assert matter.code == code + assert matter.name == 'TBD1' assert matter.both == code assert matter.size == None assert matter.fullSize == 8 @@ -2615,6 +2712,32 @@ def test_number(): 'Vast': 'U' } + assert Number.Codes == \ + { + 'Short': 'M', + 'Long': '0H', + 'Tall': 'R', + 'Big': 'N', + 'Large': 'S', + 'Great': 'T', + 'Huge': '0A', + 'Vast': 'U' + } + + + assert Number.Names == \ + { + 'M': 'Short', + '0H': 'Long', + 'R': 'Tall', + 'N': 'Big', + 'S': 'Large', + 'T': 'Great', + '0A': 'Huge', + 'U': 'Vast' + } + + with pytest.raises(EmptyMaterialError): number = Number(raw=b'') # missing code diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 39534f68d..bec518824 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -436,8 +436,7 @@ def test_counter_class(): for key, val in vminor.items(): # size table items assert Counter.Hards[key[:2]] == val.hs - ## verify all Codes have hs >= 2 and ss > 0 and fs = hs + ss and not fs % 4 - ## if hs < 2 or ss <= 0 or fs != cs or cs % 4 cs = hs + ss + # verify all Codes have hs >= 2 and ss > 0 and fs = hs + ss and not fs % 4 for vmajor in Counter.Sizes.values(): for vminor in vmajor.values(): for val in vminor.values(): # size table values @@ -507,10 +506,6 @@ def test_counter_class(): Counter.verToB64(minor=-1) - - - - """ Done Test """ def test_counter_v1(): @@ -533,6 +528,7 @@ def test_counter_v1(): counter = Counter(tag="ControllerIdxSigs", count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs == counter.hard + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -541,9 +537,13 @@ def test_counter_v1(): assert counter.fullSize == 4 assert counter.soft =='AB' assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 + assert counter.codes == counting.CtrDex_1_0 + assert counter.tags == counting.Tags_1_0 + assert counter.sizes == Counter.Sizes[1][0] counter = Counter(tag=AllTags.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -556,6 +556,7 @@ def test_counter_v1(): count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -564,6 +565,7 @@ def test_counter_v1(): counter = Counter(code=CtrDex.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default count = 1 assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -572,6 +574,7 @@ def test_counter_v1(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -580,6 +583,7 @@ def test_counter_v1(): counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -588,6 +592,7 @@ def test_counter_v1(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -624,6 +629,7 @@ def test_counter_v1(): counter = Counter(code=CtrDex.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -632,6 +638,7 @@ def test_counter_v1(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -640,6 +647,7 @@ def test_counter_v1(): counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -648,6 +656,7 @@ def test_counter_v1(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -663,6 +672,7 @@ def test_counter_v1(): counter = Counter(code=CtrDex.BigAttachmentGroup, count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.BigAttachmentGroup + assert counter.tag == AllTags.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -671,6 +681,7 @@ def test_counter_v1(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.BigAttachmentGroup + assert counter.tag == AllTags.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -679,6 +690,7 @@ def test_counter_v1(): counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.BigAttachmentGroup + assert counter.tag == AllTags.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -687,6 +699,7 @@ def test_counter_v1(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.BigAttachmentGroup + assert counter.tag == AllTags.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -700,6 +713,7 @@ def test_counter_v1(): qb2 = counter.qb2 counter._bexfil(qb2) assert counter.code == code + assert counter.tag == AllTags.BigAttachmentGroup assert counter.count == count assert counter.qb64 == qsc assert counter.qb2 == qb2 @@ -720,6 +734,7 @@ def test_counter_v1(): # strip ignored if qb64 counter = Counter(qb64=qsc, strip=True, gvrsn=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -730,6 +745,7 @@ def test_counter_v1(): counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_1_0) # strip assert not ims # deleted assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -740,6 +756,7 @@ def test_counter_v1(): counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_1_0) assert not ims # deleted assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -782,6 +799,7 @@ def test_counter_v1(): ims = bytearray(qscb) counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.BigAttachmentGroup + assert counter.tag == AllTags.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -793,6 +811,7 @@ def test_counter_v1(): ims = bytearray(qscb2) counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.BigAttachmentGroup + assert counter.tag == AllTags.BigAttachmentGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -816,6 +835,7 @@ def test_counter_v1(): count=genverint, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.KERIACDCGenusVersion + assert counter.tag == AllTags.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length @@ -823,11 +843,16 @@ def test_counter_v1(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_1_0 + assert counter.codes == counting.CtrDex_1_0 + assert counter.tags == counting.Tags_1_0 + assert counter.sizes == Counter.Sizes[1][0] + counter = Counter(code=CtrDex.KERIACDCGenusVersion, countB64=genver, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.KERIACDCGenusVersion + assert counter.tag == AllTags.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length @@ -859,6 +884,7 @@ def test_counter_v2(): # default version and default count = 1 counter = Counter(code=CtrDex.ControllerIdxSigs) assert counter.code == CtrDex.ControllerIdxSigs == counter.hard + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -867,10 +893,16 @@ def test_counter_v2(): assert counter.fullSize == 4 assert counter.soft =='AB' assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 + assert counter.codes == counting.CtrDex_2_0 + assert counter.tags == counting.Tags_2_0 + assert counter.sizes == Counter.Sizes[2][0] + + # default count = 1 counter = Counter(code=CtrDex.ControllerIdxSigs, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -879,6 +911,7 @@ def test_counter_v2(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -887,6 +920,7 @@ def test_counter_v2(): counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -895,6 +929,7 @@ def test_counter_v2(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -931,6 +966,7 @@ def test_counter_v2(): counter = Counter(tag="ControllerIdxSigs", count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -941,6 +977,7 @@ def test_counter_v2(): count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -953,6 +990,7 @@ def test_counter_v2(): count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -961,6 +999,7 @@ def test_counter_v2(): counter = Counter(code=CtrDex.ControllerIdxSigs, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -969,6 +1008,7 @@ def test_counter_v2(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -977,6 +1017,7 @@ def test_counter_v2(): counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -985,6 +1026,7 @@ def test_counter_v2(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -998,6 +1040,7 @@ def test_counter_v2(): qb2 = counter.qb2 counter._bexfil(qb2) assert counter.code == code + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64 == qsc assert counter.qb2 == qb2 @@ -1018,6 +1061,7 @@ def test_counter_v2(): # strip ignored if qb64 counter = Counter(qb64=qsc, strip=True, gvrsn=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1028,6 +1072,7 @@ def test_counter_v2(): counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) # strip assert not ims # deleted assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1038,6 +1083,7 @@ def test_counter_v2(): counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) assert not ims # deleted assert counter.code == CtrDex.ControllerIdxSigs + assert counter.tag == AllTags.ControllerIdxSigs assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1079,6 +1125,7 @@ def test_counter_v2(): counter = Counter(code=CtrDex.BigGenericGroup, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1087,6 +1134,7 @@ def test_counter_v2(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1095,6 +1143,7 @@ def test_counter_v2(): counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1103,6 +1152,7 @@ def test_counter_v2(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1119,6 +1169,7 @@ def test_counter_v2(): ims = bytearray(qscb) counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1129,6 +1180,7 @@ def test_counter_v2(): ims = bytearray(qscb2) counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1145,6 +1197,7 @@ def test_counter_v2(): counter = Counter(code=CtrDex.BigGenericGroup, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.BigGenericGroup == counter.hard + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1156,6 +1209,7 @@ def test_counter_v2(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1164,6 +1218,7 @@ def test_counter_v2(): counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1172,6 +1227,7 @@ def test_counter_v2(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1188,6 +1244,7 @@ def test_counter_v2(): ims = bytearray(qscb) counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1199,6 +1256,7 @@ def test_counter_v2(): ims = bytearray(qscb2) counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1215,6 +1273,7 @@ def test_counter_v2(): counter = Counter(code=CtrDex.GenericGroup, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.BigGenericGroup == counter.hard + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1225,6 +1284,7 @@ def test_counter_v2(): counter = Counter(tag=AllTags.GenericGroup, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.BigGenericGroup + assert counter.tag == AllTags.BigGenericGroup assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1245,6 +1305,7 @@ def test_counter_v2(): count=genverint, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.KERIACDCGenusVersion + assert counter.tag == AllTags.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length @@ -1253,11 +1314,15 @@ def test_counter_v2(): assert counter.qb2 == qscb2 assert counter.version == Vrsn_2_0 assert counter.fullSize == 8 + assert counter.codes == counting.CtrDex_2_0 + assert counter.tags == counting.Tags_2_0 + assert counter.sizes == Counter.Sizes[2][0] counter = Counter(code=CtrDex.KERIACDCGenusVersion, countB64=genver, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.KERIACDCGenusVersion + assert counter.tag == AllTags.KERIACDCGenusVersion assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length diff --git a/tests/core/test_indexing.py b/tests/core/test_indexing.py index 705e7743c..453dab432 100644 --- a/tests/core/test_indexing.py +++ b/tests/core/test_indexing.py @@ -27,15 +27,12 @@ from keri.core.coring import (Verfer,) - - -def test_indexer(): +def test_indexer_class(): """ Test Indexer class """ - assert Indexer.Codex == IdrDex - assert asdict(IdrDex) == { + assert Indexer.Codes == { 'Ed25519_Sig': 'A', 'Ed25519_Crt_Sig': 'B', 'ECDSA_256k1_Sig': 'C', @@ -57,6 +54,30 @@ def test_indexer(): 'TBD4': '4z', } + assert Indexer.Names == \ + { + 'A': 'Ed25519_Sig', + 'B': 'Ed25519_Crt_Sig', + 'C': 'ECDSA_256k1_Sig', + 'D': 'ECDSA_256k1_Crt_Sig', + 'E': 'ECDSA_256r1_Sig', + 'F': 'ECDSA_256r1_Crt_Sig', + '0A': 'Ed448_Sig', + '0B': 'Ed448_Crt_Sig', + '2A': 'Ed25519_Big_Sig', + '2B': 'Ed25519_Big_Crt_Sig', + '2C': 'ECDSA_256k1_Big_Sig', + '2D': 'ECDSA_256k1_Big_Crt_Sig', + '2E': 'ECDSA_256r1_Big_Sig', + '2F': 'ECDSA_256r1_Big_Crt_Sig', + '3A': 'Ed448_Big_Sig', + '3B': 'Ed448_Big_Crt_Sig', + '0z': 'TBD0', + '1z': 'TBD1', + '4z': 'TBD4' + } + + assert IdrDex.Ed25519_Sig == 'A' assert IdrDex.Ed25519_Crt_Sig == 'B' assert IdrDex.ECDSA_256k1_Sig == 'C' @@ -217,6 +238,15 @@ def test_indexer(): ckey = codeB64ToB2(skey) assert Indexer.Bards[ckey] == sval + """End Test""" + + +def test_indexer(): + """ + Test Indexer instance + """ + + with pytest.raises(EmptyMaterialError): indexer = Indexer() @@ -776,6 +806,7 @@ def test_siger(): if __name__ == "__main__": + test_indexer_class() test_indexer() test_siger() From 26c3ba475e27e91fa4e962ce02a2bbccce15b8e0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 16 Apr 2024 21:53:17 -0700 Subject: [PATCH 154/418] added demo tests for annot --- src/keri/core/streaming.py | 169 ++++++++++++++++++++++++++++++++++- src/keri/core/structing.py | 1 + src/keri/kering.py | 4 +- tests/core/test_streaming.py | 43 ++++++++- 4 files changed, 208 insertions(+), 9 deletions(-) diff --git a/src/keri/core/streaming.py b/src/keri/core/streaming.py index b154572d8..8461d3b9a 100644 --- a/src/keri/core/streaming.py +++ b/src/keri/core/streaming.py @@ -10,20 +10,181 @@ from collections import namedtuple from .. import kering +from ..kering import sniff, Colds, Ilks + +from ..help.helping import intToB64 -from .. import help +from .. import help + from . import coring +from .coring import (Matter, Verser, Ilker, Diger, Prefixer, Number, Tholder, + Verfer, Traitor) + +from . import counting +from .counting import Counter + +from . import structing +from .structing import Sealer + +from . import serdering +from .serdering import Serder -def annot(): - """Annotate CESR stream""" +def annot(ims): + """Annotate CESR stream + Returns: + annotation (str): annotation of input CESR stream + Parameters: + ims (str | bytes | bytearray | memoryview): CESR incoming message stream + as qb64 (maybe qb2) + + """ + oms = bytearray() + indent = 0 + + + if not isinstance(ims, bytearray): # going to strip + ims = bytearray(ims) # so make bytearray copy + + while ims: # right now just for KERI event messages + cold = sniff(ims) # check for spurious counters at front of stream + + if cold in Colds.txt: + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # Key Event Counter " + f"{val.name} count={val.count} quadlets\n".encode()) + indent += 1 + # version + val = Verser(qb64b=ims, strip=True) + versage = val.versage + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'v' version Verser {val.name} " + f"proto={versage.proto} vrsn={versage.vrsn.major}." + f"{versage.vrsn.minor:02}\n".encode()) + # ilk + val = Ilker(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 't' message type Ilker " + f"{val.name} Ilk={val.ilk}\n".encode()) + if val.ilk == Ilks.icp: # inception + # icp v='', t='', d='', i='', s='0', kt='0',k=[], nt='0', n=[], bt='0', b=[], c=[], a=[] + # said + val = Diger(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'd' SAID Diger " + f"{val.name} \n".encode()) + # aid pre + val = Prefixer(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'i' AID Prefixer " + f"{val.name} \n".encode()) + # sn + val = Number(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 's' Number " + f"{val.name} sn={val.sn}\n".encode()) + # Signing key threshold + val = Tholder(limen=ims, strip=True) # add qb64 and qb2 to tholder as aliases for limen and limen.decode() + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'kt' " + f"Tholder signing threshold={val.sith}\n".encode()) + # Signing key list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'k' Signing Key " + f"List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Verfer(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # key Verfer " + f"{val.name}\n".encode()) + indent -= 1 + # Rotation key threshold + val = Tholder(limen=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'nt' " + f"Tholder rotation threshold={val.sith}\n".encode()) + # Next key digest list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'n' Rotation Key " + f"Digest List Counter {val.name} count={val.count} " + f"quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Diger(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # key digest Diger" + f" {val.name}\n".encode()) + indent -= 1 + # Witness Backer threshold + val = Tholder(limen=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'bt' " + f"Tholder Backer (witness) threshold={val.sith}\n".encode()) + # Witness Backer list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'b' Backer (witness)" + f"List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Prefixer(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # AID Prefixer " + f"{val.name}\n".encode()) + indent -= 1 + # Config Trait List + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'c' Config Trait " + f"List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Traitor(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # trait Traitor " + f"{val.name} trait={val.trait}\n".encode()) + indent -= 1 + # Seal (anchor) List + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'a' Seal List Counter" + f" {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Counter(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # Seal Counter " + f"{val.name} count={val.count} quadlets\n".encode()) + indent += 1 + subframe = frame[:val.count*4] + del frame[:val.count*4] # strip subframe + clan = Sealer.Clans[Serder.CodeClans[val.code]] + while subframe: + val = Sealer(clan=clan, qb64=subframe, strip=True) # need to add qb64 parameter to structor + oms.extend(f"{' ' * indent * 2}{val.qb64}# seal Sealer {val.name}\n".encode()) + indent += 1 + for i, t in enumerate(val.crew._asdict().items()): + oms.extend(f"{' ' * indent * 2}# '{t[0]}' = " + f"{t[1]}\n".encode()) + indent -= 1 + indent -= 1 + indent -= 1 + + + elif cold in Colds.bny: + pass + + else: + raise kering.ColdStartError("Expecting stream tritet={}" + "".format(cold)) + + return oms.decode() # return unicode string def denot(): - """De-annotate CESR stream""" + """De-annotate CESR stream + + + + """ diff --git a/src/keri/core/structing.py b/src/keri/core/structing.py index b40b30f65..a9ef531f4 100644 --- a/src/keri/core/structing.py +++ b/src/keri/core/structing.py @@ -20,6 +20,7 @@ from .coring import (MapDom, Matter, Diger, Prefixer, Number) + # ToDo Change seal namedtuple definitions to NamedTuple subclasses so can # use typehints on field values which type hints are the primitive types. Use # union | on type hints to allow qb64, qb2, primitive instance, primitive class diff --git a/src/keri/kering.py b/src/keri/kering.py index aee3481a7..7f0b771ad 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -250,7 +250,7 @@ def __iter__(self): ColdDex = ColdCodex() # Make instance Coldage = namedtuple("Coldage", 'msg txt bny') # stream cold start status -Colds = Coldage(msg='msg', txt='txt', bny='bny') +Colds = Coldage(msg='msg', txt='txt', bny='bny') # add 'ant' for annotated def sniff(ims): @@ -290,6 +290,8 @@ def sniff(ims): return Colds.txt if tritet in (ColdDex.CtOpB2,): return Colds.bny + #if tritet in (ColdDex.AnB64, ): + raise ColdStartError("Unexpected tritet={} at stream start.".format(tritet)) diff --git a/tests/core/test_streaming.py b/tests/core/test_streaming.py index b32c43344..641553bbf 100644 --- a/tests/core/test_streaming.py +++ b/tests/core/test_streaming.py @@ -19,19 +19,54 @@ from keri.core.streaming import (annot, denot, Streamer) +def test_streamer(): + """Test streamer instance""" + pass + + """End Test""" + + def test_annot(): """Test annot function Annotate""" + ims = bytearray( + b'-FAtYKERICAAXicpEO6lMLcTbUhdpbQVXCh78MShuT_69th6tiZhEbAfPCj4DG9X' + b'hvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAAMAAB-LALDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAA-LAAMAAA-LAA-LAA-LAA') + + print(f"incoming = \n{ims}\n") + ams = annot(ims) # annotated message dream + assert not ims + print(f"annotated = \n{ams}\n") + + ims = bytearray( + b'-FDCYKERICAAXicpEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6EMEv' + b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAAAMAAC-LAhDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' + b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-LAh' + b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUGEEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZp' + b'ZEyPMAAD-LAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' + b'o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnBMOmBoddcrRHShSajb4d60S6RK34' + b'gXZ2WYbr3AiPY1M0-LABXDND-LA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'4fBJre3NGwTQMAAAEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' + b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAABEMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' + b'ZJGJqQZpZEyP') + + print(f"incoming = \n{ims}\n") + ams = annot(ims) # annotated message dream + assert not ims + print(f"annotated = \n{ams}\n") - """End Test""" -def test_streamer(): - """Test streamer instance""" """End Test""" + + if __name__ == "__main__": - test_annot() test_streamer() + test_annot() From 79159fedf8a58cb18f5b036b6df752e268e5ab6c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 17 Apr 2024 07:07:06 -0700 Subject: [PATCH 155/418] annotation working for basic KERI key events started on deannotation --- src/keri/core/streaming.py | 145 +++++++++++++++++++++++++++++++++-- src/keri/kering.py | 7 ++ tests/core/test_streaming.py | 78 ++++++++++++++++++- 3 files changed, 221 insertions(+), 9 deletions(-) diff --git a/src/keri/core/streaming.py b/src/keri/core/streaming.py index 8461d3b9a..878a76ecc 100644 --- a/src/keri/core/streaming.py +++ b/src/keri/core/streaming.py @@ -58,6 +58,10 @@ def annot(ims): oms.extend(f"{' ' * indent * 2}{val.qb64} # Key Event Counter " f"{val.name} count={val.count} quadlets\n".encode()) indent += 1 + + # should grab fill message frame here so can verify its consumed + # when done annotating + # version val = Verser(qb64b=ims, strip=True) versage = val.versage @@ -65,23 +69,32 @@ def annot(ims): f"proto={versage.proto} vrsn={versage.vrsn.major}." f"{versage.vrsn.minor:02}\n".encode()) # ilk - val = Ilker(qb64b=ims, strip=True) - oms.extend(f"{' ' * indent * 2}{val.qb64} # 't' message type Ilker " - f"{val.name} Ilk={val.ilk}\n".encode()) - if val.ilk == Ilks.icp: # inception - # icp v='', t='', d='', i='', s='0', kt='0',k=[], nt='0', n=[], bt='0', b=[], c=[], a=[] + ilker = Ilker(qb64b=ims, strip=True) + ilk = ilker.ilk + oms.extend(f"{' ' * indent * 2}{ilker.qb64} # 't' message type Ilker " + f"{ilker.name} Ilk={ilker.ilk}\n".encode()) + + + + if ilk in (Ilks.icp, Ilks.ixn, Ilks.rot, Ilks.dip, Ilks.drt): # said val = Diger(qb64b=ims, strip=True) oms.extend(f"{' ' * indent * 2}{val.qb64} # 'd' SAID Diger " f"{val.name} \n".encode()) + # aid pre val = Prefixer(qb64b=ims, strip=True) oms.extend(f"{' ' * indent * 2}{val.qb64} # 'i' AID Prefixer " f"{val.name} \n".encode()) + # sn val = Number(qb64b=ims, strip=True) oms.extend(f"{' ' * indent * 2}{val.qb64} # 's' Number " f"{val.name} sn={val.sn}\n".encode()) + + if ilk in (Ilks.icp, Ilks.dip): # inception + # v='', t='', d='', i='', s='0', kt='0',k=[], nt='0', n=[], bt='0', b=[], c=[], a=[] + # Signing key threshold val = Tholder(limen=ims, strip=True) # add qb64 and qb2 to tholder as aliases for limen and limen.decode() oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'kt' " @@ -143,6 +156,106 @@ def annot(ims): oms.extend(f"{' ' * indent * 2}{val.qb64} # trait Traitor " f"{val.name} trait={val.trait}\n".encode()) indent -= 1 + + + elif ilk == Ilks.ixn: # Interaction + # v='', t='',d='', i='', s='0', p='', a=[] + # Prior said + val = Diger(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'p' prior SAID Diger " + f"{val.name} \n".encode()) + + + + elif ilk in (Ilks.rot, Ilks.drt): # Rotation + # v='', t='',d='', i='', s='0', p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], ba=[], c=[], a=[] + # Prior said + val = Diger(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'p' prior SAID Diger " + f"{val.name} \n".encode()) + + # Signing key threshold + val = Tholder(limen=ims, strip=True) # add qb64 and qb2 to tholder as aliases for limen and limen.decode() + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'kt' " + f"Tholder signing threshold={val.sith}\n".encode()) + # Signing key list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'k' Signing Key " + f"List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Verfer(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # key Verfer " + f"{val.name}\n".encode()) + indent -= 1 + # Rotation key threshold + val = Tholder(limen=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'nt' " + f"Tholder rotation threshold={val.sith}\n".encode()) + # Next key digest list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'n' Rotation Key " + f"Digest List Counter {val.name} count={val.count} " + f"quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Diger(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # key digest Diger" + f" {val.name}\n".encode()) + indent -= 1 + # Witness Backer threshold + val = Tholder(limen=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'bt' " + f"Tholder Backer (witness) threshold={val.sith}\n".encode()) + + # Witness Backer Cut list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'b' Backer (witness)" + f"Cut List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Prefixer(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # AID Prefixer " + f"{val.name}\n".encode()) + indent -= 1 + + # Witness Backer Add list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'b' Backer (witness)" + f"Add List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Prefixer(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # AID Prefixer " + f"{val.name}\n".encode()) + indent -= 1 + + # Config Trait List + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'c' Config Trait " + f"List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Traitor(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # trait Traitor " + f"{val.name} trait={val.trait}\n".encode()) + indent -= 1 + + + else: + raise kering.IlkError(f"Unexpected message type={ilk}") + + if ilk in (Ilks.icp, Ilks.ixn, Ilks.rot, Ilks.dip, Ilks.drt): # Seal (anchor) List val = Counter(qb64b=ims, strip=True) oms.extend(f"{' ' * indent * 2}{val.qb64} # 'a' Seal List Counter" @@ -169,6 +282,14 @@ def annot(ims): indent -= 1 indent -= 1 + if ilk == Ilks.dip: + # delegator aid delpre + val = Prefixer(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'di' Delegator AID Prefixer " + f"{val.name} \n".encode()) + + if ims: + raise kering.ExtractionError(f"Unexpected remaining bytes in stream.") elif cold in Colds.bny: pass @@ -179,13 +300,23 @@ def annot(ims): return oms.decode() # return unicode string -def denot(): +def denot(ams): """De-annotate CESR stream + Returns: + deannotation (bytes): deannotation of input annotated CESR stream - + Parameters: + ams (str): CESR annotated message stream text """ + oms = bytearray() + + + + + return bytes(oms) + class Streamer: diff --git a/src/keri/kering.py b/src/keri/kering.py index 7f0b771ad..2f277ffd8 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -833,6 +833,13 @@ class KindError(ExtractionError): raise KindError("error message") """ +class IlkError(ExtractionError): + """ + Bad or Unsupported Message Type (Ilk) + + Usage: + raise IlkError("error message") + """ class ConversionError(ExtractionError): """ diff --git a/tests/core/test_streaming.py b/tests/core/test_streaming.py index 641553bbf..21e7d12aa 100644 --- a/tests/core/test_streaming.py +++ b/tests/core/test_streaming.py @@ -28,16 +28,19 @@ def test_streamer(): def test_annot(): """Test annot function Annotate""" + + # simple Inception ims = bytearray( b'-FAtYKERICAAXicpEO6lMLcTbUhdpbQVXCh78MShuT_69th6tiZhEbAfPCj4DG9X' b'hvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAAMAAB-LALDG9XhvcVryHj' b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAA-LAAMAAA-LAA-LAA-LAA') - print(f"incoming = \n{ims}\n") + print(f"incoming = \n{bytes(ims)}\n") ams = annot(ims) # annotated message dream assert not ims print(f"annotated = \n{ams}\n") + # complex inception ims = bytearray( b'-FDCYKERICAAXicpEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6EMEv' b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAAAMAAC-LAhDG9XhvcVryHj' @@ -53,7 +56,78 @@ def test_annot(): b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' b'ZJGJqQZpZEyP') - print(f"incoming = \n{ims}\n") + print(f"incoming = \n{bytes(ims)}\n") + ams = annot(ims) # annotated message dream + assert not ims + print(f"annotated = \n{ams}\n") + + + # interaction + ims = bytearray( + b'-FB6YKERICAAXixnEHeLJVa4LLNRRYVkLQsXHIDvllcmhDaahe5a_oMvXKePEMEv' + b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAABEMEvSn0o6Iv2-3gInTDM' + b'MDTV0qQEfooM-yTzkj6Kynn6-LBU-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'4fBJre3NGwTQMAACEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' + b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAAiEMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMABDEEbufBpvagqe9kijKISOoQPYFEOpy22C' + b'ZJGJqQZpZEyP-RAXDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MACA' + b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_') + + print(f"incoming = \n{bytes(ims)}\n") + ams = annot(ims) # annotated message dream + assert not ims + print(f"annotated = \n{ams}\n") + + + # Rotation + ims = bytearray( + b'-FCGYKERICAAXrotEDtBwgOB0uGrSMBJhOmnkRoCupjg-4sJApvOx04ujhKsEMEv' + b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAACEHeLJVa4LLNRRYVkLQsX' + b'HIDvllcmhDaahe5a_oMvXKePMAAC-LAhDH7p14xo09rob5cEupmo8jSDi35ZOGt1' + b'k4t2nm1C1A68DIAdqJzLWEwQbhXEMOFjvFVZ7oMCJP4XXDP_ILaTEBAQDKhYdMBe' + b'P6FoH3ajGJTf_4fH229rm_lTZXfYkfwGTMERMAAC-LAhEBvDSpcj3y0y9W2-1GzY' + b'J85KEkDIPxu4y_TxAK49k7ciEEb97lh2oOd_yM3meBaRX5xSs8mIeBoPdhOTgVkd' + b'31jbECQTrhKHgrOXJS4kdvifvOqoJ7RjfJSsN3nshclYStgaMAAD-LALBG9XhvcV' + b'ryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ-LALBH7p14xo09rob5cEupmo8jSD' + b'i35ZOGt1k4t2nm1C1A68-LAA-LAA') + + print(f"incoming = \n{bytes(ims)}\n") + ams = annot(ims) # annotated message dream + assert not ims + print(f"annotated = \n{ams}\n") + + # Delegated Inception + ims = bytearray( + b'-FDeYKERICAAXdipECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TECQs' + b'0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TMAAA4AADA1s2c1s2c1s2-LAh' + b'DIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uVDN7WiKyjLLBTK92xayCu' + b'ddZsBuwPmD2BKrl83h1xEUtiDOE5jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl' + b'5nfY4AADA1s2c1s2c1s2-LAhEKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ' + b'44DSEC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZEHgewy_ymPxtSFwu' + b'X2KaI_mPmoIUkxClviX3f-M38kCDMAAD-LAhBIR8GACw4z2GC5_XoReU4DMKbqi6' + b'-EdbgDZUAobRb8uVBN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiBOE5' + b'jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl5nfY-LAA-LBI-RAuDIR8GACw4z2G' + b'C5_XoReU4DMKbqi6-EdbgDZUAobRb8uVMAADEKFoJ9Conb37zSn8zHLKP3YwHbeQ' + b'iD1D9Qx0MagJ44DSDN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiMAAE' + b'EC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZ-QAYMAAVEHgewy_ymPxt' + b'SFwuX2KaI_mPmoIUkxClviX3f-M38kCDMD4SEKFoJ9Conb37zSn8zHLKP3YwHbeQ' + b'iD1D9Qx0MagJ44DSEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6') + + print(f"incoming = \n{bytes(ims)}\n") + ams = annot(ims) # annotated message dream + assert not ims + print(f"annotated = \n{ams}\n") + + # Delegated Rotatation + ims = bytearray( + b'-FBaYKERICAAXdrtEKwDKG0L9pAMbzV2e31-I5ObiEfkptfs8VqXYiHGCL1vECQs' + b'0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TMAABECQs0t3_GL7-B3q4kMU-' + b'qLeRCugTFjrxR15mxUwYWp8TMAAB-LALDJ0pLe3f2zGus0Va1dqWAnukWdZHGNWl' + b'K9NciJop9N4fMAAB-LALENX_LTL97uOSOkA1PEzam9vtmCLPprnbcpi71wXpmhFF' + b'MAAD-LALBIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV-LALBJ0pLe3f' + b'2zGus0Va1dqWAnukWdZHGNWlK9NciJop9N4f-LAA-LAA') + + print(f"incoming = \n{bytes(ims)}\n") ams = annot(ims) # annotated message dream assert not ims print(f"annotated = \n{ams}\n") From fd7e98a26d1e5ce42997a1afc0ee52a62da9a5aa Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 22 Apr 2024 13:36:04 -0400 Subject: [PATCH 156/418] Update Contributing.md (#765) --- Contributing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Contributing.md b/Contributing.md index ccef9a424..3f59768e9 100644 --- a/Contributing.md +++ b/Contributing.md @@ -81,7 +81,7 @@ Adapted from the Rust Code of Conduct: [https://www.rust-lang.org/policies/code- ## Getting Help For questions or clarifications, reach out via: -- Discord: [https://discord.gg/edGDD632tP](https://discord.gg/edGDD632tP) +- Discord: [See KERI repo] (https://github.com/WebOfTrust/keri) - KERI Development Meetings: [https://github.com/WebOfTrust/keri#implementors-call] - ACDC Standards Meeting@TOIP (technically must be a member of ToIP to contribute): [https://github.com/WebOfTrust/keri#specification-call] @@ -91,4 +91,4 @@ Thanks to all our wonderful contributors. ## Additional Resources - [Project Documentation and Search Engine](https://kerisse.org) -- [Related Projects](https://github.com/WebOfTrust) \ No newline at end of file +- [Related Projects](https://github.com/WebOfTrust) From ae5ed8d661fb4b90d773314b8d3c463d2d8ff594 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Tue, 30 Apr 2024 10:17:02 -0700 Subject: [PATCH 157/418] Attempt to conditionally run `brew install libsodium` on macos-latest build (#770) * Attempt to conditionally run `brew install libsodium` on macos-latest build Signed-off-by: pfeairheller * How about this? Signed-off-by: pfeairheller * Just going to pin to macos-13 Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller --- .github/workflows/python-app-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app-ci.yml b/.github/workflows/python-app-ci.yml index 359f8f64f..d0ae68b2d 100644 --- a/.github/workflows/python-app-ci.yml +++ b/.github/workflows/python-app-ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ macos-latest, ubuntu-latest ] + os: [ macos-13, ubuntu-latest ] steps: - uses: actions/checkout@v3 From 05cbbd5c28c3b7d37e6219f01edc5ffe467d3244 Mon Sep 17 00:00:00 2001 From: Philipp Schlarb <87540518+pSchlarb@users.noreply.github.com> Date: Tue, 30 Apr 2024 20:00:48 +0200 Subject: [PATCH 158/418] Fixed missing verbose output from ipex list (#761) Signed-off-by: pSchlarb --- src/keri/app/cli/commands/ipex/list.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/keri/app/cli/commands/ipex/list.py b/src/keri/app/cli/commands/ipex/list.py index 70569a52e..4dd3a50ca 100644 --- a/src/keri/app/cli/commands/ipex/list.py +++ b/src/keri/app/cli/commands/ipex/list.py @@ -8,6 +8,7 @@ import argparse import datetime import os +import json import sys from hio import help @@ -199,6 +200,12 @@ def grant(self, exn): if response is not None: print(f" Response: {responseType} ({response.qb64})") + if self.verbose: + bsad = json.dumps(sad, indent=2) + print(" Full Credential:") + for line in bsad.splitlines(): + print(f"\t{line}") + def apply(self, note, exn, pathed): pass From e414492d9f999dc20e8a07034a2ce95c19421beb Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Tue, 30 Apr 2024 16:14:37 -0400 Subject: [PATCH 159/418] issue templates (#769) Signed-off-by: Kevin Griffin --- .github/ISSUE_TEMPLATE/bug.yaml | 34 +++++++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature.yaml | 12 ++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug.yaml create mode 100644 .github/ISSUE_TEMPLATE/feature.yaml diff --git a/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug.yaml new file mode 100644 index 000000000..94a4f3bc7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yaml @@ -0,0 +1,34 @@ +name: "Bug report" +description: Bug report template +title: "A concise description" +labels: ["bug", "triage"] +body: + - type: input + id: version + attributes: + label: Version + validations: + required: true + - type: input + id: Evnironment + attributes: + label: Environment + description: (OS, Python version, deployment environment) + - type: textarea + id: expected-behavior + attributes: + label: Expected behavior + validations: + required: true + - type: textarea + id: actual-behavior + attributes: + label: Actual behavior + validations: + required: true + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to reproduce + validations: + required: true \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature.yaml b/.github/ISSUE_TEMPLATE/feature.yaml new file mode 100644 index 000000000..a73d76781 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yaml @@ -0,0 +1,12 @@ +name: "Feature request" +description: Feature request template +title: "A concise description" +labels: ["feature request"] +body: + - type: textarea + id: desired-behavior + attributes: + label: Feature request description/rationale + validations: + required: true + \ No newline at end of file From 06f19035a6616122b09b577259432e25d5542547 Mon Sep 17 00:00:00 2001 From: Rubel Hassan Date: Tue, 30 Apr 2024 16:44:33 -0500 Subject: [PATCH 160/418] Deprecate randomNonce() and refactor using Salter().qb64 (#768) Co-authored-by: Rubel Hassan Mollik --- src/keri/app/cli/commands/ipex/agree.py | 4 +- src/keri/app/cli/commands/ipex/apply.py | 4 +- src/keri/app/cli/commands/ipex/offer.py | 4 +- src/keri/app/cli/commands/nonce.py | 4 +- src/keri/core/coring.py | 15 +- src/keri/vdr/eventing.py | 3 +- tests/app/test_connecting.py | 4 +- tests/app/test_grouping.py | 14 +- tests/app/test_signing.py | 228 ++++++++++++------------ tests/conftest.py | 7 +- tests/vc/test_protocoling.py | 180 +++++++++---------- tests/vc/test_walleting.py | 32 ++-- tests/vdr/test_eventing.py | 146 +++++++-------- tests/vdr/test_txn_state.py | 98 +++++----- tests/vdr/test_verifying.py | 12 +- 15 files changed, 380 insertions(+), 375 deletions(-) diff --git a/src/keri/app/cli/commands/ipex/agree.py b/src/keri/app/cli/commands/ipex/agree.py index 5e04b49ee..34eea242f 100644 --- a/src/keri/app/cli/commands/ipex/agree.py +++ b/src/keri/app/cli/commands/ipex/agree.py @@ -7,7 +7,7 @@ from hio.base import doing -from keri.core import coring +from keri.core import signing parser = argparse.ArgumentParser(description='Reply to IPEX offer message acknowledged willingness to accept offered ' 'credential') @@ -23,4 +23,4 @@ def nonce(tymth, tock=0.0): """ _ = (yield tock) - print(coring.randomNonce()) + print(signing.Salter().qb64) diff --git a/src/keri/app/cli/commands/ipex/apply.py b/src/keri/app/cli/commands/ipex/apply.py index b5d3d8ef9..47d15fd48 100644 --- a/src/keri/app/cli/commands/ipex/apply.py +++ b/src/keri/app/cli/commands/ipex/apply.py @@ -7,7 +7,7 @@ from hio.base import doing -from keri.core import coring +from keri.core import signing parser = argparse.ArgumentParser(description='Request a credential from another party by initiating an IPEX exchange') parser.set_defaults(handler=lambda args: handler(args)) @@ -22,4 +22,4 @@ def nonce(tymth, tock=0.0): """ _ = (yield tock) - print(coring.randomNonce()) + print(signing.Salter().qb64) diff --git a/src/keri/app/cli/commands/ipex/offer.py b/src/keri/app/cli/commands/ipex/offer.py index a297b234f..ea269d40c 100644 --- a/src/keri/app/cli/commands/ipex/offer.py +++ b/src/keri/app/cli/commands/ipex/offer.py @@ -7,7 +7,7 @@ from hio.base import doing -from keri.core import coring +from keri.core import signing parser = argparse.ArgumentParser(description='Reply to IPEX apply message or initiate an IPEX exchange with an offer' ' for a credential with certain characteristics') @@ -23,4 +23,4 @@ def nonce(tymth, tock=0.0): """ _ = (yield tock) - print(coring.randomNonce()) + print(signing.Salter().qb64) diff --git a/src/keri/app/cli/commands/nonce.py b/src/keri/app/cli/commands/nonce.py index 96669c816..77a6e3fdc 100644 --- a/src/keri/app/cli/commands/nonce.py +++ b/src/keri/app/cli/commands/nonce.py @@ -8,7 +8,7 @@ import pysodium from hio.base import doing -from keri.core import coring +from keri.core import signing parser = argparse.ArgumentParser(description='Print a new random nonce') parser.set_defaults(handler=lambda args: handler(args)) @@ -23,4 +23,4 @@ def nonce(tymth, tock=0.0): """ _ = (yield tock) - print(coring.randomNonce()) + print(signing.Salter().qb64) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index a02d29fbb..f21c55829 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -184,24 +184,21 @@ def loads(raw, size=None, kind=Serials.json): return ked -# ToDo: nonces only need 128 bits of entropy. a Salt is enough -# Just use Salter().qb64. # Deprecated +# randomNonce() refactored to match Salter().qb64 and only used in coring to avoid circular dependencies +# use Salter().qb64 in other places def randomNonce(): - """ Generate a random ed25519 seed and encode as qb64 + """ Generate a random 128 bits salt and encode as qb64 Returns: - str: qb64 encoded ed25519 random seed + str: qb64 encoded 128 bits random salt """ - preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - seedqb64 = Matter(raw=preseed, code=MtrDex.Ed25519_Seed).qb64 + preseed = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + seedqb64 = Matter(raw=preseed, code=MtrDex.Salt_128).qb64 return seedqb64 - - - # secret derivation security tier Tierage = namedtuple("Tierage", 'low med high') diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 278eb4bac..9deb9a544 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -20,6 +20,7 @@ from ..core import serdering, coring, indexing from ..core.coring import (MtrDex, Serials, versify, Prefixer, Ilks, Seqner, Verfer, Number) +from ..core.signing import (Salter,) from ..core.eventing import SealEvent, ample, TraitDex, verifySigs from ..db import basing, dbing from ..db.dbing import dgKey, snKey @@ -90,7 +91,7 @@ def incept( if toad != 0: # invalid toad raise ValueError("Invalid toad = {} for baks = {}".format(toad, baks)) - nonce = nonce if nonce is not None else coring.randomNonce() + nonce = nonce if nonce is not None else Salter().qb64 ked = dict(v=vs, # version string t=ilk, d="", diff --git a/tests/app/test_connecting.py b/tests/app/test_connecting.py index 278efa891..448cbc37a 100644 --- a/tests/app/test_connecting.py +++ b/tests/app/test_connecting.py @@ -9,7 +9,7 @@ from keri import kering from keri.app import connecting, habbing -from keri.core import coring +from keri.core import signing def test_organizer(): @@ -144,7 +144,7 @@ def test_organizer(): 'zip': '08807'} # Update the Jen's data signature by signing garbage - nonce = coring.randomNonce() + nonce = signing.Salter().qb64 cigar = hby.signator.sign(ser=nonce.encode("utf-8")) hby.db.ccigs.pin(keys=(jen,), val=cigar) with pytest.raises(kering.ValidationError): diff --git a/tests/app/test_grouping.py b/tests/app/test_grouping.py index fca1bc91d..6d0cbe59e 100644 --- a/tests/app/test_grouping.py +++ b/tests/app/test_grouping.py @@ -701,14 +701,14 @@ def test_multisig_registry_incept(mockHelpingNowUTC, mockCoringRandomNonce): usage="Issue vLEI Credentials") assert exn.ked["r"] == '/multisig/vcp' - assert exn.saidb == b'ECKiNFo7fpG4vS5tUeja3EvOqT8ctq4AW8E3HKsP7dJo' - assert atc == (b'-FABEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba0AAAAAAAAAAAAAAA' - b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAABh6d0m' - b'lebT57L8o2si7DfEvPCoXJP0ekPiBqkzQns3-P7dz36MPXhjNFW6xRRdUstDLAZe' - b'BEqBxBCltMpTZGsD-LAa5AACAA-e-anc-AABAAD2mK9ICW9x1-0NZGkEDOcAbZ58' - b'VWK9LOTwyN2lSfHr2zY638P1SBStoh8mjgy7nOTGMyujOXMKvF_ZDeQ_ISYA') + assert exn.saidb == b'EJN27EYqgxTS2NCHdnjE-CU0UikV5a1Cw5OZdv-g0jQ6' + assert atc == (b'-FABEDEf72ZZ9mhpT1Xz-_YkXl7cg93sjZUFLIsxaFNTbXQO0AAAAAAAAAAAAAAA' + b'AAAAAAAAEDEf72ZZ9mhpT1Xz-_YkXl7cg93sjZUFLIsxaFNTbXQO-AABAABsPiWf' + b'UE9L7D9KBVOYMg1rt88gK9DBkiBYb21xMR0YH7sCT0hqwX9y8CM5Y6jQwzz3NqqN' + b'-aJWShzz1mbtb5AG-LAa5AACAA-e-anc-AABAABXlwkzbp_tC4MEbx1Uyny1o7dB' + b'GHrYjU3u90Mhv2GtrIGG-7va1jZnlXef2R_LM4TRN8_XjmpLv1skcJaM90UB') data = exn.ked["a"] - assert data == {'gid': 'EERn_laF0qwP8zTBGL86LbF84J0Yh2IvQSRskH3BZZiy', + assert data == {'gid': 'EEVG5a8c88Fg9vH-6zQP6gJdc4LxVbUTRydx-JhpDcob', 'usage': 'Issue vLEI Credentials'} assert "vcp" in exn.ked["e"] assert "anc" in exn.ked["e"] diff --git a/tests/app/test_signing.py b/tests/app/test_signing.py index 8d334543a..0b297d17e 100644 --- a/tests/app/test_signing.py +++ b/tests/app/test_signing.py @@ -46,85 +46,86 @@ def test_sad_signature(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): # Sign with non-transferable identifier, default to entire SAD sig0 = signing.ratify(wanHab, cred) - assert sig0 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtSKl' - b'DJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","ri":' - b'"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1rLBSMZ' - b'_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQzDXg39j1b' - b'7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0Jp6ZFRD9dTb' - b'4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"EBmRy7xMwsxUe' - b'lUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84GM83MG36","pers' - b'onal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4GswcU6ov8-","n":"Q8r' - b'NaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","personLegalName":"Anna ' - b'Jones","engagementContextRole":"Project Manager"}},"e":[{"qualifie' - b'dvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU' - b'5sHYTGFD"}]}-JAB6AABAAA--CABBAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBw' - b'bAqteP0BApbos85syKE_VgfuNTtVRYkAlw5fwb_4ZWN-V-FFO_MrSGt71luX0rt-9e' - b'hNZFPHV1EuPc1YDQvZJ1XqPumewN') + assert sig0 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtS' + b'KlDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","' + b'ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1' + b'rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQz' + b'DXg39j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0J' + b'p6ZFRD9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"E' + b'BmRy7xMwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84G' + b'M83MG36","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4Gswc' + b'U6ov8-","n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","perso' + b'nLegalName":"Anna Jones","engagementContextRole":"Project Manage' + b'r"}},"e":[{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_o' + b'zM1uAnFvSfC0N1jaQ42aKU5sHYTGFD"}]}-JAB6AABAAA--CABBEByXTm1zJZ1oI' + b'uvnad6hppC3iBrUmQLb0a7iXSaRK4p0BCCKuUzhtasI04dqYQY4x7AqHMLP7Cbn2' + b'nqKFrR34_4zjJvT3HARSLhL_iUOSyoHMVv2-9mgtWuuW0oF4XS3n0C') # sign with non-trans identifier with a specific set of paths sig1 = signing.ratify(wanHab, cred, paths=paths) - assert sig1 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtSK' - b'lDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","ri' - b'":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1rLB' - b'SMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQzDXg3' - b'9j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0Jp6ZFR' - b'D9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"EBmRy7x' - b'MwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84GM83MG36' - b'","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4GswcU6ov8-",' - b'"n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","personLegalNam' - b'e":"Anna Jones","engagementContextRole":"Project Manager"}},"e":[' - b'{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_ozM1uAnFvSfC' - b'0N1jaQ42aKU5sHYTGFD"}]}-KAD6AABAAA--JAB6AABAAA--CADBAbSj3jfaeJbpu' - b'qg0WtvHw31UoRZOnN_RZQYBwbAqteP0BApbos85syKE_VgfuNTtVRYkAlw5fwb_4Z' - b'WN-V-FFO_MrSGt71luX0rt-9ehNZFPHV1EuPc1YDQvZJ1XqPumewN-JAB5AABAA-a' - b'-CADBAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP0BC1HB3w11jDft8De' - b'EhpuRVWZknabe5omZZA0s4QqX2o1O3gSM-PrjYr6lXlvnQ4TU_PaeF70kSo--LA22' - b'kL2csA-JAB4AADA-a-personal-CADBAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQ' - b'YBwbAqteP0BAPZ92lw8F2_Ap81vFpyQsuTU9l7tOLI2ZmqanKOMVd2ar-m16JjA38' - b'PPH_mBFasyadIQgyun410RpxCUvsIBAH') + assert sig1 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtS' + b'KlDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","' + b'ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1' + b'rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQz' + b'DXg39j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0J' + b'p6ZFRD9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"E' + b'BmRy7xMwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84G' + b'M83MG36","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4Gswc' + b'U6ov8-","n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","perso' + b'nLegalName":"Anna Jones","engagementContextRole":"Project Manage' + b'r"}},"e":[{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_o' + b'zM1uAnFvSfC0N1jaQ42aKU5sHYTGFD"}]}-KAD6AABAAA--JAB6AABAAA--CADBE' + b'ByXTm1zJZ1oIuvnad6hppC3iBrUmQLb0a7iXSaRK4p0BCCKuUzhtasI04dqYQY4x' + b'7AqHMLP7Cbn2nqKFrR34_4zjJvT3HARSLhL_iUOSyoHMVv2-9mgtWuuW0oF4XS3n' + b'0C-JAB5AABAA-a-CADBEByXTm1zJZ1oIuvnad6hppC3iBrUmQLb0a7iXSaRK4p0B' + b'A0YOvEHllXmhlmv7ec_MShdDf2eZ3OvMDKE_zhfQsrFvU2ip1g-z04HZ097hN66r' + b'Lo5nqglvdCkxXJcYBqe8kD-JAB4AADA-a-personal-CADBEByXTm1zJZ1oIuvna' + b'd6hppC3iBrUmQLb0a7iXSaRK4p0BBjkD40acCOcdFuAM9d5FAgnfXqkZyI9haUTe' + b'VDO9vhPqgnRIXAUcTMxkY3eAfhX0ggy7n99sosla1nyVyBE_cK') # Sign with transferable identifier defaults to single signature on entire SAD sig2 = signing.ratify(sidHab, cred) - assert sig2 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtSK' - b'lDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","ri' - b'":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1rLB' - b'SMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQzDXg3' - b'9j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0Jp6ZFR' - b'D9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"EBmRy7x' - b'MwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84GM83MG36' - b'","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4GswcU6ov8-",' - b'"n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","personLegalNam' - b'e":"Anna Jones","engagementContextRole":"Project Manager"}},"e":[' - b'{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_ozM1uAnFvSfC' - b'0N1jaQ42aKU5sHYTGFD"}]}-JAB6AABAAA--FABEKC8085pwSwzLwUGzh-HrEoFDw' - b'ZnCJq27bVp5atdMT9o0AAAAAAAAAAAAAAAAAAAAAAAEKC8085pwSwzLwUGzh-HrEo' - b'FDwZnCJq27bVp5atdMT9o-AABAABf42ZzPJkYp6fm6yMhqYqSOg9IAf8dmxpBRq22' - b'GNZTQaalCJIuAxCVT2GRiqs7z7Z5kIPy5yhR3DYVx2Y_Rj8N') + assert sig2 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtS' + b'KlDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","' + b'ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1' + b'rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQz' + b'DXg39j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0J' + b'p6ZFRD9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"E' + b'BmRy7xMwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84G' + b'M83MG36","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4Gswc' + b'U6ov8-","n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","perso' + b'nLegalName":"Anna Jones","engagementContextRole":"Project Manage' + b'r"}},"e":[{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_o' + b'zM1uAnFvSfC0N1jaQ42aKU5sHYTGFD"}]}-JAB6AABAAA--FABEDT4jW7VYWenu1' + b'MrgJJEZWB0ScqZdVmDQw0RUdxKLWax0AAAAAAAAAAAAAAAAAAAAAAAEDT4jW7VYW' + b'enu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax-AABAAA5PVjmjZFG-1meySYd8YrYhV' + b'XubpZd9DXRDJ1uIeFxjKiy_XbJAG7Yl00KxVAcObWDmBStuWLn-qoOI1YcFYoB') # Sign with transferable identifier with specific set of paths sig3 = signing.ratify(sidHab, cred, paths=paths) - assert sig3 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtSK' - b'lDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","ri' - b'":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1rLB' - b'SMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQzDXg3' - b'9j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0Jp6ZFR' - b'D9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"EBmRy7x' - b'MwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84GM83MG36' - b'","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4GswcU6ov8-",' - b'"n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","personLegalNam' - b'e":"Anna Jones","engagementContextRole":"Project Manager"}},"e":[' - b'{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_ozM1uAnFvSfC' - b'0N1jaQ42aKU5sHYTGFD"}]}-KAD6AABAAA--JAB6AABAAA--FABEKC8085pwSwzLw' - b'UGzh-HrEoFDwZnCJq27bVp5atdMT9o0AAAAAAAAAAAAAAAAAAAAAAAEKC8085pwSw' - b'zLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o-AABAABf42ZzPJkYp6fm6yMhqYqSOg9I' - b'Af8dmxpBRq22GNZTQaalCJIuAxCVT2GRiqs7z7Z5kIPy5yhR3DYVx2Y_Rj8N-JAB5' - b'AABAA-a-FABEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o0AAAAAAAAA' - b'AAAAAAAAAAAAAAEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o-AABAAB' - b'B5IVZOhEfcH4TBQgOCyMgyQrJujtBBjT8K_zTPk0-FLMtTZuBgXV7jnLw6fDe6FWt' - b'zshh2HGCL_H_j4i1b9kF-JAB4AADA-a-personal-FABEKC8085pwSwzLwUGzh-Hr' - b'EoFDwZnCJq27bVp5atdMT9o0AAAAAAAAAAAAAAAAAAAAAAAEKC8085pwSwzLwUGzh' - b'-HrEoFDwZnCJq27bVp5atdMT9o-AABAACUs3V7O3fRGmi_-X8xo8UFw7rGEQ0CYyf' - b'K7U1URYBwtW546a1pVJZ_c1wzcTXWLDj7OPPNrAQx2em5sNrOMPcH') + assert sig3 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtS' + b'KlDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","' + b'ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1' + b'rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQz' + b'DXg39j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0J' + b'p6ZFRD9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"E' + b'BmRy7xMwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84G' + b'M83MG36","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4Gswc' + b'U6ov8-","n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","perso' + b'nLegalName":"Anna Jones","engagementContextRole":"Project Manage' + b'r"}},"e":[{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_o' + b'zM1uAnFvSfC0N1jaQ42aKU5sHYTGFD"}]}-KAD6AABAAA--JAB6AABAAA--FABED' + b'T4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax0AAAAAAAAAAAAAAAAAAAAA' + b'AAEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax-AABAAA5PVjmjZFG-1' + b'meySYd8YrYhVXubpZd9DXRDJ1uIeFxjKiy_XbJAG7Yl00KxVAcObWDmBStuWLn-q' + b'oOI1YcFYoB-JAB5AABAA-a-FABEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUd' + b'xKLWax0AAAAAAAAAAAAAAAAAAAAAAAEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw' + b'0RUdxKLWax-AABAABYXrbXx0VXn_qs1AVUzVqJ19lfnN1aBoxaLjlcgHaCgCOaY_' + b'AUPOuEyE9EFgA_VLCHnPi7TdZhbXyG09Due24P-JAB4AADA-a-personal-FABED' + b'T4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax0AAAAAAAAAAAAAAAAAAAAA' + b'AAEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax-AABAACNv9Bg6Gg9oi' + b'pvuArddJgY1uNfp7uFO8ifkwKHhQargXScvQuvYIV7-VZM4Xu0vNOWaKA-mDmfei' + b'sRYanh_gEJ') # Test multisig identifier with configing.openCF(name="mel", base="mel", temp=True) as cf, \ @@ -156,18 +157,19 @@ def test_sad_signature(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): # Sign with multisig transferable identifier defaults to single signature on entire SAD sig1 = signing.ratify(hab=hab, serder=cred) - assert sig1 == (b'{"v":"ACDC10JSON00019e_","d":"EFnCQiKLTr37bhuFrvbE88GhWyQ-LUryYNiDjO4' - b'cW1Nt","i":"EKMHlh4epBApuYP-3-A_ZldeImCa6WxLe8Nmzhy-SvWB","ri":"ELIc0' - b'Va2OSemOpuiD2Fxfzd2yXg6CWibjOCJY3vNFLb9","s":"EMQWEcCnVRk1hatTNyK3sIy' - b'kYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EEBT5231zltSwAzucatIlavzJyE43F3op2l6' - b'aB6or1FP","dt":"2021-06-09T17:35:54.169967+00:00","i":"EAwC_-ISX9helB' - b'DUSKU5j_VEU3G0Jp6ZFRD9dTb4-5c0","LEI":"254900OPPU84GM83MG36"},"e":{}}' - b'-JAB6AABAAA--FABEKMHlh4epBApuYP-3-A_ZldeImCa6WxLe8Nmzhy-SvWB0AAAAAAAA' - b'AAAAAAAAAAAAAAAEKMHlh4epBApuYP-3-A_ZldeImCa6WxLe8Nmzhy-SvWB-AADAADV_a' - b'TlO9rV3bxKCiSr4zqqu0Jg1Q3u6Fk2pWxrctkXepRRlDCfKeKmlMQHnNJ8r6vRQp8Eslv' - b'h33uSBdS_MlIPABC--losKLa_YfcwGBIabsq8g5VABrsmnXkZMT7eB-jqK6bdB4Zs_863' - b'la0mm_DeMwS1KXomLb_j1zCmgZ3RJPAPACA539yer3U8JQlcgXrdbPlR-1kADcFA4bsN_' - b'klRSu7p61y-Z2CS5d7Aitrc7yq00YIG_u-v7OToChDC3TsVCR4D') + assert sig1 == (b'{"v":"ACDC10JSON00019e_","d":"EPA-T12T4d8sHaj2IyvK2ReIs5KREYN24W' + b'tJbYL-d2HQ","i":"ELqU3H49LwniPBMVMGqWjSfh_8mFRntbXVlBQirijKMi","' + b'ri":"ENw4eVQH4l7uHSYa50yWh1AW8Wj22V77KHL7Wg55F5gB","s":"EMQWEcCn' + b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EEBT5231zltSwAzu' + b'catIlavzJyE43F3op2l6aB6or1FP","dt":"2021-06-09T17:35:54.169967+0' + b'0:00","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0Jp6ZFRD9dTb4-5c0","LEI":"' + b'254900OPPU84GM83MG36"},"e":{}}-JAB6AABAAA--FABELqU3H49LwniPBMVMG' + b'qWjSfh_8mFRntbXVlBQirijKMi0AAAAAAAAAAAAAAAAAAAAAAAELqU3H49LwniPB' + b'MVMGqWjSfh_8mFRntbXVlBQirijKMi-AADAAB7s8EP1DushDLnCeo3xcud-y_fXm' + b'X4AL93H8hUAFDv4o2x-5_GHmpUWC9lt86DW_sUX1rJA1OsPGaCYXiFFOoHABD3bW' + b'HcN4gL4tgtN9kl8737NzW18wIIE2GEEvKh2Z45Ai5sIynHzU0q6mCD3LTIRb5DfA' + b'aazZfDFlqgLYtyBVIPACDpAkl7FGW7pQ5ZLW6rCLTBYUfM_bagl7Fwo_GJhYNAiR' + b'3D52Y6QFhqQDn-F_FJZSAlQUZZGmT0ZJeIEHg5qwwH') """End Test""" @@ -215,15 +217,15 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs sig0.extend(seqner.qb64b) sig0.extend(coring.Saider(qb64=issuer.regd).qb64b) - assert sig0 == (b'{"v":"ACDC10JSON00019e_","d":"EK88fyN65bfA63o1jgeOGKeIxw6sTJEwwU' - b'3ycpjdtCUD","i":"EKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o","' - b'ri":"ENzh5cyGjFhQYuIXuheXV2wkKp23rkxYI7wbEBQIyqhP","s":"EMQWEcCn' + assert sig0 == (b'{"v":"ACDC10JSON00019e_","d":"EEs1BzDdw0h5VMhg5KUo904UyCdaMT1ut_' + b'_vN5G879pP","i":"EDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax","' + b'ri":"ECBNRQkp71gi2mTqZu2cp-xe6CrWlvJ5pRoBIBnRQbGp","s":"EMQWEcCn' b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EFyxk35e1r5G9pcu' b'vv8j5F4FWRHD8xlZ_E4rWPdlVASI","dt":"2021-06-09T17:35:54.169967+0' b'0:00","i":"EIflL4H4134zYoRM6ls6Q086RLC_BhfNFh5uk-WxvhsL","LEI":"' - b'254900OPPU84GM83MG36"},"e":{}}-IABENzh5cyGjFhQYuIXuheXV2wkKp23rk' - b'xYI7wbEBQIyqhP0AAAAAAAAAAAAAAAAAAAAAABENzh5cyGjFhQYuIXuheXV2wkKp' - b'23rkxYI7wbEBQIyqhP') + b'254900OPPU84GM83MG36"},"e":{}}-IABECBNRQkp71gi2mTqZu2cp-xe6CrWlv' + b'J5pRoBIBnRQbGp0AAAAAAAAAAAAAAAAAAAAAABECBNRQkp71gi2mTqZu2cp-xe6C' + b'rWlvJ5pRoBIBnRQbGp') iss = issuer.issue(said=cred.said) rseal = SealEvent(iss.pre, "0", iss.said)._asdict() @@ -263,23 +265,23 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs # Bad magic values here but can't figure out where looks like Sadder Said seeder # is using a bad magic value sig1 = signing.ratify(hab=hab, serder=cred, paths=[[], ["a"], ["a", "i"]]) - assert sig1 == (b'{"v":"ACDC10JSON00019e_","d":"EK88fyN65bfA63o1jgeOGKeIxw6sTJEwwU3y' - b'cpjdtCUD","i":"EKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o","ri":' - b'"ENzh5cyGjFhQYuIXuheXV2wkKp23rkxYI7wbEBQIyqhP","s":"EMQWEcCnVRk1ha' - b'tTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EFyxk35e1r5G9pcuvv8j5F4F' - b'WRHD8xlZ_E4rWPdlVASI","dt":"2021-06-09T17:35:54.169967+00:00","i":' - b'"EIflL4H4134zYoRM6ls6Q086RLC_BhfNFh5uk-WxvhsL","LEI":"254900OPPU84' - b'GM83MG36"},"e":{}}-KAD6AABAAA--JAB6AABAAA--FABEKC8085pwSwzLwUGzh-H' - b'rEoFDwZnCJq27bVp5atdMT9o0AAAAAAAAAAAAAAAAAAAAAAAEKC8085pwSwzLwUGzh' - b'-HrEoFDwZnCJq27bVp5atdMT9o-AABAACIRDrYzCyMB5jBHY9jwfT4KEb7kx_vYgHJ' - b'7LDsiQRD-Roj5bGfJXj6PAo5TS36t4kWmiBhpvqLgb2l9vUhpiUK-JAB5AABAA-a-F' - b'ABEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o0AAAAAAAAAAAAAAAAAAA' - b'AAAAEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o-AABAAB80PrmAUGj_i' - b'ATyLY-kzdpI6omm5X05EsdkRZGymwVn62-1nijoSh0dlUo6rGOoywUQWu-eZ0i5PuH' - b'skgV9nwP-JAB4AAB-a-i-FABEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT' - b'9o0AAAAAAAAAAAAAAAAAAAAAAAEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atd' - b'MT9o-AABAABsIw-EgCMnex1m7Qm8RkU4jMGAV3wNGyD_CxfetmMp-iGBLhZ5wArAw6' - b'_Qdg75K_NMTKVV4hv7bWw3OvJnNY8A') + assert sig1 == (b'{"v":"ACDC10JSON00019e_","d":"EEs1BzDdw0h5VMhg5KUo904UyCdaMT1ut_' + b'_vN5G879pP","i":"EDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax","' + b'ri":"ECBNRQkp71gi2mTqZu2cp-xe6CrWlvJ5pRoBIBnRQbGp","s":"EMQWEcCn' + b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EFyxk35e1r5G9pcu' + b'vv8j5F4FWRHD8xlZ_E4rWPdlVASI","dt":"2021-06-09T17:35:54.169967+0' + b'0:00","i":"EIflL4H4134zYoRM6ls6Q086RLC_BhfNFh5uk-WxvhsL","LEI":"' + b'254900OPPU84GM83MG36"},"e":{}}-KAD6AABAAA--JAB6AABAAA--FABEDT4jW' + b'7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax0AAAAAAAAAAAAAAAAAAAAAAAED' + b'T4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax-AABAAA0vjmyMr3yp2DPHG' + b'fWczKx3rTd0EQndtntndKMEucbuUBh9B-KOIifWpzfpvvdOl4seK_nkfLsYt1NXx' + b'EMLqIC-JAB5AABAA-a-FABEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLW' + b'ax0AAAAAAAAAAAAAAAAAAAAAAAEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUd' + b'xKLWax-AABAAAMSXJBtyWP60AH_SPThf2kW7J7VNudUEwTX6fvD-EhE-DypJogi-' + b'SAm-eJ7kyo5v1zbgeacUDo2lYK3BZ1POcE-JAB4AAB-a-i-FABEDT4jW7VYWenu1' + b'MrgJJEZWB0ScqZdVmDQw0RUdxKLWax0AAAAAAAAAAAAAAAAAAAAAAAEDT4jW7VYW' + b'enu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax-AABAADvLjrNYsoHiYyrSzaFIrPd88' + b'gqaTIWqwnsVK_HfmR27PJgLXv36zqT4lNDTHyTjFWcA7tBghCkG_7aBsDkW38P') # signing SAD with non-transferable identifier with habbing.openHab(name="wan", temp=True, salt=b'0123456789abcdef', transferable=False) as (hby, hab): @@ -291,15 +293,15 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs # Sign with non-transferable identifier sig0 = signing.ratify(hab=hab, serder=cred) - assert sig0 == (b'{"v":"ACDC10JSON00019e_","d":"EBFPlqNJ4FovUoaGiDg3TWAlabvbUv2OnV' - b'wF8zPuaey7","i":"BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP","' + assert sig0 == (b'{"v":"ACDC10JSON00019e_","d":"EDvgu38Hmuprln_wc57VYkeJ9Dh0PlcOon' + b'qHhXjh6Yql","i":"BEByXTm1zJZ1oIuvnad6hppC3iBrUmQLb0a7iXSaRK4p","' b'ri":"Eg1H4eN7P5ndJAWtcymq3ZrYZwQsBRYd3-VuZ6wMAwxE","s":"EMQWEcCn' b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EFyxk35e1r5G9pcu' b'vv8j5F4FWRHD8xlZ_E4rWPdlVASI","dt":"2021-06-09T17:35:54.169967+0' b'0:00","i":"EIflL4H4134zYoRM6ls6Q086RLC_BhfNFh5uk-WxvhsL","LEI":"' - b'254900OPPU84GM83MG36"},"e":{}}-JAB6AABAAA--CABBAbSj3jfaeJbpuqg0W' - b'tvHw31UoRZOnN_RZQYBwbAqteP0BD1mbdIQtZtdLvSdR7in7XOt-d2ALjyOTvVbQ' - b'6AMLcyd5H7m-rPM8RshHoREehbr5S82w7b844-ykD5yJHyqhIG') + b'254900OPPU84GM83MG36"},"e":{}}-JAB6AABAAA--CABBEByXTm1zJZ1oIuvna' + b'd6hppC3iBrUmQLb0a7iXSaRK4p0BBfvm3yAoWIysgYFm4nQ7zQY6_Obo50YHKRVZ' + b'0aqphSHLn4H0YuZK-0dQEsUjAOZuvPK6TkR8w1hWxOE-lXug8C') pather = coring.Pather(path=["a", "b", "c"]) cigars = hab.sign(ser=cred.raw, @@ -308,9 +310,9 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs sadcigars = [(pather, cigars)] tp = eventing.proofize(sadcigars=sadcigars, pipelined=True) - assert tp == (b'-VAm-JAB5AACAA-a-b-c-CABBAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP' - b'0BD1mbdIQtZtdLvSdR7in7XOt-d2ALjyOTvVbQ6AMLcyd5H7m-rPM8RshHoREehbr5S' - b'82w7b844-ykD5yJHyqhIG') + assert tp == (b'-VAm-JAB5AACAA-a-b-c-CABBEByXTm1zJZ1oIuvnad6hppC3iBrUmQLb0a7iXSa' + b'RK4p0BBfvm3yAoWIysgYFm4nQ7zQY6_Obo50YHKRVZ0aqphSHLn4H0YuZK-0dQEs' + b'UjAOZuvPK6TkR8w1hWxOE-lXug8C') """End Test""" diff --git a/tests/conftest.py b/tests/conftest.py index 89dbaab37..bcf7f5d0d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ from hio.base import doing from keri import kering -from keri.core import scheming, coring, routing, eventing, parsing +from keri.core import scheming, coring, routing, eventing, parsing, signing from keri.db import basing from keri.help import helping from keri import help @@ -68,7 +68,12 @@ def mockCoringRandomNonce(monkeypatch): def mockRandomNonce(): return "A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U" + @property + def mockRandomSalt(self): + return "0AAUiJMii_rPXXCiLTEEaDT7" + monkeypatch.setattr(coring, "randomNonce", mockRandomNonce) + monkeypatch.setattr(signing.Salter, "qb64", mockRandomSalt) @pytest.fixture diff --git a/tests/vc/test_protocoling.py b/tests/vc/test_protocoling.py index 356a3f329..f507f9fad 100644 --- a/tests/vc/test_protocoling.py +++ b/tests/vc/test_protocoling.py @@ -19,9 +19,9 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN """ Test IPEX exchange protocol """ sidSalt = core.Salter(raw=b'0123456789abcdef').qb64 - assert sidSalt == '0AAwMTIzNDU2Nzg5YWJjZGVm' + assert sidSalt == '0AAUiJMii_rPXXCiLTEEaDT7' wanSalt = core.Salter(raw=b'wann-the-witness').qb64 - assert wanSalt == '0AB3YW5uLXRoZS13aXRuZXNz' + assert wanSalt == '0AAUiJMii_rPXXCiLTEEaDT7' default_salt = core.Salter(raw=b'0123456789abcdef').qb64 @@ -32,11 +32,11 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN sidHab = sidHby.makeHab(name="test") sidPre = sidHab.pre - assert sidPre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" + assert sidPre == "EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl" redHab = redHby.makeHab(name="test") redPre = redHab.pre - assert redPre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" + assert redPre == "EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl" sidRgy = credentialing.Regery(hby=sidHby, name="bob", temp=True) sidVer = verifying.Verifier(hby=sidHby, reger=sidRgy.reger) @@ -71,12 +71,12 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN data=d, status=issuer.regk) - assert creder.said == "EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG" + assert creder.said == "EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH" iss = issuer.issue(said=creder.said) - assert iss.raw == (b'{"v":"KERI10JSON0000ed_","t":"iss","d":"EK2WxcpF3oL1yqS3Z8i08WDYkHDcYhJL9afq' - b'dCIZjMy3","i":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG","s":"0","ri":"E' - b'O0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4","dt":"2021-06-27T21:26:21.23325' + assert iss.raw == (b'{"v":"KERI10JSON0000ed_","t":"iss","d":"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H8-i' + b'LLAmIUPO","i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","s":"0","ri":"E' + b'B-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","dt":"2021-06-27T21:26:21.23325' b'7+00:00"}') rseal = SealEvent(iss.pre, "0", iss.said)._asdict() sidHab.interact(data=[rseal]) @@ -88,11 +88,11 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN sidRgy.processEscrows() msg = creder.raw - assert msg == (b'{"v":"ACDC10JSON000197_","d":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","ri":"EO0_SyqPS1-EVYSITak' - b'YpUHaUZZpZGsjaXFOaO_kCfS4","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC' - b'","a":{"d":"EF2__B6DiLQHpdJZ_C0bddxy2o6nXIHEwchO9yylr3xx","dt":"2021-06-27T2' - b'1:26:21.233257+00:00","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LE' + assert msg == (b'{"v":"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH",' + b'"i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJo' + b'AVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC' + b'","a":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T2' + b'1:26:21.233257+00:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LE' b'I":"254900OPPU84GM83MG36"}}') atc = bytearray(msg) @@ -101,15 +101,15 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN atc.extend(coring.Seqner(sn=0).qb64b) atc.extend(iss.saidb) - assert atc == (b'{"v":"ACDC10JSON000197_","d":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qw' - b'vHdlvgfOyG","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","' - b'ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4","s":"EMQWEcCn' - b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EF2__B6DiLQHpdJZ' - b'_C0bddxy2o6nXIHEwchO9yylr3xx","dt":"2021-06-27T21:26:21.233257+0' - b'0:00","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LEI":"' - b'254900OPPU84GM83MG36"}}-IABEDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHd' - b'lvgfOyG0AAAAAAAAAAAAAAAAAAAAAAAEK2WxcpF3oL1yqS3Z8i08WDYkHDcYhJL9' - b'afqdCIZjMy3') + assert atc == (b'{"v":"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103' + b'V3-4E-ClXH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","' + b'ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCn' + b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EO9_6NattzsFiO8F' + b'w1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T21:26:21.233257+0' + b'0:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"' + b'254900OPPU84GM83MG36"}}-IABEElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-' + b'4E-ClXH0AAAAAAAAAAAAAAAAAAAAAAAECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H' + b'8-iLLAmIUPO') parsing.Parser().parseOne(ims=bytes(atc), vry=sidVer) # Successfully parsed credential is now saved in database. @@ -120,26 +120,26 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN apply0, apply0atc = protocoling.ipexApplyExn(sidHab, message="Please give me a credential", schema=schema, recp=redPre, attrs={}) - assert apply0.raw == (b'{"v":"KERI10JSON00016d_","t":"exn","d":"EI1MnUrT0aUprMN97FabgJdxVQtoCPqamVUp' - b'3iFgnDBE","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","p":"","dt":"20' + assert apply0.raw == (b'{"v":"KERI10JSON00016d_","t":"exn","d":"EIPeVB3u7L-mEKjhY6zIu5M7LErPwlUccxIN' + b'ddwhcFrH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"","dt":"20' b'21-06-27T21:26:21.233257+00:00","r":"/ipex/apply","q":{},"a":{"m":"Please gi' b've me a credential","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{' - b'},"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3"},"e":{}}') + b'},"i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},"e":{}}') # No requirements for apply, except that its first, no `p` assert ipexhan.verify(serder=apply0) is True offer0, offer0atc = protocoling.ipexOfferExn(sidHab, "How about this", acdc=creder.raw, apply=apply0) - assert offer0.raw == (b'{"v":"KERI10JSON0002f0_","t":"exn","d":"EO_wiH5ZEikfLQb8rKBjPATnjiSOHGBvvN3m' - b'F0LDvaIC","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","p":"EI1MnUrT0a' - b'UprMN97FabgJdxVQtoCPqamVUp3iFgnDBE","dt":"2021-06-27T21:26:21.233257+00:00",' + assert offer0.raw == (b'{"v":"KERI10JSON0002f0_","t":"exn","d":"EO4NXvOU-UpwwAR67txzKrFBHGAtDu9ehg8g' + b'Ic5haJy3","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EIPeVB3u7L' + b'-mEKjhY6zIu5M7LErPwlUccxINddwhcFrH","dt":"2021-06-27T21:26:21.233257+00:00",' b'"r":"/ipex/offer","q":{},"a":{"m":"How about this"},"e":{"acdc":{"v":"ACDC10' - b'JSON000197_","d":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG","i":"EIaGMMW' - b'JFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGs' - b'jaXFOaO_kCfS4","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"' - b'EF2__B6DiLQHpdJZ_C0bddxy2o6nXIHEwchO9yylr3xx","dt":"2021-06-27T21:26:21.2332' - b'57+00:00","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LEI":"254900OP' - b'PU84GM83MG36"}},"d":"EOVRKHUAEjvfyWzQ8IL4icBiaVuy_CSTse_W_AssaAeE"}}') + b'JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","i":"EMl4Rhu' + b'R_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXe' + b'w8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"' + b'EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T21:26:21.2332' + b'57+00:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"254900OP' + b'PU84GM83MG36"}},"d":"EOG-KWyllXlb2HVIuewN1YJAOT304PaSczyt3V5Z878S"}}') # This should fail because it is not first and the apply isn't persisted yet assert ipexhan.verify(serder=offer0) is False @@ -166,9 +166,9 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # Let's see if we can spurn a message we previously accepted. spurn0, spurn0atc = protocoling.ipexSpurnExn(sidHab, "I reject you", spurned=apply0) - assert spurn0.raw == (b'{"v":"KERI10JSON00011d_","t":"exn","d":"EKvtmxPkOklgRNgWxLj-1ZW4Zb0MwZIUloWx' - b'A_dam95r","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","p":"EI1MnUrT0a' - b'UprMN97FabgJdxVQtoCPqamVUp3iFgnDBE","dt":"2021-06-27T21:26:21.233257+00:00",' + assert spurn0.raw == (b'{"v":"KERI10JSON00011d_","t":"exn","d":"ENWl0Dd-9idlgrpkFL2aA0wfnuGHzvT4bHCN' + b'G6C0WBZk","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EIPeVB3u7L' + b'-mEKjhY6zIu5M7LErPwlUccxINddwhcFrH","dt":"2021-06-27T21:26:21.233257+00:00",' b'"r":"/ipex/spurn","q":{},"a":{"m":"I reject you"},"e":{}}') # This will fail, we've already responded with an offer @@ -176,16 +176,16 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # Now lets try an offer without a pointer back to a reply offer1, offer1atc = protocoling.ipexOfferExn(sidHab, "Here a credential offer", acdc=creder.raw) - assert offer1.raw == (b'{"v":"KERI10JSON0002cd_","t":"exn","d":"EMEmoi4k9gxWu4uZyYuEK3MvFPn-5B0LHnNx' - b'uQ4vRqRA","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","p":"","dt":"20' + assert offer1.raw == (b'{"v":"KERI10JSON0002cd_","t":"exn","d":"ELnRdb-cA_rLckt9jwlSY-1nnPKwzRc4Up_7' + b'tCdzI12n","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"","dt":"20' b'21-06-27T21:26:21.233257+00:00","r":"/ipex/offer","q":{},"a":{"m":"Here a cr' - b'edential offer"},"e":{"acdc":{"v":"ACDC10JSON000197_","d":"EDkftEwWBpohjTpem' - b'h_6xkaGNuoDsRU3qwvHdlvgfOyG","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDj' - b'I3","ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4","s":"EMQWEcCnVRk1hat' - b'TNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EF2__B6DiLQHpdJZ_C0bddxy2o6nXIHEwch' - b'O9yylr3xx","dt":"2021-06-27T21:26:21.233257+00:00","i":"EIaGMMWJFPmtXznY1IIi' - b'KDIrg-vIyge6mBl2QV8dDjI3","LEI":"254900OPPU84GM83MG36"}},"d":"EOVRKHUAEjvfyW' - b'zQ8IL4icBiaVuy_CSTse_W_AssaAeE"}}') + b'edential offer"},"e":{"acdc":{"v":"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoC' + b'eOtSsNOROLuqOz103V3-4E-ClXH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiU' + b'bl","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hat' + b'TNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn' + b'-wXn9S3iB","dt":"2021-06-27T21:26:21.233257+00:00","i":"EMl4RhuR_JxpiMd1N8DE' + b'JEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"254900OPPU84GM83MG36"}},"d":"EOG-KWyllXlb2H' + b'VIuewN1YJAOT304PaSczyt3V5Z878S"}}') # Will work because it is starting a new conversation assert ipexhan.verify(serder=offer1) is True @@ -197,9 +197,9 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN assert serder.ked == offer1.ked agree, argeeAtc = protocoling.ipexAgreeExn(sidHab, "I'll accept that offer", offer=offer0) - assert agree.raw == (b'{"v":"KERI10JSON000127_","t":"exn","d":"EGpJ9S0TqIVHkRmDsbgP59NC8ZLCaSUirslB' - b'KDeYKOR7","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","p":"EO_wiH5ZEi' - b'kfLQb8rKBjPATnjiSOHGBvvN3mF0LDvaIC","dt":"2021-06-27T21:26:21.233257+00:00",' + assert agree.raw == (b'{"v":"KERI10JSON000127_","t":"exn","d":"EEtu1OAPj03IdbhMwsQtgbJJaWgG2tdYLJ_3' + b'BuJQekdP","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EO4NXvOU-U' + b'pwwAR67txzKrFBHGAtDu9ehg8gIc5haJy3","dt":"2021-06-27T21:26:21.233257+00:00",' b'"r":"/ipex/agree","q":{},"a":{"m":"I\'ll accept that offer"},"e":{}}') # Can not create an agree without an offer, so this will pass since it has an offer that has no response @@ -215,24 +215,24 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN anc = sidHab.makeOwnEvent(sn=2) grant0, grant0atc = protocoling.ipexGrantExn(sidHab, message="Here's a credential", recp=sidHab.pre, acdc=msg, iss=iss.raw, anc=anc) - assert grant0.raw == (b'{"v":"KERI10JSON000531_","t":"exn","d":"EJxM3em5fSpAIQsyXYovrr0UjblWLtmbTnFp' - b'xAUqnwG-","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","p":"","dt":"20' + assert grant0.raw == (b'{"v":"KERI10JSON000531_","t":"exn","d":"EC-EsfvXD2cNw4GNgCFp9UTl7_DgWUgk9Rnw' + b'eCxocaG-","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"","dt":"20' b'21-06-27T21:26:21.233257+00:00","r":"/ipex/grant","q":{},"a":{"m":"Here\'' - b's a credential","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3"},"e":{"ac' - b'dc":{"v":"ACDC10JSON000197_","d":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfO' - b'yG","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","ri":"EO0_SyqPS1-EVYS' - b'ITakYpUHaUZZpZGsjaXFOaO_kCfS4","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gk' - b'k1kC","a":{"d":"EF2__B6DiLQHpdJZ_C0bddxy2o6nXIHEwchO9yylr3xx","dt":"2021-06-' - b'27T21:26:21.233257+00:00","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3"' + b's a credential","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},"e":{"ac' + b'dc":{"v":"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-Cl' + b'XH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8' + b'PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gk' + b'k1kC","a":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-' + b'27T21:26:21.233257+00:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"' b',"LEI":"254900OPPU84GM83MG36"}},"iss":{"v":"KERI10JSON0000ed_","t":"iss","d"' - b':"EK2WxcpF3oL1yqS3Z8i08WDYkHDcYhJL9afqdCIZjMy3","i":"EDkftEwWBpohjTpemh_6xka' - b'GNuoDsRU3qwvHdlvgfOyG","s":"0","ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_' - b'kCfS4","dt":"2021-06-27T21:26:21.233257+00:00"},"anc":{"v":"KERI10JSON00013a' - b'_","t":"ixn","d":"EOjAxp-AMLzicGz2h-DxvMK9kicajpZEwdN8-8k54hvz","i":"EIaGMMW' - b'JFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","s":"2","p":"EGKglEgIpdHuhuwl-IiSDG9x' - b'094gMrRxVaXGgXvCzCYM","a":[{"i":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOy' - b'G","s":"0","d":"EK2WxcpF3oL1yqS3Z8i08WDYkHDcYhJL9afqdCIZjMy3"}]},"d":"EI5mZX' - b'Z84Su4DrEUOxtl-NaUURQtTJeAn12xf146beg3"}}') + b':"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H8-iLLAmIUPO","i":"EElymNmgs1u0mSaoCeOtSsN' + b'OROLuqOz103V3-4E-ClXH","s":"0","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w' + b'9mQUQ","dt":"2021-06-27T21:26:21.233257+00:00"},"anc":{"v":"KERI10JSON00013a' + b'_","t":"ixn","d":"EGhSHKIV5-nkeirdkqzqsvmeF1FXw_yH8NvPSAY1Rgyd","i":"EMl4Rhu' + b'R_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","s":"2","p":"ED1kkh5_ECYriK-j2gSv6Zjr' + b'5way88XVhwRCxk5zoTRG","a":[{"i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClX' + b'H","s":"0","d":"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H8-iLLAmIUPO"}]},"d":"EJ4-dl' + b'S9ktlb9HDWPYc0IJ2hS2NbvnCQBhUsFSkEPwIo"}}') assert ipexhan.verify(serder=grant0) is True @@ -245,9 +245,9 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # Let's see if we can spurn a message we previously accepted. spurn1, spurn1atc = protocoling.ipexSpurnExn(sidHab, "I reject you", spurned=grant0) - assert spurn1.raw == (b'{"v":"KERI10JSON00011d_","t":"exn","d":"EEs0bIGplWsjSOw5BMhAdFmgv-jm3-4nPgcK' - b'-LDv8tdB","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","p":"EJxM3em5fS' - b'pAIQsyXYovrr0UjblWLtmbTnFpxAUqnwG-","dt":"2021-06-27T21:26:21.233257+00:00",' + assert spurn1.raw == (b'{"v":"KERI10JSON00011d_","t":"exn","d":"EJA0LVnMxOdrOEArWudVtdonorCUa4nCIRgX' + b'vibhxdp3","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EC-EsfvXD2' + b'cNw4GNgCFp9UTl7_DgWUgk9RnweCxocaG-","dt":"2021-06-27T21:26:21.233257+00:00",' b'"r":"/ipex/spurn","q":{},"a":{"m":"I reject you"},"e":{}}') smsg = bytearray(spurn1.raw) smsg.extend(spurn1atc) @@ -258,25 +258,25 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # Now we'll run a grant pointing back to the agree all the way to the database grant1, grant1atc = protocoling.ipexGrantExn(sidHab, message="Here's a credential", acdc=msg, iss=iss.raw, recp=sidHab.pre, anc=anc, agree=agree) - assert grant1.raw == (b'{"v":"KERI10JSON00055d_","t":"exn","d":"EIqh-L9GnnVSdNLeqwmx-vpE9V1DvOQAlVWf' - b'wENpm8sW","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","p":"EGpJ9S0TqI' - b'VHkRmDsbgP59NC8ZLCaSUirslBKDeYKOR7","dt":"2021-06-27T21:26:21.233257+00:00",' - b'"r":"/ipex/grant","q":{},"a":{"m":"Here\'s a credential","i":"EIaGMMWJFPm' - b'tXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3"},"e":{"acdc":{"v":"ACDC10JSON000197_","d"' - b':"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG","i":"EIaGMMWJFPmtXznY1IIiKDI' - b'rg-vIyge6mBl2QV8dDjI3","ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4","' - b's":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EF2__B6DiLQHpdJZ' - b'_C0bddxy2o6nXIHEwchO9yylr3xx","dt":"2021-06-27T21:26:21.233257+00:00","i":"E' - b'IaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LEI":"254900OPPU84GM83MG36"}},' - b'"iss":{"v":"KERI10JSON0000ed_","t":"iss","d":"EK2WxcpF3oL1yqS3Z8i08WDYkHDcYh' - b'JL9afqdCIZjMy3","i":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG","s":"0","' - b'ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4","dt":"2021-06-27T21:26:21' - b'.233257+00:00"},"anc":{"v":"KERI10JSON00013a_","t":"ixn","d":"EOjAxp-AMLzicG' - b'z2h-DxvMK9kicajpZEwdN8-8k54hvz","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8' - b'dDjI3","s":"2","p":"EGKglEgIpdHuhuwl-IiSDG9x094gMrRxVaXGgXvCzCYM","a":[{"i":' - b'"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG","s":"0","d":"EK2WxcpF3oL1yqS3' - b'Z8i08WDYkHDcYhJL9afqdCIZjMy3"}]},"d":"EI5mZXZ84Su4DrEUOxtl-NaUURQtTJeAn12xf1' - b'46beg3"}}') + assert grant1.raw == (b'{"v":"KERI10JSON00055d_","t":"exn","d":"EBAhSTTx3--xDZZRoCIemBzxybFV_FoicvPf' + b'4hSfeyAJ","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EEtu1OAPj0' + b'3IdbhMwsQtgbJJaWgG2tdYLJ_3BuJQekdP","dt":"2021-06-27T21:26:21.233257+00:00",' + b'"r":"/ipex/grant","q":{},"a":{"m":"Here\'s a credential","i":"EMl4RhuR_Jx' + b'piMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},"e":{"acdc":{"v":"ACDC10JSON000197_","d"' + b':"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","i":"EMl4RhuR_JxpiMd1N8DEJEh' + b'TxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","' + b's":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EO9_6NattzsFiO8F' + b'w1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T21:26:21.233257+00:00","i":"E' + b'Ml4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"254900OPPU84GM83MG36"}},' + b'"iss":{"v":"KERI10JSON0000ed_","t":"iss","d":"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTT' + b'a_H8-iLLAmIUPO","i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","s":"0","' + b'ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","dt":"2021-06-27T21:26:21' + b'.233257+00:00"},"anc":{"v":"KERI10JSON00013a_","t":"ixn","d":"EGhSHKIV5-nkei' + b'rdkqzqsvmeF1FXw_yH8NvPSAY1Rgyd","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-' + b'tiUbl","s":"2","p":"ED1kkh5_ECYriK-j2gSv6Zjr5way88XVhwRCxk5zoTRG","a":[{"i":' + b'"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","s":"0","d":"ECUw7AdWEE3fvr7d' + b'gbFDXj0CEZuJTTa_H8-iLLAmIUPO"}]},"d":"EJ4-dlS9ktlb9HDWPYc0IJ2hS2NbvnCQBhUsFS' + b'kEPwIo"}}') assert ipexhan.verify(serder=grant1) is True gmsg = bytearray(grant1.raw) @@ -287,9 +287,9 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # And now the last... admit the granted credential to complete the full flow admit0, admit0atc = protocoling.ipexAdmitExn(sidHab, "Thanks for the credential", grant=grant1) - assert admit0.raw == (b'{"v":"KERI10JSON00012a_","t":"exn","d":"ELNz82kqV94vlbT7lJulVFWtf6_jhGRgH556' - b'Z-xYRaGY","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","p":"EIqh-L9Gnn' - b'VSdNLeqwmx-vpE9V1DvOQAlVWfwENpm8sW","dt":"2021-06-27T21:26:21.233257+00:00",' + assert admit0.raw == (b'{"v":"KERI10JSON00012a_","t":"exn","d":"EDkQZpUOKKOzsQ50yoZxzLz8tDaChAid_llr' + b'QNvKVAdO","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EBAhSTTx3-' + b'-xDZZRoCIemBzxybFV_FoicvPf4hSfeyAJ","dt":"2021-06-27T21:26:21.233257+00:00",' b'"r":"/ipex/admit","q":{},"a":{"m":"Thanks for the credential"},"e":{}}') assert ipexhan.verify(serder=admit0) is True diff --git a/tests/vc/test_walleting.py b/tests/vc/test_walleting.py index d514171c6..7049477fa 100644 --- a/tests/vc/test_walleting.py +++ b/tests/vc/test_walleting.py @@ -19,7 +19,7 @@ def test_wallet(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): with habbing.openHby(name="sid", base="test", salt=sidSalt) as sidHby: sidHab = sidHby.makeHab(name="test") seeder.seedSchema(db=sidHby.db) - assert sidHab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" + assert sidHab.pre == "EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl" schema = "EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC" credSubject = dict( @@ -43,7 +43,7 @@ def test_wallet(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): schema=schema, data=credSubject, status=issuer.regk) - assert creder.said == "EOavcpdGvk4sTXjOQiNxHeNf3HYMjMINMhar4R5a3OfB" + assert creder.said == "EAP1MTFwoSZ7P9Ym9yIqBvihjqZYpilpFpZj2oPTc7vM" iss = issuer.issue(said=creder.said) rseal = SealEvent(iss.pre, "0", iss.said)._asdict() @@ -61,21 +61,21 @@ def test_wallet(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): msg.extend(coring.Seqner(sn=0).qb64b) msg.extend(iss.saidb) - assert msg == (b'{"v":"ACDC10JSON000197_","d":"EOavcpdGvk4sTXjOQiNxHeNf3HYMjMINMh' - b'ar4R5a3OfB","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","' - b'ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4","s":"EMQWEcCn' - b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EFKsAdq9CZF_w9yv' - b'ia8RiRdDeXLMjR6q7Lp7FKKIgJx-","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIy' - b'ge6mBl2QV8dDjI3","dt":"2021-06-27T21:26:21.233257+00:00","LEI":"' - b'254900OPPU84GM83MG36"}}-IABEOavcpdGvk4sTXjOQiNxHeNf3HYMjMINMhar4' - b'R5a3OfB0AAAAAAAAAAAAAAAAAAAAAAAEIMoFDXHR3cNF0fADC5nLPme34n-ZsMEu' - b'n6eDFvN8Jgc') + assert msg == (b'{"v":"ACDC10JSON000197_","d":"EAP1MTFwoSZ7P9Ym9yIqBvihjqZYpilpFp' + b'Zj2oPTc7vM","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","' + b'ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCn' + b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EMyHBc5ZujkNFm9t' + b'nrwBU2nmp_qodcV4aDW28pwbDdgb","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ov' + b'vn9Xya8AN-tiUbl","dt":"2021-06-27T21:26:21.233257+00:00","LEI":"' + b'254900OPPU84GM83MG36"}}-IABEAP1MTFwoSZ7P9Ym9yIqBvihjqZYpilpFpZj2' + b'oPTc7vM0AAAAAAAAAAAAAAAAAAAAAAAEAOV3Ie3yAvGU1MwbIHr816qewYzRLlvX' + b'NreKmXtJShe') - ser = (b'{"v":"ACDC10JSON000197_","d":"EOavcpdGvk4sTXjOQiNxHeNf3HYMjMINMhar4R5a3OfB",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","ri":"EO0_SyqPS1-EVYSITak' - b'YpUHaUZZpZGsjaXFOaO_kCfS4","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC' - b'","a":{"d":"EFKsAdq9CZF_w9yvia8RiRdDeXLMjR6q7Lp7FKKIgJx-","i":"EIaGMMWJFPmtX' - b'znY1IIiKDIrg-vIyge6mBl2QV8dDjI3","dt":"2021-06-27T21:26:21.233257+00:00","LE' + ser = (b'{"v":"ACDC10JSON000197_","d":"EAP1MTFwoSZ7P9Ym9yIqBvihjqZYpilpFpZj2oPTc7vM",' + b'"i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJo' + b'AVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC' + b'","a":{"d":"EMyHBc5ZujkNFm9tnrwBU2nmp_qodcV4aDW28pwbDdgb","i":"EMl4RhuR_Jxpi' + b'Md1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","dt":"2021-06-27T21:26:21.233257+00:00","LE' b'I":"254900OPPU84GM83MG36"}}') parsing.Parser().parse(ims=msg, vry=verifier) diff --git a/tests/vdr/test_eventing.py b/tests/vdr/test_eventing.py index 5978edc90..124030cb3 100644 --- a/tests/vdr/test_eventing.py +++ b/tests/vdr/test_eventing.py @@ -32,17 +32,17 @@ def test_incept(mockCoringRandomNonce): # no backers, allowed to add later serder = eventing.incept(pre, baks=[], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON00010f_","t":"vcp","d":"ELkr1d1qLyIXVPfuaEjkDJIfgxXarUjoB0RN' - b'mswxHnvD","i":"ELkr1d1qLyIXVPfuaEjkDJIfgxXarUjoB0RNmswxHnvD","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"0","b":[],"n":"A9X' - b'fpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON0000fb_","t":"vcp","d":"EE7_RC-GkGy_N_IbOjFHLhqDekFNeSxygvIZ' + b'JBf1SPbA","i":"EE7_RC-GkGy_N_IbOjFHLhqDekFNeSxygvIZJBf1SPbA","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"0","b":[],"n":"0AA' + b'UiJMii_rPXXCiLTEEaDT7"}') # no backers allowed serder = eventing.incept(pre, baks=[], cnfg=[keventing.TraitDex.NoBackers], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON000113_","t":"vcp","d":"EBoBPh3N5nr1tItAUCkXNx3vShB_Be6iiQPX' - b'Bsg2LvxA","i":"EBoBPh3N5nr1tItAUCkXNx3vShB_Be6iiQPXBsg2LvxA","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":["NB"],"bt":"0","b":[],"n":' - b'"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON0000ff_","t":"vcp","d":"EM2g-TbMWL98wHMiY7fs1eBRE0uVNL21xnwa' + b'ys_i9K1Q","i":"EM2g-TbMWL98wHMiY7fs1eBRE0uVNL21xnways_i9K1Q","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":["NB"],"bt":"0","b":[],"n":' + b'"0AAUiJMii_rPXXCiLTEEaDT7"}') # no backers allows, one attempted with pytest.raises(ValueError): @@ -68,45 +68,43 @@ def test_incept(mockCoringRandomNonce): serder = eventing.incept(pre, baks=[bak1], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON00013d_","t":"vcp","d":"EHYybz8zckAN_4rj8JfuKMogmWr7gRLKxf_I' - b'6AwmpZ6x","i":"EHYybz8zckAN_4rj8JfuKMogmWr7gRLKxf_I6AwmpZ6x","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"1","b":["EAvR3p8V9' - b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc"],"n":"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK' - b'3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON000129_","t":"vcp","d":"EP_pM8RMJ2K25lepbhe5vYJnxYf4OrebU_eg' + b'8uYupLCq","i":"EP_pM8RMJ2K25lepbhe5vYJnxYf4OrebU_eg8uYupLCq","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"1","b":["EAvR3p8V9' + b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc"],"n":"0AAUiJMii_rPXXCiLTEEaDT7"}') # 3 backers serder = eventing.incept(pre, baks=[bak1, bak2, bak3], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON00019b_","t":"vcp","d":"ECw3lpxt56pRMeh7VnZStljriimMfm5or0Ka' - b'aNv7PSHl","i":"ECw3lpxt56pRMeh7VnZStljriimMfm5or0KaaNv7PSHl","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"3","b":["EAvR3p8V9' - b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc","DBEpNJeSJjxo6oAxkNE8eCOJg2HRPstqkeHWBA' - b'vN9XNU","DCxo-P4W_Z0xXTfoA3_4DMPn7oi0mLCElOWJDpC0nQXw"],"n":"A9XfpxIl1LcIkMh' - b'USCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON000187_","t":"vcp","d":"EI7cIxSfVdi8WhU3dj-laWIkNI1a-7MS6F-5' + b'Ss758cv9","i":"EI7cIxSfVdi8WhU3dj-laWIkNI1a-7MS6F-5Ss758cv9","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"3","b":["EAvR3p8V9' + b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc","DBEpNJeSJjxo6oAxkNE8eCOJg2HRPstqkeHWBA' + b'vN9XNU","DCxo-P4W_Z0xXTfoA3_4DMPn7oi0mLCElOWJDpC0nQXw"],"n":"0AAUiJMii_rPXXC' + b'iLTEEaDT7"}') # one backer, with threshold serder = eventing.incept(pre, toad=1, baks=[bak1], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON00013d_","t":"vcp","d":"EHYybz8zckAN_4rj8JfuKMogmWr7gRLKxf_I' - b'6AwmpZ6x","i":"EHYybz8zckAN_4rj8JfuKMogmWr7gRLKxf_I6AwmpZ6x","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"1","b":["EAvR3p8V9' - b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc"],"n":"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK' - b'3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON000129_","t":"vcp","d":"EP_pM8RMJ2K25lepbhe5vYJnxYf4OrebU_eg' + b'8uYupLCq","i":"EP_pM8RMJ2K25lepbhe5vYJnxYf4OrebU_eg8uYupLCq","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"1","b":["EAvR3p8V9' + b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc"],"n":"0AAUiJMii_rPXXCiLTEEaDT7"}') # 3 backers, with threshold serder = eventing.incept(pre, toad=2, baks=[bak1, bak2, bak3], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON00019b_","t":"vcp","d":"EGqiHHLDIMr7VYNhDSnmsjOMpaLUOBRAgxvb' - b'rXYSFDfk","i":"EGqiHHLDIMr7VYNhDSnmsjOMpaLUOBRAgxvbrXYSFDfk","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"2","b":["EAvR3p8V9' - b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc","DBEpNJeSJjxo6oAxkNE8eCOJg2HRPstqkeHWBA' - b'vN9XNU","DCxo-P4W_Z0xXTfoA3_4DMPn7oi0mLCElOWJDpC0nQXw"],"n":"A9XfpxIl1LcIkMh' - b'USCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON000187_","t":"vcp","d":"EDnhbkM1_hVVC_df54giqQ0qq6722QCXM9BE' + b'AK-GFqiF","i":"EDnhbkM1_hVVC_df54giqQ0qq6722QCXM9BEAK-GFqiF","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"2","b":["EAvR3p8V9' + b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc","DBEpNJeSJjxo6oAxkNE8eCOJg2HRPstqkeHWBA' + b'vN9XNU","DCxo-P4W_Z0xXTfoA3_4DMPn7oi0mLCElOWJDpC0nQXw"],"n":"0AAUiJMii_rPXXC' + b'iLTEEaDT7"}') """ End Test """ @@ -428,7 +426,7 @@ def test_tever_escrow(mockCoringRandomNonce): cnfg=[], code=MtrDex.Blake3_256) regk = vcp.pre - assert regk == 'EJfe0JaRiSWTCnasczAPkvZ-M2dd5nnhxcTcS59ssXg5' + assert regk == 'EEu4cX0EqO9mTqsNgxDgCT9lJbj9qmuPvD7BwNUl6wms' assert vcp.said == vcp.pre assert vcp.ked["ii"] == hab.pre @@ -457,13 +455,13 @@ def test_tever_escrow(mockCoringRandomNonce): dgkey = dgKey(pre=regk, dig=vcp.said) vcp = reg.getTvt(dgkey) - assert bytes(vcp) == (b'{"v":"KERI10JSON00010f_","t":"vcp","d":"EJfe0JaRiSWTCnasczAPkvZ-M2dd5nnhxcTc' - b'S59ssXg5","i":"EJfe0JaRiSWTCnasczAPkvZ-M2dd5nnhxcTcS59ssXg5","ii":"EPst_DQ1d' - b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"0","b":[],"n":"A9X' - b'fpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert bytes(vcp) == (b'{"v":"KERI10JSON0000fb_","t":"vcp","d":"EEu4cX0EqO9mTqsNgxDgCT9lJbj9qmuPvD7B' + b'wNUl6wms","i":"EEu4cX0EqO9mTqsNgxDgCT9lJbj9qmuPvD7BwNUl6wms","ii":"EPst_DQ1d' + b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"0","b":[],"n":"0AA' + b'UiJMii_rPXXCiLTEEaDT7"}') dig = reg.getTae(snKey(pre=regk, sn=0)) - assert bytes(dig) == b'EJfe0JaRiSWTCnasczAPkvZ-M2dd5nnhxcTcS59ssXg5' + assert bytes(dig) == b'EEu4cX0EqO9mTqsNgxDgCT9lJbj9qmuPvD7BwNUl6wms' # registry with backers, no signatures. should escrow with basing.openDB() as db, keeping.openKS() as kpr, viring.openReger() as reg: @@ -490,17 +488,16 @@ def test_tever_escrow(mockCoringRandomNonce): dgkey = dgKey(pre=regk, dig=vcp.said) vcp = reg.getTvt(dgkey) - assert bytes(vcp) == (b'{"v":"KERI10JSON00013d_","t":"vcp","d":"EGMjDTKwtL24UE06lU1ZR6LGBQFZTRa5bFiA' - b'3WDTa8GO","i":"EGMjDTKwtL24UE06lU1ZR6LGBQFZTRa5bFiA3WDTa8GO","ii":"EPst_DQ1d' - b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"1","b":["BAOcciw30' - b'IVQsaenKXpiyMVrjtPDW3KeD_6KFnSfoaqI"],"n":"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK' - b'3SM-S8a8Y_U"}') + assert bytes(vcp) == (b'{"v":"KERI10JSON000129_","t":"vcp","d":"EBkUjPBzZuFeSTP-Quuz0Exr6jdUNd8VDa5h' + b'oNvnS1Jo","i":"EBkUjPBzZuFeSTP-Quuz0Exr6jdUNd8VDa5hoNvnS1Jo","ii":"EPst_DQ1d' + b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"1","b":["BAOcciw30' + b'IVQsaenKXpiyMVrjtPDW3KeD_6KFnSfoaqI"],"n":"0AAUiJMii_rPXXCiLTEEaDT7"}') anc = reg.getAnc(dgkey) - assert bytes(anc) == b'0AAAAAAAAAAAAAAAAAAAAAABEAfW0zq1WbQ7e6fYQ8EvEZUpZ6nn1RX_Zjmczeke6JTx' + assert bytes(anc) == b'0AAAAAAAAAAAAAAAAAAAAAABEErlKvdXWTpOg9s5AasfABIk17o_EaM2uAuGe-c_LEbc' assert reg.getTel(snKey(pre=regk, sn=0)) is None dig = reg.getTwe(snKey(pre=regk, sn=0)) - assert bytes(dig) ==b'EGMjDTKwtL24UE06lU1ZR6LGBQFZTRa5bFiA3WDTa8GO' + assert bytes(dig) ==b'EBkUjPBzZuFeSTP-Quuz0Exr6jdUNd8VDa5hoNvnS1Jo' def test_tever_no_backers(mockHelpingNowUTC, mockCoringRandomNonce): @@ -532,14 +529,15 @@ def test_tever_no_backers(mockHelpingNowUTC, mockCoringRandomNonce): assert tev.sn == 0 dgkey = dgKey(pre=regk, dig=vcp.said) - assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON000113_","t":"vcp","d":"ECRIn7b_Y6aB-G3x45-fPb_LQgke8CDq0taW' - b'GplUj03s","i":"ECRIn7b_Y6aB-G3x45-fPb_LQgke8CDq0taWGplUj03s","ii":"EPst_DQ1d' - b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":["NB"],"bt":"0","b":[],"n":' - b'"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert bytes(reg.getTvt(dgkey)) == ( + b'{"v":"KERI10JSON0000ff_","t":"vcp","d":"EKWuqbpBPglFWnzZuD3f_DTCLwYd4ub1bWUZ' + b'XdRB2g6C","i":"EKWuqbpBPglFWnzZuD3f_DTCLwYd4ub1bWUZXdRB2g6C","ii":"EPst_DQ1d' + b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":["NB"],"bt":"0","b":[],"n":' + b'"0AAUiJMii_rPXXCiLTEEaDT7"}') - assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAABEFEUzOixM-Avz6VPmQoB59eYQ2oan9ltJ1JOSe-QQFRq' - assert bytes(reg.getTel(snKey(pre=regk, sn=0))) == b'ECRIn7b_Y6aB-G3x45-fPb_LQgke8CDq0taWGplUj03s' + assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAABEGe5uFh3t0JglSPQtJJoxCV1RlFoTBept1BPRk3o6hgh' + assert bytes(reg.getTel(snKey(pre=regk, sn=0))) == b'EKWuqbpBPglFWnzZuD3f_DTCLwYd4ub1bWUZXdRB2g6C' assert reg.getTibs(dgkey) == [] assert reg.getTwe(snKey(pre=regk, sn=0)) is None @@ -572,11 +570,12 @@ def test_tever_no_backers(mockHelpingNowUTC, mockCoringRandomNonce): vci = vcdig dgkey = dgKey(pre=vci, dig=iss.said) - assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON0000ed_","t":"iss","d":"EPlZDm4GgNl2aZNvHyfbk-B8mN6dNMMdJxrR' - b'LovbJSCS","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","s":"0","ri":"E' - b'CRIn7b_Y6aB-G3x45-fPb_LQgke8CDq0taWGplUj03s","dt":"2021-01-01T00:00:00.00000' - b'0+00:00"}') - assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAADEOMHrVBAHEJ-5qHzoBLLvBj_m9761m7CavMZmVOXgZpT' + assert bytes(reg.getTvt(dgkey)) == ( + b'{"v":"KERI10JSON0000ed_","t":"iss","d":"EFQdb41xs1FwGz0m-Ekzwv9gwnpD8hKc4XGJ' + b'3-jPUA6I","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","s":"0","ri":"E' + b'KWuqbpBPglFWnzZuD3f_DTCLwYd4ub1bWUZXdRB2g6C","dt":"2021-01-01T00:00:00.00000' + b'0+00:00"}') + assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAADEHzX4VXW_xCDf_pFhFvkDdc6TXTxiSiaEpHHCr1tDVtD' # revoke vc with no backers rev = eventing.revoke(vcdig=vcdig.decode("utf-8"), regk=regk, dig=iss.said) @@ -591,10 +590,11 @@ def test_tever_no_backers(mockHelpingNowUTC, mockCoringRandomNonce): tev.update(rev, seqner=seqner, saider=saider) dgkey = dgKey(pre=vci, dig=rev.said) - assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON000120_","t":"rev","d":"EGjtu2bIII28dwxA0BH8KeXCN03U7TN3SkLD' - b'1KZi77pj","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","s":"1","ri":"E' - b'CRIn7b_Y6aB-G3x45-fPb_LQgke8CDq0taWGplUj03s","p":"EPlZDm4GgNl2aZNvHyfbk-B8mN' - b'6dNMMdJxrRLovbJSCS","dt":"2021-01-01T00:00:00.000000+00:00"}') + assert bytes(reg.getTvt(dgkey)) == ( + b'{"v":"KERI10JSON000120_","t":"rev","d":"EEXxqWHCGw1XAkzEX_32xyRSboJDIwCKZUA9' + b'WfBzn-jx","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","s":"1","ri":"E' + b'KWuqbpBPglFWnzZuD3f_DTCLwYd4ub1bWUZXdRB2g6C","p":"EFQdb41xs1FwGz0m-Ekzwv9gwn' + b'pD8hKc4XGJ3-jPUA6I","dt":"2021-01-01T00:00:00.000000+00:00"}') # assert reg.getAnc(dgkey) == b'0AAAAAAAAAAAAAAAAAAAAABAECgc6yHeTRhsKh1M7k65feWZGCf_MG0dWoei5Q6SwgqU' @@ -632,15 +632,15 @@ def test_tever_backers(mockHelpingNowUTC, mockCoringRandomNonce): tev = Tever(serder=vcp, seqner=seqner, saider=saider, bigers=[valCigar], db=db, reger=reg) dgkey = dgKey(pre=regk, dig=vcp.said) - assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON00013d_","t":"vcp","d":"EBgdJt_ASWeq7HjOmut2E8vQL8P1c9VTPDA0' - b'Pdh4KsZX","i":"EBgdJt_ASWeq7HjOmut2E8vQL8P1c9VTPDA0Pdh4KsZX","ii":"EPst_DQ1d' - b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"1","b":["BPmRWtx8n' - b'wSzRdJ0zTvP5uBb0t3BSjjstDk0gTayFfjV"],"n":"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK' - b'3SM-S8a8Y_U"}') - assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAABELUmsgBmQ-OwMp2Zi7NpnRvZPwcWqld49zvTP9r-I4vp' - assert bytes(reg.getTel(snKey(pre=regk, sn=0))) == b'EBgdJt_ASWeq7HjOmut2E8vQL8P1c9VTPDA0Pdh4KsZX' - assert [bytes(tib) for tib in reg.getTibs(dgkey)] == [b'AAAzew389hwVg7TzgHucIdznjjgWh9A9w3T2YAe6A1U7mQCG_tEhQ_px7G0ZO39GdtyDx4Z890e0' - b'WBPKdTyKbqIE'] + assert bytes(reg.getTvt(dgkey)) == ( + b'{"v":"KERI10JSON000129_","t":"vcp","d":"ECfzJv1hIYAF68tEDDSelka5aPNKg_pmdcZO' + b'Ts0aubF-","i":"ECfzJv1hIYAF68tEDDSelka5aPNKg_pmdcZOTs0aubF-","ii":"EPst_DQ1d' + b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"1","b":["BPmRWtx8n' + b'wSzRdJ0zTvP5uBb0t3BSjjstDk0gTayFfjV"],"n":"0AAUiJMii_rPXXCiLTEEaDT7"}') + assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAABEDD2vrz4Eg3iLOOlZw5-d3ioZ1q703IC0M0LJP_3v-PT' + assert bytes(reg.getTel(snKey(pre=regk, sn=0))) == b'ECfzJv1hIYAF68tEDDSelka5aPNKg_pmdcZOTs0aubF-' + assert [bytes(tib) for tib in reg.getTibs(dgkey)] == [b'AAAUr5RHYiDH8RU0ig-2Dp5h7rVKx89StH5M3CL60-cWEbgG-XmtW31pZlFicYgSPduJZUnD838_' + b'QLbASSQLAZcC'] assert reg.getTwe(snKey(pre=regk, sn=0)) is None debSecret = 'AKUotEE0eAheKdDJh9QvNmSEmO_bjIav8V_GmctGpuCQ' @@ -686,11 +686,11 @@ def test_tever_backers(mockHelpingNowUTC, mockCoringRandomNonce): vci = vcdig dgkey = dgKey(pre=vci, dig=bis.said) - assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON000162_","t":"bis","d":"EN01O_jV46iSPtJMFXLNB83OWHtUl0wEDnMT' - b'eHSWXJwf","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","ii":"EBgdJt_AS' - b'Weq7HjOmut2E8vQL8P1c9VTPDA0Pdh4KsZX","s":"0","ra":{"i":"EBgdJt_ASWeq7HjOmut2' - b'E8vQL8P1c9VTPDA0Pdh4KsZX","s":"1","d":"EEc1UURU3liIomWOFeDVqCo-3QbZHFZCUorx8' - b'MdZFZvy"},"dt":"2021-01-01T00:00:00.000000+00:00"}') + assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON000162_","t":"bis","d":"ECtlJanGHVM3-xcy7nrw06FyqqM-OZaewC28' + b'3XpVjep3","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","ii":"ECfzJv1hI' + b'YAF68tEDDSelka5aPNKg_pmdcZOTs0aubF-","s":"0","ra":{"i":"ECfzJv1hIYAF68tEDDSe' + b'lka5aPNKg_pmdcZOTs0aubF-","s":"1","d":"EOP4g7675ItLSVGbYsmpalH_UBQaJE-Ir5I83' + b'carNegC"},"dt":"2021-01-01T00:00:00.000000+00:00"}') def test_tevery(): @@ -778,7 +778,7 @@ def test_tevery_process_escrow(mockCoringRandomNonce): seqner = Seqner(sn=1) # said of rotation - rotsaid = b'EFEUzOixM-Avz6VPmQoB59eYQ2oan9ltJ1JOSe-QQFRq' + rotsaid = b'EGe5uFh3t0JglSPQtJJoxCV1RlFoTBept1BPRk3o6hgh' diger = coring.Diger(qb64b=rotsaid) tvy = Tevery(reger=reg, db=db) diff --git a/tests/vdr/test_txn_state.py b/tests/vdr/test_txn_state.py index c9ae51a32..8daa46b0e 100644 --- a/tests/vdr/test_txn_state.py +++ b/tests/vdr/test_txn_state.py @@ -21,7 +21,7 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): bobHab = bobHby.makeHab(name="bob", isith='1', icount=1,) - assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + assert bobHab.pre == 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj' regery = credentialing.Regery(hby=bobHby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -34,7 +34,7 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert issuer.regk == 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' # Gather up Bob's key event log msgs = bytearray() @@ -53,11 +53,11 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): assert asdict(rsr) == {'b': [], 'bt': '0', 'c': ['NB'], - 'd': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', + 'd': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', 'dt': '2021-01-01T00:00:00.000000+00:00', 'et': 'vcp', - 'i': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', - 'ii': 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH', + 'i': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', + 'ii': 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj', 's': '0', 'vn': [1, 0]} @@ -74,7 +74,7 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): assert cue['q']['ri'] == issuer.regk saider = bamReger.txnsb.escrowdb.get(keys=("registry-ooo", issuer.regk, bobHab.pre)) - assert saider[0].qb64b == b'ELprJNCkha3b3t7YKOPzfv-prTZvgJFM_xTnaaJop-3A' + assert saider[0].qb64b == b'ECZWYxq_Qgs0J0ls_imRWRYxrojzTKL2REjqe0rN8kWy' tmsgs = bytearray() cloner = regery.reger.clonePreIter(pre=issuer.regk, fn=0) # create iterator at 0 @@ -89,7 +89,7 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): assert bamReger.txnsb.escrowdb.get(keys=(issuer.regk, bobHab.pre)) == [] # check to make sure the tsn has been saved saider = bamReger.txnsb.saiderdb.get(keys=(issuer.regk, bobHab.pre)) - assert saider.qb64b == b'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert saider.qb64b == b'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): @@ -100,7 +100,7 @@ def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): bobHab = bobHby.makeHab(name="bob", isith='1', icount=1,) - assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + assert bobHab.pre == 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj' regery = credentialing.Regery(hby=bobHby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -113,7 +113,7 @@ def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert issuer.regk == 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' # pass key event log to Bam bamRtr = routing.Router() @@ -126,11 +126,11 @@ def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): assert asdict(tsn) == {'b': [], 'bt': '0', 'c': ['NB'], - 'd': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', + 'd': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', 'dt': '2021-01-01T00:00:00.000000+00:00', 'et': 'vcp', - 'i': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', - 'ii': 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH', + 'i': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', + 'ii': 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj', 's': '0', 'vn': [1, 0]} @@ -142,7 +142,7 @@ def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): parsing.Parser().parse(ims=bytearray(rpy), tvy=bamTvy, rvy=bamRvy) saider = bamReger.txnsb.escrowdb.get(keys=("registry-mae", issuer.regk, bobHab.pre)) - said = b'ELprJNCkha3b3t7YKOPzfv-prTZvgJFM_xTnaaJop-3A' + said = b'ECZWYxq_Qgs0J0ls_imRWRYxrojzTKL2REjqe0rN8kWy' assert saider[0].qb64b == said assert len(bamTvy.cues) == 1 cue = bamTvy.cues.popleft() @@ -180,7 +180,7 @@ def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): assert bamReger.txnsb.escrowdb.get(keys=(issuer.regk, bobHab.pre)) == [] # check to make sure the tsn has been saved saider = bamReger.txnsb.saiderdb.get(keys=(issuer.regk, bobHab.pre)) - assert saider.qb64b == b'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert saider.qb64b == b'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): @@ -195,10 +195,10 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): # setup Wes's habitat nontrans wesHab = wesHby.makeHab(name="wes", isith='1', icount=1,transferable=False,) - assert wesHab.pre == 'BCuDiSPCTq-qBBFDHkhf1_kmysrH8KSsFvoaOSgEbx-X' + assert wesHab.pre == 'BJX05FKbj6M7EoUp53nKJNdG5eDZMGBatlDjg_QcpuqE' bobHab = bobHby.makeHab(name="bob", isith='1', icount=1, wits=[wesHab.pre]) - assert bobHab.pre == 'EDroh9lTel0P1YQaiL7shXG63SRSzKSDek7PaceOs6bY' + assert bobHab.pre == 'EPa6GLVG4lFV9oi28WQbC7UfcSoDb7kMlZkaa3qaj4UA' regery = credentialing.Regery(hby=bobHby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -211,7 +211,7 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'EBrr1pxZoY5nY38YifrGvn5HSMv0sAwvTTAQ5e3_-ivP' + assert issuer.regk == 'EH3hN33719ybSg21Kboy-V2jafwvQSHnY1HUGzzBqqk6' # Create Bob's icp, pass to Wes. wesKvy = Kevery(db=wesHby.db, lax=False, local=False) @@ -243,11 +243,11 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): assert asdict(tsn) == {'b': [], 'bt': '0', 'c': ['NB'], - 'd': 'EBrr1pxZoY5nY38YifrGvn5HSMv0sAwvTTAQ5e3_-ivP', + 'd': 'EH3hN33719ybSg21Kboy-V2jafwvQSHnY1HUGzzBqqk6', 'dt': '2021-01-01T00:00:00.000000+00:00', 'et': 'vcp', - 'i': 'EBrr1pxZoY5nY38YifrGvn5HSMv0sAwvTTAQ5e3_-ivP', - 'ii': 'EDroh9lTel0P1YQaiL7shXG63SRSzKSDek7PaceOs6bY', + 'i': 'EH3hN33719ybSg21Kboy-V2jafwvQSHnY1HUGzzBqqk6', + 'ii': 'EPa6GLVG4lFV9oi28WQbC7UfcSoDb7kMlZkaa3qaj4UA', 's': '0', 'vn': [1, 0]} @@ -262,7 +262,7 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): parsing.Parser().parse(ims=bytearray(rpy), tvy=bamTvy, rvy=bamRvy, local=True) saider = bamReger.txnsb.escrowdb.get(keys=("registry-mae", issuer.regk, wesHab.pre)) - said = b'EMoqBJpoPJCkCejSUZ8IShTeGd_WQkF0zOQc2l0HQusn' + said = b'EPKAvk7JJzylDFPjOxfanPjqDvUk_a-hzjNTT9Svhik1' assert saider[0].qb64b == said assert len(bamTvy.cues) == 1 cue = bamTvy.cues.popleft() @@ -300,7 +300,7 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): assert bamReger.txnsb.escrowdb.get(keys=(issuer.regk, wesHab.pre)) == [] # check to make sure the tsn has been saved saider = bamReger.txnsb.saiderdb.get(keys=(issuer.regk, wesHab.pre)) - assert saider.qb64b == b'EBrr1pxZoY5nY38YifrGvn5HSMv0sAwvTTAQ5e3_-ivP' + assert saider.qb64b == b'EH3hN33719ybSg21Kboy-V2jafwvQSHnY1HUGzzBqqk6' def test_tsn_from_no_one(mockHelpingNowUTC, mockCoringRandomNonce): @@ -319,10 +319,10 @@ def test_tsn_from_no_one(mockHelpingNowUTC, mockCoringRandomNonce): # setup Wes's habitat nontrans wesHab = wesHby.makeHab(name="wes", isith='1', icount=1,transferable=False,) - assert wesHab.pre == 'BCuDiSPCTq-qBBFDHkhf1_kmysrH8KSsFvoaOSgEbx-X' + assert wesHab.pre == 'BJX05FKbj6M7EoUp53nKJNdG5eDZMGBatlDjg_QcpuqE' bobHab = bobHby.makeHab(name="bob", isith='1', icount=1) - assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + assert bobHab.pre == 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj' regery = credentialing.Regery(hby=bobHby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -335,7 +335,7 @@ def test_tsn_from_no_one(mockHelpingNowUTC, mockCoringRandomNonce): saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert issuer.regk == 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' # Create Bob's icp, pass to Wes. @@ -366,11 +366,11 @@ def test_tsn_from_no_one(mockHelpingNowUTC, mockCoringRandomNonce): assert asdict(tsn) == {'b': [], 'bt': '0', 'c': ['NB'], - 'd': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', + 'd': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', 'dt': '2021-01-01T00:00:00.000000+00:00', 'et': 'vcp', - 'i': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', - 'ii': 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH', + 'i': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', + 'ii': 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj', 's': '0', 'vn': [1, 0]} @@ -410,7 +410,7 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): bobHab = bobHby.makeHab(name="bob", isith='1', icount=1,) - assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + assert bobHab.pre == 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj' regery = credentialing.Regery(hby=bobHby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -423,7 +423,7 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert issuer.regk == 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' # pass key event log to Bam bamRtr = routing.Router() @@ -453,25 +453,25 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe tever = issuer.tevers[issuer.regk] rsr = tever.state() - assert rsr._asdict() == {'vn': [1, 0], - 'i': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', - 's': '0', - 'd': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', - 'ii': 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH', - 'dt': '2021-06-27T21:26:21.233257+00:00', - 'et': 'vcp', - 'bt': '0', - 'b': [], - 'c': ['NB']} + assert rsr._asdict() == {'b': [], + 'bt': '0', + 'c': ['NB'], + 'd': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', + 'dt': '2021-06-27T21:26:21.233257+00:00', + 'et': 'vcp', + 'i': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', + 'ii': 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj', + 's': '0', + 'vn': [1, 0]} ctsn = tever.vcState(vci=creder.said) - assert asdict(ctsn) == {'a': {'d': 'EGJIFzWexy6LQbW4-IQqhGLD6wA9yR7pcLzxomjb40Ku', 's': 2}, - 'd': 'EIxhyBA8h6BMmtWEJzqNkoAquIkMucpXbdY3kQX25GQu', + assert asdict(ctsn) == {'a': {'d': 'EJEsBdnprtcNR-QaFM-h1OPaiGDBikgDk7aqVXPHAhPL', 's': 2}, + 'd': 'EGE5B5vYhHNYHN0g8YHIcAIu3LR0vJQ9fCXqjJunEZf1', 'dt': '2021-06-27T21:26:21.233257+00:00', 'et': 'iss', - 'i': 'EEqcwL-ew_OaQphSQvy8bRGtnKlL_g_SkXjsAGWgtFGl', + 'i': 'EPQNM4YNlVZ6nGu7ElGC6GhAHdLBLnoxYXKFP4R_ZbGz', 'ra': {}, - 'ri': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', + 'ri': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', 's': '0', 'vn': [1, 0]} @@ -483,7 +483,7 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe parsing.Parser().parse(ims=bytearray(rpy), tvy=bamTvy, rvy=bamRvy) saider = bamReger.txnsb.escrowdb.get(keys=("credential-mre", creder.said, bobHab.pre)) - assert saider[0].qb64b == b'EAslzIz1FKVBY1zd_0gF-gclpvHMf1V4arW3puZlPv4K' + assert saider[0].qb64b == b'EHERhBLfaMik0Ne9ysU3UICXWge0yobK0FQv3QhyeqF7' assert len(bamTvy.cues) == 1 cue = bamTvy.cues.popleft() assert cue["kin"] == "telquery" @@ -512,7 +512,7 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe assert cue['q']['ri'] == issuer.regk saider = bamReger.txnsb.escrowdb.get(keys=("credential-ooo", creder.said, bobHab.pre)) - assert saider[0].qb64b == b'EAslzIz1FKVBY1zd_0gF-gclpvHMf1V4arW3puZlPv4K' + assert saider[0].qb64b == b'EHERhBLfaMik0Ne9ysU3UICXWge0yobK0FQv3QhyeqF7' vci = creder.said tmsgs = bytearray() @@ -529,13 +529,13 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe # check to make sure the tsn has been saved keys = (creder.said, bobHab.pre) saider = bamReger.txnsb.saiderdb.get(keys=keys) - assert saider.qb64b == b'EIxhyBA8h6BMmtWEJzqNkoAquIkMucpXbdY3kQX25GQu' + assert saider.qb64b == b'EGE5B5vYhHNYHN0g8YHIcAIu3LR0vJQ9fCXqjJunEZf1' def test_tever_reload(mockHelpingNowUTC, mockCoringRandomNonce, mockHelpingNowIso8601): with habbing.openHby(name="bob", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: bobHab = hby.makeHab(name="bob", isith='1', icount=1,) - assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + assert bobHab.pre == 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj' regery = credentialing.Regery(hby=hby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -548,7 +548,7 @@ def test_tever_reload(mockHelpingNowUTC, mockCoringRandomNonce, mockHelpingNowIs saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert issuer.regk == 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' rsr = regery.reger.states.get(keys=issuer.regk) tever = eventing.Tever(rsr=rsr, reger=regery.reger) diff --git a/tests/vdr/test_verifying.py b/tests/vdr/test_verifying.py index 05c4d496b..ea7bb1bc9 100644 --- a/tests/vdr/test_verifying.py +++ b/tests/vdr/test_verifying.py @@ -25,13 +25,13 @@ def test_verifier_query(mockHelpingNowUTC, mockCoringRandomNonce): msg = verfer.query(hab.pre, issuer.regk, "EA8Ih8hxLi3mmkyItXK1u55cnHl4WgNZ_RE-gKXqgcX4", route="tels") - assert msg == (b'{"v":"KERI10JSON0000fe_","t":"qry","d":"EHraBkp-XMf1x_bo70O2x3br' - b'BCHlJHa7q_MzsBNeYz2_","dt":"2021-01-01T00:00:00.000000+00:00","r' + assert msg == (b'{"v":"KERI10JSON0000fe_","t":"qry","d":"EFa6oMZA5bgpALIc7yykT6O6' + b'ovdbDQnRFeTPDI4zaOhr","dt":"2021-01-01T00:00:00.000000+00:00","r' b'":"tels","rr":"","q":{"i":"EA8Ih8hxLi3mmkyItXK1u55cnHl4WgNZ_RE-g' - b'KXqgcX4","ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4"}}-V' - b'Aj-HABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAABUWETZTw' - b'TVuh0mNvN5KsJ_9V1epoP5wqgW32x8nUnGB20aI8xQBAhQ-aVP61ZEq97BDGSnxO' - b'hU6tGCfDmvtugI') + b'KXqgcX4","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ"}}-V' + b'Aj-HABEMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl-AABAABGnrnayV' + b'yK1siivaffGHpWWhcVThPN_dsePQvMXrlsOYNf0UdT0e6ch-0bN-UuOJCd1behue' + b'Zs_0V9FQ9vw0wK') def test_verifier(seeder): From 61f333734249a141ccf9bc3607e39149ffd6d92c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 1 May 2024 07:43:19 -0600 Subject: [PATCH 161/418] fixed typo in new count code table and fixed related code and tests. wrong code for list. --- src/keri/core/counting.py | 4 +- src/keri/core/streaming.py | 20 ++-- tests/core/test_counting.py | 4 +- tests/core/test_serdering.py | 121 ++++++++++--------- tests/core/test_streaming.py | 224 ++++++++++++++++++++++------------- 5 files changed, 217 insertions(+), 156 deletions(-) diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 9dee9fe14..cd14e01cd 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -102,8 +102,8 @@ class CounterCodex_2_0(MapDom): BigMapMessageBodyGroup: str = '-0G' # Big Field Map Message Body Group (Universal). GenericMapGroup: str = '-H' # Generic Field Map Group (Universal). BigGenericMapGroup: str = '-0H' # Big Generic Field Map Group (Universal). - GenericListGroup: str = '-L' # Generic List Group (Universal). - BigGenericListGroup: str = '-0L' # Big Generic List Group (Universal). + GenericListGroup: str = '-I' # Generic List Group (Universal). + BigGenericListGroup: str = '-0I' # Big Generic List Group (Universal). ControllerIdxSigs: str = '-J' # Controller Indexed Signature(s) of qb64. BigControllerIdxSigs: str = '-0J' # Big Controller Indexed Signature(s) of qb64. WitnessIdxSigs: str = '-K' # Witness Indexed Signature(s) of qb64. diff --git a/src/keri/core/streaming.py b/src/keri/core/streaming.py index 878a76ecc..617a74d21 100644 --- a/src/keri/core/streaming.py +++ b/src/keri/core/streaming.py @@ -304,19 +304,21 @@ def denot(ams): """De-annotate CESR stream Returns: - deannotation (bytes): deannotation of input annotated CESR stream + dms (bytes): deannotation of input annotated CESR message stream Parameters: ams (str): CESR annotated message stream text """ - - oms = bytearray() - - - - - return bytes(oms) - + dms = bytearray() # deannotated message stream + lines = ams.splitlines() + for line in lines: + line = line.strip() + front, sep, back = line.partition('#') # finde comment if any + front = front.strip() # non-commented portion strip white space + if front: + dms.extend(front.encode()) + + return bytes(dms) class Streamer: diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index bec518824..7887a610f 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -98,8 +98,8 @@ def test_codexes_tags(): 'BigMapMessageBodyGroup': '-0G', 'GenericMapGroup': '-H', 'BigGenericMapGroup': '-0H', - 'GenericListGroup': '-L', - 'BigGenericListGroup': '-0L', + 'GenericListGroup': '-I', + 'BigGenericListGroup': '-0I', 'ControllerIdxSigs': '-J', 'BigControllerIdxSigs': '-0J', 'WitnessIdxSigs': '-K', diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 6cdddfc31..5ca21c30c 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -2698,7 +2698,7 @@ def test_cesr_native_dumps(): { 'v': 'KERICAACESRAAAA.', 't': 'icp', - 'd': 'EO6lMLcTbUhdpbQVXCh78MShuT_69th6tiZhEbAfPCj4', + 'd': 'EEx4oxGYbNrd6nZsdGu2KdN4MSDGD5IWS7hXjST7r8ew', 'i': 'DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ', 's': '0', 'kt': '1', @@ -2711,9 +2711,10 @@ def test_cesr_native_dumps(): 'a': [] } - assert serder.raw == (b'-FAtYKERICAAXicpEO6lMLcTbUhdpbQVXCh78MShuT_69th6tiZhEbAfPCj4DG9X' - b'hvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAAMAAB-LALDG9XhvcVryHj' - b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAA-LAAMAAA-LAA-LAA-LAA') + + assert serder.raw == (b'-FAtYKERICAAXicpEEx4oxGYbNrd6nZsdGu2KdN4MSDGD5IWS7hXjST7r8ewDG9X' + b'hvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAAMAAB-IALDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAA-IAAMAAA-IAA-IAA-IAA') assert len(serder.raw) == serder.size == 184 sizeh = serder.raw[2:4] assert sizeh == b"At" @@ -2779,7 +2780,7 @@ def test_cesr_native_dumps(): kind=kering.Serials.cesr) pre = serder.pre - assert pre == 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6' + assert pre == 'EKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi' said = serder.said assert said == pre @@ -2787,8 +2788,8 @@ def test_cesr_native_dumps(): { 'v': 'KERICAACESRAAAA.', 't': 'icp', - 'd': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6', - 'i': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6', + 'd': 'EKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi', + 'i': 'EKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi', 's': '0', 'kt': '2', 'k': @@ -2831,15 +2832,15 @@ def test_cesr_native_dumps(): ] } - assert serder.raw == (b'-FDCYKERICAAXicpEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6EMEv' - b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAAAMAAC-LAhDG9XhvcVryHj' + assert serder.raw == (b'-FDCYKERICAAXicpEKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiEKIu' + b'A20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiMAAAMAAC-IAhDG9XhvcVryHj' b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' - b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-LAh' + b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-IAh' b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' b'TPUtc7uSAvrchIPN3I8S6vUGEEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZp' - b'ZEyPMAAD-LAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' + b'ZEyPMAAD-IAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' b'o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnBMOmBoddcrRHShSajb4d60S6RK34' - b'gXZ2WYbr3AiPY1M0-LABXDND-LA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'gXZ2WYbr3AiPY1M0-IABXDND-IA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' b'4fBJre3NGwTQMAAAEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAABEMrowWRk6u1imR32ZNHn' b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' @@ -2852,21 +2853,22 @@ def test_cesr_native_dumps(): rawqb64 = serder._dumps() # default is it dumps self.sad assert rawqb64 == serder.raw - assert rawqb64 == (b'-FDCYKERICAAXicpEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6EMEv' - b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAAAMAAC-LAhDG9XhvcVryHj' + assert rawqb64 == (b'-FDCYKERICAAXicpEKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiEKIu' + b'A20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiMAAAMAAC-IAhDG9XhvcVryHj' b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' - b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-LAh' + b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-IAh' b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' b'TPUtc7uSAvrchIPN3I8S6vUGEEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZp' - b'ZEyPMAAD-LAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' + b'ZEyPMAAD-IAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' b'o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnBMOmBoddcrRHShSajb4d60S6RK34' - b'gXZ2WYbr3AiPY1M0-LABXDND-LA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'gXZ2WYbr3AiPY1M0-IABXDND-IA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' b'4fBJre3NGwTQMAAAEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAABEMrowWRk6u1imR32ZNHn' b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' b'ZJGJqQZpZEyP') + assert len(rawqb64) == 780 rawqb2 = decodeB64(rawqb64) @@ -2931,16 +2933,16 @@ def test_cesr_native_dumps(): kind=kering.Serials.cesr) said = serder.said - assert said == 'EHeLJVa4LLNRRYVkLQsXHIDvllcmhDaahe5a_oMvXKeP' + assert said == 'EHXLwMJsZLyG643VW8Do1cqqiMxD_E65Mc3Z1we6vTaR' assert serder.sad == \ { 'v': 'KERICAACESRAAAA.', 't': 'ixn', - 'd': 'EHeLJVa4LLNRRYVkLQsXHIDvllcmhDaahe5a_oMvXKeP', - 'i': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6', + 'd': 'EHXLwMJsZLyG643VW8Do1cqqiMxD_E65Mc3Z1we6vTaR', + 'i': 'EKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi', 's': '1', - 'p': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6', + 'p': 'EKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi', 'a': [ { @@ -2965,9 +2967,9 @@ def test_cesr_native_dumps(): ] } - assert serder.raw == (b'-FB6YKERICAAXixnEHeLJVa4LLNRRYVkLQsXHIDvllcmhDaahe5a_oMvXKePEMEv' - b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAABEMEvSn0o6Iv2-3gInTDM' - b'MDTV0qQEfooM-yTzkj6Kynn6-LBU-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + assert serder.raw == (b'-FB6YKERICAAXixnEHXLwMJsZLyG643VW8Do1cqqiMxD_E65Mc3Z1we6vTaREKIu' + b'A20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiMAABEKIuA20I5q6IrgAHrX-g' + b'kAt4Og17Ebu5CDBrRvh8RToi-IBU-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' b'4fBJre3NGwTQMAACEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAAiEMrowWRk6u1imR32ZNHn' b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMABDEEbufBpvagqe9kijKISOoQPYFEOpy22C' @@ -3025,20 +3027,23 @@ def test_cesr_native_dumps(): kind=kering.Serials.cesr) said = serder.said - assert said == 'EDtBwgOB0uGrSMBJhOmnkRoCupjg-4sJApvOx04ujhKs' + assert said == 'EDHlTlOcSXZInbTE4iXzb1iFjZcxJZn3C3UXhckb3uQm' assert serder.sad == \ { 'v': 'KERICAACESRAAAA.', 't': 'rot', - 'd': 'EDtBwgOB0uGrSMBJhOmnkRoCupjg-4sJApvOx04ujhKs', - 'i': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6', + 'd': 'EDHlTlOcSXZInbTE4iXzb1iFjZcxJZn3C3UXhckb3uQm', + 'i': 'EKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi', 's': '2', - 'p': 'EHeLJVa4LLNRRYVkLQsXHIDvllcmhDaahe5a_oMvXKeP', + 'p': 'EHXLwMJsZLyG643VW8Do1cqqiMxD_E65Mc3Z1we6vTaR', 'kt': '2', - 'k': ['DH7p14xo09rob5cEupmo8jSDi35ZOGt1k4t2nm1C1A68', - 'DIAdqJzLWEwQbhXEMOFjvFVZ7oMCJP4XXDP_ILaTEBAQ', - 'DKhYdMBeP6FoH3ajGJTf_4fH229rm_lTZXfYkfwGTMER'], + 'k': + [ + 'DH7p14xo09rob5cEupmo8jSDi35ZOGt1k4t2nm1C1A68', + 'DIAdqJzLWEwQbhXEMOFjvFVZ7oMCJP4XXDP_ILaTEBAQ', + 'DKhYdMBeP6FoH3ajGJTf_4fH229rm_lTZXfYkfwGTMER' + ], 'nt': '2', 'n': [ @@ -3054,15 +3059,15 @@ def test_cesr_native_dumps(): } - assert serder.raw == (b'-FCGYKERICAAXrotEDtBwgOB0uGrSMBJhOmnkRoCupjg-4sJApvOx04ujhKsEMEv' - b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAACEHeLJVa4LLNRRYVkLQsX' - b'HIDvllcmhDaahe5a_oMvXKePMAAC-LAhDH7p14xo09rob5cEupmo8jSDi35ZOGt1' + assert serder.raw == (b'-FCGYKERICAAXrotEDHlTlOcSXZInbTE4iXzb1iFjZcxJZn3C3UXhckb3uQmEKIu' + b'A20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiMAACEHXLwMJsZLyG643VW8Do' + b'1cqqiMxD_E65Mc3Z1we6vTaRMAAC-IAhDH7p14xo09rob5cEupmo8jSDi35ZOGt1' b'k4t2nm1C1A68DIAdqJzLWEwQbhXEMOFjvFVZ7oMCJP4XXDP_ILaTEBAQDKhYdMBe' - b'P6FoH3ajGJTf_4fH229rm_lTZXfYkfwGTMERMAAC-LAhEBvDSpcj3y0y9W2-1GzY' + b'P6FoH3ajGJTf_4fH229rm_lTZXfYkfwGTMERMAAC-IAhEBvDSpcj3y0y9W2-1GzY' b'J85KEkDIPxu4y_TxAK49k7ciEEb97lh2oOd_yM3meBaRX5xSs8mIeBoPdhOTgVkd' - b'31jbECQTrhKHgrOXJS4kdvifvOqoJ7RjfJSsN3nshclYStgaMAAD-LALBG9XhvcV' - b'ryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ-LALBH7p14xo09rob5cEupmo8jSD' - b'i35ZOGt1k4t2nm1C1A68-LAA-LAA') + b'31jbECQTrhKHgrOXJS4kdvifvOqoJ7RjfJSsN3nshclYStgaMAAD-IALBG9XhvcV' + b'ryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ-IALBH7p14xo09rob5cEupmo8jSD' + b'i35ZOGt1k4t2nm1C1A68-IAA-IAA') assert len(serder.raw) == serder.size == 540 sizeh = serder.raw[2:4] @@ -3138,7 +3143,7 @@ def test_cesr_native_dumps(): kind=kering.Serials.cesr) pre = serder.pre - assert pre == 'ECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8T' + assert pre == 'EKCFMk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjC' said = serder.said assert said == pre @@ -3146,8 +3151,8 @@ def test_cesr_native_dumps(): { 'v': 'KERICAACESRAAAA.', 't': 'dip', - 'd': 'ECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8T', - 'i': 'ECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8T', + 'd': 'EKCFMk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjC', + 'i': 'EKCFMk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjC', 's': '0', 'kt': ['1/2', '1/2', '1/2'], 'k': @@ -3192,24 +3197,24 @@ def test_cesr_native_dumps(): 'd': 'EKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ44DS' } ], - 'di': 'EMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6' + 'di': 'EKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi' } - assert serder.raw == (b'-FDeYKERICAAXdipECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TECQs' - b'0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TMAAA4AADA1s2c1s2c1s2-LAh' + assert serder.raw == (b'-FDeYKERICAAXdipEKCFMk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjCEKCF' + b'Mk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjCMAAA4AADA1s2c1s2c1s2-IAh' b'DIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uVDN7WiKyjLLBTK92xayCu' b'ddZsBuwPmD2BKrl83h1xEUtiDOE5jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl' - b'5nfY4AADA1s2c1s2c1s2-LAhEKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ' + b'5nfY4AADA1s2c1s2c1s2-IAhEKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ' b'44DSEC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZEHgewy_ymPxtSFwu' - b'X2KaI_mPmoIUkxClviX3f-M38kCDMAAD-LAhBIR8GACw4z2GC5_XoReU4DMKbqi6' + b'X2KaI_mPmoIUkxClviX3f-M38kCDMAAD-IAhBIR8GACw4z2GC5_XoReU4DMKbqi6' b'-EdbgDZUAobRb8uVBN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiBOE5' - b'jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl5nfY-LAA-LBI-RAuDIR8GACw4z2G' + b'jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl5nfY-IAA-IBI-RAuDIR8GACw4z2G' b'C5_XoReU4DMKbqi6-EdbgDZUAobRb8uVMAADEKFoJ9Conb37zSn8zHLKP3YwHbeQ' b'iD1D9Qx0MagJ44DSDN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiMAAE' b'EC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZ-QAYMAAVEHgewy_ymPxt' b'SFwuX2KaI_mPmoIUkxClviX3f-M38kCDMD4SEKFoJ9Conb37zSn8zHLKP3YwHbeQ' - b'iD1D9Qx0MagJ44DSEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6') + b'iD1D9Qx0MagJ44DSEKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi') assert len(serder.raw) == serder.size == 892 sizeh = serder.raw[2:4] @@ -3264,16 +3269,16 @@ def test_cesr_native_dumps(): kind=kering.Serials.cesr) said = serder.said - assert said == 'EKwDKG0L9pAMbzV2e31-I5ObiEfkptfs8VqXYiHGCL1v' + assert said == 'EKfRY6YrpqUU0HyKWMGvNtzuZCaeMcIBrdKzHAqpmtTA' assert serder.sad == \ { 'v': 'KERICAACESRAAAA.', 't': 'drt', - 'd': 'EKwDKG0L9pAMbzV2e31-I5ObiEfkptfs8VqXYiHGCL1v', - 'i': 'ECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8T', + 'd': 'EKfRY6YrpqUU0HyKWMGvNtzuZCaeMcIBrdKzHAqpmtTA', + 'i': 'EKCFMk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjC', 's': '1', - 'p': 'ECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8T', + 'p': 'EKCFMk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjC', 'kt': '1', 'k': ['DJ0pLe3f2zGus0Va1dqWAnukWdZHGNWlK9NciJop9N4f'], 'nt': '1', @@ -3285,13 +3290,13 @@ def test_cesr_native_dumps(): 'a': {} } + assert serder.raw == (b'-FBaYKERICAAXdrtEKfRY6YrpqUU0HyKWMGvNtzuZCaeMcIBrdKzHAqpmtTAEKCF' + b'Mk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjCMAABEKCFMk4nmn3t8jdC1pB_' + b'-Qmp7w8EROvdYaxgru7vHOjCMAAB-IALDJ0pLe3f2zGus0Va1dqWAnukWdZHGNWl' + b'K9NciJop9N4fMAAB-IALENX_LTL97uOSOkA1PEzam9vtmCLPprnbcpi71wXpmhFF' + b'MAAD-IALBIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV-IALBJ0pLe3f' + b'2zGus0Va1dqWAnukWdZHGNWlK9NciJop9N4f-IAA-IAA') - assert serder.raw == (b'-FBaYKERICAAXdrtEKwDKG0L9pAMbzV2e31-I5ObiEfkptfs8VqXYiHGCL1vECQs' - b'0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TMAABECQs0t3_GL7-B3q4kMU-' - b'qLeRCugTFjrxR15mxUwYWp8TMAAB-LALDJ0pLe3f2zGus0Va1dqWAnukWdZHGNWl' - b'K9NciJop9N4fMAAB-LALENX_LTL97uOSOkA1PEzam9vtmCLPprnbcpi71wXpmhFF' - b'MAAD-LALBIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV-LALBJ0pLe3f' - b'2zGus0Va1dqWAnukWdZHGNWlK9NciJop9N4f-LAA-LAA') assert len(serder.raw) == serder.size == 364 sizeh = serder.raw[2:4] diff --git a/tests/core/test_streaming.py b/tests/core/test_streaming.py index 21e7d12aa..60b3ecc47 100644 --- a/tests/core/test_streaming.py +++ b/tests/core/test_streaming.py @@ -3,7 +3,7 @@ tests.core.test_streaming module """ - +from binascii import unhexlify import pytest @@ -12,7 +12,8 @@ from keri.help import helping -from keri.core import (Matter, ) +from keri.core import (Matter,) +from keri.core.coring import dumps from keri.core import streaming @@ -30,107 +31,160 @@ def test_annot(): """Test annot function Annotate""" # simple Inception - ims = bytearray( - b'-FAtYKERICAAXicpEO6lMLcTbUhdpbQVXCh78MShuT_69th6tiZhEbAfPCj4DG9X' - b'hvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAAMAAB-LALDG9XhvcVryHj' - b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAA-LAAMAAA-LAA-LAA-LAA') - - print(f"incoming = \n{bytes(ims)}\n") - ams = annot(ims) # annotated message dream - assert not ims + ims = (b'-FAtYKERICAAXicpEEx4oxGYbNrd6nZsdGu2KdN4MSDGD5IWS7hXjST7r8ewDG9X' + b'hvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAAMAAB-IALDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAA-IAAMAAA-IAA-IAA-IAA') + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims # complex inception - ims = bytearray( - b'-FDCYKERICAAXicpEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6EMEv' - b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAAAMAAC-LAhDG9XhvcVryHj' - b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' - b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-LAh' - b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' - b'TPUtc7uSAvrchIPN3I8S6vUGEEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZp' - b'ZEyPMAAD-LAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' - b'o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnBMOmBoddcrRHShSajb4d60S6RK34' - b'gXZ2WYbr3AiPY1M0-LABXDND-LA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' - b'4fBJre3NGwTQMAAAEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' - b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAABEMrowWRk6u1imR32ZNHn' - b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' - b'ZJGJqQZpZEyP') - - print(f"incoming = \n{bytes(ims)}\n") - ams = annot(ims) # annotated message dream - assert not ims + ims = (b'-FDCYKERICAAXicpEKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiEKIu' + b'A20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiMAAAMAAC-IAhDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' + b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-IAh' + b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUGEEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZp' + b'ZEyPMAAD-IAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' + b'o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnBMOmBoddcrRHShSajb4d60S6RK34' + b'gXZ2WYbr3AiPY1M0-IABXDND-IA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'4fBJre3NGwTQMAAAEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' + b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAABEMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' + b'ZJGJqQZpZEyP') + + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims # interaction - ims = bytearray( - b'-FB6YKERICAAXixnEHeLJVa4LLNRRYVkLQsXHIDvllcmhDaahe5a_oMvXKePEMEv' - b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAABEMEvSn0o6Iv2-3gInTDM' - b'MDTV0qQEfooM-yTzkj6Kynn6-LBU-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' - b'4fBJre3NGwTQMAACEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' - b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAAiEMrowWRk6u1imR32ZNHn' - b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMABDEEbufBpvagqe9kijKISOoQPYFEOpy22C' - b'ZJGJqQZpZEyP-RAXDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MACA' - b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_') - - print(f"incoming = \n{bytes(ims)}\n") - ams = annot(ims) # annotated message dream - assert not ims + ims = (b'-FB6YKERICAAXixnEHXLwMJsZLyG643VW8Do1cqqiMxD_E65Mc3Z1we6vTaREKIu' + b'A20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiMAABEKIuA20I5q6IrgAHrX-g' + b'kAt4Og17Ebu5CDBrRvh8RToi-IBU-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'4fBJre3NGwTQMAACEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' + b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAAiEMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMABDEEbufBpvagqe9kijKISOoQPYFEOpy22C' + b'ZJGJqQZpZEyP-RAXDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MACA' + b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_') + + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims # Rotation - ims = bytearray( - b'-FCGYKERICAAXrotEDtBwgOB0uGrSMBJhOmnkRoCupjg-4sJApvOx04ujhKsEMEv' - b'Sn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6MAACEHeLJVa4LLNRRYVkLQsX' - b'HIDvllcmhDaahe5a_oMvXKePMAAC-LAhDH7p14xo09rob5cEupmo8jSDi35ZOGt1' - b'k4t2nm1C1A68DIAdqJzLWEwQbhXEMOFjvFVZ7oMCJP4XXDP_ILaTEBAQDKhYdMBe' - b'P6FoH3ajGJTf_4fH229rm_lTZXfYkfwGTMERMAAC-LAhEBvDSpcj3y0y9W2-1GzY' - b'J85KEkDIPxu4y_TxAK49k7ciEEb97lh2oOd_yM3meBaRX5xSs8mIeBoPdhOTgVkd' - b'31jbECQTrhKHgrOXJS4kdvifvOqoJ7RjfJSsN3nshclYStgaMAAD-LALBG9XhvcV' - b'ryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ-LALBH7p14xo09rob5cEupmo8jSD' - b'i35ZOGt1k4t2nm1C1A68-LAA-LAA') - - print(f"incoming = \n{bytes(ims)}\n") - ams = annot(ims) # annotated message dream - assert not ims + ims = (b'-FCGYKERICAAXrotEDHlTlOcSXZInbTE4iXzb1iFjZcxJZn3C3UXhckb3uQmEKIu' + b'A20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiMAACEHXLwMJsZLyG643VW8Do' + b'1cqqiMxD_E65Mc3Z1we6vTaRMAAC-IAhDH7p14xo09rob5cEupmo8jSDi35ZOGt1' + b'k4t2nm1C1A68DIAdqJzLWEwQbhXEMOFjvFVZ7oMCJP4XXDP_ILaTEBAQDKhYdMBe' + b'P6FoH3ajGJTf_4fH229rm_lTZXfYkfwGTMERMAAC-IAhEBvDSpcj3y0y9W2-1GzY' + b'J85KEkDIPxu4y_TxAK49k7ciEEb97lh2oOd_yM3meBaRX5xSs8mIeBoPdhOTgVkd' + b'31jbECQTrhKHgrOXJS4kdvifvOqoJ7RjfJSsN3nshclYStgaMAAD-IALBG9XhvcV' + b'ryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ-IALBH7p14xo09rob5cEupmo8jSD' + b'i35ZOGt1k4t2nm1C1A68-IAA-IAA') + + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims + # Delegated Inception - ims = bytearray( - b'-FDeYKERICAAXdipECQs0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TECQs' - b'0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TMAAA4AADA1s2c1s2c1s2-LAh' - b'DIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uVDN7WiKyjLLBTK92xayCu' - b'ddZsBuwPmD2BKrl83h1xEUtiDOE5jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl' - b'5nfY4AADA1s2c1s2c1s2-LAhEKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ' - b'44DSEC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZEHgewy_ymPxtSFwu' - b'X2KaI_mPmoIUkxClviX3f-M38kCDMAAD-LAhBIR8GACw4z2GC5_XoReU4DMKbqi6' - b'-EdbgDZUAobRb8uVBN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiBOE5' - b'jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl5nfY-LAA-LBI-RAuDIR8GACw4z2G' - b'C5_XoReU4DMKbqi6-EdbgDZUAobRb8uVMAADEKFoJ9Conb37zSn8zHLKP3YwHbeQ' - b'iD1D9Qx0MagJ44DSDN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiMAAE' - b'EC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZ-QAYMAAVEHgewy_ymPxt' - b'SFwuX2KaI_mPmoIUkxClviX3f-M38kCDMD4SEKFoJ9Conb37zSn8zHLKP3YwHbeQ' - b'iD1D9Qx0MagJ44DSEMEvSn0o6Iv2-3gInTDMMDTV0qQEfooM-yTzkj6Kynn6') - - print(f"incoming = \n{bytes(ims)}\n") - ams = annot(ims) # annotated message dream - assert not ims + ims = (b'-FDeYKERICAAXdipEKCFMk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjCEKCF' + b'Mk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjCMAAA4AADA1s2c1s2c1s2-IAh' + b'DIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uVDN7WiKyjLLBTK92xayCu' + b'ddZsBuwPmD2BKrl83h1xEUtiDOE5jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl' + b'5nfY4AADA1s2c1s2c1s2-IAhEKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ' + b'44DSEC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZEHgewy_ymPxtSFwu' + b'X2KaI_mPmoIUkxClviX3f-M38kCDMAAD-IAhBIR8GACw4z2GC5_XoReU4DMKbqi6' + b'-EdbgDZUAobRb8uVBN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiBOE5' + b'jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl5nfY-IAA-IBI-RAuDIR8GACw4z2G' + b'C5_XoReU4DMKbqi6-EdbgDZUAobRb8uVMAADEKFoJ9Conb37zSn8zHLKP3YwHbeQ' + b'iD1D9Qx0MagJ44DSDN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiMAAE' + b'EC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZ-QAYMAAVEHgewy_ymPxt' + b'SFwuX2KaI_mPmoIUkxClviX3f-M38kCDMD4SEKFoJ9Conb37zSn8zHLKP3YwHbeQ' + b'iD1D9Qx0MagJ44DSEKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi') + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims + # Delegated Rotatation - ims = bytearray( - b'-FBaYKERICAAXdrtEKwDKG0L9pAMbzV2e31-I5ObiEfkptfs8VqXYiHGCL1vECQs' - b'0t3_GL7-B3q4kMU-qLeRCugTFjrxR15mxUwYWp8TMAABECQs0t3_GL7-B3q4kMU-' - b'qLeRCugTFjrxR15mxUwYWp8TMAAB-LALDJ0pLe3f2zGus0Va1dqWAnukWdZHGNWl' - b'K9NciJop9N4fMAAB-LALENX_LTL97uOSOkA1PEzam9vtmCLPprnbcpi71wXpmhFF' - b'MAAD-LALBIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV-LALBJ0pLe3f' - b'2zGus0Va1dqWAnukWdZHGNWlK9NciJop9N4f-LAA-LAA') - - print(f"incoming = \n{bytes(ims)}\n") - ams = annot(ims) # annotated message dream - assert not ims + ims = (b'-FBaYKERICAAXdrtEKfRY6YrpqUU0HyKWMGvNtzuZCaeMcIBrdKzHAqpmtTAEKCF' + b'Mk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjCMAABEKCFMk4nmn3t8jdC1pB_' + b'-Qmp7w8EROvdYaxgru7vHOjCMAAB-IALDJ0pLe3f2zGus0Va1dqWAnukWdZHGNWl' + b'K9NciJop9N4fMAAB-IALENX_LTL97uOSOkA1PEzam9vtmCLPprnbcpi71wXpmhFF' + b'MAAD-IALBIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV-IALBJ0pLe3f' + b'2zGus0Va1dqWAnukWdZHGNWlK9NciJop9N4f-IAA-IAA') + + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims + + # JWK https://connect2id.com/products/nimbus-jose-jwt/examples/jwk-generation#okp + # ED25519 EDDSA + djwk = { + "kty" : "OKP", + "crv" : "Ed25519", + "x" : "11qYAYKxCrfVS_7TyWQHOg7hcvPapiMlrwIaaPcHURo", + "d" : "nWGxne_9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A", + "use" : "sig", + "kid" : "FdFYFzERwC2uCBB46pZQi4GG85LujR8obt-KWRBICVQ" + } + + jwk = dumps(djwk, kering.Serials.json) + assert jwk == (b'{"kty":"OKP","crv":"Ed25519","x":"11qYAYKxCrfVS_7TyWQHOg7hcvPapiMlrwIaaPcHUR' + b'o","d":"nWGxne_9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A","use":"sig","kid":"FdFY' + b'FzERwC2uCBB46pZQi4GG85LujR8obt-KWRBICVQ"}') + + assert len(jwk) == 193 + assert 193 - 44 == 149 + + # ECDSA Example + # https://pycose.readthedocs.io/en/latest/pycose/keys/ec2.html + + ckat = { + 'KTY': 'EC2', + 'CURVE': 'P_256', + 'ALG': 'ES256', + 'D': unhexlify(b'57c92077664146e876760c9520d054aa93c3afb04e306705db6090308507b4d3') + } + + + ck = (b'\xa6\x01\x02\x03&\x01!X\xba\xc5\xb1\x1c\xad\x8f\x99\xf9\xc7+\x05\xcfK\x9e&\xd2D\xdc\x18\x9ftR(%Z!\x9a\x86\xd6\xa0\x9e\xffX\x13\x8b\xf8-\xc1\xb6\xd5b\xbe\x0f\xa5J\xb7\x80J:d\xb6\xd7,\xcf\xedko\xb6\xed(\xbb\xfc\x11~#XW\xc9 wfAF\xe8vv\x0c\x95\xd0T\xaa\x93\xc3\xaf\xb0N0g\x05\xdb`\x900\x85\x07\xb4\xd3') + assert len(ck) == 105 + assert 105 - 36 == 69 + From 94b674452e320176384e67d4dda3cb00a69c3a86 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Wed, 1 May 2024 18:45:44 -0700 Subject: [PATCH 162/418] New introduce rpy handler and updates to create HTTP server (#772) * Fix OOBI resolution (#766) * Fix OOBI resolution to account for knowing about a delegated AID without having the delegation approval. Added test. Closes #762 Signed-off-by: pfeairheller * Testing older mac test running on GitHub Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller * Merging from conflict Signed-off-by: pfeairheller * New /introduce rpy message handler to process OOBIs presented as introductions. Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller --- src/keri/app/cli/commands/watcher/__init__.py | 6 + src/keri/app/cli/commands/watcher/add.py | 132 ++++++++++++++++++ .../app/cli/commands/witness/authenticate.py | 7 +- src/keri/app/habbing.py | 2 +- src/keri/app/indirecting.py | 12 +- src/keri/app/oobiing.py | 97 ++++++++++++- src/keri/core/__init__.py | 3 - src/keri/core/eventing.py | 23 ++- src/keri/core/parsing.py | 8 +- src/keri/db/basing.py | 3 + src/keri/db/escrowing.py | 2 - src/keri/end/ending.py | 4 + tests/app/test_indirecting.py | 7 +- tests/app/test_oobiing.py | 72 +++++++++- tests/end/test_ending.py | 43 ++++++ 15 files changed, 375 insertions(+), 46 deletions(-) create mode 100644 src/keri/app/cli/commands/watcher/__init__.py create mode 100644 src/keri/app/cli/commands/watcher/add.py diff --git a/src/keri/app/cli/commands/watcher/__init__.py b/src/keri/app/cli/commands/watcher/__init__.py new file mode 100644 index 000000000..dbf28aaaf --- /dev/null +++ b/src/keri/app/cli/commands/watcher/__init__.py @@ -0,0 +1,6 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.app.cli.commands.watcher Package +""" + diff --git a/src/keri/app/cli/commands/watcher/add.py b/src/keri/app/cli/commands/watcher/add.py new file mode 100644 index 000000000..7f3c8410f --- /dev/null +++ b/src/keri/app/cli/commands/watcher/add.py @@ -0,0 +1,132 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app import connecting, habbing, forwarding +from keri.app.cli.common import existing +from keri.core import eventing, serdering + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Add AID or Alias to list of AIDs for a watcher to watch') +parser.set_defaults(handler=lambda args: add(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--watcher", '-w', help="the watcher AID or alias to add to", required=True) +parser.add_argument("--watched", '-W', help="the watched AID or alias to add") + + +def add(args): + """ Command line handler for adding an aid to a watcher's list of AIds to watch + + Parameters: + args(Namespace): parsed command line arguments + + """ + + ed = AddDoer(name=args.name, + alias=args.alias, + base=args.base, + bran=args.bran, + watcher=args.watcher, + watched=args.watched) + return [ed] + + +class AddDoer(doing.DoDoer): + + def __init__(self, name, alias, base, bran, watcher, watched): + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hab = self.hby.habByName(alias) + self.org = connecting.Organizer(hby=self.hby) + + if watcher in self.hby.kevers: + wat = watcher + else: + wat = self.org.find("alias", watcher) + if len(wat) != 1: + raise ValueError(f"invalid recipient {watcher}") + wat = wat[0]['id'] + + if not wat: + raise ValueError(f"unknown watcher {watcher}") + + if watched in self.hby.kevers: + watd = watched + else: + watd = self.org.find("alias", watched) + if len(watd) != 1: + raise ValueError(f"invalid recipient {watched}") + watd = watd[0]['id'] + + if not watd: + raise ValueError(f"unknown watched {watched}") + + self.watcher = wat + self.watched = watd + + self.oobi = None + for (key,), obr in self.hby.db.roobi.getItemIter(): + if obr.cid == watd: + self.oobi = key + + if not self.oobi: + raise ValueError(f"no valid oobi for watched {self.watched}") + + doers = [doing.doify(self.addDo)] + + super(AddDoer, self).__init__(doers=doers) + + def addDo(self, tymth, tock=0.0): + """ Grant credential by creating /ipex/grant exn message + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + if isinstance(self.hab, habbing.GroupHab): + raise ValueError("watchers for multisig AIDs not currently supported") + + postman = forwarding.StreamPoster(hby=self.hby, hab=self.hab, recp=self.watcher, topic="reply") + for msg in self.hab.db.clonePreIter(pre=self.hab.pre): + serder = serdering.SerderKERI(raw=msg) + postman.send(serder=serder, attachment=msg[serder.size:]) + + data = dict(cid=self.hab.pre, + wid=self.watched, + oobi=self.oobi) + + route = "/watcher/aid/add" + msg = self.hab.reply(route=route, data=data) + rpy = serdering.SerderKERI(raw=msg) + postman.send(serder=rpy, attachment=msg[rpy.size:]) + + doer = doing.DoDoer(doers=postman.deliver()) + self.extend([doer]) + + while not doer.done: + yield self.tock + + print(f"Request to add {self.watched} to watcher {self.watcher} submitted.") diff --git a/src/keri/app/cli/commands/witness/authenticate.py b/src/keri/app/cli/commands/witness/authenticate.py index bea4c8a6d..126fc0417 100644 --- a/src/keri/app/cli/commands/witness/authenticate.py +++ b/src/keri/app/cli/commands/witness/authenticate.py @@ -35,7 +35,10 @@ def auth(args): - """ Command line list credential registries handler + """ Command line handler for authenticating against a witness by retrieving the secret or a TOTP + + Parameters: + args(Namespace): parsed command line arguments """ @@ -65,7 +68,7 @@ def __init__(self, name, alias, base, bran, witness): if not wit: raise ValueError(f"unknown witness {witness}") - self.witness = witness + self.witness = wit self.clienter = httping.Clienter() doers = [doing.doify(self.authDo), self.clienter] diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 0bf3f2c24..0d4b3d235 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -2077,7 +2077,7 @@ def processCuesIter(self, cues): """ while cues: # iteratively process each cue in cues msgs = bytearray() - cue = cues.pull() #cues.popleft() + cue = cues.pull() # cues.popleft() cueKin = cue["kin"] # type or kind of cue if cueKin in ("receipt",): # cue to receipt a received event from other pre diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 031c9cf53..32dc0b190 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -40,6 +40,7 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo Setup witness controller and doers """ + host = "0.0.0.0" cues = decking.Deck() doers = [] @@ -87,7 +88,7 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo receiptEnd = ReceiptEnd(hab=hab, inbound=cues, aids=aids) app.add_route("/receipts", receiptEnd) - server = createHttpServer(httpPort, app, keypath, certpath, cafilepath) + server = createHttpServer(host, httpPort, app, keypath, certpath, cafilepath) if not server.reopen(): raise RuntimeError(f"cannot create http server on port {httpPort}") httpServerDoer = http.ServerDoer(server=server) @@ -112,10 +113,11 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo return doers -def createHttpServer(port, app, keypath=None, certpath=None, cafilepath=None): +def createHttpServer(host, port, app, keypath=None, certpath=None, cafilepath=None): """ Create an HTTP or HTTPS server depending on whether TLS key material is present Parameters: + host(str) : host to bind to for this server, or None for default of '0.0.0.0', all ifaces port (int) : port to listen on for all HTTP(s) server instances app (falcon.App) : application instance to pass to the http.Server instance keypath (string) : the file path to the TLS private key @@ -130,9 +132,9 @@ def createHttpServer(port, app, keypath=None, certpath=None, cafilepath=None): certpath=certpath, cafilepath=cafilepath, port=port) - server = http.Server(port=port, app=app, servant=servant) + server = http.Server(host=host, port=port, app=app, servant=servant) else: - server = http.Server(port=port, app=app) + server = http.Server(host=host, port=port, app=app) return server @@ -246,7 +248,7 @@ def cueDo(self, tymth=None, tock=0.0): while True: while self.cues: - cue = self.cues.pull() # self.cues.popleft() + cue = self.cues.pull() # self.cues.popleft() cueKin = cue["kin"] if cueKin == "stream": self.queries.append(cue) diff --git a/src/keri/app/oobiing.py b/src/keri/app/oobiing.py index 41a5e6c92..89f87a526 100644 --- a/src/keri/app/oobiing.py +++ b/src/keri/app/oobiing.py @@ -13,18 +13,18 @@ import falcon from hio.base import doing from hio.help import decking -from keri.core import coring +from keri.core import coring from . import httping -from .habbing import GroupHab from .. import help from .. import kering -from ..app import forwarding, connecting +from ..app import connecting from ..core import routing, eventing, parsing, scheming, serdering from ..db import basing from ..end import ending from ..end.ending import OOBI_RE, DOOBI_RE from ..help import helping +from ..kering import Ilks, ValidationError, UnverifiedReplyError, ConfigurationError from ..peer import exchanging logger = help.ogler.getLogger() @@ -276,7 +276,7 @@ class Oobiery: RetryDelay = 30 - def __init__(self, hby, clienter=None, cues=None): + def __init__(self, hby, rvy=None, clienter=None, cues=None): """ DoDoer to handle the request and parsing of OOBIs Parameters: @@ -286,8 +286,14 @@ def __init__(self, hby, clienter=None, cues=None): """ self.hby = hby + self.rvy = rvy + if self.rvy is not None: + self.registerReplyRoutes(self.rvy.rtr) + self.clienter = clienter or httping.Clienter() self.org = connecting.Organizer(hby=self.hby) + + # Set up a local parser for returned events from OOBI queries. rtr = routing.Router() rvy = routing.Revery(db=self.hby.db, rtr=rtr) kvy = eventing.Kevery(db=self.hby.db, lax=True, local=False, rvy=rvy) @@ -298,6 +304,89 @@ def __init__(self, hby, clienter=None, cues=None): self.clients = dict() self.doers = [self.clienter, doing.doify(self.scoobiDo)] + def registerReplyRoutes(self, router): + """ Register the routes for processing messages embedded in `rpy` event messages + + The Oobiery handles rpy messages with the /introduce route by processing the contained oobi + + Parameters: + router(Router): reply message router + + """ + router.addRoute("/introduce", self) + + def processReply(self, *, serder, saider, route, cigars=None, tsgs=None, **kwargs): + """ + Process one reply message for route = /introduce + with either attached nontrans receipt couples in cigars or attached trans + indexed sig groups in tsgs. + Assumes already validated saider, dater, and route from serder.ked + + Parameters: + serder (SerderKERI): instance of reply msg (SAD) + saider (Saider): instance from said in serder (SAD) + route (str): reply route + cigars (list): of Cigar instances that contain nontrans signing couple + signature in .raw and public key in .verfer + tsgs (list): tuples (quadruples) of form + (prefixer, seqner, diger, [sigers]) where: + prefixer is pre of trans endorser + seqner is sequence number of trans endorser's est evt for keys for sigs + diger is digest of trans endorser's est evt for keys for sigs + [sigers] is list of indexed sigs from trans endorser's keys from est evt + + OobiRecord: + date: str = date time of reply message of the introduction + + Reply Message: + { + "v" : "KERI10JSON00011c_", + "t" : "rpy", + "d": "EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM", + "dt": "2020-08-22T17:50:12.988921+00:00", + "r" : "/introduce", + "a" : + { + "cid": "ENcOes8_t2C7tck4X4j61fSm0sWkLbZrEZffq7mSn8On", + "oobi": "http://localhost:5632/oobi/ENcOes8_t2C7tck4X4j61fSm0sWkLbZrEZffq7mSn8On/witness", + } + } + + """ + if route != "/introduce": + raise ValidationError(f"Usupported route={route} in {Ilks.rpy} " + f"msg={serder.ked}.") + + data = serder.ked['a'] + dt = serder.ked["dt"] + + for k in ("cid", "oobi"): + if k not in data: + raise ValidationError(f"Missing element={k} from attributes in" + f" {Ilks.rpy} msg={serder.ked}.") + + cider = coring.Prefixer(qb64=data["cid"]) # raises error if unsupported code + cid = cider.qb64 # controller authorizing eid at role + aid = cid # authorizing attribution id + + oobi = data["oobi"] + url = urlparse(oobi) + if url.scheme not in ("http", "https"): + raise ValidationError(f"Invalid url scheme for introduced OOBI scheme={url.scheme}") + + if self.rvy is None: + raise ConfigurationError("this oobiery is not configured to handle rpy introductions") + + # Process BADA RUN but with no previous reply message, always process introductions + accepted = self.rvy.acceptReply(serder=serder, saider=saider, route=route, + aid=aid, osaider=None, cigars=cigars, + tsgs=tsgs) + if not accepted: + raise UnverifiedReplyError(f"Unverified introduciton reply. {serder.ked}") + + obr = basing.OobiRecord(cid=cid, date=dt) + self.hby.db.oobis.put(keys=(oobi,), val=obr) + def scoobiDo(self, tymth=None, tock=0.0): """ Returns doifiable Doist compatibile generator method (doer dog) to process diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index ea1b52de1..7f8f9af76 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -4,9 +4,6 @@ keri.core Package """ -#__all__ = ["coring", "eventing", "parsing", "scheming"] - - # Constants etc from .coring import (Tiers, ) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index f7c9da324..a1ff8553f 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1162,15 +1162,6 @@ def reply(route="", serder._verify() # raises error if fails verifications return serder - #_, sad = coring.Saider.saidify(sad=sad, kind=kind, label=label) - - #saider = coring.Saider(qb64=sad[label]) - #if not saider.verify(sad=sad, kind=kind, label=label, prefixed=True): - #raise ValidationError("Invalid said = {} for reply msg={}." - #"".format(saider.qb64, sad)) - - #return Serder(ked=sad) # return serialized Self-Addressed Data (SAD) - def prod(route="", replyRoute="", @@ -4115,7 +4106,6 @@ def removeStaleReplyLocScheme(self, saider): """ pass - def registerReplyRoutes(self, router): """ Register the routes for processing messages embedded in `rpy` event messages @@ -4127,9 +4117,7 @@ def registerReplyRoutes(self, router): router.addRoute("/loc/scheme", self, suffix="LocScheme") router.addRoute("/ksn/{aid}", self, suffix="KeyStateNotice") - - def processReplyEndRole(self, *, serder, saider, route, - cigars=None, tsgs=None, **kwargs): + def processReplyEndRole(self, *, serder, saider, route, cigars=None, tsgs=None, **kwargs): """ Process one reply message for route = /end/role/add or /end/role/cut with either attached nontrans receipt couples in cigars or attached trans @@ -4532,6 +4520,11 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): # do signature validation and replay attack prevention logic here # src, dt, route + if source is None and cigars: + dest = cigars[0].verfer.qb64 + else: + dest = source.qb64 + if route == "logs": pre = qry["i"] src = qry["src"] @@ -4563,7 +4556,7 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): msgs.append(msg) if msgs: - self.cues.push(dict(kin="replay", src=src, msgs=msgs, dest=source.qb64)) + self.cues.push(dict(kin="replay", pre=pre, src=src, msgs=msgs, dest=dest)) elif route == "ksn": pre = qry["i"] @@ -4585,7 +4578,7 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): rserder = reply(route=f"/ksn/{src}", data=kever.state()._asdict()) self.cues.push(dict(kin="reply", src=src, route="/ksn", serder=rserder, - dest=source.qb64)) + dest=dest)) elif route == "mbx": pre = qry["i"] diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index e1275075c..e5f7eb1a0 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -1038,8 +1038,8 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, rvy.processReply(serder, tsgs=tsgs) # trans except AttributeError as e: - raise kering.ValidationError("No kevery to process so dropped msg" - "= {}.".format(serder.pretty())) + raise kering.ValidationError("No revery to process so dropped msg" + "= {}.".format(serder.pretty())) from e elif ilk in (Ilks.qry,): # query message args = dict(serder=serder) @@ -1059,9 +1059,9 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, if route in ["logs", "ksn", "mbx"]: try: kvy.processQuery(**args) - except AttributeError: + except AttributeError as e: raise kering.ValidationError("No kevery to process so dropped msg" - "= {}.".format(serder.pretty())) + "= {} from e = {}".format(serder.pretty(), e)) elif route in ["tels", "tsn"]: try: diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 8a28698cf..db24fbd53 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1500,6 +1500,9 @@ def cloneEvtMsg(self, pre, fn, dig): atc.extend(coring.Counter(code=coring.CtrDex.SealSourceCouples, count=1).qb64b) atc.extend(couple) + elif self.kevers[pre].delegated: + if coring.SerderKERI(raw=raw).estive: + raise kering.MissingEntryError("Missing delegator anchor seal for dig={}.".format(dig)) # add trans endorsement quadruples to attachments not controller # may have been originally key event attachments or receipted endorsements diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index e1818b993..19b8432f5 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -115,8 +115,6 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow attempt failed: %s", ex.args[0]) except Exception as ex: # other error so remove from reply escrow self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow diff --git a/src/keri/end/ending.py b/src/keri/end/ending.py index 624a615d7..6ff62ef65 100644 --- a/src/keri/end/ending.py +++ b/src/keri/end/ending.py @@ -577,6 +577,10 @@ def on_get(self, req, rep, aid=None, role=None, eid=None): rep.status = falcon.HTTP_NOT_FOUND return + if kever.delegated and kever.delpre not in self.hby.kevers: + rep.status = falcon.HTTP_NOT_FOUND + return + owits = oset(kever.wits) if kever.prefixer.qb64 in self.hby.prefixes: # One of our identifiers hab = self.hby.habs[kever.prefixer.qb64] diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index 018f1a55e..07377d761 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -158,20 +158,21 @@ def __init__(self, certify, keypath, certpath, cafilepath, port): class MockHttpServer: - def __init__(self, port, app, servant=None): + def __init__(self, host, port, app, servant=None): self.servant = servant def test_createHttpServer(monkeypatch): + host = "0.0.0.0" port = 5632 app = falcon.App() - server = indirecting.createHttpServer(port, app) + server = indirecting.createHttpServer(host, port, app) assert isinstance(server, http.Server) monkeypatch.setattr(hio.core.tcp, 'ServerTls', MockServerTls) monkeypatch.setattr(hio.core.http, 'Server', MockHttpServer) - server = indirecting.createHttpServer(port, app, keypath='keypath', certpath='certpath', cafilepath='cafilepath') + server = indirecting.createHttpServer(host, port, app, keypath='keypath', certpath='certpath', cafilepath='cafilepath') assert isinstance(server, MockHttpServer) assert isinstance(server.servant, MockServerTls) diff --git a/tests/app/test_oobiing.py b/tests/app/test_oobiing.py index 348393a5d..2e81a3050 100644 --- a/tests/app/test_oobiing.py +++ b/tests/app/test_oobiing.py @@ -3,24 +3,20 @@ tests.app.test_multisig module """ -import json import falcon -from falcon import testing from hio.base import doing +from hio.core import http import keri -from hio.core import http +from keri import help, kering, core from keri.app import habbing, oobiing, notifying -from keri.core import coring, parsing, serdering +from keri.core import serdering, eventing, parsing, routing from keri.db import basing from keri.end import ending from keri.help import helping -from keri import help, kering from keri.peer import exchanging -from tests.app import openMultiSig - def test_oobi_share(mockHelpingNowUTC): oobi = "http://127.0.0.1:5642/oobi/Egw3N07Ajdkjvv4LB2Mhx2qxl6TOCFdWNJU6cYR_ImFg/witness" \ @@ -133,6 +129,68 @@ def test_oobiery(): """Done Test""" +def test_introduce(mockHelpingNowUTC): + raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' + salt = core.Salter(raw=raw).qb64 + assert salt == '0AAFqo8tU5rp-lWcApybCEh1' + # makHab uses stem=name to make different names have differnt AID pre + with (habbing.openHby(name="wat", base="test", salt=salt) as watHby, + habbing.openHby(name="wit", base="test", salt=salt) as witHby): + # setup Wes's habitat nontrans + watHab = watHby.makeHab(name='wes', isith="1", icount=1, transferable=False) + assert not watHab.kever.prefixer.transferable + assert watHab.pre == "BBVDlgWic_rAf-m_v7vz_VvIYAUPErvZgLTfXGNrFRom" + watKvy = eventing.Kevery(db=watHab.db, lax=False, local=False) + watPsr = parsing.Parser(kvy=watKvy) + + # setup Wok's habitat nontrans + witHab = witHby.makeHab(name='wok', isith="1", icount=1, transferable=False) + assert not witHab.kever.prefixer.transferable + assert witHab.pre == "BKVb58uITf48YoMPz8SBOTVwLgTO9BY4oEXRPoYIOErX" + witKvy = eventing.Kevery(db=witHab.db, lax=False, local=False) + + rtr = routing.Router() + rvy = routing.Revery(db=witHby.db, rtr=rtr) + oobiing.Oobiery(hby=witHby, rvy=rvy) + witPsr = parsing.Parser(kvy=witKvy, rvy=rvy) + assert witHby.db.oobis.cntAll() == 0 + + oobi = f"https://localhost:8989/oobi/{watHab.pre}/controller" + data = dict( + cid=watHab.pre, + oobi=oobi + ) + + msg = watHab.reply(route="/introduce", data=data) + assert msg == (b'{"v":"KERI10JSON000127_","t":"rpy","d":"EPEU3V7e2d2mhMWVFDS-oC9z' + b'Q8DX8t6ELkhINIaYGFNZ","dt":"2021-01-01T00:00:00.000000+00:00","r' + b'":"/introduce","a":{"cid":"BBVDlgWic_rAf-m_v7vz_VvIYAUPErvZgLTfX' + b'GNrFRom","oobi":"https://localhost:8989/oobi/BBVDlgWic_rAf-m_v7v' + b'z_VvIYAUPErvZgLTfXGNrFRom/controller"}}-VAi-CABBBVDlgWic_rAf-m_v' + b'7vz_VvIYAUPErvZgLTfXGNrFRom0BBqF8yHDeXpzDUNIOsDBGezNBdgHafmOYbQ7' + b'qw0R5t89FbLA26RwaA3NF9-dU0JpbNuJs7jiEBYbSGeDkbBDDEM') + + witPsr.parseOne(ims=msg) + assert witHby.db.oobis.cntAll() == 1 + obr = witHby.db.oobis.get(keys=(oobi,)) + assert obr.cid == watHab.pre + + # Send one missing fields + data = dict(cid=watHab.pre) + msg = watHab.reply(route="/introduce", data=data) + witPsr.parseOne(ims=msg) + assert witHby.db.oobis.cntAll() == 1 # Still one because of the missing 'oobi' field + + # Send one bad scheme + data = dict(cid=watHab.pre, oobi="ftp://localhost") + msg = watHab.reply(route="/introduce", data=data) + witPsr.parseOne(ims=msg) + assert witHby.db.oobis.cntAll() == 1 # Still one because of the missing 'oobi' field + + + + + class MOOBIEnd: """ Test endpoint returning a static MOOBI """ def __init__(self, hab, url): diff --git a/tests/end/test_ending.py b/tests/end/test_ending.py index 60b3d0811..b27a9378b 100644 --- a/tests/end/test_ending.py +++ b/tests/end/test_ending.py @@ -438,6 +438,49 @@ def test_get_oobi(): assert serder.ked['t'] == coring.Ilks.icp assert serder.ked['i'] == "EOaICQwhOy3wMwecjAuHQTbv_Cmuu1azTMnHi4QtUmEU" + delname = "delegator" + with habbing.openHby(name=name, base=base, salt=salt) as hby, \ + habbing.openHby(name=delname, base=base, salt=salt) as delhby: + delhab = delhby.makeHab(name=delname) + hab = hby.makeHab(name=name, delpre=delhab.pre) + + assert hab.pre == "EPERMS4wKU7ejhCdhI2qQR8snEx1cislR9C9bSEs0kS5" + assert hab.kever.delpre == delhab.pre + + msgs.extend(hab.makeEndRole(eid=hab.pre, + role=kering.Roles.controller, + stamp=help.nowIso8601())) + + msgs.extend(hab.makeLocScheme(url='http://127.0.0.1:5555', + scheme=kering.Schemes.http, + stamp=help.nowIso8601())) + hab.psr.parse(ims=msgs) + + # must do it here to inject into Falcon endpoint resource instances + tymist = tyming.Tymist(tyme=0.0) + + app = falcon.App() # falcon.App instances are callable WSGI apps + ending.loadEnds(app, tymth=tymist.tymen(), hby=hby, default=hab.pre) + + client = testing.TestClient(app=app) + + # This should fail with 404 because we haven't been approved yet so we don't exist + rep = client.simulate_get('/oobi', ) + assert rep.status == falcon.HTTP_NOT_FOUND + + # Approve the delegation manually + delhab.interact(data=[dict(i=hab.pre, s="0", d=hab.pre)]) + for msg in delhab.db.clonePreIter(pre=delhab.pre, fn=0): + hab.psr.parse(ims=msg) + + rep = client.simulate_get('/oobi', ) + assert rep.status == falcon.HTTP_OK + + # We'll get the delegator first + serder = serdering.SerderKERI(raw=rep.text.encode("utf-8")) + assert serder.ked['t'] == coring.Ilks.icp + assert serder.ked['i'] == "EKL3to0Q059vtxKi7wWmaNFJ3NKE1nQsOPasRXqPzpjS" + """Done Test""" From 057de1182d087b8cdac44ba2b1ab17505e56ab5d Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Wed, 1 May 2024 19:07:27 -0700 Subject: [PATCH 163/418] Version Bump --- Makefile | 4 ++-- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index a99d41d9b..0ea899105 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev1 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev1-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev2 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev2-arm64 . .PHONY: build-witness-demo build-witness-demo: diff --git a/setup.py b/setup.py index 9a0cda9a5..7fc406286 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev1', # also change in src/keri/__init__.py + version='1.2.0-dev2', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index ea009cc64..5b3d355f5 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev1' # also change in setup.py +__version__ = '1.2.0-dev2' # also change in setup.py From bb201e022df54b2235ccff94d08d193545b8eefa Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Thu, 2 May 2024 20:26:19 -0700 Subject: [PATCH 164/418] Fix multisig rotation (#774) --- .../demo/basic/multisig-rotation-in-third.sh | 9 +++-- scripts/demo/test_scripts.sh | 8 ++--- src/keri/app/cli/commands/multisig/join.py | 6 ++-- src/keri/app/grouping.py | 6 +--- src/keri/core/eventing.py | 33 ++++++++----------- 5 files changed, 28 insertions(+), 34 deletions(-) diff --git a/scripts/demo/basic/multisig-rotation-in-third.sh b/scripts/demo/basic/multisig-rotation-in-third.sh index 55d384a43..79f3b5382 100755 --- a/scripts/demo/basic/multisig-rotation-in-third.sh +++ b/scripts/demo/basic/multisig-rotation-in-third.sh @@ -17,8 +17,6 @@ kli oobi resolve --name multisig1 --oobi-alias multisig2 --oobi http://127.0.0.1 kli oobi resolve --name multisig1 --oobi-alias multisig3 --oobi http://127.0.0.1:5642/oobi/ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha kli oobi resolve --name multisig2 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha kli oobi resolve --name multisig2 --oobi-alias multisig3 --oobi http://127.0.0.1:5642/oobi/ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha -kli oobi resolve --name multisig3 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha -kli oobi resolve --name multisig3 --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha # Follow commands run in parallel kli multisig incept --name multisig1 --alias multisig1 --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-sample.json & @@ -34,6 +32,13 @@ kli status --name multisig1 --alias multisig PID_LIST="" +kli rotate --name multisig1 --alias multisig1 +kli query --name multisig2 --alias multisig2 --prefix EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 +kli rotate --name multisig2 --alias multisig2 +kli query --name multisig1 --alias multisig1 --prefix EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 +kli oobi resolve --name multisig3 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name multisig3 --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha + kli multisig rotate --name multisig1 --alias multisig --smids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --smids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --smids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --rmids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --rmids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U & pid=$! PID_LIST+=" $pid" diff --git a/scripts/demo/test_scripts.sh b/scripts/demo/test_scripts.sh index 8d7a3aad0..a3f186db3 100755 --- a/scripts/demo/test_scripts.sh +++ b/scripts/demo/test_scripts.sh @@ -41,7 +41,7 @@ printf "\n************************************\n" isSuccess printf "\n************************************\n" -printf "Running delegate.sh" +printf "Skipping delegate.sh" printf "\n************************************\n" #"${script_dir}/basic/delegate.sh" #isSuccess @@ -65,7 +65,7 @@ printf "\n************************************\n" isSuccess printf "\n************************************\n" -printf "Skipping multisig-join.sh" +printf "Running multisig-join.sh" printf "\n************************************\n" -#"${script_dir}/basic/multisig-join.sh" -#isSuccess +"${script_dir}/basic/multisig-join.sh" +isSuccess diff --git a/src/keri/app/cli/commands/multisig/join.py b/src/keri/app/cli/commands/multisig/join.py index b1e9d9f60..d7ac76270 100644 --- a/src/keri/app/cli/commands/multisig/join.py +++ b/src/keri/app/cli/commands/multisig/join.py @@ -177,8 +177,8 @@ def incept(self, attrs): inits["isith"] = oicp.ked["kt"] inits["nsith"] = oicp.ked["nt"] - inits["estOnly"] = eventing.TraitCodex.EstOnly in oicp.ked["c"] - inits["DnD"] = eventing.TraitCodex.DoNotDelegate in oicp.ked["c"] + inits["estOnly"] = kering.TraitCodex.EstOnly in oicp.ked["c"] + inits["DnD"] = kering.TraitCodex.DoNotDelegate in oicp.ked["c"] inits["toad"] = oicp.ked["bt"] inits["wits"] = oicp.ked["b"] @@ -405,7 +405,7 @@ def rotate(self, attrs): ghab = self.hby.joinGroupHab(pre, group=alias, mhab=mhab, smids=smids, rmids=rmids) try: - ghab.rotate(serder=orot) + ghab.rotate(serder=orot, smids=smids, rmids=rmids) except ValueError: return False diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index bbefab2da..46d85a3de 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -47,11 +47,7 @@ def start(self, ghab, prefixer, seqner, saider): saider (Saider): saider of event of group identifier """ - evt = ghab.makeOwnEvent(sn=seqner.sn, allowPartiallySigned=True) - serder = serdering.SerderKERI(raw=evt) - del evt[:serder.size] - - logger.info(f"Waiting for other signatures for {serder.pre}:{seqner.sn}...") + print(f"Waiting for other signatures for {prefixer.qb64}:{seqner.sn}...") return self.hby.db.gpse.add(keys=(prefixer.qb64,), val=(seqner, saider)) def complete(self, prefixer, seqner, saider=None): diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index a1ff8553f..5285e5b3d 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1709,7 +1709,6 @@ def locallyMembered(self, pre: str | None = None): pre = pre if pre is not None else self.prefixer.qb64 return pre in self.groups # groups - def locallyContributedIndices(self, verfers: list[Verfer]): """Returns list of indices of public keys contributed by local members to the KEL with current signing keys represented by verfers @@ -1723,19 +1722,11 @@ def locallyContributedIndices(self, verfers: list[Verfer]): indices list[int]: list of indices of keys contributed by local members """ - indices = [] - - for i, verfer in enumerate(verfers): - if (couples := self.pubs.get(keys=(verfer.qb64,))) is None: - continue - - for (prefixer, seqner) in couples: - if self.locallyOwned(prefixer.qb64): # only member not group aid - indices.append(i) - break # only need one local member to exclude signature - - return indices + habord = self.db.habs.get(keys=(self.prefixer.qb64,)) + kever = self.kevers[habord.mid] + idx = [verfer.qb64 for verfer in verfers].index(kever.verfers[0].qb64) + return [idx] def reload(self, state): """ @@ -2227,13 +2218,15 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # compromised signature remotely to satisfy threshold. if not local and self.locallyMembered(): # is this Kever's pre a local group - if (indices := self.locallyContributedIndices(verfers)): + if indices := self.locallyContributedIndices(verfers): for siger in list(sigers): # copy so clean del on original elements if siger.index in indices: - del sigers[siger.index] - self.cues.push(dict(kin="remoteMemberedSig", - serder=serder, - index=siger.index)) + sigers.remove(siger) + if self.cues: + self.cues.push(dict(kin="remoteMemberedSig", + serder=serder, + index=siger.index)) + # get unique verified sigers and indices lists from sigers list sigers, indices = verifySigs(raw=serder.raw, sigers=sigers, verfers=verfers) @@ -2255,7 +2248,6 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f"or locally witnessed event" f" = {serder.ked}.") - werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers # get unique verified wigers and windices lists from wigers list wigers, windices = verifySigs(raw=serder.raw, sigers=wigers, verfers=werfers) @@ -2270,6 +2262,7 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f" on sigs for {[siger.qb64 for siger in sigers]}" f" for evt = {serder.ked}.") + # escrow if not fully signed vs prior next rotation threshold if serder.ilk in (Ilks.rot, Ilks.drt): # rotation so check prior next threshold # prior next threshold in .ntholder and digers in .ndigers @@ -2306,7 +2299,7 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # short circuit witness validation when either locallyOwned or locallyWitnessed # otherwise must validate fully witnessed - if not (self.locallyOwned() or self.locallyWitnessed(wits=wits)): + if not (self.locallyOwned() or self.locallyMembered() or self.locallyWitnessed(wits=wits)): if wits: # is witnessed if toader.num < 1 or toader.num > len(wits): # out of bounds toad raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") From e691cf01c2bdb3ce9512b157e3f525bce2c573eb Mon Sep 17 00:00:00 2001 From: Kent Bull <65027257+kentbull@users.noreply.github.com> Date: Fri, 3 May 2024 13:59:39 -0600 Subject: [PATCH 165/418] test: three stooges multisig rotate and interact (#776) * test: three stooges multisig rotate and interact * fix: important details --- .../basic/multisig-rotate-three-stooges.sh | 125 ++++++++++++++++++ scripts/demo/basic/script-utils.sh | 22 +++ scripts/demo/data/multisig-three-aids.json | 16 +++ 3 files changed, 163 insertions(+) create mode 100755 scripts/demo/basic/multisig-rotate-three-stooges.sh create mode 100644 scripts/demo/basic/script-utils.sh create mode 100644 scripts/demo/data/multisig-three-aids.json diff --git a/scripts/demo/basic/multisig-rotate-three-stooges.sh b/scripts/demo/basic/multisig-rotate-three-stooges.sh new file mode 100755 index 000000000..0c92af823 --- /dev/null +++ b/scripts/demo/basic/multisig-rotate-three-stooges.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# three stooges + +source ${KERI_SCRIPT_DIR}/demo/basic/script-utils.sh + +# WITNESSES +# To run the following scripts, open another console window and run: +# $ kli witness demo + +print_yellow "Multisig rotation with three AIDs" +echo + +kli init --name larry --salt 0ACDEyMzQ1Njc4OWxtbm9aBc --nopasscode --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +kli incept --name larry --alias larry --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-1-sample.json + +kli init --name moe --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +kli incept --name moe --alias moe --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-2-sample.json + +kli init --name curly --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +kli incept --name curly --alias curly --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-3-sample.json + +# OOBI resolution does the initial discovery of key state +echo +print_yellow "Resolve OOBIs" +kli oobi resolve --name larry --oobi-alias moe --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name larry --oobi-alias curly --oobi http://127.0.0.1:5642/oobi/ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha + +kli oobi resolve --name moe --oobi-alias larry --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name moe --oobi-alias curly --oobi http://127.0.0.1:5642/oobi/ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha + +kli oobi resolve --name curly --oobi-alias larry --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name curly --oobi-alias moe --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha + +# Multisig Inception +echo +print_yellow "Multisig Inception" +# Follow commands run in parallel +kli multisig incept --name larry --alias larry --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-three-aids.json & +pid=$! +PID_LIST+=" $pid" +kli multisig incept --name moe --alias moe --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-three-aids.json & +pid=$! +PID_LIST+=" $pid" +kli multisig incept --name curly --alias curly --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-three-aids.json & +pid=$! +PID_LIST+=" $pid" + +echo +print_yellow "Multisig Inception - wait" +wait $PID_LIST + +echo +print_green "Multisig Inception - status" +kli status --name larry --alias multisig + +# Rotate keys for each multisig - required before rotating the multisig +echo +print_yellow "Rotate keys for each multisig" +kli rotate --name larry --alias larry +kli rotate --name moe --alias moe +kli rotate --name curly --alias curly + +# Pull key state in from other multisig group participant identifiers so they have the next digest +echo +print_yellow "Pull key state in from other multisig group participant identifiers" +# 2 about 1 +kli query --name moe --alias moe --prefix EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 +# 2 about 3 +kli query --name moe --alias moe --prefix ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U +# 1 about 2 +kli query --name larry --alias larry --prefix EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 +# 1 about 3 +kli query --name larry --alias larry --prefix ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U +# 3 about 1 +kli query --name curly --alias curly --prefix EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 +# 3 about 2 +kli query --name curly --alias curly --prefix EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 + + +echo +print_yellow "Multisig rotation" + +PID_LIST="" + +kli multisig rotate --name larry --alias multisig --smids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --smids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --smids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --rmids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --rmids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U & +pid=$! +PID_LIST+=" $pid" +kli multisig rotate --name moe --alias multisig --smids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --smids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --smids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --rmids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --rmids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U & +pid=$! +PID_LIST+=" $pid" +kli multisig rotate --name curly --alias multisig --smids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --smids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --smids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --rmids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --rmids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U & +pid=$! +PID_LIST+=" $pid" + +echo +print_yellow "Multisig rotation - wait" +wait $PID_LIST + +echo +print_green "Multisig rotation - status" +kli status --name larry --alias multisig + +echo +print_yellow "Multisig interact" + +PID_LIST="" + +kli multisig interact --name larry --alias multisig --data "{'tagline': 'three lost souls'}" & +pid=$! +PID_LIST+=" $pid" +kli multisig interact --name moe --alias multisig --data "{'tagline': 'three lost souls'}" & +pid=$! +PID_LIST+=" $pid" +kli multisig interact --name curly --alias multisig --data "{'tagline': 'three lost souls'}" & +pid=$! +PID_LIST+=" $pid" + +echo +print_yellow "Multisig interact - wait" +wait $PID_LIST + +echo +print_green "Multisig interact - status" +kli status --name larry --alias multisig +print_lcyan "Multisig rotate three stooges - done." diff --git a/scripts/demo/basic/script-utils.sh b/scripts/demo/basic/script-utils.sh new file mode 100644 index 000000000..f0585afdc --- /dev/null +++ b/scripts/demo/basic/script-utils.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Utility functions +print_green() { + text=$1 + printf "\e[32m${text}\e[0m\n" +} + +print_yellow(){ + text=$1 + printf "\e[33m${text}\e[0m\n" +} + +print_red() { + text=$1 + printf "\e[31m${text}\e[0m\n" +} + +print_lcyan() { + text=$1 + printf "\e[96m${text}\e[0m\n" +} \ No newline at end of file diff --git a/scripts/demo/data/multisig-three-aids.json b/scripts/demo/data/multisig-three-aids.json new file mode 100644 index 000000000..feb30e801 --- /dev/null +++ b/scripts/demo/data/multisig-three-aids.json @@ -0,0 +1,16 @@ +{ + "aids": [ + "EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1", + "EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4", + "ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U" + ], + "transferable": true, + "wits": [ + "BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha", + "BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM", + "BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX" + ], + "toad": 3, + "isith": "3", + "nsith": "3" +} From 3f624aa553e1e113fe507f3d3ac959825bf7c7e7 Mon Sep 17 00:00:00 2001 From: Kent Bull <65027257+kentbull@users.noreply.github.com> Date: Fri, 3 May 2024 14:27:48 -0600 Subject: [PATCH 166/418] fix: stooge AID Prefixes and run in test_scripts.sh (#777) --- .../basic/multisig-rotate-three-stooges.sh | 39 ++++++++++--------- scripts/demo/data/multisig-three-aids.json | 6 +-- scripts/demo/test_scripts.sh | 7 ++++ 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/scripts/demo/basic/multisig-rotate-three-stooges.sh b/scripts/demo/basic/multisig-rotate-three-stooges.sh index 0c92af823..5db39bfa6 100755 --- a/scripts/demo/basic/multisig-rotate-three-stooges.sh +++ b/scripts/demo/basic/multisig-rotate-three-stooges.sh @@ -11,25 +11,28 @@ print_yellow "Multisig rotation with three AIDs" echo kli init --name larry --salt 0ACDEyMzQ1Njc4OWxtbm9aBc --nopasscode --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +# Prefix EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U kli incept --name larry --alias larry --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-1-sample.json kli init --name moe --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +# Prefix ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW kli incept --name moe --alias moe --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-2-sample.json kli init --name curly --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +# Prefix EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu kli incept --name curly --alias curly --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-3-sample.json # OOBI resolution does the initial discovery of key state echo print_yellow "Resolve OOBIs" -kli oobi resolve --name larry --oobi-alias moe --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha -kli oobi resolve --name larry --oobi-alias curly --oobi http://127.0.0.1:5642/oobi/ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name larry --oobi-alias moe --oobi http://127.0.0.1:5642/oobi/ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name larry --oobi-alias curly --oobi http://127.0.0.1:5642/oobi/EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha -kli oobi resolve --name moe --oobi-alias larry --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha -kli oobi resolve --name moe --oobi-alias curly --oobi http://127.0.0.1:5642/oobi/ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name moe --oobi-alias larry --oobi http://127.0.0.1:5642/oobi/EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name moe --oobi-alias curly --oobi http://127.0.0.1:5642/oobi/EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha -kli oobi resolve --name curly --oobi-alias larry --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha -kli oobi resolve --name curly --oobi-alias moe --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name curly --oobi-alias larry --oobi http://127.0.0.1:5642/oobi/EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name curly --oobi-alias moe --oobi http://127.0.0.1:5642/oobi/ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha # Multisig Inception echo @@ -64,17 +67,17 @@ kli rotate --name curly --alias curly echo print_yellow "Pull key state in from other multisig group participant identifiers" # 2 about 1 -kli query --name moe --alias moe --prefix EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 +kli query --name moe --alias moe --prefix EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U # 2 about 3 -kli query --name moe --alias moe --prefix ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U +kli query --name moe --alias moe --prefix EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu # 1 about 2 -kli query --name larry --alias larry --prefix EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 +kli query --name larry --alias larry --prefix ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW # 1 about 3 -kli query --name larry --alias larry --prefix ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U +kli query --name larry --alias larry --prefix EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu # 3 about 1 -kli query --name curly --alias curly --prefix EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 +kli query --name curly --alias curly --prefix EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U # 3 about 2 -kli query --name curly --alias curly --prefix EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 +kli query --name curly --alias curly --prefix ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW echo @@ -82,13 +85,13 @@ print_yellow "Multisig rotation" PID_LIST="" -kli multisig rotate --name larry --alias multisig --smids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --smids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --smids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --rmids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --rmids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U & +kli multisig rotate --name larry --alias multisig --smids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --smids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --smids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --rmids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --rmids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu & pid=$! PID_LIST+=" $pid" -kli multisig rotate --name moe --alias multisig --smids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --smids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --smids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --rmids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --rmids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U & +kli multisig rotate --name moe --alias multisig --smids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --smids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --smids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --rmids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --rmids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu & pid=$! PID_LIST+=" $pid" -kli multisig rotate --name curly --alias multisig --smids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --smids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --smids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --rmids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --rmids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U & +kli multisig rotate --name curly --alias multisig --smids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --smids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --smids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --rmids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --rmids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu & pid=$! PID_LIST+=" $pid" @@ -105,13 +108,13 @@ print_yellow "Multisig interact" PID_LIST="" -kli multisig interact --name larry --alias multisig --data "{'tagline': 'three lost souls'}" & +kli multisig interact --name larry --alias multisig --data "{\"tagline\":\"three lost souls\"}" & pid=$! PID_LIST+=" $pid" -kli multisig interact --name moe --alias multisig --data "{'tagline': 'three lost souls'}" & +kli multisig interact --name moe --alias multisig --data "{\"tagline\":\"three lost souls\"}" & pid=$! PID_LIST+=" $pid" -kli multisig interact --name curly --alias multisig --data "{'tagline': 'three lost souls'}" & +kli multisig interact --name curly --alias multisig --data "{\"tagline\":\"three lost souls\"}" & pid=$! PID_LIST+=" $pid" diff --git a/scripts/demo/data/multisig-three-aids.json b/scripts/demo/data/multisig-three-aids.json index feb30e801..c17b8b2a7 100644 --- a/scripts/demo/data/multisig-three-aids.json +++ b/scripts/demo/data/multisig-three-aids.json @@ -1,8 +1,8 @@ { "aids": [ - "EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1", - "EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4", - "ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U" + "EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U", + "ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW", + "EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu" ], "transferable": true, "wits": [ diff --git a/scripts/demo/test_scripts.sh b/scripts/demo/test_scripts.sh index a3f186db3..8fe0a4c29 100755 --- a/scripts/demo/test_scripts.sh +++ b/scripts/demo/test_scripts.sh @@ -4,6 +4,7 @@ script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${script_dir}"/demo-scripts.sh +source "${script_dir}"/basic/script-utils.sh # Launch witnesses in background kli witness demo & @@ -52,6 +53,12 @@ printf "\n************************************\n" "${script_dir}/basic/multisig.sh" isSuccess +printf "\n************************************\n" +printf "Running multisig-rotate-three-stooges.sh" +printf "\n************************************\n" +"${script_dir}/basic/multisig-rotate-three-stooges.sh" +isSuccess + printf "\n************************************\n" printf "Skipping multisig-delegate-delegator.sh" printf "\n************************************\n" From 462238817fdaa3d4651b1ef1a45544a8994f50a2 Mon Sep 17 00:00:00 2001 From: Rubel Hassan Date: Tue, 7 May 2024 11:35:56 -0500 Subject: [PATCH 167/418] Remove generateSigners function and refactor with Salter().signers() (#778) Co-authored-by: Rubel Hassan Mollik --- src/keri/core/__init__.py | 3 +-- src/keri/core/signing.py | 38 ------------------------------------- tests/core/test_eventing.py | 8 ++++---- tests/core/test_signing.py | 17 +++++------------ 4 files changed, 10 insertions(+), 56 deletions(-) diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index 7f8f9af76..e71721fc2 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -14,5 +14,4 @@ from .coring import Tholder from .indexing import Indexer, Siger, IdrDex, IdxSigDex -from .signing import (Signer, Salter, Cipher, Encrypter, Decrypter, - generateSigners, ) +from .signing import (Signer, Salter, Cipher, Encrypter, Decrypter, ) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index f956a4d02..875ce179b 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -25,44 +25,6 @@ ECDSA_256k1_SEEDBYTES = 32 -# deprecated use Salter.signers instead -def generateSigners(raw=None, count=8, transferable=True): - """Returns list of Signers for Ed25519 - - Deprecated, use Salter.signers instead. - - Use this when simply need valid AIDs but not when need valid controller - contexts. In the latter case use openHby or openHab which create databases. - - Parameters: - raw (bytes): 16 byte long salt cryptomatter from which seeds - for Signers in list are derived - random salt created if not provided - count is number of signers in list - transferable is boolean true means signer.verfer code is transferable - non-transferable otherwise - """ - if not raw: - raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - - signers = [] - for i in range(count): - path = f"{i:x}" - # algorithm default is argon2id - seed = pysodium.crypto_pwhash(outlen=32, - passwd=path, - salt=raw, - opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, - memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, - alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) - - signers.append(Signer(raw=seed, transferable=transferable)) - - return signers - - - - class Signer(Matter): """ Signer is Matter subclass with method to create signature of serialization diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index d2a330850..942a20cac 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -2453,7 +2453,7 @@ def test_keyeventsequence_0(): """ # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = core.generateSigners(raw=salt, count=8, transferable=True) + signers = core.Salter(raw=salt).signers(count=8) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2723,7 +2723,7 @@ def test_keyeventsequence_1(): # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = core.generateSigners(raw=salt, count=8, transferable=True) + signers = core.Salter(raw=salt).signers(count=8) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2820,7 +2820,7 @@ def test_multisig_digprefix(): # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = core.generateSigners(raw=salt, count=8, transferable=True) + signers = core.Salter(raw=salt).signers(count=8) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2974,7 +2974,7 @@ def test_recovery(): """ # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = core.generateSigners(raw=salt, count=8, transferable=True) + signers = core.Salter(raw=salt).signers(count=8) with openDB(name="controller") as conlgr, openDB(name="validator") as vallgr: event_digs = [] # list of event digs in sequence to verify against database diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index 4465c58d0..f0e519db2 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -6,20 +6,14 @@ import pysodium - import pytest from keri import kering - -from keri.help import helping - -from keri import core -from keri.core import (Tiers, ) -from keri.core import (Matter, MtrDex, Cigar, Verfer, Prefixer) from keri.core import (Indexer, IdrDex, ) -from keri.core import (Signer, generateSigners, Salter, +from keri.core import (Matter, MtrDex, Cigar, Verfer, Prefixer) +from keri.core import (Signer, Salter, Cipher, Encrypter, Decrypter, ) - +from keri.core import (Tiers, ) def test_signer(): @@ -381,13 +375,12 @@ def test_signer(): """ Done Test """ -# deprecated uses Salter.signers() instead def test_generatesigners(): """ Test the support function genSigners """ - signers = generateSigners(count=2, transferable=False) + signers = Salter().signers(count=2, temp=True, transferable=False) assert len(signers) == 2 for signer in signers: assert signer.verfer.code == MtrDex.Ed25519N @@ -395,7 +388,7 @@ def test_generatesigners(): # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) # raw salt raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' assert len(raw) == 16 - signers = generateSigners(raw=raw, count=4) # default is transferable + signers = Salter(raw=raw).signers(count=4) assert len(signers) == 4 for signer in signers: assert signer.code == MtrDex.Ed25519_Seed From 1f12ff97c534ca18faf0010b61188b2c71329f03 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Thu, 9 May 2024 11:08:56 -0400 Subject: [PATCH 168/418] removes bran from GroupMultisigRotate constructor inplace of an existing Habery (#780) Signed-off-by: Kevin Griffin --- src/keri/app/cli/commands/multisig/rotate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/keri/app/cli/commands/multisig/rotate.py b/src/keri/app/cli/commands/multisig/rotate.py index fa73ea724..e79e1f6d3 100644 --- a/src/keri/app/cli/commands/multisig/rotate.py +++ b/src/keri/app/cli/commands/multisig/rotate.py @@ -53,9 +53,9 @@ def rotateGroupIdentifier(args): """ data = config.parseData(args.data) if args.data is not None else None - - rotDoer = GroupMultisigRotate(name=args.name, base=args.base, alias=args.alias, smids=args.smids, rmids=args.rmids, - bran=args.bran, wits=args.witnesses, cuts=args.cuts, adds=args.witness_add, + hby = existing.setupHby(name=args.name, base=args.base, bran=args.bran) + rotDoer = GroupMultisigRotate(hby=hby, alias=args.alias, smids=args.smids, rmids=args.rmids, + wits=args.witnesses, cuts=args.cuts, adds=args.witness_add, isith=args.isith, nsith=args.nsith, toad=args.toad, data=data) doers = [rotDoer] @@ -70,7 +70,7 @@ class GroupMultisigRotate(doing.DoDoer): """ - def __init__(self, name, base, bran, alias, smids=None, rmids=None, isith=None, nsith=None, + def __init__(self, hby, alias, smids=None, rmids=None, isith=None, nsith=None, toad=None, wits=None, cuts=None, adds=None, data: list = None): self.alias = alias @@ -80,12 +80,12 @@ def __init__(self, name, base, bran, alias, smids=None, rmids=None, isith=None, self.smids = smids self.rmids = rmids self.data = data + self.hby = hby self.wits = wits if wits is not None else [] self.cuts = cuts if cuts is not None else [] self.adds = adds if adds is not None else [] - self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer notifier = Notifier(self.hby) mux = grouping.Multiplexor(self.hby, notifier=notifier) From b3e801b23bc92d2c9aa977b01725ed6e38148025 Mon Sep 17 00:00:00 2001 From: Kent Bull <65027257+kentbull@users.noreply.github.com> Date: Tue, 14 May 2024 18:25:27 -0600 Subject: [PATCH 169/418] feat: multisig join --group arg (#783) * feat: multisig join --alias arg * fix: alias->group if auto True else default --- src/keri/app/cli/commands/multisig/join.py | 59 +++++++++++++--------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/src/keri/app/cli/commands/multisig/join.py b/src/keri/app/cli/commands/multisig/join.py index d7ac76270..e3c1e7850 100644 --- a/src/keri/app/cli/commands/multisig/join.py +++ b/src/keri/app/cli/commands/multisig/join.py @@ -21,47 +21,52 @@ logger = help.ogler.getLogger() parser = argparse.ArgumentParser(description='Join group multisig inception, rotation or interaction event.') -parser.set_defaults(handler=lambda args: confirm(args)) +parser.set_defaults(handler=lambda args: join(args)) parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") +parser.add_argument('--group', '-a', help='human-readable name for the multisig group identifier prefix', required=False, default=None) parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--auto", "-Y", help="auto approve any delegation request non-interactively", action="store_true") -def confirm(args): - """ Wait for and provide interactive confirmation of group multisig inception, rotation or interaction events +def join(args): + """ Wait for and provide interactive confirmation of group multisig inception, rotation or interaction events Parameters: - args(Namespace): parsed arguements namespace object + args(Namespace): parsed arguments namespace object """ name = args.name base = args.base bran = args.bran auto = args.auto + group = args.group - confirmDoer = ConfirmDoer(name=name, base=base, bran=bran, auto=auto) + joinDoer = JoinDoer(name=name, base=base, bran=bran, group=group, auto=auto) - doers = [confirmDoer] + doers = [joinDoer] return doers -class ConfirmDoer(doing.DoDoer): - """ Doist doer capable of polling for group multisig events and prompting user for action +class JoinDoer(doing.DoDoer): + """ Doist doer capable of polling for group multisig events and prompting user for action """ - def __init__(self, name, base, bran, auto=False): + def __init__(self, name, base, bran, group, auto=False): """ Create doer for polling for group multisig events and either approve automatically or prompt user Parameters: name (str): database environment name base (str): database directory prefix bran (str): passcode to unlock keystore - + group (str): human-readable name for the multisig identifier prefix + auto (bool): non-interactively auto approve any inception, rotation, interaction, or other event + while using the default group of "default-group" """ + self.group = group self.hby = existing.setupHby(name=name, base=base, bran=bran) self.rgy = credentialing.Regery(hby=self.hby, name=name, base=base) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer @@ -88,11 +93,11 @@ def __init__(self, name, base, bran, auto=False): doers = [self.hbyDoer, self.witq, self.mbx, self.counselor, self.registrar, self.credentialer, self.postman] self.toRemove = list(doers) - doers.extend([doing.doify(self.confirmDo)]) + doers.extend([doing.doify(self.joinDo)]) self.auto = auto - super(ConfirmDoer, self).__init__(doers=doers) + super(JoinDoer, self).__init__(doers=doers) - def confirmDo(self, tymth, tock=0.0): + def joinDo(self, tymth, tock=0.0): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -146,7 +151,7 @@ def confirmDo(self, tymth, tock=0.0): self.remove(self.toRemove) def incept(self, attrs): - """ Incept group multisig + """ Join a group multisig inception event """ said = attrs["d"] @@ -196,17 +201,20 @@ def incept(self, attrs): if approve: if self.auto: - alias = "test alias" + if self.group is None: + group = "default-group" + else: + group = self.group else: while True: - alias = input(f"\nEnter alias for new AID: ") - if self.hby.habByName(alias) is not None: - print(f"AID alias {alias} is already in use, please try again") + group = input(f"\nEnter group name for new AID: ") + if self.hby.habByName(group) is not None: + print(f"AID group name {group} is already in use, please try again") else: break try: - ghab = self.hby.makeGroupHab(group=alias, mhab=mhab, + ghab = self.hby.makeGroupHab(group=group, mhab=mhab, smids=smids, rmids=rmids, **inits) except ValueError as e: return False @@ -393,16 +401,19 @@ def rotate(self, attrs): ghab = self.hby.habs[pre] else: if self.auto: - alias = "test alias" + if self.group is None: + group = "default-group" + else: + group = self.group else: while True: - alias = input(f"\nEnter alias for new AID: ") - if self.hby.habByName(alias) is not None: - print(f"AID alias {alias} is already in use, please try again") + group = input(f"\nEnter group name for new AID: ") + if self.hby.habByName(group) is not None: + print(f"AID group name {group} is already in use, please try again") else: break - ghab = self.hby.joinGroupHab(pre, group=alias, mhab=mhab, smids=smids, rmids=rmids) + ghab = self.hby.joinGroupHab(pre, group=group, mhab=mhab, smids=smids, rmids=rmids) try: ghab.rotate(serder=orot, smids=smids, rmids=rmids) From 67fb1804f322cb44796a5291fd0031c26168e0d1 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Sat, 18 May 2024 13:58:48 -0700 Subject: [PATCH 170/418] Updates to delegation processing to fix delegation. Reversed the order of witness receipting and delegation approval request and updated command line tools to work correctly. Re-enabled the delegate.sh script in CI/CD. (#784) Signed-off-by: pfeairheller --- scripts/demo/basic/delegate.sh | 4 +- scripts/demo/test_scripts.sh | 6 +- src/keri/app/cli/commands/delegate/confirm.py | 16 ++--- src/keri/app/cli/commands/incept.py | 13 ++-- src/keri/app/cli/commands/rotate.py | 4 +- src/keri/app/delegating.py | 67 ++++++++++--------- src/keri/core/eventing.py | 22 ------ src/keri/core/parsing.py | 4 +- src/keri/db/basing.py | 5 +- 9 files changed, 56 insertions(+), 85 deletions(-) diff --git a/scripts/demo/basic/delegate.sh b/scripts/demo/basic/delegate.sh index cefc2fa3e..4a1e40dfb 100755 --- a/scripts/demo/basic/delegate.sh +++ b/scripts/demo/basic/delegate.sh @@ -9,7 +9,6 @@ kli incept --name delegate --alias delegate --proxy proxy --file ${KERI_DEMO_SCR pid=$! PID_LIST+=" $pid" -sleep 2 kli delegate confirm --name delegator --alias delegator -Y & pid=$! PID_LIST+=" $pid" @@ -23,10 +22,11 @@ kli rotate --name delegate --alias delegate --proxy proxy & pid=$! PID_LIST="$pid" -sleep 2 echo "Checking for delegate rotate..." kli delegate confirm --name delegator --alias delegator -Y & pid=$! PID_LIST+=" $pid" wait $PID_LIST + +kli status --name delegate --alias delegate diff --git a/scripts/demo/test_scripts.sh b/scripts/demo/test_scripts.sh index 8fe0a4c29..dd2c3cc88 100755 --- a/scripts/demo/test_scripts.sh +++ b/scripts/demo/test_scripts.sh @@ -42,10 +42,10 @@ printf "\n************************************\n" isSuccess printf "\n************************************\n" -printf "Skipping delegate.sh" +printf "Running delegate.sh" printf "\n************************************\n" -#"${script_dir}/basic/delegate.sh" -#isSuccess +"${script_dir}/basic/delegate.sh" +isSuccess printf "\n************************************\n" printf "Running multisig.sh" diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index 0161a2eec..a595630e4 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -96,9 +96,8 @@ def confirmDo(self, tymth, tock=0.0): while True: esc = self.escrowed() - for ekey, edig in esc: - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item - dgkey = dbing.dgKey(pre, bytes(edig)) + for pre, sn, edig in esc: + dgkey = dbing.dgKey(pre, edig) eraw = self.hby.db.getEvt(dgkey) if eraw is None: continue @@ -204,6 +203,7 @@ def confirmDo(self, tymth, tock=0.0): print(f"Delegate {eserder.pre} {typ} event committed.") + self.hby.db.delegables.rem(keys=(pre, sn)) self.remove(self.toRemove) return True @@ -213,12 +213,6 @@ def confirmDo(self, tymth, tock=0.0): def escrowed(self): esc = [] - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in self.hby.db.getPseItemsNextIter(key=key): - esc.append((ekey, edig)) - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey - + for (pre, sn), edig in self.hby.db.delegables.getItemIter(): + esc.append((pre, sn, edig)) return esc diff --git a/src/keri/app/cli/commands/incept.py b/src/keri/app/cli/commands/incept.py index 68e771c45..3576f797b 100644 --- a/src/keri/app/cli/commands/incept.py +++ b/src/keri/app/cli/commands/incept.py @@ -137,16 +137,15 @@ def __init__(self, name, base, alias, bran, endpoint, proxy=None, cnfg=None, **k clear=False) self.endpoint = endpoint self.proxy = proxy - hby = existing.setupHby(name=name, base=base, bran=bran, cf=cf) - self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer - self.swain = delegating.Anchorer(hby=hby) - self.postman = forwarding.Poster(hby=hby) - self.mbx = indirecting.MailboxDirector(hby=hby, topics=['/receipt', "/replay", "/reply"]) + self.hby = existing.setupHby(name=name, base=base, bran=bran, cf=cf) + self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer + self.swain = delegating.Anchorer(hby=self.hby, proxy=self.hby.habByName(self.proxy)) + self.postman = forwarding.Poster(hby=self.hby) + self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=['/receipt', "/replay", "/reply"]) doers = [self.hbyDoer, self.postman, self.mbx, self.swain, doing.doify(self.inceptDo)] self.inits = kwa self.alias = alias - self.hby = hby super(InceptDoer, self).__init__(doers=doers) def inceptDo(self, tymth, tock=0.0): @@ -169,7 +168,7 @@ def inceptDo(self, tymth, tock=0.0): self.extend([witDoer, receiptor]) if hab.kever.delpre: - self.swain.delegation(pre=hab.pre, sn=0, proxy=self.hby.habByName(self.proxy)) + self.swain.delegation(pre=hab.pre, sn=0) print("Waiting for delegation approval...") while not self.swain.complete(hab.kever.prefixer, coring.Seqner(sn=hab.kever.sn)): yield self.tock diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index c5eb3b586..530288b94 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -151,7 +151,7 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer - self.swain = delegating.Anchorer(hby=self.hby) + self.swain = delegating.Anchorer(hby=self.hby, proxy=self.hby.habByName(self.proxy)) self.postman = forwarding.Poster(hby=self.hby) self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=['/receipt', "/replay", "/reply"]) doers = [self.hbyDoer, self.mbx, self.swain, self.postman, doing.doify(self.rotateDo)] @@ -198,7 +198,7 @@ def rotateDo(self, tymth, tock=0.0): auths[wit] = f"{code}#{helping.nowIso8601()}" if hab.kever.delpre: - self.swain.delegation(pre=hab.pre, sn=hab.kever.sn, proxy=self.hby.habByName(self.proxy)) + self.swain.delegation(pre=hab.pre, sn=hab.kever.sn) print("Waiting for delegation approval...") while not self.swain.complete(hab.kever.prefixer, coring.Seqner(sn=hab.kever.sn)): yield self.tock diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 2ffa80ce6..fcbdb053b 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -47,50 +47,30 @@ def __init__(self, hby, proxy=None, **kwa): self.proxy = proxy super(Anchorer, self).__init__(doers=[self.witq, self.witDoer, self.postman, doing.doify(self.escrowDo)], - **kwa) + **kwa) def delegation(self, pre, sn=None, proxy=None): if pre not in self.hby.habs: raise kering.ValidationError(f"{pre} is not a valid local AID for delegation") + if proxy is not None: + self.proxy = proxy + # load the hab of the delegated identifier to anchor hab = self.hby.habs[pre] delpre = hab.kever.delpre # get the delegator identifier if delpre not in hab.kevers: raise kering.ValidationError(f"delegator {delpre} not found, unable to process delegation") - dkever = hab.kevers[delpre] # and the delegator's kever sn = sn if sn is not None else hab.kever.sner.num # load the event and signatures evt = hab.makeOwnEvent(sn=sn) - smids = [] - if isinstance(hab, GroupHab): - phab = hab.mhab - smids = hab.smids - elif proxy is not None: - phab = proxy - elif hab.kever.sn > 0: - phab = hab - elif self.proxy is not None: - phab = self.proxy - else: - raise kering.ValidationError("no proxy to send messages for delegation") - # Send exn message for notification purposes - exn, atc = delegateRequestExn(phab, delpre=delpre, evt=bytes(evt), aids=smids) - - self.postman.send(hab=phab, dest=hab.kever.delpre, topic="delegate", serder=exn, attachment=atc) - srdr = serdering.SerderKERI(raw=evt) - del evt[:srdr.size] - self.postman.send(hab=phab, dest=delpre, topic="delegate", serder=srdr, attachment=evt) - - seal = dict(i=srdr.pre, s=srdr.snh, d=srdr.said) - self.witq.query(hab=phab, pre=dkever.prefixer.qb64, anchor=seal) - - self.hby.db.dune.pin(keys=(srdr.pre, srdr.said), val=srdr) + self.witDoer.msgs.append(dict(pre=pre, sn=srdr.sn)) + self.hby.db.dpwe.pin(keys=(srdr.pre, srdr.said), val=srdr) def complete(self, prefixer, seqner, saider=None): """ Check for completed delegation protocol for the specific event @@ -139,8 +119,8 @@ def escrowDo(self, tymth, tock=1.0): yield 0.5 def processEscrows(self): - self.processUnanchoredEscrow() self.processPartialWitnessEscrow() + self.processUnanchoredEscrow() def processUnanchoredEscrow(self): """ @@ -159,11 +139,10 @@ def processUnanchoredEscrow(self): couple = seqner.qb64b + dserder.saidb dgkey = dbing.dgKey(kever.prefixer.qb64b, kever.serder.saidb) self.hby.db.setAes(dgkey, couple) # authorizer event seal (delegator/issuer) - self.witDoer.msgs.append(dict(pre=pre, sn=serder.sn)) # Move to escrow waiting for witness receipts - logger.info(f"Waiting for fully signed witness receipts for {serder.sn}") - self.hby.db.dpwe.pin(keys=(pre, said), val=serder) + logger.info(f"Delegation approval received, {serder.pre} confirmed") + self.hby.db.cdel.put(keys=(pre, coring.Seqner(sn=serder.sn).qb64), val=coring.Saider(qb64=serder.said)) self.hby.db.dune.rem(keys=(pre, said)) def processPartialWitnessEscrow(self): @@ -188,9 +167,33 @@ def processPartialWitnessEscrow(self): witnessed = True if not witnessed: continue - logger.info(f"Witness receipts complete, {pre} confirmed.") + logger.info(f"Witness receipts complete, waiting for delegation approval.") + hab = self.hby.habs[pre] + delpre = hab.kever.delpre # get the delegator identifier + dkever = hab.kevers[delpre] # and the delegator's kever + smids = [] + if isinstance(hab, GroupHab): + phab = hab.mhab + smids = hab.smids + elif self.proxy is not None: + phab = self.proxy + else: + raise kering.ValidationError("no proxy to send messages for delegation") + + evt = hab.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said) + exn, atc = delegateRequestExn(phab, delpre=delpre, evt=bytes(evt), aids=smids) + + self.postman.send(hab=phab, dest=hab.kever.delpre, topic="delegate", serder=exn, attachment=atc) + + srdr = serdering.SerderKERI(raw=evt) + del evt[:srdr.size] + self.postman.send(hab=phab, dest=delpre, topic="delegate", serder=srdr, attachment=evt) + + seal = dict(i=srdr.pre, s=srdr.snh, d=srdr.said) + self.witq.query(hab=phab, pre=dkever.prefixer.qb64, anchor=seal) + self.hby.db.dpwe.rem(keys=(pre, said)) - self.hby.db.cdel.put(keys=(pre, seqner.qb64), val=coring.Saider(qb64=serder.said)) + self.hby.db.dune.pin(keys=(srdr.pre, srdr.said), val=srdr) def loadHandlers(hby, exc, notifier): diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 5285e5b3d..3e809dd6d 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2632,38 +2632,16 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, f" delegation by {delpre} of" f"event = {serder.ked}.") - # ToDo XXXX This logic moves to the Delegable escrow processing - # ToDo XXXX create process escrow for delegable events "dees." - #in order to get delegator approval - # any virtual delegation or sandboxing logic happens there - # create virtual anchor seal so local delegator can evaluate - # superseding logic with provisional virtual seal - #dkever = self.kevers[delpre] - #dseal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said) - #dserder = interact(pre=dkever.prefixer.qb64, - #dig=dkever.serder.said, - #sn=dkever.sner.num + 1, - #data=[dseal._asdict()]) - #delseqner = coring.Seqner(snh=dserder.snh) - #delsaider = coring.Saider(qb64=dserder.said) - # ToDo XXXX need to cue task here to approve delegation by generating - # an anchoring SealEvent of serder in delegators KEL - # may include MFA and or business logic for the delegator i.e. is local - # event that designates this controller as delegator triggers - # this cue to approave delegation #self.cues.push(dict(kin="approveDelegation", #delegator=kever.delpre, #serder=serder)) - else: # not local delegator so escrow self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) raise MissingDelegationError(f"No delegation seal for delegator " "{delpre} of evt = {serder.ked}.") - #ssn = validateSN(sn=delseqner.snh, inceptive=False) # delseqner Number should already do this ssn = Number(num=delseqner.sn).validate(inceptive=False).sn - #ssn = sner.num sner is Number seqner is Seqner # ToDo XXXX need to replace Seqners with Numbers # get the dig of the delegating event. Using getKeLast ensures delegating diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index e5f7eb1a0..9273f6255 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -994,10 +994,10 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, if cigars: kvy.processAttachedReceiptCouples(serder, cigars, - firner=firner, local=local) + firner=firner, local=local) if trqs: kvy.processAttachedReceiptQuadruples(serder, trqs, - firner=firner, local=local) + firner=firner, local=local) except AttributeError as ex: raise kering.ValidationError("No kevery to process so dropped msg" diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index db24fbd53..0a1ec1f0a 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -949,7 +949,7 @@ def reopen(self, **kwa): self.misfits = subing.IoSetSuber(db=self, subkey='mfes.') # delegable events escrows. events with local delegator that need approval - self.delegables = subing.CesrIoSetSuber(db=self, subkey='dees.', klas=coring.Diger) + self.delegables = subing.IoSetSuber(db=self, subkey='dees.') # events as ordered by first seen ordinals self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=core.Number) @@ -1500,9 +1500,6 @@ def cloneEvtMsg(self, pre, fn, dig): atc.extend(coring.Counter(code=coring.CtrDex.SealSourceCouples, count=1).qb64b) atc.extend(couple) - elif self.kevers[pre].delegated: - if coring.SerderKERI(raw=raw).estive: - raise kering.MissingEntryError("Missing delegator anchor seal for dig={}.".format(dig)) # add trans endorsement quadruples to attachments not controller # may have been originally key event attachments or receipted endorsements From 80e547c64c8fae0c062b2b183fce184ace851866 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Sat, 18 May 2024 14:17:00 -0700 Subject: [PATCH 171/418] Version Bump Signed-off-by: pfeairheller --- Makefile | 4 ++-- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0ea899105..f0a9a0ed0 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev2 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev2-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev3 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev3-arm64 . .PHONY: build-witness-demo build-witness-demo: diff --git a/setup.py b/setup.py index 7fc406286..a098b5990 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev2', # also change in src/keri/__init__.py + version='1.2.0-dev3', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 5b3d355f5..fe141b304 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev2' # also change in setup.py +__version__ = '1.2.0-dev3' # also change in setup.py From feff6660972e097169d756b89acf16e048007a00 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Tue, 21 May 2024 12:19:53 -0700 Subject: [PATCH 172/418] Add processing method for delegables escrow in Kevers. (#786) Signed-off-by: pfeairheller --- src/keri/app/cli/commands/delegate/confirm.py | 4 +- src/keri/core/eventing.py | 118 ++++++++++++++++++ tests/core/test_delegating.py | 51 ++++++++ 3 files changed, 171 insertions(+), 2 deletions(-) diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index a595630e4..374f017ac 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -130,7 +130,7 @@ def confirmDo(self, tymth, tock=0.0): if isinstance(hab, GroupHab): aids = hab.smids - #seqner = coring.Seqner(sn=eserder.sn) + anchor = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) if self.interact: msg = hab.interact(data=[anchor]) @@ -166,7 +166,7 @@ def confirmDo(self, tymth, tock=0.0): else: cur = hab.kever.sner.num - #seqner = coring.Seqner(sn=eserder.sn) + anchor = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) if self.interact: hab.interact(data=[anchor]) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 3e809dd6d..53a59ea1b 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5714,6 +5714,124 @@ def processEscrowUnverNonTrans(self): break key = ekey # setup next while iteration, with key after ekey + def processEscrowDelegables(self): + """ + Process events escrowed by Kever that require delegation. + + Escrowed items are indexed in database table keyed by prefix and + sn with duplicates given by different dig inserted in insertion order. + This allows FIFO processing of events with same prefix and sn but different + digest. + + Uses .db.delegables.add(key, val) which is IOVal with dups. + + Value is dgkey for event stored in .Evt where .Evt has serder.raw of event. + + Steps: + Each pass (walk index table) + For each prefix,sn + For each escrow item dup at prefix,sn: + Get Event + Get and Attach Signatures + Process event as if it came in over the wire + If successful then remove from escrow table + """ + + for (pre, sn), dig in self.db.delegables.getItemIter(): + try: + edig = dig.encode("utf-8") + dgkey = dgKey(pre.encode("utf-8"), edig) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) + + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) + + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutOOE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) + + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) + + # get the escrowed event using edig + eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + if eraw is None: + # no event so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) + + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + + # get sigs and attach + sigs = self.db.getSigs(dgKey(pre, bytes(edig))) + if not sigs: # otherwise its a list of sigs + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) + + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + + # get wigs + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + + # get delgate seal + couple = self.db.getAes(dgkey) + if couple is not None: # Only try to parse the event if we have the del seal + raw = bytearray(couple) + seqner = coring.Seqner(qb64b=raw, strip=True) + saider = coring.Saider(qb64b=raw) + + # process event + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=seqner, + delsaider=saider, local=esr.local) + else: + raise MissingDelegableApprovalError() + + except MissingDelegableApprovalError as ex: + # still waiting on missing delegation approval + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) + else: + logger.error("Kevery unescrow failed: %s", ex.args[0]) + + except Exception as ex: # log diagnostics errors etc + # error other than out of order so remove from OO escrow + self.db.delegables.rem(keys=(pre, sn,), val=edig) # removes one escrow at key val + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) + + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delegables.rem(keys=(pre, sn,), val=edig) # removes one escrow at key val + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") + def processQueryNotFound(self): """ Process qry events escrowed by Kevery for KELs that have not yet met the criteria of the query. diff --git a/tests/core/test_delegating.py b/tests/core/test_delegating.py index 5d659f5ce..59f4cbecd 100644 --- a/tests/core/test_delegating.py +++ b/tests/core/test_delegating.py @@ -13,6 +13,7 @@ from keri.app import keeping, habbing from keri.db import dbing, basing +from keri.db.dbing import snKey logger = help.ogler.getLogger() @@ -723,6 +724,56 @@ def test_load_event(mockHelpingNowUTC): """End Test""" +def test_delegables_escrow(): + gateSalt = core.Salter(raw=b'0123456789abcdef').qb64 + torSalt = core.Salter(raw=b'0123456789defabc').raw + + with habbing.openHby(name="delegate", temp=True, salt=gateSalt) as gateHby, \ + habbing.openHab(name="delegator", temp=True, salt=torSalt) as (torHby, torHab): + + gateHab = gateHby.makeHab(name="repTest", transferable=True, delpre=torHab.pre) + assert gateHab.pre == "EFqw1EgGdd2B6MgNLJaNO13_JoQpxAtasIjySDzGm9pd" + + gateIcp = gateHab.makeOwnEvent(sn=0) + torKvy = eventing.Kevery(db=torHab.db, lax=False, local=False) + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) + assert gateHab.pre not in torKvy.kevers + assert len(torHab.db.delegables.get(keys=snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn))) == 1 + + # Now create delegating interaction event + seal = eventing.SealEvent(i=gateHab.pre, + s="0", + d=gateHab.pre) + ixn = torHab.interact(data=[seal._asdict()]) + assert ixn == (b'{"v":"KERI10JSON00013a_","t":"ixn","d":"EPUCIjCibL-VeT3n6PYIkbyP' + b'qpioIFT79NRqxboFv0Os","i":"EJTtW40aDl0aKDZ09v-o6uDz_VwLJGplp6WTI' + b'BGCoVog","s":"1","p":"EJTtW40aDl0aKDZ09v-o6uDz_VwLJGplp6WTIBGCoV' + b'og","a":[{"i":"EFqw1EgGdd2B6MgNLJaNO13_JoQpxAtasIjySDzGm9pd","s"' + b':"0","d":"EFqw1EgGdd2B6MgNLJaNO13_JoQpxAtasIjySDzGm9pd"}]}-AABAA' + b'BRR9HDRx_7KdWJ7uokLzREP3c1Hg7Grq5fwoGl_EXA-reR05aYPjDdZ4CIZTnqDo' + b'EN2hqNbHfq4zMaDlR8Ja4D') + + # Make sure that our anchoring ixn event is in our own KEL + assert torHab.kever.sn == 1 + + # Place the anchor seal in the database... this will be retrieved from the fully committed delegate event + serder = torHab.kever.serder + seqner = coring.Seqner(sn=serder.sn) + couple = seqner.qb64b + serder.saidb + dgkey = dbing.dgKey(gateHab.kever.prefixer.qb64b, gateHab.kever.serder.saidb) + torHab.db.setAes(dgkey, couple) # authorizer event seal (delegator/issuer) + + # delegate still not kevers + assert gateHab.pre not in torKvy.kevers + assert len(torHab.db.delegables.get(keys=snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn))) == 1 + + # run the delegables escrow processor to make get delegate in our Kevers + torKvy.processEscrowDelegables() + assert len(torHab.db.delegables.get(keys=snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn))) == 0 + assert gateHab.pre in torKvy.kevers + + if __name__ == "__main__": test_delegation() test_delegation_supersede() + From a2457e5b32ffde7e648a04048019767b81996ab8 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Tue, 21 May 2024 12:25:40 -0700 Subject: [PATCH 173/418] Version Bump --- Makefile | 4 ++-- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f0a9a0ed0..30bb20a0f 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev3 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev3-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev4 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev4-arm64 . .PHONY: build-witness-demo build-witness-demo: diff --git a/setup.py b/setup.py index a098b5990..28b8876c9 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev3', # also change in src/keri/__init__.py + version='1.2.0-dev4', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index fe141b304..d2b0b42ac 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev3' # also change in setup.py +__version__ = '1.2.0-dev4' # also change in setup.py From 1c09b39780f76e292d1b3bc8617b2f51b91b354d Mon Sep 17 00:00:00 2001 From: Lance Date: Wed, 29 May 2024 16:40:26 -0400 Subject: [PATCH 174/418] fixes MissingDelegableApprovalError logging bug (#789) Signed-off-by: 2byrds <2byrds@gmail.com> --- src/keri/core/eventing.py | 2 +- tests/core/test_delegating.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 53a59ea1b..6b4e743f5 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5806,7 +5806,7 @@ def processEscrowDelegables(self): self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=seqner, delsaider=saider, local=esr.local) else: - raise MissingDelegableApprovalError() + raise MissingDelegableApprovalError("No delegation seal found for event.") except MissingDelegableApprovalError as ex: # still waiting on missing delegation approval diff --git a/tests/core/test_delegating.py b/tests/core/test_delegating.py index 59f4cbecd..0c05f27c8 100644 --- a/tests/core/test_delegating.py +++ b/tests/core/test_delegating.py @@ -739,6 +739,8 @@ def test_delegables_escrow(): parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) assert gateHab.pre not in torKvy.kevers assert len(torHab.db.delegables.get(keys=snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn))) == 1 + # Exercise the MissingDelegableApprovalError case + torKvy.processEscrowDelegables() # Now create delegating interaction event seal = eventing.SealEvent(i=gateHab.pre, From 2b31d065bc58e71af2a4941cb4dd70521d691545 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Fri, 31 May 2024 11:25:51 -0400 Subject: [PATCH 175/418] cleans up some usage of coring.serder (#790) * cleans up some usage of coring.serder Signed-off-by: Kevin Griffin * cleans up some usage of coring.serder Signed-off-by: Kevin Griffin --------- Signed-off-by: Kevin Griffin --- src/keri/app/cli/commands/delegate/request.py | 2 +- src/keri/app/cli/commands/ipex/grant.py | 2 +- src/keri/app/cli/commands/multisig/notice.py | 4 ++-- src/keri/app/cli/commands/vc/create.py | 2 +- src/keri/app/cli/commands/vc/registry/incept.py | 2 +- src/keri/app/grouping.py | 4 ++-- src/keri/app/habbing.py | 2 +- src/keri/app/notifying.py | 2 +- src/keri/vdr/credentialing.py | 8 ++++---- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/keri/app/cli/commands/delegate/request.py b/src/keri/app/cli/commands/delegate/request.py index 272856d83..80e152feb 100644 --- a/src/keri/app/cli/commands/delegate/request.py +++ b/src/keri/app/cli/commands/delegate/request.py @@ -96,7 +96,7 @@ def requestDo(self, tymth, tock=0.0): exn, atc = delegating.delegateRequestExn(hab.mhab, delpre=delpre, evt=bytes(evt), aids=hab.smids) # delegate AID ICP and exn of delegation request EXN - srdr = serdering.SerderKERI(raw=evt) # coring.Serder(raw=evt) + srdr = serdering.SerderKERI(raw=evt) del evt[:srdr.size] self.postman.send(src=phab.pre, dest=delpre, topic="delegate", serder=srdr, attachment=evt) self.postman.send(src=phab.pre, dest=hab.kever.delpre, topic="delegate", serder=exn, attachment=atc) diff --git a/src/keri/app/cli/commands/ipex/grant.py b/src/keri/app/cli/commands/ipex/grant.py index dad3a79ab..d6e309d72 100644 --- a/src/keri/app/cli/commands/ipex/grant.py +++ b/src/keri/app/cli/commands/ipex/grant.py @@ -108,7 +108,7 @@ def grantDo(self, tymth, tock=0.0): iss = self.rgy.reger.cloneTvtAt(creder.said) - iserder = serdering.SerderKERI(raw=bytes(iss)) # coring.Serder(raw=bytes(iss)) + iserder = serdering.SerderKERI(raw=bytes(iss)) seqner = coring.Seqner(sn=iserder.sn) serder = self.hby.db.findAnchoringSealEvent(creder.sad['i'], diff --git a/src/keri/app/cli/commands/multisig/notice.py b/src/keri/app/cli/commands/multisig/notice.py index 3ba464797..e866984e4 100644 --- a/src/keri/app/cli/commands/multisig/notice.py +++ b/src/keri/app/cli/commands/multisig/notice.py @@ -11,8 +11,8 @@ from keri.app import habbing, forwarding, grouping from keri.app.cli.common import existing -from keri.core import coring from keri.core.coring import Ilks +from keri.core import serdering logger = help.ogler.getLogger() @@ -86,7 +86,7 @@ def noticeDo(self, tymth, tock=0.0): (smids, rmids) = hab.members() serder = hab.kever.serder rot = hab.makeOwnEvent(sn=hab.kever.sn) - eserder = coring.Serder(raw=rot) + eserder = serdering.SerderKERI(raw=rot) del rot[:eserder.size] ilk = serder.ked['t'] diff --git a/src/keri/app/cli/commands/vc/create.py b/src/keri/app/cli/commands/vc/create.py index c8a36e049..b16e9a178 100644 --- a/src/keri/app/cli/commands/vc/create.py +++ b/src/keri/app/cli/commands/vc/create.py @@ -220,7 +220,7 @@ def createDo(self, tymth, tock=0.0): else: anc = hab.interact(data=[rseal]) - aserder = serdering.SerderKERI(raw=anc) # coring.Serder(raw=anc) + aserder = serdering.SerderKERI(raw=anc) self.credentialer.issue(self.creder, iserder) self.registrar.issue(self.creder, iserder, aserder) diff --git a/src/keri/app/cli/commands/vc/registry/incept.py b/src/keri/app/cli/commands/vc/registry/incept.py index 83dd6b320..5a2411359 100644 --- a/src/keri/app/cli/commands/vc/registry/incept.py +++ b/src/keri/app/cli/commands/vc/registry/incept.py @@ -132,7 +132,7 @@ def inceptDo(self, tymth, tock=0.0, **kwa): else: anc = hab.interact(data=[rseal]) - aserder = serdering.SerderKERI(raw=bytes(anc)) # coring.Serder(raw=bytes(anc)) + aserder = serdering.SerderKERI(raw=bytes(anc)) self.registrar.incept(iserder=registry.vcp, anc=aserder) if isinstance(hab, GroupHab): diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 46d85a3de..5cf10dd63 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -275,7 +275,7 @@ def multisigInceptExn(hab, smids, rmids, icp, delegator=None): """ rmids = rmids if rmids is not None else smids - serder = serdering.SerderKERI(raw=icp) # coring.Serder(raw=icp) + serder = serdering.SerderKERI(raw=icp) data = dict( gid=serder.pre, smids=smids, @@ -585,7 +585,7 @@ def add(self, serder): events so that any addition signatures can be processed. Parameters: - serder (coring.Serder): peer-to-peer exn "/multisig" message to coordinate from other participants + serder (serdering.SerderKERI): peer-to-peer exn "/multisig" message to coordinate from other participants Returns: diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 0d4b3d235..ef3674b07 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -942,7 +942,7 @@ class BaseHab: kevers (dict): of eventing.Kever instances from KELs in local db keyed by qb64 prefix. Read through cache of of kevers of states for KELs in db.states - iserder (coring.Serder): own inception event + iserder (serdering.SerderKERI): own inception event prefixes (OrderedSet): local prefixes for .db accepted (bool): True means accepted into local KEL. False otherwise diff --git a/src/keri/app/notifying.py b/src/keri/app/notifying.py index 37128049d..371815f9b 100644 --- a/src/keri/app/notifying.py +++ b/src/keri/app/notifying.py @@ -181,7 +181,7 @@ def getItemIter(self, keys: Union[str, Iterable] = b""): keys (tuple): of key strs to be combined in order to form key Returns: - iterator: of tuples of keys tuple and val coring.Serder for + iterator: of tuples of keys tuple and val serdering.SerderKERI for each entry in db """ diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index 7200f8192..6c9d906fc 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -985,7 +985,7 @@ def sendArtifacts(hby, reger, postman, creder, recp): postman.send(serder=serder, attachment=atc) for msg in reger.clonePreIter(pre=creder.said): - serder = serdering.SerderKERI(raw=msg) # coring.Serder(raw=msg) + serder = serdering.SerderKERI(raw=msg) atc = msg[serder.size:] postman.send(serder=serder, attachment=atc) @@ -999,16 +999,16 @@ def sendRegistry(hby, reger, postman, creder, sender, recp): ikever = hby.db.kevers[issr] for msg in hby.db.cloneDelegation(ikever): - serder = serdering.SerderKERI(raw=msg) # coring.Serder(raw=msg) + serder = serdering.SerderKERI(raw=msg) atc = msg[serder.size:] postman.send(serder=serder, attachment=atc) for msg in hby.db.clonePreIter(pre=issr): - serder = serdering.SerderKERI(raw=msg) # coring.Serder(raw=msg) + serder = serdering.SerderKERI(raw=msg) atc = msg[serder.size:] postman.send(serder=serder, attachment=atc) for msg in reger.clonePreIter(pre=regk): - serder = serdering.SerderKERI(raw=msg) # coring.Serder(raw=msg) + serder = serdering.SerderKERI(raw=msg) atc = msg[serder.size:] postman.send(serder=serder, attachment=atc) From 4606d0f35e3192f8151d95578f5935f8ea2974c8 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 3 Jun 2024 09:20:20 -0400 Subject: [PATCH 176/418] updates dockerfile to new alpine (#791) Signed-off-by: Kevin Griffin --- images/keripy.dockerfile | 41 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/images/keripy.dockerfile b/images/keripy.dockerfile index f04bb811d..9cc82212c 100644 --- a/images/keripy.dockerfile +++ b/images/keripy.dockerfile @@ -1,18 +1,21 @@ -# Builder layer -FROM python:3.12-alpine as builder +ARG BASE=python:3.12.3-alpine3.20 -# Install compilation dependencies -RUN apk --no-cache add \ - bash \ +FROM ${BASE} as builder + +RUN apk add --no-cache bash + +SHELL ["/bin/bash", "-c"] + +RUN apk add --no-cache \ + curl \ + build-base \ alpine-sdk \ libffi-dev \ libsodium \ - libsodium-dev - -SHELL ["/bin/bash", "-c"] + libsodium-dev -# Setup Rust for blake3 dependency build -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" WORKDIR /keripy @@ -20,19 +23,16 @@ RUN python -m venv venv ENV PATH=/keripy/venv/bin:${PATH} -RUN pip install --upgrade pip && \ - mkdir /keripy/src +RUN pip install --upgrade pip +RUN mkdir /keripy/src -# Copy Python dependency files in COPY requirements.txt setup.py ./ -# Set up Rust environment and install Python dependencies -# Must source the Cargo environment for the blake3 library to see -# the Rust intallation during requirements install -RUN . ${HOME}/.cargo/env && \ - pip install -r requirements.txt + +RUN . ${HOME}/.cargo/env +RUN pip install -r requirements.txt # Runtime layer -FROM python:3.12.2-alpine3.18 +FROM ${BASE} RUN apk --no-cache add \ bash \ @@ -44,7 +44,6 @@ WORKDIR /keripy COPY --from=builder /keripy /keripy COPY src/ src/ -ENV PATH=/keripy/venv/bin:${PATH} - +ENV PATH="/keripy/venv/bin:${PATH}" ENTRYPOINT [ "kli" ] From 38cc2891286afce6c7426b0bd14abca287f229fc Mon Sep 17 00:00:00 2001 From: Fergal Date: Wed, 5 Jun 2024 15:42:56 +0100 Subject: [PATCH 177/418] fix: re-cue filtered kvy/tvy cues from respondant (#797) --- src/keri/app/storing.py | 125 +++++++++++++++++++++------------------- 1 file changed, 65 insertions(+), 60 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index 99b38cf37..297dc97e7 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -219,65 +219,70 @@ def cueDo(self, tymth=None, tock=0.0): _ = (yield self.tock) while True: - while self.cues: # iteratively process each cue in cues - cue = self.cues.pull() # self.cues.popleft() - cueKin = cue["kin"] # type or kind of cue - if cueKin in ("receipt",): # cue to receipt a received event from other pre - serder = cue["serder"] # Serder of received event for other pre - cuedKed = serder.ked - cuedPrefixer = coring.Prefixer(qb64=cuedKed["i"]) - - # If respondant configured with list of acceptable AIDs to witness for, check them here - if self.aids is not None and cuedPrefixer.qb64 not in self.aids: - continue - - if cuedPrefixer.qb64 in self.hby.kevers: - kever = self.hby.kevers[cuedPrefixer.qb64] - owits = oset(kever.wits) - if match := owits.intersection(self.hby.prefixes): - pre = match.pop() - hab = self.hby.habByPre(pre) - if hab is None: - continue - - raw = hab.receipt(serder) - rserder = serdering.SerderKERI(raw=raw) - del raw[:rserder.size] - self.postman.send(serder.pre, topic="receipt", serder=rserder, hab=hab, attachment=raw) - - elif cueKin in ("replay",): - src = cue["src"] - dest = cue["dest"] - msgs = cue["msgs"] - - hab = self.hby.habByPre(src) - if hab is None: - continue - - if dest not in self.hby.kevers: - continue - - for msg in msgs: - raw = bytearray(msg) - serder = serdering.SerderKERI(raw=raw) - del raw[:serder.size] - self.postman.send(dest, topic="replay", serder=serder, hab=hab, attachment=raw) - - elif cueKin in ("reply",): - src = cue["src"] - serder = cue["serder"] - - dest = cue["dest"] - - if dest not in self.hby.kevers: - continue - - hab = self.hby.habByPre(src) - if hab is None: - continue - - atc = hab.endorse(serder) - del atc[:serder.size] - self.postman.send(hab=hab, dest=dest, topic="reply", serder=serder, attachment=atc) + while not self.cues: + yield self.tock + + cue = self.cues.pull() # self.cues.popleft() + cueKin = cue["kin"] # type or kind of cue + if cueKin in ("receipt",): # cue to receipt a received event from other pre + serder = cue["serder"] # Serder of received event for other pre + cuedKed = serder.ked + cuedPrefixer = coring.Prefixer(qb64=cuedKed["i"]) + + # If respondant configured with list of acceptable AIDs to witness for, check them here + if self.aids is not None and cuedPrefixer.qb64 not in self.aids: + continue + + if cuedPrefixer.qb64 in self.hby.kevers: + kever = self.hby.kevers[cuedPrefixer.qb64] + owits = oset(kever.wits) + if match := owits.intersection(self.hby.prefixes): + pre = match.pop() + hab = self.hby.habByPre(pre) + if hab is None: + continue + + raw = hab.receipt(serder) + rserder = serdering.SerderKERI(raw=raw) + del raw[:rserder.size] + self.postman.send(serder.pre, topic="receipt", serder=rserder, hab=hab, attachment=raw) + + elif cueKin in ("replay",): + src = cue["src"] + dest = cue["dest"] + msgs = cue["msgs"] + + hab = self.hby.habByPre(src) + if hab is None: + continue + + if dest not in self.hby.kevers: + continue + + for msg in msgs: + raw = bytearray(msg) + serder = serdering.SerderKERI(raw=raw) + del raw[:serder.size] + self.postman.send(dest, topic="replay", serder=serder, hab=hab, attachment=raw) + + elif cueKin in ("reply",): + src = cue["src"] + serder = cue["serder"] + + dest = cue["dest"] + + if dest not in self.hby.kevers: + continue + + hab = self.hby.habByPre(src) + if hab is None: + continue + + atc = hab.endorse(serder) + del atc[:serder.size] + self.postman.send(hab=hab, dest=dest, topic="reply", serder=serder, attachment=atc) + + else: + self.cues.push(cue) yield self.tock From 1146a74a8bca870884c457350c2d5502b2095464 Mon Sep 17 00:00:00 2001 From: arilieb Date: Wed, 12 Jun 2024 14:17:28 -0700 Subject: [PATCH 178/418] Added new script modeling revokable authorization with a distributed multisig. Re-ordered GroupHab.make such that HabitatRecord is created, saved, and prefixes are added before event processing begins, accounts for instances similar to multisig-for-revoke-auth.sh. (#800) --- .../demo/basic/multisig-for-revoke-auth.sh | 65 +++++++++++++++++++ scripts/demo/data/multisig-sign-auth.json | 16 +++++ src/keri/app/habbing.py | 18 ++--- 3 files changed, 90 insertions(+), 9 deletions(-) create mode 100755 scripts/demo/basic/multisig-for-revoke-auth.sh create mode 100644 scripts/demo/data/multisig-sign-auth.json diff --git a/scripts/demo/basic/multisig-for-revoke-auth.sh b/scripts/demo/basic/multisig-for-revoke-auth.sh new file mode 100755 index 000000000..29599e89d --- /dev/null +++ b/scripts/demo/basic/multisig-for-revoke-auth.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# This script models revokable authority + +# Initialize and incept the 3 parties +kli init -n multisig1 --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis +kli incept -n multisig1 --alias multisig1 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-1-sample.json + +kli init -n multisig2 --salt 0ACDEyMzQ1Njc4OWdoaWphea --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis +kli incept -n multisig2 --alias multisig2 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-2-sample.json + +kli init -n multisig3 --salt 0ACDEyMzQ1Njc4OWdoaWpomw --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis +kli incept -n multisig3 --alias multisig3 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-3-sample.json + +# Resolve OOBIs to establish connections +kli oobi resolve -n multisig1 --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC/witness +kli oobi resolve -n multisig2 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EI0jXuw_V_zjj_mFgJLJWgFtbpVRNdUmv01WoM4na1ek/witness + +kli oobi resolve -n multisig1 --oobi-alias multisig3 --oobi http://127.0.0.1:5642/oobi/EIwtBwakOchYfReVjZnou_ZR9pA9Sjd877Y4pegfOGSC/witness +kli oobi resolve -n multisig3 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EI0jXuw_V_zjj_mFgJLJWgFtbpVRNdUmv01WoM4na1ek/witness + +kli oobi resolve -n multisig2 --oobi-alias multisig3 --oobi http://127.0.0.1:5642/oobi/EIwtBwakOchYfReVjZnou_ZR9pA9Sjd877Y4pegfOGSC/witness +kli oobi resolve -n multisig3 --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC/witness + +# Incept a multisig group for multisig1 and multisig2 using a shared configuration file +kli multisig incept -n multisig1 --alias multisig1 --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-sign-auth.json & +pid=$! +PID_LIST+=" $pid" + +kli multisig incept -n multisig2 --alias multisig2 --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-sign-auth.json & +pid=$! +PID_LIST+=" $pid" + +wait $PID_LIST + +# Rotate the keys for multisig2 +# This models the ability of the second party to rotate keys and potentially exclude the first party's key +kli rotate -n multisig2 --alias multisig2 + +# Query the state of multisig3 to check the current configuration +kli query --name multisig3 --alias multisig3 --prefix EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC + +# Resolve OOBIs for multisig3 to update its state with the latest information +kli oobi resolve -n multisig3 --oobi-alias multisig --oobi http://127.0.0.1:5642/oobi/EPKgQWXeBFsE9DjyqvspoPX1JVmRbRlNkCCaqvEeppM6/witness + +# Perform a multisig rotate operation for multisig2 +# The smids (signing member identifiers) and rmids (rotation member identifiers) are used to configure the new state +# The new state excludes the first party's key, effectively revoking its signing privileges +kli multisig rotate -n multisig2 --alias multisig \ + --smids EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC:1 \ + --smids EIwtBwakOchYfReVjZnou_ZR9pA9Sjd877Y4pegfOGSC:0 \ + --isith '["0","1"]' \ + --rmids EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC:1 \ + --rmids EIwtBwakOchYfReVjZnou_ZR9pA9Sjd877Y4pegfOGSC:0 \ + --nsith '["1","0"]' & +pid=$! +PID_LIST="$pid" + +# Join the multisig group for multisig3 to synchronize its state +kli multisig join --name multisig3 --auto & +pid=$! +PID_LIST+=" $pid" + +# Wait for all background processes to complete +wait $PID_LIST \ No newline at end of file diff --git a/scripts/demo/data/multisig-sign-auth.json b/scripts/demo/data/multisig-sign-auth.json new file mode 100644 index 000000000..ebcc168de --- /dev/null +++ b/scripts/demo/data/multisig-sign-auth.json @@ -0,0 +1,16 @@ +{ + "aids": [ + "EI0jXuw_V_zjj_mFgJLJWgFtbpVRNdUmv01WoM4na1ek", + "EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC" + ], + "transferable": true, + "wits": [ + "BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha", + "BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM", + "BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX" + ], + "toad": 2, + "isith": ["1", "0"], + "nsith": ["0", "1"] + } + \ No newline at end of file diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index ef3674b07..ab9c2c9ee 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -2719,6 +2719,15 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, # sign handles group hab with .mhab case sigers = self.sign(ser=serder.raw, verfers=verfers) + habord = basing.HabitatRecord(hid=self.pre, + mid=self.mhab.pre, + name=self.name, + domain=self.ns, + smids=self.smids, + rmids=self.rmids) + self.save(habord) + self.prefixes.add(self.pre) + # during delegation initialization of a habitat we ignore the MissingDelegationError and # MissingSignatureError try: @@ -2729,15 +2738,6 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, raise kering.ConfigurationError("Improper Habitat inception for " "pre={} {}".format(self.pre, ex)) - habord = basing.HabitatRecord(hid=self.pre, - mid=self.mhab.pre, - name=self.name, - domain=self.ns, - smids=self.smids, - rmids=self.rmids) - - self.save(habord) - self.prefixes.add(self.pre) self.inited = True From b6e4df92cec208dc836d1a07b838da803006fcff Mon Sep 17 00:00:00 2001 From: Kent Bull <65027257+kentbull@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:46:03 -0600 Subject: [PATCH 179/418] fix: correct short arg for --group from -a to -g (#802) --- src/keri/app/cli/commands/multisig/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/app/cli/commands/multisig/join.py b/src/keri/app/cli/commands/multisig/join.py index e3c1e7850..829b8a393 100644 --- a/src/keri/app/cli/commands/multisig/join.py +++ b/src/keri/app/cli/commands/multisig/join.py @@ -25,7 +25,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--group', '-a', help='human-readable name for the multisig group identifier prefix', required=False, default=None) +parser.add_argument('--group', '-g', help='human-readable name for the multisig group identifier prefix', required=False, default=None) parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--auto", "-Y", help="auto approve any delegation request non-interactively", action="store_true") From c1163dd5557723a2493df1164c0823fb7fdf1a35 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Wed, 12 Jun 2024 19:40:38 -0700 Subject: [PATCH 180/418] Many additions and fixes to kli commands, new ESSR payload attachment support (#803) * New mailbox commands for using bespoke mailbox services. Signed-off-by: pfeairheller * Resolving merge conflicts Signed-off-by: pfeairheller * Resolving merge conflicts Signed-off-by: pfeairheller * Improvements to kli commands `mailbox add` and `witness authenticate` to work in all cases (like delegated AIDs). New `kli aid` command for printing the AID of an alias (useful in scripts). Added `--delpre` as a command line argument to `kli icncept` (useful in scripts). Fix for witness publisher to account for multiple events in one cue'ed message. Added support for witness auth codes for delegated AIDs. Removed logging messages in the INFO case for escrow reprocessing. Signed-off-by: pfeairheller * Updated test Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller --- images/keripy.base.dockerfile | 15 ++ src/keri/app/agenting.py | 13 +- src/keri/app/cli/commands/aid.py | 58 +++++++ src/keri/app/cli/commands/delegate/confirm.py | 28 ++- src/keri/app/cli/commands/incept.py | 2 + src/keri/app/cli/commands/mailbox/add.py | 135 ++++++++++++++ src/keri/app/cli/commands/oobi/generate.py | 13 ++ src/keri/app/cli/commands/rotate.py | 17 +- src/keri/app/cli/commands/status.py | 2 +- .../app/cli/commands/witness/authenticate.py | 31 +++- src/keri/app/cli/common/incepting.py | 2 + src/keri/app/delegating.py | 8 +- src/keri/app/httping.py | 3 + src/keri/app/indirecting.py | 2 +- src/keri/core/coring.py | 8 +- src/keri/core/counting.py | 3 + src/keri/core/eventing.py | 18 -- src/keri/core/parsing.py | 41 +++-- src/keri/core/serdering.py | 4 +- src/keri/db/basing.py | 2 + src/keri/peer/exchanging.py | 67 +++++-- tests/app/test_delegating.py | 14 +- tests/app/test_grouping.py | 32 ++-- tests/app/test_oobiing.py | 11 +- tests/core/test_coring.py | 46 ++++- tests/core/test_counting.py | 71 +++++++- tests/core/test_serdering.py | 28 +-- tests/peer/test_exchanging.py | 164 ++++++++++++------ tests/vc/test_protocoling.py | 158 ++++++++--------- 29 files changed, 736 insertions(+), 260 deletions(-) create mode 100644 images/keripy.base.dockerfile create mode 100644 src/keri/app/cli/commands/aid.py create mode 100644 src/keri/app/cli/commands/mailbox/add.py diff --git a/images/keripy.base.dockerfile b/images/keripy.base.dockerfile new file mode 100644 index 000000000..23de45f8d --- /dev/null +++ b/images/keripy.base.dockerfile @@ -0,0 +1,15 @@ +# Builder layer +FROM python:3.12-alpine as builder + +# Install compilation dependencies +RUN apk --no-cache add \ + bash \ + alpine-sdk \ + libffi-dev \ + libsodium \ + libsodium-dev + +SHELL ["/bin/bash", "-c"] + +# Setup Rust for blake3 dependency build +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 2fd13cf49..ad164a9b3 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -227,8 +227,9 @@ def witDo(self, tymth=None, tock=0.0): msg = self.msgs.popleft() pre = msg["pre"] sn = msg["sn"] if "sn" in msg else None + auths = msg["auths"] if "auths" in msg else None - yield from self.receipt(pre, sn) + yield from self.receipt(pre, sn, auths) self.cues.push(msg) yield self.tock @@ -616,12 +617,10 @@ def sendDo(self, tymth=None, tock=0.0, **opts): _ = (yield self.tock) - total = len(witers) - count = 0 - while count < total: - for witer in witers: - count += len(witer.sent) - _ = (yield self.tock) + while witers: + witer = witers.pop() + while not witer.idle: + _ = (yield self.tock) self.remove(witers) self.cues.push(evt) diff --git a/src/keri/app/cli/commands/aid.py b/src/keri/app/cli/commands/aid.py new file mode 100644 index 000000000..856f65dda --- /dev/null +++ b/src/keri/app/cli/commands/aid.py @@ -0,0 +1,58 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Print the AID for a given alias') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None, + required=True) +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(status, **kwa)] + + +def status(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + alias = args.alias + base = args.base + bran = args.bran + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + if alias is None: + alias = existing.aliasInput(hby) + + hab = hby.habByName(alias) + if hab is None: + print(f"{alias} is not a valid alias for an identifier") + + print(hab.pre) + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index 374f017ac..a0ddb7525 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -16,6 +16,7 @@ from keri import core from keri.core import coring, serdering from keri.db import dbing +from keri.help import helping from keri.peer import exchanging logger = help.ogler.getLogger() @@ -32,6 +33,10 @@ parser.add_argument("--interact", "-i", help="anchor the delegation approval in an interaction event. " "Default is to use a rotation event.", action="store_true") parser.add_argument("--auto", "-Y", help="auto approve any delegation request non-interactively", action="store_true") +parser.add_argument("--authenticate", '-z', help="Prompt the controller for authentication codes for each witness", + action='store_true') +parser.add_argument('--code', help=': formatted witness auth codes. Can appear multiple times', + default=[], action="append", required=False) def confirm(args): @@ -47,15 +52,18 @@ def confirm(args): alias = args.alias interact = args.interact auto = args.auto + authenticate = args.authenticate + codes = args.code - confirmDoer = ConfirmDoer(name=name, base=base, alias=alias, bran=bran, interact=interact, auto=auto) + confirmDoer = ConfirmDoer(name=name, base=base, alias=alias, bran=bran, interact=interact, auto=auto, + authenticate=authenticate, codes=codes) doers = [confirmDoer] return doers class ConfirmDoer(doing.DoDoer): - def __init__(self, name, base, alias, bran, interact=False, auto=False): + def __init__(self, name, base, alias, bran, interact=False, auto=False, authenticate=False, codes=None): hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer self.witq = agenting.WitnessInquisitor(hby=hby) @@ -63,6 +71,8 @@ def __init__(self, name, base, alias, bran, interact=False, auto=False): self.counselor = grouping.Counselor(hby=hby) self.notifier = notifying.Notifier(hby=hby) self.mux = grouping.Multiplexor(hby=hby, notifier=self.notifier) + self.authenticate = authenticate + self.codes = codes if codes is not None else [] exc = exchanging.Exchanger(hby=hby, handlers=[]) delegating.loadHandlers(hby=hby, exc=exc, notifier=self.notifier) @@ -173,7 +183,19 @@ def confirmDo(self, tymth, tock=0.0): else: hab.rotate(data=[anchor]) - witDoer = agenting.WitnessReceiptor(hby=self.hby) + auths = {} + if self.authenticate: + for arg in self.codes: + (wit, code) = arg.split(":") + auths[wit] = f"{code}#{helping.nowIso8601()}" + + for wit in hab.kever.wits: + if wit in auths: + continue + code = input(f"Entire code for {wit}: ") + auths[wit] = f"{code}#{helping.nowIso8601()}" + + witDoer = agenting.WitnessReceiptor(hby=self.hby, auths=auths) self.extend(doers=[witDoer]) self.toRemove.append(witDoer) yield self.tock diff --git a/src/keri/app/cli/commands/incept.py b/src/keri/app/cli/commands/incept.py index 3576f797b..4af084c73 100644 --- a/src/keri/app/cli/commands/incept.py +++ b/src/keri/app/cli/commands/incept.py @@ -117,6 +117,8 @@ def mergeArgsWithFile(args): incept_opts.estOnly = args.est_only if args.data is not None: incept_opts.data = config.parseData(args.data) + if args.delpre is not None: + incept_opts.delpre = args.delpre return incept_opts diff --git a/src/keri/app/cli/commands/mailbox/add.py b/src/keri/app/cli/commands/mailbox/add.py new file mode 100644 index 000000000..67abf1990 --- /dev/null +++ b/src/keri/app/cli/commands/mailbox/add.py @@ -0,0 +1,135 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing +from hio.help import Hict + +from keri import kering +from keri.app import connecting, habbing, forwarding +from keri.app.agenting import httpClient, WitnessPublisher +from keri.app.cli.common import existing +from keri.core import serdering + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Add mailbox role') +parser.set_defaults(handler=lambda args: add(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--mailbox", '-w', help="the mailbox AID or alias to add", required=True) + + +def add(args): + """ Command line handler for adding an aid to a watcher's list of AIds to watch + + Parameters: + args(Namespace): parsed command line arguments + + """ + + ed = AddDoer(name=args.name, + alias=args.alias, + base=args.base, + bran=args.bran, + mailbox=args.mailbox) + return [ed] + + +class AddDoer(doing.DoDoer): + + def __init__(self, name, alias, base, bran, mailbox): + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hab = self.hby.habByName(alias) + self.org = connecting.Organizer(hby=self.hby) + self.witpub = WitnessPublisher(hby=self.hby) + + if mailbox in self.hby.kevers: + mbx = mailbox + else: + mbx = self.org.find("alias", mailbox) + if len(mbx) != 1: + raise ValueError(f"invalid mailbox {mailbox}") + mbx = mbx[0]['id'] + + if not mbx: + raise ValueError(f"unknown mailbox {mailbox}") + + self.mailbox = mbx + + doers = [doing.doify(self.addDo), self.witpub] + + super(AddDoer, self).__init__(doers=doers) + + def addDo(self, tymth, tock=0.0): + """ Grant credential by creating /ipex/grant exn message + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + if isinstance(self.hab, habbing.GroupHab): + raise ValueError("watchers for multisig AIDs not currently supported") + + kel = self.hab.replay() + data = dict(cid=self.hab.pre, + role=kering.Roles.mailbox, + eid=self.mailbox) + + route = "/end/role/add" + msg = self.hab.reply(route=route, data=data) + self.hab.psr.parseOne(ims=(bytes(msg))) # make copy to preserve + + fargs = dict([("kel", kel.decode("utf-8")), + ("rpy", msg.decode("utf-8"))]) + + headers = (Hict([ + ("Content-Type", "multipart/form-data"), + ])) + + client, clientDoer = httpClient(self.hab, self.mailbox) + self.extend([clientDoer]) + + client.request( + method="POST", + path=f"{client.requester.path}/mailboxes", + headers=headers, + fargs=fargs + ) + while not client.responses: + yield self.tock + + rep = client.respond() + if rep.status == 200: + msg = self.hab.replyEndRole(cid=self.hab.pre, role=kering.Roles.mailbox) + self.witpub.msgs.append(dict(pre=self.hab.pre, msg=bytes(msg))) + + while not self.witpub.cues: + yield self.tock + + print(f"Mailbox {self.mailbox} added for {self.hab.name}") + + else: + print(rep.status, rep.data) + + self.remove([clientDoer, self.witpub]) diff --git a/src/keri/app/cli/commands/oobi/generate.py b/src/keri/app/cli/commands/oobi/generate.py index f487f3546..e35a53354 100644 --- a/src/keri/app/cli/commands/oobi/generate.py +++ b/src/keri/app/cli/commands/oobi/generate.py @@ -81,3 +81,16 @@ def generate(tymth, tock=0.0, **opts): url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] up = urlparse(url) print(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + elif role in (kering.Roles.mailbox,): + for (_, _, eid), end in hab.db.ends.getItemIter(keys=(hab.pre, kering.Roles.mailbox, )): + if not (end.allowed and end.enabled is not False): + continue + + urls = hab.fetchUrls(eid=eid, scheme=kering.Schemes.http) or hab.fetchUrls(eid=hab.pre, + scheme=kering.Schemes.https) + if not urls: + print(f"{alias} identifier {hab.pre} does not have any mailbox endpoints") + return + url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] + up = urlparse(url) + print(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/mailbox/{eid}") diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index 530288b94..6adbac029 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -29,6 +29,9 @@ dest="endpoint", action='store_true') parser.add_argument("--authenticate", '-z', help="Prompt the controller for authentication codes for each witness", action='store_true') +parser.add_argument('--code', help=': formatted witness auth codes. Can appear multiple times', + default=[], action="append", required=False) + parser.add_argument("--proxy", help="alias for delegation communication proxy", default="") rotating.addRotationArgs(parser) @@ -63,7 +66,7 @@ def rotate(args): cuts=opts.witsCut, adds=opts.witsAdd, isith=opts.isith, nsith=opts.nsith, count=opts.ncount, toad=opts.toad, - data=opts.data, proxy=args.proxy, authenticate=args.authenticate) + data=opts.data, proxy=args.proxy, authenticate=args.authenticate, codes=args.code) doers = [rotDoer] @@ -118,7 +121,8 @@ class RotateDoer(doing.DoDoer): """ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=None, count=None, - toad=None, wits=None, cuts=None, adds=None, data: list = None, proxy=None, authenticate=False): + toad=None, wits=None, cuts=None, adds=None, data: list = None, proxy=None, authenticate=False, + codes=None): """ Returns DoDoer with all registered Doers needed to perform rotation. @@ -144,6 +148,7 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.endpoint = endpoint self.proxy = proxy self.authenticate = authenticate + self.codes = codes if codes is not None else [] self.wits = wits if wits is not None else [] self.cuts = cuts if cuts is not None else [] @@ -193,12 +198,18 @@ def rotateDo(self, tymth, tock=0.0): auths = {} if self.authenticate: + for arg in self.codes: + (wit, code) = arg.split(":") + auths[wit] = f"{code}#{helping.nowIso8601()}" + for wit in hab.kever.wits: + if wit in auths: + continue code = input(f"Entire code for {wit}: ") auths[wit] = f"{code}#{helping.nowIso8601()}" if hab.kever.delpre: - self.swain.delegation(pre=hab.pre, sn=hab.kever.sn) + self.swain.delegation(pre=hab.pre, sn=hab.kever.sn, auths=auths) print("Waiting for delegation approval...") while not self.swain.complete(hab.kever.prefixer, coring.Seqner(sn=hab.kever.sn)): yield self.tock diff --git a/src/keri/app/cli/commands/status.py b/src/keri/app/cli/commands/status.py index 662a6b625..96f531828 100644 --- a/src/keri/app/cli/commands/status.py +++ b/src/keri/app/cli/commands/status.py @@ -10,7 +10,7 @@ from hio.base import doing from keri.app.cli.common import displaying, existing -from keri.core import coring, serdering +from keri.core import serdering from keri.kering import ConfigurationError logger = help.ogler.getLogger() diff --git a/src/keri/app/cli/commands/witness/authenticate.py b/src/keri/app/cli/commands/witness/authenticate.py index 126fc0417..1758093ce 100644 --- a/src/keri/app/cli/commands/witness/authenticate.py +++ b/src/keri/app/cli/commands/witness/authenticate.py @@ -16,7 +16,6 @@ from keri.app import httping, connecting from keri.app.agenting import httpClient from keri.app.cli.common import existing -from keri.app.httping import CESR_CONTENT_TYPE from keri.core import coring logger = help.ogler.getLogger() @@ -32,6 +31,8 @@ parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--witness", '-w', help="the witness AID or alias to authenticate against", required=True) +parser.add_argument("--url-only", '-u', dest="url", help="display only the URL (no QR Code).", required=False, + action="store_true") def auth(args): @@ -46,16 +47,18 @@ def auth(args): alias=args.alias, base=args.base, bran=args.bran, - witness=args.witness) + witness=args.witness, + urlOnly=args.url) return [ed] class AuthDoer(doing.DoDoer): - def __init__(self, name, alias, base, bran, witness): + def __init__(self, name, alias, base, bran, witness, urlOnly): self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hab = self.hby.habByName(alias) self.org = connecting.Organizer(hby=self.hby) + self.urlOnly = urlOnly if witness in self.hby.kevers: wit = witness @@ -94,9 +97,17 @@ def authDo(self, tymth, tock=0.0): for msg in self.hab.db.clonePreIter(pre=self.hab.pre): body.extend(msg) + fargs = dict([("kel", body.decode("utf-8"))]) + + if self.hab.kever.delegated: + delkel = bytearray() + for msg in self.hab.db.clonePreIter(self.hab.kever.delpre): + delkel.extend(msg) + + fargs['delkel'] = delkel.decode("utf-8") + headers = (Hict([ - ("Content-Type", "application/cesr"), - ("Content-Length", len(body)), + ("Content-Type", "multipart/form-data") ])) client, clientDoer = httpClient(self.hab, self.witness) @@ -106,7 +117,7 @@ def authDo(self, tymth, tock=0.0): method="POST", path=f"{client.requester.path}/aids", headers=headers, - body=bytes(body) + fargs=fargs ) while not client.responses: yield self.tock @@ -120,10 +131,12 @@ def authDo(self, tymth, tock=0.0): d = coring.Matter(qb64=self.hab.decrypt(m.raw)) otpurl = f"otpauth://totp/KERIpy:{self.witness}?secret={d.raw.decode('utf-8')}&issuer=KERIpy" - qr = qrcode.QRCode() - qr.add_data(otpurl) + if not self.urlOnly: + qr = qrcode.QRCode() + qr.add_data(otpurl) + + qr.print_ascii() - qr.print_ascii() print(otpurl) else: diff --git a/src/keri/app/cli/common/incepting.py b/src/keri/app/cli/common/incepting.py index 6a0789c91..6fe211fa3 100644 --- a/src/keri/app/cli/common/incepting.py +++ b/src/keri/app/cli/common/incepting.py @@ -28,4 +28,6 @@ def addInceptingArgs(parser): help='only allow establishment events in KEL for this prefix') parser.add_argument('--data', '-d', default=None, required=False, action="store", help='Anchor data, \'@\' allowed',) + parser.add_argument('--delpre', '-di', default=None, required=False, action="store", + help='Delegator AID for delegated identfiers',) diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index fcbdb053b..8d9e4c194 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -28,7 +28,7 @@ class Anchorer(doing.DoDoer): """ - def __init__(self, hby, proxy=None, **kwa): + def __init__(self, hby, proxy=None, auths=None, **kwa): """ For the current event, gather the current set of witnesses, send the event, gather all receipts and send them to all other witnesses @@ -45,11 +45,12 @@ def __init__(self, hby, proxy=None, **kwa): self.witq = agenting.WitnessInquisitor(hby=hby) self.witDoer = agenting.Receiptor(hby=self.hby) self.proxy = proxy + self.auths = auths super(Anchorer, self).__init__(doers=[self.witq, self.witDoer, self.postman, doing.doify(self.escrowDo)], **kwa) - def delegation(self, pre, sn=None, proxy=None): + def delegation(self, pre, sn=None, proxy=None, auths=None): if pre not in self.hby.habs: raise kering.ValidationError(f"{pre} is not a valid local AID for delegation") @@ -63,13 +64,14 @@ def delegation(self, pre, sn=None, proxy=None): raise kering.ValidationError(f"delegator {delpre} not found, unable to process delegation") sn = sn if sn is not None else hab.kever.sner.num + self.auths = auths if auths is not None else self.auths # load the event and signatures evt = hab.makeOwnEvent(sn=sn) # Send exn message for notification purposes srdr = serdering.SerderKERI(raw=evt) - self.witDoer.msgs.append(dict(pre=pre, sn=srdr.sn)) + self.witDoer.msgs.append(dict(pre=pre, sn=srdr.sn, auths=self.auths)) self.hby.db.dpwe.pin(keys=(srdr.pre, srdr.said), val=srdr) def complete(self, prefixer, seqner, saider=None): diff --git a/src/keri/app/httping.py b/src/keri/app/httping.py index d3ac5e8ac..346b870e0 100644 --- a/src/keri/app/httping.py +++ b/src/keri/app/httping.py @@ -233,6 +233,9 @@ def request(self, method, url, body=None, headers=None): print(f"error establishing client connection={e}") return None + if hasattr(body, "encode"): + body = body.encode("utf-8") + client.request( method=method, path=f"{purl.path}?{purl.query}", diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 32dc0b190..4dedb432a 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -971,7 +971,7 @@ def __iter__(self): def __next__(self): if self.iter is None: if self.cues: - cue = self.cues.pull() # self.cues.popleft() + cue = self.cues.pull() serder = cue["serder"] if serder.said == self.said: kin = cue["kin"] diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 5bea6584b..fc2f51009 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -989,8 +989,8 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, code = f"{s}{code[1:hs]}" ss = 4 else: - raise InvalidVarRawSizeError(f"Unsupported raw size for " - f"{code=}.") + raise InvalidVarRawSizeError(f"Unsupported raw size for large " + f"{code=}. {size} <= {64 ** 4 - 1}") else: raise InvalidVarRawSizeError(f"Unsupported variable raw size " f"{code=}.") @@ -3926,8 +3926,10 @@ class CounterCodex: SadPathSigGroups: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content RootSadPathSigGroups: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples PathedMaterialGroup: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) + BigPathedMaterialGroup: str = '-0L' # Composed Grouped Pathed Material Quadlet (4 char each) AttachmentGroup: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) BigAttachmentGroup: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) + ESSRPayloadGroup: str = '-Z' # ESSR Payload Group, dig of content+Texter group KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Protocol Stack CESR Version def __iter__(self): @@ -3995,7 +3997,9 @@ class Counter: '-J': Sizage(hs=2, ss=2, fs=4, ls=0), '-K': Sizage(hs=2, ss=2, fs=4, ls=0), '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), } diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index cd14e01cd..404b9b69b 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -66,8 +66,10 @@ class CounterCodex_1_0(MapDom): SadPathSigGroups: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content RootSadPathSigGroups: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples PathedMaterialGroup: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) + BigPathedMaterialGroup: str = '-0L' # Composed Grouped Pathed Material Quadlet (4 char each) AttachmentGroup: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) BigAttachmentGroup: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) + ESSRPayloadGroup: str = '-Z' # ESSR Payload Group, dig of content+Texter group KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Protocol Stack CESR Version @@ -345,6 +347,7 @@ class Counter: '-J': Sizage(hs=2, ss=2, fs=4, ls=0), '-K': Sizage(hs=2, ss=2, fs=4, ls=0), '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), '-V': Sizage(hs=2, ss=2, fs=4, ls=0), '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 6b4e743f5..dd9cb2244 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5065,8 +5065,6 @@ def processEscrowOutOfOrders(self): # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow @@ -5221,8 +5219,6 @@ def processEscrowPartialSigs(self): # still waiting on missing sigs or missing seal to validate if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow @@ -5393,8 +5389,6 @@ def processEscrowPartialWigs(self): # still waiting on missing witness sigs if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow @@ -5518,8 +5512,6 @@ def processEscrowUnverWitness(self): # only happens if we process above if logger.isEnabledFor(logging.DEBUG): # adds exception data logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow @@ -5691,8 +5683,6 @@ def processEscrowUnverNonTrans(self): # only happens if we process above if logger.isEnabledFor(logging.DEBUG): # adds exception data logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow @@ -5812,8 +5802,6 @@ def processEscrowDelegables(self): # still waiting on missing delegation approval if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow @@ -5930,8 +5918,6 @@ def processQueryNotFound(self): # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow @@ -6221,8 +6207,6 @@ def processEscrowUnverTrans(self): # only happens if we process above if logger.isEnabledFor(logging.DEBUG): # adds exception data logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow @@ -6355,8 +6339,6 @@ def processEscrowDuplicitous(self): # still can't determine if duplicitous if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than likely duplicitous so remove from escrow diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index 9273f6255..4faa7f36a 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -8,7 +8,7 @@ import logging from .coring import (Ilks, CtrDex, Counter, Seqner, Cigar, - Dater, Verfer, Prefixer, Saider, Pather) + Dater, Verfer, Prefixer, Saider, Pather, Texter) from .indexing import (Siger, ) from . import serdering from .. import help @@ -445,7 +445,7 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, if logger.isEnabledFor(logging.DEBUG): logger.exception("Parser msg non-extraction error: %s", ex) else: - logger.error("Parser msg non-extraction error: %s", ex) + logger.exception("Parser msg non-extraction error: %s", ex) yield return True @@ -530,7 +530,7 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery msg non-extraction error: %s", ex.args[0]) else: - logger.error("Kevery msg non-extraction error: %s", ex.args[0]) + logger.exception("Kevery msg non-extraction error: %s", ex.args[0]) finally: done = True @@ -620,7 +620,7 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, if logger.isEnabledFor(logging.DEBUG): logger.exception("Parser msg non-extraction error: %s", ex.args[0]) else: - logger.error("Parser msg non-extraction error: %s", ex.args[0]) + logger.exception("Parser msg non-extraction error: %s", ex.args[0]) yield return True # should never return @@ -721,6 +721,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, # List of tuples from extracted SAD path sig groups from non-trans identifiers sadcigs = [] # each converted group is path plus list of non-trans sigs pathed = [] # grouped attachments targetting a subpath + essrs = [] # group texter pipelined = False # all attachments in one big pipeline counted group # extract and deserialize attachments try: # catch errors here to flush only counted part of stream @@ -919,16 +920,6 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, else: sadcigs.append(sigs) - elif ctr.code == CtrDex.SadPathSigGroups: - for code, sigs in self._sadPathSigGroup(ctr=ctr, - ims=ims, - cold=cold, - pipelined=pipelined): - if code == CtrDex.TransIdxSigGroups: - sadtsgs.append(sigs) - else: - sadcigs.append(sigs) - elif ctr.code == CtrDex.PathedMaterialGroup: # pathed ctr? # compute pipelined attached group size based on txt or bny pags = ctr.count * 4 if cold == Colds.txt else ctr.count * 3 @@ -939,6 +930,25 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, del ims[:pags] # strip off from ims pathed.append(pims) + elif ctr.code == CtrDex.BigPathedMaterialGroup: # pathed ctr? + # compute pipelined attached group size based on txt or bny + pags = ctr.count * 4 if cold == Colds.txt else ctr.count * 3 + while len(ims) < pags: # wait until rx full pipelned group + yield + + pims = ims[:pags] # copy out substream pipeline group + del ims[:pags] # strip off from ims + pathed.append(pims) + + elif ctr.code == CtrDex.ESSRPayloadGroup: + for i in range(ctr.count): + texter = yield from self._extractor(ims, + klas=Texter, + cold=cold, + abort=pipelined) + essrs.append(texter) + + else: raise kering.UnexpectedCountCodeError("Unsupported count" " code={}.".format(ctr.code)) @@ -1079,6 +1089,9 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, if pathed: args["pathed"] = pathed + if essrs: + args["essrs"] = essrs + try: if cigars: exc.processEvent(cigars=cigars, **args) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 312bc4b5b..9c46826d0 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -427,8 +427,8 @@ class Serder: Ilks.bar: FieldDom(alls=dict(v='', t='',d='', dt='', r='', a=[]), saids={Saids.d: DigDex.Blake3_256}), - Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", p="", - dt='', r='',q={}, a=[], e={}), + Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", rp="", + p="", dt='', r='',q={}, a=[], e={}), saids={Saids.d: DigDex.Blake3_256}), Ilks.vcp: FieldDom(alls=dict(v='', t='',d='', i='', ii='', s='0', c=[], bt='0', b=[], n=''), diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 0a1ec1f0a..d911eeff2 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1063,6 +1063,8 @@ def reopen(self, **kwa): # TODO: clean self.epath = subing.IoSetSuber(db=self, subkey=".epath") + self.essrs = subing.CesrIoSetSuber(db=self, subkey=".essrs", klass=coring.Texter) + # accepted signed 12-word challenge response exn messages keys by prefix of signer # TODO: clean self.chas = subing.CesrIoSetSuber(db=self, subkey='chas.', klas=coring.Saider) diff --git a/src/keri/peer/exchanging.py b/src/keri/peer/exchanging.py index fcbefa8a8..ea96b7d41 100644 --- a/src/keri/peer/exchanging.py +++ b/src/keri/peer/exchanging.py @@ -71,6 +71,7 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): route = serder.ked["r"] sender = serder.ked["i"] pathed = kwargs["pathed"] if "pathed" in kwargs else [] + essrs = kwargs["essrs"] if "essrs" in kwargs else [] behavior = self.routes[route] if route in self.routes else None if tsgs is not None: @@ -110,6 +111,8 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): " for evt = {}.".format(serder.ked)) e = coring.Pather(path=["e"]) + + kwargs = dict() attachments = [] for p in pathed: pattach = bytearray(p) @@ -118,9 +121,23 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): np = pather.strip(e) attachments.append((np, pattach)) + kwargs["attachments"] = attachments + if essrs: + kwargs["essr"] = b''.join([texter.raw for texter in essrs]) + + if isinstance(serder.seals, str): + if 'essr' not in kwargs: + raise ValidationError("at least one essr attachment is required") + + essr = kwargs['essr'] + dig = serder.seals + diger = coring.Diger(qb64=dig) + if not diger.verify(ser=essr): + raise ValidationError(f"essr diger={diger.qb64} is invalid against content") + # Perform behavior specific verification, think IPEX chaining requirements try: - if not behavior.verify(serder=serder, attachments=attachments): + if not behavior.verify(serder=serder, **kwargs): logger.info(f"exn event for route {route} failed behavior verfication. said={serder.said}") logger.debug(f"event=\n{serder.pretty()}\n") return @@ -129,13 +146,13 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): logger.info(f"Behavior for {route} missing or does not have verify for said={serder.said}") logger.debug(f"event=\n{serder.pretty()}\n") - # Always persis events - self.logEvent(serder, pathed, tsgs, cigars) + # Always persist events + self.logEvent(serder, pathed, tsgs, cigars, essrs) self.cues.append(dict(kin="saved", said=serder.said)) # Execute any behavior specific handling, not sure if this should be different than verify try: - behavior.handle(serder=serder, attachments=attachments) + behavior.handle(serder=serder, **kwargs) except AttributeError: logger.info(f"Behavior for {route} missing or does not have handle for said={serder.said}") logger.debug(f"event=\n{serder.pretty()}\n") @@ -187,9 +204,13 @@ def processEscrowPartialSigned(self): tsgs.append((prefixer, seqner, saider, sigers)) pathed = [bytearray(p.encode("utf-8")) for p in self.hby.db.epath.get(keys=(dig,))] + essrs = [texter for texter in self.hby.db.essrs.get(keys=(dig,))] try: - self.processEvent(serder=serder, tsgs=tsgs, pathed=pathed) + kwargs = dict() + if essrs: + kwargs["essrs"] = essrs + self.processEvent(serder=serder, tsgs=tsgs, pathed=pathed, **kwargs) except MissingSignatureError as ex: if logger.isEnabledFor(logging.DEBUG): @@ -210,12 +231,13 @@ def processEscrowPartialSigned(self): "creder=%s", serder.said) logger.debug(f"event=\n{serder.pretty()}\n") - def logEvent(self, serder, pathed=None, tsgs=None, cigars=None): + def logEvent(self, serder, pathed=None, tsgs=None, cigars=None, essrs=None): dig = serder.said pdig = serder.ked['p'] pathed = pathed or [] tsgs = tsgs or [] cigars = cigars or [] + essrs = essrs or [] for prefixer, seqner, ssaider, sigers in tsgs: # iterate over each tsg quadkeys = (serder.said, prefixer.qb64, f"{seqner.sn:032x}", ssaider.qb64) @@ -226,6 +248,8 @@ def logEvent(self, serder, pathed=None, tsgs=None, cigars=None): saider = coring.Saider(qb64=serder.said) self.hby.db.epath.pin(keys=(dig,), vals=[bytes(p) for p in pathed]) + for texter in essrs: + self.hby.db.essrs.add(keys=(dig,), val=texter) if pdig: self.hby.db.erpy.pin(keys=(pdig,), val=saider) @@ -278,8 +302,9 @@ def complete(self, said): def exchange(route, - payload, sender, + payload=None, + diger=None, recipient=None, date=None, dig=None, @@ -292,6 +317,7 @@ def exchange(route, Parameters: route (str): to destination route of the message payload (list | dict): body of message to deliver to route + diger (Diger): qb64 digest of payload sender (str): qb64 AID of sender of the exn recipient (str) optional qb64 AID recipient of exn date (str): Iso8601 formatted date string to use for this request @@ -307,6 +333,7 @@ def exchange(route, ilk = eventing.Ilks.exn dt = date if date is not None else helping.nowIso8601() p = dig if dig is not None else "" + rp = recipient if recipient is not None else "" embeds = embeds if embeds is not None else {} e = dict() @@ -322,26 +349,38 @@ def exchange(route, pather = coring.Pather(path=["e", label]) pathed.extend(pather.qb64b) pathed.extend(atc) - end.extend(coring.Counter(code=coring.CtrDex.PathedMaterialGroup, - count=(len(pathed) // 4)).qb64b) + if len(pathed) // 4 < 4096: + end.extend(coring.Counter(code=coring.CtrDex.PathedMaterialGroup, + count=(len(pathed) // 4)).qb64b) + else: + end.extend(coring.Counter(code=coring.CtrDex.BigPathedMaterialGroup, + count=(len(pathed) // 4)).qb64b) end.extend(pathed) if e: e["d"] = "" _, e = coring.Saider.saidify(sad=e, label=coring.Saids.d) - attrs = dict( - ) + modifiers = modifiers if modifiers is not None else {} + + if diger is None: + attrs = dict() - if recipient is not None: - attrs['i'] = recipient + if recipient is not None: + attrs['i'] = recipient - attrs |= payload + attrs |= payload + + else: + attrs = diger.qb64 + # Attr field 'a' can be either a said or a nested block and the fields + # of the nested block can be saids of further nested block or nested blocks ked = dict(v=vs, t=ilk, d="", i=sender, + rp=rp, p=p, dt=dt, r=route, diff --git a/tests/app/test_delegating.py b/tests/app/test_delegating.py index 132a588c5..f5c83870f 100644 --- a/tests/app/test_delegating.py +++ b/tests/app/test_delegating.py @@ -129,16 +129,16 @@ def test_delegation_request(mockHelpingNowUTC): exn, atc = delegating.delegateRequestExn(hab=hab, delpre=delpre, evt=evt) assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAA' - b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAADECnBl' - b'0c14SVi7Keh__sd1PVhinSy-itPr33ZxvSjJYFastqXw9ZTFGNKsY6iALUk5xP3S' - b'399tJrPFe7PtuNAN') + b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAACzeUyP' + b'6__0oDca-Oiv2iGXKghBw_8sI4ZHyyeMedvz0iZIIQYqJd2Zt7cDHRh7xBGWI85J' + b'_oOixLET3mFZUu0A') assert exn.ked["r"] == '/delegate/request' - assert exn.saidb == b'EOiDc2wEmhHc7sbLG64y2gveCIRlFe4BuISaz0mlOuZz' + assert exn.saidb == b'EHPkcmdLGql9_1WD0wl0OalYk8PcF4HMMd7gGi-iqfSe' assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAA' - b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAADECnBl' - b'0c14SVi7Keh__sd1PVhinSy-itPr33ZxvSjJYFastqXw9ZTFGNKsY6iALUk5xP3S' - b'399tJrPFe7PtuNAN') + b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAACzeUyP' + b'6__0oDca-Oiv2iGXKghBw_8sI4ZHyyeMedvz0iZIIQYqJd2Zt7cDHRh7xBGWI85J' + b'_oOixLET3mFZUu0A') data = exn.ked["a"] assert data["delpre"] == delpre embeds = exn.ked['e'] diff --git a/tests/app/test_grouping.py b/tests/app/test_grouping.py index 6d0cbe59e..5fd0090a7 100644 --- a/tests/app/test_grouping.py +++ b/tests/app/test_grouping.py @@ -638,11 +638,11 @@ def test_multisig_incept(mockHelpingNowUTC): icp=hab.makeOwnEvent(sn=hab.kever.sn)) assert exn.ked["r"] == '/multisig/icp' - assert exn.saidb == b'EGDEBUZW--n-GqOOwRflzBeqoQsYWKMOQVU_1YglG-BL' + assert exn.saidb == b'EJ6Kl50IBicAa8zND_3wMSQ5itw555V7NKid9y1SKobe' assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAA' - b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAAC84-o2' - b'HKwKxhL1ttzykB9zuFaGV6OpQ05b1ZJYAeBFrR7kVON1aNjpLgQCG_0bY4FUiP7F' - b'GTVDrBjuFhbeDKAH-LAa5AACAA-e-icp-AABAACihaKoLnoXxRoxGbFfOy67YSh6' + b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAACL4cf7' + b'LxzKJgaJbb7wWHLuTfj3wManDV0SW7euFNZDiEhD1kUiP3_wtOIfqB_ZsEceE4oI' + b'gOOZwFROyrcf9ScB-LAa5AACAA-e-icp-AABAACihaKoLnoXxRoxGbFfOy67YSh6' b'UxtgjT2oxupnLDz2FlhevGJKTMObbdex9f0Hqob6uTavSJvsXf5RzitskkkC') data = exn.ked["a"] assert data["smids"] == aids @@ -662,11 +662,11 @@ def test_multisig_rotate(mockHelpingNowUTC): exn, atc = grouping.multisigRotateExn(ghab=ghab1, smids=ghab1.smids, rmids=ghab1.rmids, rot=rot) assert exn.ked["r"] == '/multisig/rot' - assert exn.saidb == b'EAE5RLyDb_3W8fUOzDOHWwpMAvHePaz8Jpz1XWL0b0lQ' + assert exn.saidb == b'EL4LeEHvTiOxs1UDNTv5qWxCYVYojdpEMfKI62O-UsPm' assert atc == (b'-FABEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba0AAAAAAAAAAAAAAA' - b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAACzqlzb' - b'yRO_M5Kp6nyN1K5JZ9s9D4UAreliSs4OiyCe_y9KnDeP65tub3DW5hUKlVQWnPv4' - b'TrOBHk2vJNOEJtAF') + b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAACH_qI1' + b'JebS_iehZT6XmvxylpOy2hS2BjO41e4mNmscSBdun2MyGk82SC-rHfQfvDJZlRRw' + b'NhLw-pKKKxql8wUF') data = exn.ked["a"] assert data["smids"] == ghab1.smids @@ -681,11 +681,11 @@ def test_multisig_interact(mockHelpingNowUTC): ixn=ixn) assert exn.ked["r"] == '/multisig/ixn' - assert exn.saidb == b'EGQ_DqGlSBx2MKJfHr6liXAngFpQ0UCtV1cdVMUtJHdN' + assert exn.saidb == b'EDF8o6SK-s2jxUVnlGtqAVtXTF-wyZ26c0dUsS5p766q' assert atc == (b'-FABEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba0AAAAAAAAAAAAAAA' - b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAAB3yX6b' - b'EXb8N63PKaMaFqijZVT5TqVtoO8q1BFnoJW3rDkNuJ9lEMpEN-44HKGtvniWZ6-d' - b'CVPS4fsEXKZAKGkB-LAa5AACAA-e-ixn-AABAABG58m7gibjdrQ8YU-8WQ8A70nc' + b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAABFfU5s' + b'o86inNogCPN7Ko8WXvkMKeiUKPScQ3FYrVmngNpVmW8xmhOTfixuWFlLcQPjEf3b' + b'RQhvNvx7azcI_vwB-LAa5AACAA-e-ixn-AABAABG58m7gibjdrQ8YU-8WQ8A70nc' b'tYekYr3xdfZ5WgDQOD0bb9pI7SuuaJvzfAQisLAYQnztA82pAo1Skhf1vQwD') data = exn.ked["a"] assert data["smids"] == ghab1.smids @@ -701,11 +701,11 @@ def test_multisig_registry_incept(mockHelpingNowUTC, mockCoringRandomNonce): usage="Issue vLEI Credentials") assert exn.ked["r"] == '/multisig/vcp' - assert exn.saidb == b'EJN27EYqgxTS2NCHdnjE-CU0UikV5a1Cw5OZdv-g0jQ6' + assert exn.saidb == b'EBum6f9SwkUUjQTl_vDplKs7L-shzQT6fS5jJlzdP9PP' assert atc == (b'-FABEDEf72ZZ9mhpT1Xz-_YkXl7cg93sjZUFLIsxaFNTbXQO0AAAAAAAAAAAAAAA' - b'AAAAAAAAEDEf72ZZ9mhpT1Xz-_YkXl7cg93sjZUFLIsxaFNTbXQO-AABAABsPiWf' - b'UE9L7D9KBVOYMg1rt88gK9DBkiBYb21xMR0YH7sCT0hqwX9y8CM5Y6jQwzz3NqqN' - b'-aJWShzz1mbtb5AG-LAa5AACAA-e-anc-AABAABXlwkzbp_tC4MEbx1Uyny1o7dB' + b'AAAAAAAAEDEf72ZZ9mhpT1Xz-_YkXl7cg93sjZUFLIsxaFNTbXQO-AABAAAS5k5D' + b'9jH0rbS6jCtZIPyTJRS2l8TZBnChwG8try3kZUJuiAPoBLo7UuhFYmZlpTZ6MfSg' + b'cDS7XNg0ETj6L3QF-LAa5AACAA-e-anc-AABAABXlwkzbp_tC4MEbx1Uyny1o7dB' b'GHrYjU3u90Mhv2GtrIGG-7va1jZnlXef2R_LM4TRN8_XjmpLv1skcJaM90UB') data = exn.ked["a"] assert data == {'gid': 'EEVG5a8c88Fg9vH-6zQP6gJdc4LxVbUTRydx-JhpDcob', diff --git a/tests/app/test_oobiing.py b/tests/app/test_oobiing.py index 2e81a3050..b9280f7d9 100644 --- a/tests/app/test_oobiing.py +++ b/tests/app/test_oobiing.py @@ -53,19 +53,20 @@ def test_oobi_share(mockHelpingNowUTC): oobi="http://127.0.0.1/oobi") assert exn.ked == {'a': {'dest': 'EO2kxXW0jifQmuPevqg6Zpi3vE-WYoj65i_XhpruWtOg', 'oobi': 'http://127.0.0.1/oobi'}, - 'd': 'EMAhEMPbBU2B-Ha-yLxMEZk49KHYkzZgMv9aZS8gDl1m', + 'd': 'EII7EvdWFqv0jkjRv10t01zAUcRYbjVhZ_yo3VPZEbpS', 'dt': '2021-01-01T00:00:00.000000+00:00', 'e': {}, 'i': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', 'p': '', 'q': {}, 'r': '/oobis', + 'rp': '', 't': 'exn', - 'v': 'KERI10JSON00012e_'} + 'v': 'KERI10JSON000136_'} assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAA' - b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAACsgmsu' - b'VJoY5a7vicZQ7pT_MZqCe-0psgReRxyoBfFaAPxZ7Vss2eteFuvwDWBeyKc1B-yc' - b'p-2QZzIZJ94_9hIP') + b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAABdw3eS' + b'w_7BW2o3z1ufxxs1CPgX1TgtJzn-MxvMjLYTidUd8KSxNKbPU9M3A4orYJDMGMIz' + b'habHJmKA4ZIGbcgK') def test_oobiery(): diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 0933c0e89..e8eee29b4 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -2179,7 +2179,9 @@ def test_counter(): 'SadPathSigGroups': '-J', 'RootSadPathSigGroups': '-K', 'PathedMaterialGroup': '-L', + 'BigPathedMaterialGroup': '-0L', 'AttachmentGroup': '-V', + 'ESSRPayloadGroup': '-Z', 'BigAttachmentGroup': '-0V', 'KERIACDCGenusVersion': '--AAA', } @@ -2215,7 +2217,9 @@ def test_counter(): '-J': Sizage(hs=2, ss=2, fs=4, ls=0), '-K': Sizage(hs=2, ss=2, fs=4, ls=0), '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), '-V': Sizage(hs=2, ss=2, fs=4, ls=0), + '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) } @@ -2333,10 +2337,10 @@ def test_counter(): assert counter.qb64 == qsc assert counter.qb2 == qscb2 - # test with big codes index=1024 - count = 1024 + # test with big codes index=1024000 + count = 1024000 qsc = CtrDex.BigAttachmentGroup + intToB64(count, l=5) - assert qsc == '-0VAAAQA' + assert qsc == '-0VAD6AA' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) @@ -2383,6 +2387,42 @@ def test_counter(): test = counter._binfil() assert test == qb2 + # Test limits of PathedMaterialGroup + count = 255 + qsc = CtrDex.PathedMaterialGroup + intToB64(count, l=2) + assert qsc == '-LD_' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.PathedMaterialGroup, count=count) + assert counter.code == CtrDex.PathedMaterialGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + counter = Counter(qb64='-L__') + assert counter.count == 4095 + + with pytest.raises(kering.InvalidVarIndexError): + Counter(code=CtrDex.PathedMaterialGroup, count=4096) # Too big + + # Test BigPathedMaterialGroup + # test with big codes index=1024000 + count = 1024000 + qsc = CtrDex.BigPathedMaterialGroup + intToB64(count, l=5) + assert qsc == '-0LAD6AA' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.BigPathedMaterialGroup, count=count) + assert counter.code == CtrDex.BigPathedMaterialGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + + # Test with strip # create code manually count = 1 diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 7887a610f..e94d18e5e 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -74,7 +74,9 @@ def test_codexes_tags(): 'SadPathSigGroups': '-J', 'RootSadPathSigGroups': '-K', 'PathedMaterialGroup': '-L', + 'BigPathedMaterialGroup': '-0L', 'AttachmentGroup': '-V', + 'ESSRPayloadGroup': '-Z', 'BigAttachmentGroup': '-0V', 'KERIACDCGenusVersion': '--AAA' } @@ -151,8 +153,10 @@ def test_codexes_tags(): 'SadPathSigGroups': 'SadPathSigGroups', 'RootSadPathSigGroups': 'RootSadPathSigGroups', 'PathedMaterialGroup': 'PathedMaterialGroup', + 'BigPathedMaterialGroup': 'BigPathedMaterialGroup', 'AttachmentGroup': 'AttachmentGroup', 'BigAttachmentGroup': 'BigAttachmentGroup', + 'ESSRPayloadGroup': 'ESSRPayloadGroup', 'KERIACDCGenusVersion': 'KERIACDCGenusVersion' } @@ -353,6 +357,7 @@ def test_counter_class(): '-J': Sizage(hs=2, ss=2, fs=4, ls=0), '-K': Sizage(hs=2, ss=2, fs=4, ls=0), '-L': Sizage(hs=2, ss=2, fs=4, ls=0), + '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), '-V': Sizage(hs=2, ss=2, fs=4, ls=0), '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) @@ -663,10 +668,10 @@ def test_counter_v1(): assert counter.qb2 == qscb2 assert counter.version == Vrsn_1_0 - # test with big codes index=1024 - count = 1024 + # test with big codes index=100024000 + count = 100024000 qsc = CtrDex.BigAttachmentGroup + intToB64(count, l=5) - assert qsc == '-0VAAAQA' + assert qsc == '-0VF9j7A' qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) @@ -723,6 +728,66 @@ def test_counter_v1(): test = counter._binfil() assert test == qb2 + # test BigPathedMaterialGroup with big codes index=100024000 + count = 100024000 + qsc = CtrDex.BigPathedMaterialGroup + intToB64(count, l=5) + assert qsc == '-0LF9j7A' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.BigPathedMaterialGroup, count=count, gvrsn=Vrsn_1_0) + assert counter.code == CtrDex.BigPathedMaterialGroup + assert counter.tag == AllTags.BigPathedMaterialGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str + assert counter.code == CtrDex.BigPathedMaterialGroup + assert counter.tag == AllTags.BigPathedMaterialGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes + assert counter.code == CtrDex.BigPathedMaterialGroup + assert counter.tag == AllTags.BigPathedMaterialGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 + assert counter.code == CtrDex.BigPathedMaterialGroup + assert counter.tag == AllTags.BigPathedMaterialGroup + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + # Test ._bexfil + counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # + code = counter.code + count = counter.count + qb2 = counter.qb2 + counter._bexfil(qb2) + assert counter.code == code + assert counter.tag == AllTags.BigPathedMaterialGroup + assert counter.count == count + assert counter.qb64 == qsc + assert counter.qb2 == qb2 + assert counter.version == Vrsn_1_0 + + # Test ._binfil + test = counter._binfil() + assert test == qb2 + # Test with strip # create code manually count = 1 diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 5ca21c30c..ef7cbd07e 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -2225,20 +2225,20 @@ def test_serderkeri_exn(): # Test KERI JSON with makify defaults for self bootstrap with ilk ixn serder = SerderKERI(makify=True, ilk=kering.Ilks.exn) # make with defaults - assert serder.sad == {'v': 'KERI10JSON000088_', - 't': 'exn', - 'd': 'EMuAoRSE4zREKKYyvuNeYCDM9_MwPQIh1WL0cFC4e-bU', - 'i': '', - 'p': '', - 'dt': '', - 'r': '', - 'q': {}, - 'a': [], - 'e': {}} - - assert serder.raw == (b'{"v":"KERI10JSON000088_","t":"exn",' - b'"d":"EMuAoRSE4zREKKYyvuNeYCDM9_MwPQIh1WL0' - b'cFC4e-bU","i":"","p":"","dt":"","r":"","q":{},"a":[],"e":{}}') + assert serder.sad == {'a': [], + 'd': 'EPx9pShQTfv2FoISZJAZ4dlUcekG8-CSkgJh0i0q_iJn', + 'dt': '', + 'e': {}, + 'i': '', + 'p': '', + 'q': {}, + 'r': '', + 'rp': '', + 't': 'exn', + 'v': 'KERI10JSON000090_'} + + assert serder.raw == (b'{"v":"KERI10JSON000090_","t":"exn","d":"EPx9pShQTfv2FoISZJAZ4dlUcekG8-CSkgJh' + b'0i0q_iJn","i":"","rp":"","p":"","dt":"","r":"","q":{},"a":[],"e":{}}') diff --git a/tests/peer/test_exchanging.py b/tests/peer/test_exchanging.py index 6cb20d739..5e7e96949 100644 --- a/tests/peer/test_exchanging.py +++ b/tests/peer/test_exchanging.py @@ -3,10 +3,15 @@ tests.peer.test_exchanging module """ +import pysodium +from base64 import urlsafe_b64encode as encodeB64 +from base64 import urlsafe_b64decode as decodeB64 + from keri import core -from keri.core import coring, serdering +from keri.core import coring, serdering, MtrDex, parsing from keri.app import habbing, forwarding, storing, signing +from keri.core.coring import CtrDex from keri.peer import exchanging from keri.vdr.eventing import incept @@ -46,6 +51,51 @@ def test_nesting(): assert pathed == {} +def test_essrs(): + with habbing.openHab(name="sid", base="test", salt=b'0123456789abcdef') as (hby, hab), \ + habbing.openHab(name="rec", base="test", salt=b'0123456789abcdef') as (recHby, recHab): + + ims = hab.makeOwnInception() + parsing.Parser().parse(ims=ims, kvy=recHby.kvy) + # create the test message with essr attachment + msg = "This is a test message that must be secured" + rkever = recHab.kever + pubkey = pysodium.crypto_sign_pk_to_box_pk(rkever.verfers[0].raw) + raw = pysodium.crypto_box_seal(msg.encode("utf-8"), pubkey) + + texter = coring.Texter(raw=raw) + diger = coring.Diger(ser=raw, code=MtrDex.Blake3_256) + essr, _ = exchanging.exchange(route='/essr/req', sender=hab.pre, diger=diger, + modifiers=dict(src=hab.pre, dest=recHab.pre)) + ims = hab.endorse(serder=essr, pipelined=False) + ims.extend(coring.Counter(code=CtrDex.ESSRPayloadGroup, count=1).qb64b) + ims.extend(texter.qb64b) + + exc = exchanging.Exchanger(hby=recHby, handlers=[]) + parsing.Parser().parse(ims=ims, kvy=recHby.kvy, exc=exc) + + # Pull the logged exn and verify the attributes digest matches the attachment + serder = recHby.db.exns.get(keys=(essr.said,)) + assert serder.ked['a'] == diger.qb64 + + # Pull the logged ESSR attachment and verify it is the one attached + texter = recHby.db.essrs.get(keys=(serder.said,)) + raw = recHab.decrypt(texter[0].raw) + assert raw.decode("utf-8") == msg + + # Test with invalid diger + diger = coring.Diger(qb64="EKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o") + essr, _ = exchanging.exchange(route='/essr/req', sender=hab.pre, diger=diger, + modifiers=dict(src=hab.pre, dest=recHab.pre)) + ims = hab.endorse(serder=essr, pipelined=False) + ims.extend(coring.Counter(code=CtrDex.ESSRPayloadGroup, count=1).qb64b) + ims.extend(texter[0].qb64b) + + parsing.Parser().parse(ims=ims, kvy=recHby.kvy, exc=exc) + assert recHby.db.exns.get(keys=(essr.said,)) is None + + + def test_exchanger(): with habbing.openHab(name="sid", base="test", salt=b'0123456789abcdef') as (hby, hab), \ habbing.openHab(name="rec", base="test", salt=b'0123456789abcdef') as (recHby, recHab): @@ -101,26 +151,26 @@ def test_hab_exchange(mockHelpingNowUTC): data = dict(m="Let's create a registry") msg = hab.exchange(route="/multisig/registry/incept", recipient="", payload=data, embeds=embeds) - assert msg == (b'{"v":"KERI10JSON000398_","t":"exn","d":"ECcmfGnlqnc5-1_oXNpbfowv' - b'RsEa-V8tfeKmQDRJJ50i","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2Q' - b'V8dDjI3","p":"","dt":"2021-01-01T00:00:00.000000+00:00","r":"/mu' - b'ltisig/registry/incept","q":{},"a":{"i":"","m":"Let\'s create a r' - b'egistry"},"e":{"vcp":{"v":"KERI10JSON00010f_","t":"vcp","d":"EI6' - b'hBlgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB","i":"EI6hBlgkWoJgkZyf' - b'LW35_UyM4nIK44OgsSwFR_WOfvVB","ii":"EIaGMMWJFPmtXznY1IIiKDIrg-vI' - b'yge6mBl2QV8dDjI3","s":"0","c":[],"bt":"0","b":[],"n":"AH3-1EZWXU' - b'9I0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},"ixn":{"v":"KERI10JSON00013' - b'8_","t":"ixn","d":"EFuFnevyDFfpWG6il-6Qcv0ne0ZIItLwanCwI-SU8A9j"' - b',"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","s":"1","p":' - b'"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","a":[{"i":"EI6hBl' - b'gkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB","s":0,"d":"EI6hBlgkWoJgk' - b'ZyfLW35_UyM4nIK44OgsSwFR_WOfvVB"}]},"d":"EL5Nkm6T7HG_0GW6uwqYSZw' - b'lH23khtXvsVE-dq8eO_eE"}}-FABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2' - b'QV8dDjI30AAAAAAAAAAAAAAAAAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6' - b'mBl2QV8dDjI3-AABAACahD6g7IwjUyQRyGUPGLvlr5-DsvLxeJtCUVIIECYfAQ_q' - b'p3Z2pe__HRqIl-NrUv85oQrZBm0kpKn8LBQtQfkO-LAa5AACAA-e-ixn-AABAADp' - b'rTWp4llIzVzBM7VVsDOgXVJdoiVXutsWJEbDJ2pMdjXjNi1xKALBSZ1ZgRoUsD--' - b'LgUQkXIdjLoQ19XPvJMJ') + assert msg == (b'{"v":"KERI10JSON0003a0_","t":"exn","d":"ELkHqph-Tj4LGHYfFfoVmJJo' + b'09S2gp6ci8rK96upIAKE","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2Q' + b'V8dDjI3","rp":"","p":"","dt":"2021-01-01T00:00:00.000000+00:00",' + b'"r":"/multisig/registry/incept","q":{},"a":{"i":"","m":"Let\'s cr' + b'eate a registry"},"e":{"vcp":{"v":"KERI10JSON00010f_","t":"vcp",' + b'"d":"EI6hBlgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB","i":"EI6hBlgk' + b'WoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB","ii":"EIaGMMWJFPmtXznY1IIi' + b'KDIrg-vIyge6mBl2QV8dDjI3","s":"0","c":[],"bt":"0","b":[],"n":"AH' + b'3-1EZWXU9I0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},"ixn":{"v":"KERI10J' + b'SON000138_","t":"ixn","d":"EFuFnevyDFfpWG6il-6Qcv0ne0ZIItLwanCwI' + b'-SU8A9j","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","s":' + b'"1","p":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","a":[{"i"' + b':"EI6hBlgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB","s":0,"d":"EI6hB' + b'lgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB"}]},"d":"EL5Nkm6T7HG_0GW' + b'6uwqYSZwlH23khtXvsVE-dq8eO_eE"}}-FABEIaGMMWJFPmtXznY1IIiKDIrg-vI' + b'yge6mBl2QV8dDjI30AAAAAAAAAAAAAAAAAAAAAAAEIaGMMWJFPmtXznY1IIiKDIr' + b'g-vIyge6mBl2QV8dDjI3-AABAAB-teJc_7zot5TAZT6lQi2-GlBzMHXICvt3tIYo' + b'Po2gYXF7PpWDozo3y3wVW9mgHln-1DvQlqn9Aip1YnBgKUQB-LAa5AACAA-e-ixn' + b'-AABAADprTWp4llIzVzBM7VVsDOgXVJdoiVXutsWJEbDJ2pMdjXjNi1xKALBSZ1Z' + b'gRoUsD--LgUQkXIdjLoQ19XPvJMJ') exn = serdering.SerderKERI(raw=msg) @@ -143,27 +193,27 @@ def test_hab_exchange(mockHelpingNowUTC): data = dict(m="Lets create this registry instead") msg = hab2.exchange(route="/multisig/registry/incept", payload=data, recipient="", dig=exn.said, embeds=embeds) - assert msg == (b'{"v":"KERI10JSON0003ce_","t":"exn","d":"EEMxkjO9HzZoekfzmjrkE19y' - b'pU259apUWuY7alFu_GmE","i":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVli' - b'I61Bcc2","p":"ECcmfGnlqnc5-1_oXNpbfowvRsEa-V8tfeKmQDRJJ50i","dt"' - b':"2021-01-01T00:00:00.000000+00:00","r":"/multisig/registry/ince' - b'pt","q":{},"a":{"i":"","m":"Lets create this registry instead"},' - b'"e":{"vcp":{"v":"KERI10JSON00010f_","t":"vcp","d":"EB5mts6qrWOZr' - b'xjma6lSTjAdPZ0NSHM1HC3IndbS_giB","i":"EB5mts6qrWOZrxjma6lSTjAdPZ' - b'0NSHM1HC3IndbS_giB","ii":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI' - b'61Bcc2","s":"0","c":[],"bt":"0","b":[],"n":"AH3-1EZWXU9I0fv3Iz_9' - b'ZIhjj13JO7u4GNFYC3-l8_K-"},"ixn":{"v":"KERI10JSON000138_","t":"i' - b'xn","d":"EOek9JVKNeuW-5UNeHYCTDe70_GtvRwP672oWMNBJpA5","i":"EIRE' - b'QlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","s":"1","p":"EIREQlatU' - b'JODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","a":[{"i":"EB5mts6qrWOZrxjm' - b'a6lSTjAdPZ0NSHM1HC3IndbS_giB","s":0,"d":"EB5mts6qrWOZrxjma6lSTjA' - b'dPZ0NSHM1HC3IndbS_giB"}]},"d":"EM3gLTzQ9GmKd50Rlm_kiIkeYkxb004eo' - b'OsWahz70TqJ"}}-FABEIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc20A' - b'AAAAAAAAAAAAAAAAAAAAAAEIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bc' - b'c2-AABAAAxpwQLr9-D7hOZYHvvDB_ffo5sRgBf0NufowF0g_YMI1wdnttlYA2o_d' - b'wtK_WNbfh_iAytFw9nHZziCED13AwH-LAa5AACAA-e-ixn-AABAACaoxfQp5L_Gd' - b'0nKqJXMbLTXzkrJJDd8RFxWdTSesAMydUzmJQlGt0T9h8L7SwIrq8yBinj990PLJ' - b'Hl7sXmq04I') + assert msg == (b'{"v":"KERI10JSON0003d6_","t":"exn","d":"EPO_XC9nwSixqSoOvsHymFr-' + b'l3udclHBdOh4OUEqZ33P","i":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVli' + b'I61Bcc2","rp":"","p":"ELkHqph-Tj4LGHYfFfoVmJJo09S2gp6ci8rK96upIA' + b'KE","dt":"2021-01-01T00:00:00.000000+00:00","r":"/multisig/regis' + b'try/incept","q":{},"a":{"i":"","m":"Lets create this registry in' + b'stead"},"e":{"vcp":{"v":"KERI10JSON00010f_","t":"vcp","d":"EB5mt' + b's6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","i":"EB5mts6qrWOZrxjma6' + b'lSTjAdPZ0NSHM1HC3IndbS_giB","ii":"EIREQlatUJODbKogZfa3IqXZ90XdZA' + b'0qJMVliI61Bcc2","s":"0","c":[],"bt":"0","b":[],"n":"AH3-1EZWXU9I' + b'0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},"ixn":{"v":"KERI10JSON000138_' + b'","t":"ixn","d":"EOek9JVKNeuW-5UNeHYCTDe70_GtvRwP672oWMNBJpA5","' + b'i":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","s":"1","p":"E' + b'IREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","a":[{"i":"EB5mts6q' + b'rWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","s":0,"d":"EB5mts6qrWOZrxj' + b'ma6lSTjAdPZ0NSHM1HC3IndbS_giB"}]},"d":"EM3gLTzQ9GmKd50Rlm_kiIkeY' + b'kxb004eoOsWahz70TqJ"}}-FABEIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI' + b'61Bcc20AAAAAAAAAAAAAAAAAAAAAAAEIREQlatUJODbKogZfa3IqXZ90XdZA0qJM' + b'VliI61Bcc2-AABAADY5nUsBgL23ulcrTgkV09hSzktNHZSlEH1zmVpEggrGgQUq0' + b'tLQeOXztUFDxNQ4Kq2ddIYDVz6d_y0kkU3__YJ-LAa5AACAA-e-ixn-AABAACaox' + b'fQp5L_Gd0nKqJXMbLTXzkrJJDd8RFxWdTSesAMydUzmJQlGt0T9h8L7SwIrq8yBi' + b'nj990PLJHl7sXmq04I') # Test exn from non-transferable AID hab = hby.makeHab(name="test1", transferable=False) @@ -174,18 +224,18 @@ def test_hab_exchange(mockHelpingNowUTC): ) msg = hab.exchange(route="/multisig/registry/incept", payload=data, embeds=embeds, recipient="") - assert msg == (b'{"v":"KERI10JSON000263_","t":"exn","d":"ENRFAVDU_ZbcVpx6l6lrC5Mu' - b'UqHXfT3N9VjUkvU4t29S","i":"BJZ_LF61JTCCSCIw2Q4ozE2MsbRC4m-N6-tFV' - b'lCeiZPG","p":"","dt":"2021-01-01T00:00:00.000000+00:00","r":"/mu' - b'ltisig/registry/incept","q":{},"a":{"i":"","m":"Lets create this' - b' registry instead"},"e":{"vcp":{"v":"KERI10JSON00010f_","t":"vcp' - b'","d":"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","i":"EB5mts' - b'6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","ii":"EIREQlatUJODbKogZf' - b'a3IqXZ90XdZA0qJMVliI61Bcc2","s":"0","c":[],"bt":"0","b":[],"n":"' - b'AH3-1EZWXU9I0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},"d":"ENC6w8wUj-Gp' - b'_RpAJN5q4Lf00IHstzNLUvkh3ZvgHGP_"}}-CABBJZ_LF61JTCCSCIw2Q4ozE2Ms' - b'bRC4m-N6-tFVlCeiZPG0BCxLApuSnk1MF9IUq1RJNjVmr6s-fLwvP6aAPa0ag34t' - b'4G7EKKk-UFwy74-0StSlHcS8KBkN5ZbtuHvV9tXRqUJ-LAl5AACAA-e-vcp-CABB' - b'JZ_LF61JTCCSCIw2Q4ozE2MsbRC4m-N6-tFVlCeiZPG0BDjOC4j0Co6P0giMylR4' - b'7149eJ8Yf_hO-32_TpY77KMVCWCf0U8GuZPIN76R2zsyT_eARvS_zQsX1ebjl3PM' - b'P0D') + assert msg == (b'{"v":"KERI10JSON00026b_","t":"exn","d":"EMBm0p7fCIqJrP4Z-PBI-yEv' + b'Xin_-eY1dU4XTCM9ykRC","i":"BJZ_LF61JTCCSCIw2Q4ozE2MsbRC4m-N6-tFV' + b'lCeiZPG","rp":"","p":"","dt":"2021-01-01T00:00:00.000000+00:00",' + b'"r":"/multisig/registry/incept","q":{},"a":{"i":"","m":"Lets cre' + b'ate this registry instead"},"e":{"vcp":{"v":"KERI10JSON00010f_",' + b'"t":"vcp","d":"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","i"' + b':"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","ii":"EIREQlatUJ' + b'ODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","s":"0","c":[],"bt":"0","b":' + b'[],"n":"AH3-1EZWXU9I0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},"d":"ENC6' + b'w8wUj-Gp_RpAJN5q4Lf00IHstzNLUvkh3ZvgHGP_"}}-CABBJZ_LF61JTCCSCIw2' + b'Q4ozE2MsbRC4m-N6-tFVlCeiZPG0BB-sQs0WS9wsyuT4hXQD7rbczSfpnQz21wZG' + b'YucRkE0ynKy5draELEKBsckeD0Im1i-kIfMEdbY08YqVfSrEoAA-LAl5AACAA-e-' + b'vcp-CABBJZ_LF61JTCCSCIw2Q4ozE2MsbRC4m-N6-tFVlCeiZPG0BDjOC4j0Co6P' + b'0giMylR47149eJ8Yf_hO-32_TpY77KMVCWCf0U8GuZPIN76R2zsyT_eARvS_zQsX' + b'1ebjl3PMP0D') diff --git a/tests/vc/test_protocoling.py b/tests/vc/test_protocoling.py index f507f9fad..f6dde10bd 100644 --- a/tests/vc/test_protocoling.py +++ b/tests/vc/test_protocoling.py @@ -120,26 +120,26 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN apply0, apply0atc = protocoling.ipexApplyExn(sidHab, message="Please give me a credential", schema=schema, recp=redPre, attrs={}) - assert apply0.raw == (b'{"v":"KERI10JSON00016d_","t":"exn","d":"EIPeVB3u7L-mEKjhY6zIu5M7LErPwlUccxIN' - b'ddwhcFrH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"","dt":"20' - b'21-06-27T21:26:21.233257+00:00","r":"/ipex/apply","q":{},"a":{"m":"Please gi' - b've me a credential","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{' - b'},"i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},"e":{}}') + assert apply0.raw == (b'{"v":"KERI10JSON000175_","t":"exn","d":"EHVK5cO32UQJCkpK9RqRP_ONViK8u3JNXn73' + b'nJ8hdmXr","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"",' + b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/apply","q":{},"a":{"m":"P' + b'lease give me a credential","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1k' + b'C","a":{},"i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},"e":{}}') # No requirements for apply, except that its first, no `p` assert ipexhan.verify(serder=apply0) is True offer0, offer0atc = protocoling.ipexOfferExn(sidHab, "How about this", acdc=creder.raw, apply=apply0) - assert offer0.raw == (b'{"v":"KERI10JSON0002f0_","t":"exn","d":"EO4NXvOU-UpwwAR67txzKrFBHGAtDu9ehg8g' - b'Ic5haJy3","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EIPeVB3u7L' - b'-mEKjhY6zIu5M7LErPwlUccxINddwhcFrH","dt":"2021-06-27T21:26:21.233257+00:00",' - b'"r":"/ipex/offer","q":{},"a":{"m":"How about this"},"e":{"acdc":{"v":"ACDC10' - b'JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","i":"EMl4Rhu' - b'R_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXe' - b'w8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"' - b'EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T21:26:21.2332' - b'57+00:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"254900OP' - b'PU84GM83MG36"}},"d":"EOG-KWyllXlb2HVIuewN1YJAOT304PaSczyt3V5Z878S"}}') + assert offer0.raw == (b'{"v":"KERI10JSON0002f8_","t":"exn","d":"ENdVOCsP5Xz57qs1xa_msznozvBs6Ii0_JRo' + b'i6tp2NBu","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EH' + b'VK5cO32UQJCkpK9RqRP_ONViK8u3JNXn73nJ8hdmXr","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/offer","q":{},"a":{"m":"How about this"},"e":{"acdc":{"v"' + b':"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","i":' + b'"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJoAVHv' + b'5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a' + b'":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T21:26' + b':21.233257+00:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"' + b'254900OPPU84GM83MG36"}},"d":"EOG-KWyllXlb2HVIuewN1YJAOT304PaSczyt3V5Z878S"}}') # This should fail because it is not first and the apply isn't persisted yet assert ipexhan.verify(serder=offer0) is False @@ -166,26 +166,26 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # Let's see if we can spurn a message we previously accepted. spurn0, spurn0atc = protocoling.ipexSpurnExn(sidHab, "I reject you", spurned=apply0) - assert spurn0.raw == (b'{"v":"KERI10JSON00011d_","t":"exn","d":"ENWl0Dd-9idlgrpkFL2aA0wfnuGHzvT4bHCN' - b'G6C0WBZk","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EIPeVB3u7L' - b'-mEKjhY6zIu5M7LErPwlUccxINddwhcFrH","dt":"2021-06-27T21:26:21.233257+00:00",' - b'"r":"/ipex/spurn","q":{},"a":{"m":"I reject you"},"e":{}}') + assert spurn0.raw == (b'{"v":"KERI10JSON000125_","t":"exn","d":"EHijfrof83z7JeFR-wJO9Ptgl-PieQHhKC-F' + b'bZIDvGvM","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EH' + b'VK5cO32UQJCkpK9RqRP_ONViK8u3JNXn73nJ8hdmXr","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/spurn","q":{},"a":{"m":"I reject you"},"e":{}}') # This will fail, we've already responded with an offer assert ipexhan.verify(spurn0) is False # Now lets try an offer without a pointer back to a reply offer1, offer1atc = protocoling.ipexOfferExn(sidHab, "Here a credential offer", acdc=creder.raw) - assert offer1.raw == (b'{"v":"KERI10JSON0002cd_","t":"exn","d":"ELnRdb-cA_rLckt9jwlSY-1nnPKwzRc4Up_7' - b'tCdzI12n","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"","dt":"20' - b'21-06-27T21:26:21.233257+00:00","r":"/ipex/offer","q":{},"a":{"m":"Here a cr' - b'edential offer"},"e":{"acdc":{"v":"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoC' - b'eOtSsNOROLuqOz103V3-4E-ClXH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiU' - b'bl","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hat' - b'TNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn' - b'-wXn9S3iB","dt":"2021-06-27T21:26:21.233257+00:00","i":"EMl4RhuR_JxpiMd1N8DE' - b'JEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"254900OPPU84GM83MG36"}},"d":"EOG-KWyllXlb2H' - b'VIuewN1YJAOT304PaSczyt3V5Z878S"}}') + assert offer1.raw == (b'{"v":"KERI10JSON0002d5_","t":"exn","d":"EC8fiu3IoCex-7uhTskkEodJOiQYQpO61l3Y' + b'HCXWuuFi","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"",' + b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/offer","q":{},"a":{"m":"H' + b'ere a credential offer"},"e":{"acdc":{"v":"ACDC10JSON000197_","d":"EElymNmgs' + b'1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xy' + b'a8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcC' + b'nVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOs' + b'KKSbootn-wXn9S3iB","dt":"2021-06-27T21:26:21.233257+00:00","i":"EMl4RhuR_Jxp' + b'iMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"254900OPPU84GM83MG36"}},"d":"EOG-KW' + b'yllXlb2HVIuewN1YJAOT304PaSczyt3V5Z878S"}}') # Will work because it is starting a new conversation assert ipexhan.verify(serder=offer1) is True @@ -197,10 +197,11 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN assert serder.ked == offer1.ked agree, argeeAtc = protocoling.ipexAgreeExn(sidHab, "I'll accept that offer", offer=offer0) - assert agree.raw == (b'{"v":"KERI10JSON000127_","t":"exn","d":"EEtu1OAPj03IdbhMwsQtgbJJaWgG2tdYLJ_3' - b'BuJQekdP","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EO4NXvOU-U' - b'pwwAR67txzKrFBHGAtDu9ehg8gIc5haJy3","dt":"2021-06-27T21:26:21.233257+00:00",' - b'"r":"/ipex/agree","q":{},"a":{"m":"I\'ll accept that offer"},"e":{}}') + assert agree.raw == (b'{"v":"KERI10JSON00012f_","t":"exn","d":"ECU3UjnSY1_6Wl3aYEW19jaGiKuyFh_chIQQ' + b'w48bcT_X","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EN' + b'dVOCsP5Xz57qs1xa_msznozvBs6Ii0_JRoi6tp2NBu","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/agree","q":{},"a":{"m":"I\'ll accept that offer"},"e":' + b'{}}') # Can not create an agree without an offer, so this will pass since it has an offer that has no response assert ipexhan.verify(serder=agree) is True @@ -215,24 +216,24 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN anc = sidHab.makeOwnEvent(sn=2) grant0, grant0atc = protocoling.ipexGrantExn(sidHab, message="Here's a credential", recp=sidHab.pre, acdc=msg, iss=iss.raw, anc=anc) - assert grant0.raw == (b'{"v":"KERI10JSON000531_","t":"exn","d":"EC-EsfvXD2cNw4GNgCFp9UTl7_DgWUgk9Rnw' - b'eCxocaG-","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"","dt":"20' - b'21-06-27T21:26:21.233257+00:00","r":"/ipex/grant","q":{},"a":{"m":"Here\'' - b's a credential","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},"e":{"ac' - b'dc":{"v":"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-Cl' - b'XH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8' - b'PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gk' - b'k1kC","a":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-' - b'27T21:26:21.233257+00:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"' - b',"LEI":"254900OPPU84GM83MG36"}},"iss":{"v":"KERI10JSON0000ed_","t":"iss","d"' - b':"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H8-iLLAmIUPO","i":"EElymNmgs1u0mSaoCeOtSsN' - b'OROLuqOz103V3-4E-ClXH","s":"0","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w' - b'9mQUQ","dt":"2021-06-27T21:26:21.233257+00:00"},"anc":{"v":"KERI10JSON00013a' - b'_","t":"ixn","d":"EGhSHKIV5-nkeirdkqzqsvmeF1FXw_yH8NvPSAY1Rgyd","i":"EMl4Rhu' - b'R_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","s":"2","p":"ED1kkh5_ECYriK-j2gSv6Zjr' - b'5way88XVhwRCxk5zoTRG","a":[{"i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClX' - b'H","s":"0","d":"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H8-iLLAmIUPO"}]},"d":"EJ4-dl' - b'S9ktlb9HDWPYc0IJ2hS2NbvnCQBhUsFSkEPwIo"}}') + assert grant0.raw == (b'{"v":"KERI10JSON000539_","t":"exn","d":"ELnjKvzdgO57JZwG3giIScoOeTB0rLuevniv' + b'zRE5DbTE","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"",' + b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/grant","q":{},"a":{"m":"H' + b'ere\'s a credential","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},' + b'"e":{"acdc":{"v":"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103' + b'V3-4E-ClXH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VA' + b'F7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvaf' + b'X3bHQ9Gkk1kC","a":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"' + b'2021-06-27T21:26:21.233257+00:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8A' + b'N-tiUbl","LEI":"254900OPPU84GM83MG36"}},"iss":{"v":"KERI10JSON0000ed_","t":"' + b'iss","d":"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H8-iLLAmIUPO","i":"EElymNmgs1u0mSa' + b'oCeOtSsNOROLuqOz103V3-4E-ClXH","s":"0","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXe' + b'w8Yo6Z3w9mQUQ","dt":"2021-06-27T21:26:21.233257+00:00"},"anc":{"v":"KERI10JS' + b'ON00013a_","t":"ixn","d":"EGhSHKIV5-nkeirdkqzqsvmeF1FXw_yH8NvPSAY1Rgyd","i":' + b'"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","s":"2","p":"ED1kkh5_ECYriK-j' + b'2gSv6Zjr5way88XVhwRCxk5zoTRG","a":[{"i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V' + b'3-4E-ClXH","s":"0","d":"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H8-iLLAmIUPO"}]},"d"' + b':"EJ4-dlS9ktlb9HDWPYc0IJ2hS2NbvnCQBhUsFSkEPwIo"}}') assert ipexhan.verify(serder=grant0) is True @@ -245,10 +246,10 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # Let's see if we can spurn a message we previously accepted. spurn1, spurn1atc = protocoling.ipexSpurnExn(sidHab, "I reject you", spurned=grant0) - assert spurn1.raw == (b'{"v":"KERI10JSON00011d_","t":"exn","d":"EJA0LVnMxOdrOEArWudVtdonorCUa4nCIRgX' - b'vibhxdp3","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EC-EsfvXD2' - b'cNw4GNgCFp9UTl7_DgWUgk9RnweCxocaG-","dt":"2021-06-27T21:26:21.233257+00:00",' - b'"r":"/ipex/spurn","q":{},"a":{"m":"I reject you"},"e":{}}') + assert spurn1.raw == (b'{"v":"KERI10JSON000125_","t":"exn","d":"ELHFilyCgYvVq6vczgPQc7ZuRMs7Cv10U_h-' + b'2LIvJ1-Z","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EL' + b'njKvzdgO57JZwG3giIScoOeTB0rLuevnivzRE5DbTE","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/spurn","q":{},"a":{"m":"I reject you"},"e":{}}') smsg = bytearray(spurn1.raw) smsg.extend(spurn1atc) parsing.Parser().parse(ims=smsg, exc=sidExc) @@ -258,25 +259,25 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # Now we'll run a grant pointing back to the agree all the way to the database grant1, grant1atc = protocoling.ipexGrantExn(sidHab, message="Here's a credential", acdc=msg, iss=iss.raw, recp=sidHab.pre, anc=anc, agree=agree) - assert grant1.raw == (b'{"v":"KERI10JSON00055d_","t":"exn","d":"EBAhSTTx3--xDZZRoCIemBzxybFV_FoicvPf' - b'4hSfeyAJ","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EEtu1OAPj0' - b'3IdbhMwsQtgbJJaWgG2tdYLJ_3BuJQekdP","dt":"2021-06-27T21:26:21.233257+00:00",' - b'"r":"/ipex/grant","q":{},"a":{"m":"Here\'s a credential","i":"EMl4RhuR_Jx' - b'piMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},"e":{"acdc":{"v":"ACDC10JSON000197_","d"' - b':"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","i":"EMl4RhuR_JxpiMd1N8DEJEh' - b'TxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","' - b's":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EO9_6NattzsFiO8F' - b'w1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T21:26:21.233257+00:00","i":"E' - b'Ml4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"254900OPPU84GM83MG36"}},' - b'"iss":{"v":"KERI10JSON0000ed_","t":"iss","d":"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTT' - b'a_H8-iLLAmIUPO","i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","s":"0","' - b'ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","dt":"2021-06-27T21:26:21' - b'.233257+00:00"},"anc":{"v":"KERI10JSON00013a_","t":"ixn","d":"EGhSHKIV5-nkei' - b'rdkqzqsvmeF1FXw_yH8NvPSAY1Rgyd","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-' - b'tiUbl","s":"2","p":"ED1kkh5_ECYriK-j2gSv6Zjr5way88XVhwRCxk5zoTRG","a":[{"i":' - b'"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","s":"0","d":"ECUw7AdWEE3fvr7d' - b'gbFDXj0CEZuJTTa_H8-iLLAmIUPO"}]},"d":"EJ4-dlS9ktlb9HDWPYc0IJ2hS2NbvnCQBhUsFS' - b'kEPwIo"}}') + assert grant1.raw == (b'{"v":"KERI10JSON000565_","t":"exn","d":"EHAY_L6Ig4k_5qIw6uH-QZwBswWLfwVzCqaG' + b'sjtdnubK","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EC' + b'U3UjnSY1_6Wl3aYEW19jaGiKuyFh_chIQQw48bcT_X","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/grant","q":{},"a":{"m":"Here\'s a credential","i":"EMl' + b'4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},"e":{"acdc":{"v":"ACDC10JSON0001' + b'97_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","i":"EMl4RhuR_JxpiMd' + b'1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w' + b'9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EO9_6Nat' + b'tzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T21:26:21.233257+00:00' + b'","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"254900OPPU84GM83' + b'MG36"}},"iss":{"v":"KERI10JSON0000ed_","t":"iss","d":"ECUw7AdWEE3fvr7dgbFDXj' + b'0CEZuJTTa_H8-iLLAmIUPO","i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","' + b's":"0","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","dt":"2021-06-27T' + b'21:26:21.233257+00:00"},"anc":{"v":"KERI10JSON00013a_","t":"ixn","d":"EGhSHK' + b'IV5-nkeirdkqzqsvmeF1FXw_yH8NvPSAY1Rgyd","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn' + b'9Xya8AN-tiUbl","s":"2","p":"ED1kkh5_ECYriK-j2gSv6Zjr5way88XVhwRCxk5zoTRG","a' + b'":[{"i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","s":"0","d":"ECUw7AdW' + b'EE3fvr7dgbFDXj0CEZuJTTa_H8-iLLAmIUPO"}]},"d":"EJ4-dlS9ktlb9HDWPYc0IJ2hS2Nbvn' + b'CQBhUsFSkEPwIo"}}') assert ipexhan.verify(serder=grant1) is True gmsg = bytearray(grant1.raw) @@ -287,10 +288,11 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # And now the last... admit the granted credential to complete the full flow admit0, admit0atc = protocoling.ipexAdmitExn(sidHab, "Thanks for the credential", grant=grant1) - assert admit0.raw == (b'{"v":"KERI10JSON00012a_","t":"exn","d":"EDkQZpUOKKOzsQ50yoZxzLz8tDaChAid_llr' - b'QNvKVAdO","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","p":"EBAhSTTx3-' - b'-xDZZRoCIemBzxybFV_FoicvPf4hSfeyAJ","dt":"2021-06-27T21:26:21.233257+00:00",' - b'"r":"/ipex/admit","q":{},"a":{"m":"Thanks for the credential"},"e":{}}') + assert admit0.raw == (b'{"v":"KERI10JSON000132_","t":"exn","d":"EMxU5rfeqKnZzrbqnL7weXQGaC8Zum4qowj2' + b'eiL6GxqL","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EH' + b'AY_L6Ig4k_5qIw6uH-QZwBswWLfwVzCqaGsjtdnubK","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/admit","q":{},"a":{"m":"Thanks for the credential"},"e":{' + b'}}') assert ipexhan.verify(serder=admit0) is True amsg = bytearray(admit0.raw) From 3696623be8f010b402480efd7319ff722e7ebb07 Mon Sep 17 00:00:00 2001 From: Lance Date: Thu, 13 Jun 2024 10:05:12 -0400 Subject: [PATCH 181/418] fix grouping multisig parser to process events in local mode (#804) Signed-off-by: 2byrds <2byrds@gmail.com> --- src/keri/app/grouping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 5cf10dd63..af7faab22 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -662,7 +662,7 @@ def add(self, serder): ims.extend(atc) # ... and parse - self.psr.parse(ims=ims) + self.psr.parse(ims=ims, local=True) else: # Should we prod the user with another submission if we haven't already approved it? From 587f7e903a37f7f33b297b9fb6ed27106e4621aa Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Thu, 13 Jun 2024 09:51:44 -0700 Subject: [PATCH 182/418] Version bump to release 1.2.0-dev6 --- Makefile | 4 ++-- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 30bb20a0f..72f319cf0 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev4 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev4-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev6 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev6-arm64 . .PHONY: build-witness-demo build-witness-demo: diff --git a/setup.py b/setup.py index 28b8876c9..b58a1b8c7 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev4', # also change in src/keri/__init__.py + version='1.2.0-dev6', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index d2b0b42ac..8f8441c53 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev4' # also change in setup.py +__version__ = '1.2.0-dev6' # also change in setup.py From 05b9bf812f2941a6eb6c232d72e68b7a78ee17a7 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Fri, 21 Jun 2024 07:48:45 -0700 Subject: [PATCH 183/418] New and updated kli commands for witnesss, watchers and mailboxes. Delegation fixes (#805) * New kli commands for listing witnesss, watchers and mailboxes. Update to kli command `kli watcher add` to use new BADA-RUN protected data structures for tracking AIDs being observed by a watcher on behalf of a controller New kli command to perform key event adjudication across the set of watchers watching an AID. This command will retrieve updated key state if a threshold satisfying number watchers response with non-duplicitous key state change. Addition to rotate and delegate confirm to all the specification of witness auth code time stamps. Update to delegation processing to propagate the delegator anchor event to witnesses after delegation approval New databases to track observed AIDs (observed by watchers) and delegation propogation escrow. Bug fix in `kli mailbox debug` to account for empty local state. Signed-off-by: pfeairheller * Factoring several classes and methods into a watching package including a new Adjudicator class that can be used outside of the KERIpy command line. Signed-off-by: pfeairheller * Added tests for Adjudicator and diffState Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller --- src/keri/app/agenting.py | 6 + src/keri/app/cli/commands/delegate/confirm.py | 12 +- src/keri/app/cli/commands/local/watch.py | 44 +--- src/keri/app/cli/commands/mailbox/debug.py | 11 +- src/keri/app/cli/commands/mailbox/list.py | 68 ++++++ src/keri/app/cli/commands/rotate.py | 10 +- src/keri/app/cli/commands/watcher/add.py | 20 +- .../app/cli/commands/watcher/adjudicate.py | 159 +++++++++++++ src/keri/app/cli/commands/watcher/list.py | 68 ++++++ src/keri/app/cli/commands/witness/list.py | 56 +++++ src/keri/app/delegating.py | 37 ++- src/keri/app/querying.py | 4 +- src/keri/app/watching.py | 217 ++++++++++++++++++ src/keri/core/eventing.py | 137 +++++++++-- src/keri/core/routing.py | 3 - src/keri/db/basing.py | 66 +++++- tests/app/test_storing.py | 2 +- tests/app/test_watching.py | 159 +++++++++++++ tests/core/test_reply.py | 163 ++++++++++++- 19 files changed, 1151 insertions(+), 91 deletions(-) create mode 100644 src/keri/app/cli/commands/mailbox/list.py create mode 100644 src/keri/app/cli/commands/watcher/adjudicate.py create mode 100644 src/keri/app/cli/commands/watcher/list.py create mode 100644 src/keri/app/cli/commands/witness/list.py create mode 100644 src/keri/app/watching.py create mode 100644 tests/app/test_watching.py diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index ad164a9b3..456e3f4bc 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -581,6 +581,7 @@ def __init__(self, hby, msgs=None, cues=None, **kwa): """ self.hby = hby + self.posted = 0 self.msgs = msgs if msgs is not None else decking.Deck() self.cues = cues if cues is not None else decking.Deck() super(WitnessPublisher, self).__init__(doers=[doing.doify(self.sendDo)], **kwa) @@ -599,6 +600,7 @@ def sendDo(self, tymth=None, tock=0.0, **opts): while True: while self.msgs: evt = self.msgs.popleft() + self.posted += 1 pre = evt["pre"] msg = evt["msg"] @@ -642,6 +644,10 @@ def sent(self, said): return False + @property + def idle(self): + return len(self.msgs) == 0 and self.posted == len(self.cues) + class TCPMessenger(doing.DoDoer): """ Send events to witnesses for receipting using TCP direct connection diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index a0ddb7525..25eb27764 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -37,6 +37,7 @@ action='store_true') parser.add_argument('--code', help=': formatted witness auth codes. Can appear multiple times', default=[], action="append", required=False) +parser.add_argument('--code-time', help='Time the witness codes were captured.', default=None, required=False) def confirm(args): @@ -54,16 +55,18 @@ def confirm(args): auto = args.auto authenticate = args.authenticate codes = args.code + codeTime = args.code_time confirmDoer = ConfirmDoer(name=name, base=base, alias=alias, bran=bran, interact=interact, auto=auto, - authenticate=authenticate, codes=codes) + authenticate=authenticate, codes=codes, codeTime=codeTime) doers = [confirmDoer] return doers class ConfirmDoer(doing.DoDoer): - def __init__(self, name, base, alias, bran, interact=False, auto=False, authenticate=False, codes=None): + def __init__(self, name, base, alias, bran, interact=False, auto=False, authenticate=False, codes=None, + codeTime=None): hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer self.witq = agenting.WitnessInquisitor(hby=hby) @@ -73,6 +76,7 @@ def __init__(self, name, base, alias, bran, interact=False, auto=False, authenti self.mux = grouping.Multiplexor(hby=hby, notifier=self.notifier) self.authenticate = authenticate self.codes = codes if codes is not None else [] + self.codeTime = codeTime exc = exchanging.Exchanger(hby=hby, handlers=[]) delegating.loadHandlers(hby=hby, exc=exc, notifier=self.notifier) @@ -185,9 +189,11 @@ def confirmDo(self, tymth, tock=0.0): auths = {} if self.authenticate: + codeTime = helping.fromIso8601( + self.codeTime) if self.codeTime is not None else helping.nowIso8601() for arg in self.codes: (wit, code) = arg.split(":") - auths[wit] = f"{code}#{helping.nowIso8601()}" + auths[wit] = f"{code}#{codeTime}" for wit in hab.kever.wits: if wit in auths: diff --git a/src/keri/app/cli/commands/local/watch.py b/src/keri/app/cli/commands/local/watch.py index 9e17841a7..b93aeba3c 100644 --- a/src/keri/app/cli/commands/local/watch.py +++ b/src/keri/app/cli/commands/local/watch.py @@ -7,14 +7,13 @@ import random import sys import time -from collections import namedtuple from hio import help from hio.base import doing from keri.app import agenting, indirecting, habbing, forwarding from keri.app.cli.common import existing, terming from keri.app.habbing import GroupHab -from keri.core import coring +from keri.app.watching import States, diffState logger = help.ogler.getLogger() @@ -31,17 +30,6 @@ parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) -Stateage = namedtuple("Stateage", 'even ahead behind duplicitous') - -States = Stateage(even="even", ahead="ahead", behind="behind", duplicitous="duplicitous") - - -class WitnessState: - wit: str - state: Stateage - sn: int - dig: str - def watch(args): name = args.name @@ -135,7 +123,7 @@ def watchDo(self, tymth, tock=0.0, **opts): mystate = hab.kever.state() witstate = hab.db.ksns.get((saider.qb64,)) - states.append(self.diffState(wit, mystate, witstate)) + states.append(diffState(wit, mystate, witstate)) # First check for any duplicity, if so get out of here dups = [state for state in states if state.state == States.duplicitous] @@ -213,31 +201,3 @@ def cueDo(self, tymth, tock=0.0, **opts): yield self.tock yield self.tock - - @staticmethod - def diffState(wit, preksn, witksn): - - witstate = WitnessState() - witstate.wit = wit - mysn = int(preksn.s, 16) - mydig = preksn.d - witstate.sn = int(witksn.f, 16) - witstate.dig = witksn.d - - # At the same sequence number, check the DIGs - if mysn == witstate.sn: - if mydig == witstate.dig: - witstate.state = States.even - else: - witstate.state = States.duplicitous - - # This witness is behind and will need to be caught up. - elif mysn > witstate.sn: - witstate.state = States.behind - - # mysn < witstate.sn - We are behind this witness (multisig or restore situation). - # Must ensure that controller approves this event or a recovery rotation is needed - else: - witstate.state = States.ahead - - return witstate diff --git a/src/keri/app/cli/commands/mailbox/debug.py b/src/keri/app/cli/commands/mailbox/debug.py index d305a1ad1..873c9ad05 100644 --- a/src/keri/app/cli/commands/mailbox/debug.py +++ b/src/keri/app/cli/commands/mailbox/debug.py @@ -85,7 +85,7 @@ def readDo(self, tymth, tock=0.0): hab = self.hby.habByName(name=self.alias) topics = {"/receipt": 0, "/replay": 0, "/multisig": 0, "/credential": 0, "/delegate": 0, "/challenge": 0, - "/oobi": 0} + "/oobi": 0, "/reply": 0} try: client, clientDoer = agenting.httpClient(hab, self.witness) except kering.MissingEntryError as e: @@ -95,8 +95,11 @@ def readDo(self, tymth, tock=0.0): print("Local Index per Topic") witrec = hab.db.tops.get((hab.pre, self.witness)) - for topic in witrec.topics: - print(f" Topic {topic}: {witrec.topics[topic]}") + if witrec: + for topic in witrec.topics: + print(f" Topic {topic}: {witrec.topics[topic]}") + else: + print("\tNo local index") print() q = dict(pre=hab.pre, topics=topics) @@ -107,7 +110,7 @@ def readDo(self, tymth, tock=0.0): httping.createCESRRequest(msg, client, dest=self.witness) - while client.requests: + while client.requests or (not client.events and not client.requests): yield self.tock yield 1.0 diff --git a/src/keri/app/cli/commands/mailbox/list.py b/src/keri/app/cli/commands/mailbox/list.py new file mode 100644 index 000000000..369e749df --- /dev/null +++ b/src/keri/app/cli/commands/mailbox/list.py @@ -0,0 +1,68 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError, Roles + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='List current mailboxes') +parser.set_defaults(handler=lambda args: handle(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran + + +def handle(args): + """ Command line handler for adding an aid to a watcher's list of AIds to watch + + Parameters: + args(Namespace): parsed command line arguments + + """ + + kwa = dict(args=args) + return [doing.doify(listMailboxes, **kwa)] + + +def listMailboxes(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + alias = args.alias + base = args.base + bran = args.bran + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + if alias is None: + alias = existing.aliasInput(hby) + + hab = hby.habByName(alias) + + for (aid, role, eid), ender in hab.db.ends.getItemIter(keys=(hab.pre, Roles.mailbox)): + if ender.allowed: + contact = org.get(eid) + print(f"{contact['alias']}: {eid}") + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index 6adbac029..d6f615bcd 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -31,6 +31,7 @@ action='store_true') parser.add_argument('--code', help=': formatted witness auth codes. Can appear multiple times', default=[], action="append", required=False) +parser.add_argument('--code-time', help='Time the witness codes were captured.', default=None, required=False) parser.add_argument("--proxy", help="alias for delegation communication proxy", default="") @@ -66,7 +67,8 @@ def rotate(args): cuts=opts.witsCut, adds=opts.witsAdd, isith=opts.isith, nsith=opts.nsith, count=opts.ncount, toad=opts.toad, - data=opts.data, proxy=args.proxy, authenticate=args.authenticate, codes=args.code) + data=opts.data, proxy=args.proxy, authenticate=args.authenticate, + codes=args.code, codeTime=args.code_time) doers = [rotDoer] @@ -122,7 +124,7 @@ class RotateDoer(doing.DoDoer): def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=None, count=None, toad=None, wits=None, cuts=None, adds=None, data: list = None, proxy=None, authenticate=False, - codes=None): + codes=None, codeTime=None): """ Returns DoDoer with all registered Doers needed to perform rotation. @@ -149,6 +151,7 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.proxy = proxy self.authenticate = authenticate self.codes = codes if codes is not None else [] + self.codeTime = codeTime self.wits = wits if wits is not None else [] self.cuts = cuts if cuts is not None else [] @@ -198,9 +201,10 @@ def rotateDo(self, tymth, tock=0.0): auths = {} if self.authenticate: + codeTime = helping.fromIso8601(self.codeTime) if self.codeTime is not None else helping.nowIso8601() for arg in self.codes: (wit, code) = arg.split(":") - auths[wit] = f"{code}#{helping.nowIso8601()}" + auths[wit] = f"{code}#{codeTime}" for wit in hab.kever.wits: if wit in auths: diff --git a/src/keri/app/cli/commands/watcher/add.py b/src/keri/app/cli/commands/watcher/add.py index 7f3c8410f..c993f8e90 100644 --- a/src/keri/app/cli/commands/watcher/add.py +++ b/src/keri/app/cli/commands/watcher/add.py @@ -11,7 +11,8 @@ from keri.app import connecting, habbing, forwarding from keri.app.cli.common import existing -from keri.core import eventing, serdering +from keri.core import serdering +from keri.kering import Roles logger = help.ogler.getLogger() @@ -91,7 +92,7 @@ def __init__(self, name, alias, base, bran, watcher, watched): super(AddDoer, self).__init__(doers=doers) def addDo(self, tymth, tock=0.0): - """ Grant credential by creating /ipex/grant exn message + """ Add an AID to a watcher's list of AIDs to watch Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -109,17 +110,28 @@ def addDo(self, tymth, tock=0.0): if isinstance(self.hab, habbing.GroupHab): raise ValueError("watchers for multisig AIDs not currently supported") + ender = self.hab.db.ends.get(keys=(self.hab.pre, Roles.watcher, self.watcher)) + if not ender or not ender.allowed: + msg = self.hab.reply(route="/end/role/add", + data=dict(cid=self.hab.pre, role=Roles.watcher, eid=self.watcher)) + self.hab.psr.parseOne(ims=msg) + postman = forwarding.StreamPoster(hby=self.hby, hab=self.hab, recp=self.watcher, topic="reply") + for msg in self.hab.db.cloneDelegation(self.hab.kever): + serder = serdering.SerderKERI(raw=msg) + postman.send(serder=serder, attachment=msg[serder.size:]) + for msg in self.hab.db.clonePreIter(pre=self.hab.pre): serder = serdering.SerderKERI(raw=msg) postman.send(serder=serder, attachment=msg[serder.size:]) data = dict(cid=self.hab.pre, - wid=self.watched, + oid=self.watched, oobi=self.oobi) - route = "/watcher/aid/add" + route = f"/watcher/{self.watcher}/add" msg = self.hab.reply(route=route, data=data) + self.hab.psr.parseOne(ims=bytes(msg)) rpy = serdering.SerderKERI(raw=msg) postman.send(serder=rpy, attachment=msg[rpy.size:]) diff --git a/src/keri/app/cli/commands/watcher/adjudicate.py b/src/keri/app/cli/commands/watcher/adjudicate.py new file mode 100644 index 000000000..82e368e83 --- /dev/null +++ b/src/keri/app/cli/commands/watcher/adjudicate.py @@ -0,0 +1,159 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse +import datetime +import random +import sys + +from hio import help +from hio.base import doing + +from keri.app import connecting, indirecting, querying, watching +from keri.app.cli.common import existing +from keri.app.watching import diffState, States +from keri.help import helping +from keri.kering import ConfigurationError +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Perform key event adjudication on any new key state from watchers.') +parser.set_defaults(handler=lambda args: handle(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--toad', '-t', default=None, required=False, type=int, + help='int of watcher threshold (threshold of acceptable duplicity)', ) +parser.add_argument("--watched", '-W', help="the watched AID or alias to add", required=True) +parser.add_argument("--poll", "-P", help="Poll mailboxes for any issued credentials", action="store_true") + + +def handle(args): + """ Command line handler for adding an aid to a watcher's list of AIds to watch + + Parameters: + args(Namespace): parsed command line arguments + + """ + + kwa = dict(args=args) + adjudicator = AdjudicationDoer(**kwa) + + return [adjudicator] + + +class AdjudicationDoer(doing.DoDoer): + + def __init__(self, **kwa): + args = kwa["args"] + base = args.base + bran = args.bran + self.name = args.name + self.alias = args.alias + self.watched = args.watched + self.poll = args.poll + self.toad = args.toad + + self.hby = existing.setupHby(name=self.name, base=base, bran=bran) + self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=['/reply', '/replay']) + doers = [doing.doify(self.adjudicate, **kwa), self.mbx] + + super(AdjudicationDoer, self).__init__(**kwa, doers=doers) + + def adjudicate(self, tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + + try: + org = connecting.Organizer(hby=self.hby) + + if self.poll: + end = helping.nowUTC() + datetime.timedelta(seconds=5) + sys.stdout.write(f"Polling mailboxes") + sys.stdout.flush() + while helping.nowUTC() < end: + sys.stdout.write(".") + sys.stdout.flush() + yield 1.0 + print("\n") + + if self.watched in self.hby.kevers: + watd = self.watched + else: + watd = org.find("alias", self.watched) + if len(watd) != 1: + raise ValueError(f"invalid recipient {self.watched}") + watd = watd[0]['id'] + + if not watd: + raise ValueError(f"unknown watched {self.watched}") + + if self.alias is None: + self.alias = existing.aliasInput(self.hby) + + hab = self.hby.habByName(self.alias) + if hab is None: + raise ValueError(f"unknown alias {self.alias}") + + adj = watching.Adjudicator(hby=self.hby, hab=hab) + adjDoer = watching.AdjudicationDoer(adj) + self.extend([adjDoer]) + + adj.msgs.append(dict(oid=self.watched, toad=self.toad)) + + while not adj.cues: + yield self.tock + + cue = adj.cues.pull() + kin = cue['kin'] + + match kin: + case "keyStateConsistent": + states = cue['states'] + wids = cue["wids"] + print(f"Local key state is consistent with the {len(states)} (out of " + f"{len(wids)} total) watchers that responded") + + case "keyStateLagging": + bhds = cue["behinds"] + print("The following watchers are behind the local KEL:") + for state in bhds: + print(f"\tWatcher {state.wit} at seq No. {state.sn} with digest: {state.dig}") + + print(f"Recommend the checking those watchers for access to {self.watched} witnesses") + + case "keyStateUpdate": + ahds = cue["aheads"] + logger.info(f"Threshold ({self.toad}) satisfying number of watchers ({len(ahds)}) are ahead") + for state in ahds: + logger.info(f"\tWatcher {state.wit} at Seq No. {state.sn} with digest: {state.dig}") + + state = random.choice(ahds) + querier = querying.SeqNoQuerier(hby=self.hby, hab=hab, pre=self.watched, sn=state.sn, + wits=[state.wit]) + self.extend([querier]) + + while not querier.done: + yield self.tock + + case "keyStateDuplicitous": + dups = cue["dups"] + print(f"Duplicity detected for AID {self.watched}, local key state remains intact.") + for state in dups: + print(f"\tWatcher {state.wit} at seq No. {state.sn} with digest: {state.dig}") + + self.remove([self.mbx, adjDoer]) + + except ConfigurationError as e: + print(f"identifier prefix for {self.name} does not exist, incept must be run first", ) + return -1 + diff --git a/src/keri/app/cli/commands/watcher/list.py b/src/keri/app/cli/commands/watcher/list.py new file mode 100644 index 000000000..ff54dc0ff --- /dev/null +++ b/src/keri/app/cli/commands/watcher/list.py @@ -0,0 +1,68 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError, Roles + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='List current watchers') +parser.set_defaults(handler=lambda args: handle(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran + + +def handle(args): + """ Command line handler for adding an aid to a watcher's list of AIds to watch + + Parameters: + args(Namespace): parsed command line arguments + + """ + + kwa = dict(args=args) + return [doing.doify(listWatchers, **kwa)] + + +def listWatchers(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + alias = args.alias + base = args.base + bran = args.bran + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + if alias is None: + alias = existing.aliasInput(hby) + + hab = hby.habByName(alias) + + for (aid, role, eid), ender in hab.db.ends.getItemIter(keys=(hab.pre, Roles.watcher, )): + if ender.allowed: + contact = org.get(eid) + print(f"{contact['alias']}: {eid}") + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 diff --git a/src/keri/app/cli/commands/witness/list.py b/src/keri/app/cli/commands/witness/list.py new file mode 100644 index 000000000..9ab11e6e5 --- /dev/null +++ b/src/keri/app/cli/commands/witness/list.py @@ -0,0 +1,56 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app.cli.common import displaying, existing +from keri.core import serdering +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='List AIDs of witness for the provided AID') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(listWitnesses, **kwa)] + + +def listWitnesses(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + alias = args.alias + base = args.base + bran = args.bran + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + if alias is None: + alias = existing.aliasInput(hby) + + hab = hby.habByName(alias) + for idx, wit in enumerate(hab.kever.wits): + print(f'{wit}') + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 8d9e4c194..6b90c53ef 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -44,11 +44,11 @@ def __init__(self, hby, proxy=None, auths=None, **kwa): self.postman = forwarding.Poster(hby=hby) self.witq = agenting.WitnessInquisitor(hby=hby) self.witDoer = agenting.Receiptor(hby=self.hby) + self.publishers = dict() self.proxy = proxy self.auths = auths - super(Anchorer, self).__init__(doers=[self.witq, self.witDoer, self.postman, doing.doify(self.escrowDo)], - **kwa) + super(Anchorer, self).__init__(doers=[self.witq, self.witDoer, self.postman, doing.doify(self.escrowDo)], **kwa) def delegation(self, pre, sn=None, proxy=None, auths=None): if pre not in self.hby.habs: @@ -57,6 +57,7 @@ def delegation(self, pre, sn=None, proxy=None, auths=None): if proxy is not None: self.proxy = proxy + self.publishers[pre] = agenting.WitnessPublisher(hby=self.hby) # load the hab of the delegated identifier to anchor hab = self.hby.habs[pre] delpre = hab.kever.delpre # get the delegator identifier @@ -123,6 +124,7 @@ def escrowDo(self, tymth, tock=1.0): def processEscrows(self): self.processPartialWitnessEscrow() self.processUnanchoredEscrow() + self.processWitnessPublication() def processUnanchoredEscrow(self): """ @@ -143,8 +145,9 @@ def processUnanchoredEscrow(self): self.hby.db.setAes(dgkey, couple) # authorizer event seal (delegator/issuer) # Move to escrow waiting for witness receipts - logger.info(f"Delegation approval received, {serder.pre} confirmed") - self.hby.db.cdel.put(keys=(pre, coring.Seqner(sn=serder.sn).qb64), val=coring.Saider(qb64=serder.said)) + logger.info(f"Delegation approval received, {serder.pre} confirmed, publishing to my witnesses") + self.publishDelegator(pre) + self.hby.db.dpub.put(keys=(pre, said), val=serder) self.hby.db.dune.rem(keys=(pre, said)) def processPartialWitnessEscrow(self): @@ -197,6 +200,32 @@ def processPartialWitnessEscrow(self): self.hby.db.dpwe.rem(keys=(pre, said)) self.hby.db.dune.pin(keys=(srdr.pre, srdr.said), val=srdr) + def processWitnessPublication(self): + """ + Process escrow of partially signed multisig group KEL events. Message + processing will send this local controllers signature to all other participants + then this escrow waits for signatures from all other participants + + """ + for (pre, said), serder in self.hby.db.dpub.getItemIter(): # group partial witness escrow + publisher = self.publishers[pre] + + if not publisher.idle: + continue + + self.remove([publisher]) + del self.publishers[pre] + + self.hby.db.dpub.rem(keys=(pre, said)) + self.hby.db.cdel.put(keys=(pre, coring.Seqner(sn=serder.sn).qb64), val=coring.Saider(qb64=serder.said)) + + def publishDelegator(self, pre): + hab = self.hby.habs[pre] + publisher = self.publishers[pre] + self.extend([publisher]) + for msg in hab.db.cloneDelegation(hab.kever): + publisher.msgs.append(dict(pre=hab.pre, msg=bytes(msg))) + def loadHandlers(hby, exc, notifier): """ Load handlers for the peer-to-peer delegation protocols diff --git a/src/keri/app/querying.py b/src/keri/app/querying.py index 2dc4ce17d..11541640f 100644 --- a/src/keri/app/querying.py +++ b/src/keri/app/querying.py @@ -90,13 +90,13 @@ def recur(self, tyme, deeds=None): class SeqNoQuerier(doing.DoDoer): - def __init__(self, hby, hab, pre, sn, **opts): + def __init__(self, hby, hab, pre, sn, wits=None, **opts): self.hby = hby self.hab = hab self.pre = pre self.sn = sn self.witq = agenting.WitnessInquisitor(hby=self.hby) - self.witq.query(src=self.hab.pre, pre=self.pre, sn="{:x}".format(self.sn)) + self.witq.query(src=self.hab.pre, pre=self.pre, sn="{:x}".format(self.sn), wits=wits) super(SeqNoQuerier, self).__init__(doers=[self.witq], **opts) def recur(self, tyme, deeds=None): diff --git a/src/keri/app/watching.py b/src/keri/app/watching.py new file mode 100644 index 000000000..3a720d594 --- /dev/null +++ b/src/keri/app/watching.py @@ -0,0 +1,217 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.app.watching module + +""" +import random +from collections import namedtuple +from dataclasses import dataclass + +from hio.base import doing +from hio.help import decking + +from keri import help + +logger = help.ogler.getLogger() + +Stateage = namedtuple("Stateage", 'even ahead behind duplicitous') + +States = Stateage(even="even", ahead="ahead", behind="behind", duplicitous="duplicitous") + + +@dataclass +class DiffState: + """ Difference between a remote KeyStateRecord and local for the same AID. + + Uses Stateage to represent whether the remote KSR is even, ahead, behind or duplicitous + + """ + wit: str # The entity reporting the KSR (non-local) + state: Stateage # The state of the remote KSR relative to local + sn: int # The sequence number of the remote KSR + dig: str # The digest of the latest event of the remote KSR + + +class Adjudicator: + """ The Adjudicator of Key State + + This class performs key state adjudication by checking any key state reported by the watcher set for a given + watched AID and compares the reported values against the local key state for the watched AID and the key state + of all other responding watchers. It uses a per-adjudication threshold to determine what is acceptable duplicity + for each adjudication. + + Cues are sent out for each round of adjudication with the following kins: + + keyStateConsistent - Key state of all queries watchers is consistent with local key state + keyStateLagging - Key state from some watchers is behind local key state and other watchers + keyStateUpdate - A threshold satisfying number of watchers report new key state for watched AID + keyStateDuplicitous - Duplicity has been detected on some set of watchers (provided in the cue) + + Consumers of the Adjudicator's cues are safe to retrieve new key state from one of the Watchers listed in the + cue of `keyStateUpdated` is received. All other kins require controller intervention and should be bubbled up. + + """ + + def __init__(self, hby, hab, msgs=None, cues=None): + """ Create instance of Adjudicator for adjudicating key state + + Parameters: + hby (Habery): database and Habitat environment + hab (Hab): identifier database environment + msgs (Deck): incoming requests to adjudicate key state + cues (Deck): outgoing responses to adjudication of key state + + """ + self.hby = hby + self.hab = hab + self.msgs = msgs if msgs is not None else decking.Deck() + self.cues = cues if cues is not None else decking.Deck() + + def performAdjudications(self): + """ Process loop of existing messages requesting key state adjudication """ + while self.msgs: + msg = self.msgs.pull() + + watched = msg["oid"] + toad = msg["toad"] if "toad" in msg else None + + self.adjudicate(watched, toad) + + def adjudicate(self, watched, toad=None): + """ Perform key state adjudication against the `watched` AID and provided threshold + + If `toad` is not provided, the full set of watchers must come to consensus before `keyStateUpdate` + will be reported. + + Parameters: + watched (str): qb64 AID to adjudicate for key state duplicity + toad (int): threshold of acceptable duplicity amongst available watchers + + + """ + watchers = set() + for (cid, aid, oid), observed in self.hab.db.obvs.getItemIter(keys=(self.hab.pre,)): + if observed.enabled and oid == watched: + watchers.add(aid) + + toad = int(toad) if toad else len(watchers) + if toad > len(watchers): + raise ValueError(f"Threshold of {toad} is greater than number watchers {len(watchers)}") + + states = [] + mystate = self.hab.kever.state() + for watcher in watchers: + saider = self.hab.db.knas.get(keys=(watched, watcher)) + if saider is None: + print(f"No key state from watcher {watcher} for {watched}") + continue + + ksn = self.hab.db.ksns.get(keys=(saider.qb64,)) + states.append(diffState(watcher, mystate, ksn)) + + dups = [state for state in states if state.state == States.duplicitous] + ahds = [state for state in states if state.state == States.ahead] + bhds = [state for state in states if state.state == States.behind] + + if len(dups) > 0: + cue = dict(kin="keyStateDuplicitous", cid=self.hab.pre, oid=watched, wids=watchers, dups=dups) + self.cues.append(cue) + + logger.error(f"Duplicity detected for AID {watched}, local key state remains intact.") + for state in dups: + logger.error(f"\tWatcher {state.wit} at seq No. {state.sn} with digest: {state.dig}") + + elif len(ahds) > 0: + # Only group habs can be behind their watchers + # First check for duplicity among the watchers that are ahead (possible only if toad is below + # super majority) + digs = set([state.dig for state in ahds]) + if len(digs) > 1: # Duplicity across watcher sets + cue = dict(kin="keyStateDuplicitous", cid=self.hab.pre, oid=watched, wids=watchers, dups=ahds) + self.cues.append(cue) + + logger.error(f"There are multiple duplicitous events on watcher for {watched}") + for state in ahds: + logger.error(f"\tWatcher {state.wit} at seq No. {state.sn} with digest: {state.dig}") + + elif len(ahds) >= toad: # all witnesses that are ahead agree on the event + logger.info(f"Threshold ({toad}) satisfying number of watchers ({len(ahds)}) are ahead") + for state in ahds: + logger.info(f"\tWatcher {state.wit} at Seq No. {state.sn} with digest: {state.dig}") + + state = random.choice(ahds) + cue = dict(kin="keyStateUpdate", cid=self.hab.pre, oid=watched, wids=watchers, sn=state.sn, aheads=ahds) + self.cues.append(cue) + + elif len(bhds) > 0: + cue = dict(kin="keyStateLagging", cid=self.hab.pre, oid=watched, wids=watchers, behind=bhds) + self.cues.append(cue) + + logger.info("The following watchers are behind the local KEL:") + for state in bhds: + logger.info(f"\tWatcher {state.wit} at seq No. {state.sn} with digest: {state.dig}") + + logger.info(f"Recommend the checking those watchers for access to {watched} witnesses") + + else: + cue = dict(kin="keyStateConsistent", cid=self.hab.pre, oid=watched, wids=watchers, states=states) + self.cues.append(cue) + logger.info(f"Local key state is consistent with the {len(states)} (out of " + f"{len(watchers)} total) watchers that responded") + + +class AdjudicationDoer(doing.Doer): + """ Doer class responsible for process adjudication requests in an Adjudicator's msgs """ + + def __init__(self, adjudicator): + """ Create instance of Doer for performing key state adjudications """ + self.adjudicator = adjudicator + super(AdjudicationDoer, self).__init__() + + def recur(self, tyme): + """ Perform one pass over all adjudication requests + + Parameters: + tyme (float): relative cycle time + + Returns: + + """ + self.adjudicator.performAdjudications() + + +def diffState(wit, preksn, witksn): + """ Return a record of the differences between the states provided by `wit` and local state + + Parameters: + wit (str): qb64 AID of entity reporting key state + preksn (KeyStateRecord): Local key state of AID + witksn (KeyStateRecord): Key state of AID as provided by `wit` + + Returns: + state (WitnessState): record indicating the differenced between the two provided KSN records + + """ + mysn = int(preksn.s, 16) + mydig = preksn.d + sn = int(witksn.s, 16) + dig = witksn.d + + # At the same sequence number, check the DIGs + if mysn == sn: + if mydig == dig: + state = States.even + else: + state = States.duplicitous + + # This witness is behind and will need to be caught up. + elif mysn > sn: + state = States.behind + + # mysn < witstate.sn - We are behind this witness (multisig or restore situation). + # Must ensure that controller approves this event or a recovery rotation is needed + else: + state = States.ahead + + return DiffState(wit, state, sn, dig) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index dd9cb2244..7b552b072 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -7,6 +7,7 @@ import json import logging from collections import namedtuple +from dataclasses import asdict from urllib.parse import urlsplit from math import ceil from ordered_set import OrderedSet as oset @@ -39,7 +40,7 @@ from . import serdering from ..db import basing, dbing -from ..db.basing import KeyStateRecord, StateEERecord +from ..db.basing import KeyStateRecord, StateEERecord, OobiRecord from ..db.dbing import dgKey, snKey, fnKey, splitKeySN, splitKey @@ -2098,8 +2099,6 @@ def rotate(self, serder): raise ValidationError(f"Invalid sith = {serder.tholder} for keys = " f"{keys} for evt = {ked}.") - - # compute wits from existing .wits with new cuts and adds from event # use ordered set math ops to verify and ensure strict ordering of wits # cuts and add to ensure that indexed signatures on indexed witness @@ -2118,7 +2117,6 @@ def rotate(self, serder): return tholder, toader, wits, cuts, adds - def deriveBacks(self, serder): """Derives and return tuple of (wits, cuts, adds) for backers given current set and any changes provided by serder. @@ -2243,10 +2241,10 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, (self.locallyOwned() or self.locallyWitnessed(wits=wits))): self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) + seqner=delseqner, saider=delsaider, local=local) raise MisfitEventSourceError(f"Nonlocal source for locally owned" - f"or locally witnessed event" - f" = {serder.ked}.") + f" or locally witnessed event" + f" = {serder.ked}, {wits}, {self.prefixes}") werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers # get unique verified wigers and windices lists from wigers list @@ -2627,19 +2625,15 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # misfit escrow first. Mistfit escrow must first # promote to local and reprocess event before we get to here self.escrowDelegableEvent(serder=serder, sigers=sigers, - wigers=wigers,local=local) + wigers=wigers, local=local) raise MissingDelegableApprovalError(f"Missing approval for " f" delegation by {delpre} of" - f"event = {serder.ked}.") - - #self.cues.push(dict(kin="approveDelegation", - #delegator=kever.delpre, - #serder=serder)) + f"event = {serder.ked}.") else: # not local delegator so escrow self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) raise MissingDelegationError(f"No delegation seal for delegator " - "{delpre} of evt = {serder.ked}.") + f"{delpre} of evt = {serder.ked}.") ssn = Number(num=delseqner.sn).validate(inceptive=False).sn # ToDo XXXX need to replace Seqners with Numbers @@ -2949,8 +2943,7 @@ def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): self.db.delegables.add(snKey(serder.preb, serder.sn), serder.saidb) # log escrowed logger.debug("Kever state: escrowed delegable event=\n%s\n", - json.dumps(serder.ked, indent=1)) - + json.dumps(serder.ked, indent=1)) def escrowPSEvent(self, serder, sigers, wigers=None, local=True): """ @@ -2985,9 +2978,9 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): self.db.esrs.put(keys=dgkey, val=esr) snkey = snKey(serder.preb, serder.sn) - self.db.addPse(snkey, serder.saidb) # b'EOWwyMU3XA7RtWdelFt-6waurOTH_aW_Z9VTaU-CshGk.00000000000000000000000000000001' + self.db.addPse(snkey, serder.saidb) logger.debug("Kever state: Escrowed partially signed or delegated " - "event = %s\n", serder.ked) + "event = %s\n", serder.ked) def escrowPACouple(self, serder, seqner, saider, local=True): @@ -4087,6 +4080,7 @@ def registerReplyRoutes(self, router): router.addRoute("/end/role/{action}", self, suffix="EndRole") router.addRoute("/loc/scheme", self, suffix="LocScheme") router.addRoute("/ksn/{aid}", self, suffix="KeyStateNotice") + router.addRoute("/watcher/{aid}/{action}", self, suffix="AddWatched") def processReplyEndRole(self, *, serder, saider, route, cigars=None, tsgs=None, **kwargs): """ @@ -4406,7 +4400,7 @@ def processReplyKeyStateNotice(self, *, serder, saider, route, ksaider = coring.Saider(qb64=diger.qb64) self.updateKeyState(aid=aid, ksr=ksr, saider=ksaider, dater=dater) - self.cues.push(dict(kin="keyStateSaved", ksn=ksr._asdict())) + self.cues.push(dict(kin="keyStateSaved", ksn=asdict(ksr))) def updateEnd(self, keys, saider, allowed=None): """ @@ -4470,6 +4464,100 @@ def removeKeyState(self, saider): self.db.ksns.rem(keys=keys) self.db.kdts.rem(keys=keys) + def processReplyAddWatched(self, *, serder, saider, route, + cigars=None, tsgs=None, **kwargs): + """ Process one reply message for adding an AID for a watcher to watch + + Process one reply message for adding an AID for a watcher to watch = /watcher/{aid}/add + with either attached nontrans receipt couples in cigars or attached trans + indexed sig groups in tsgs. + Assumes already validated saider, dater, and route from serder.ked + + Parameters: + serder (SerderKERI): instance of reply msg (SAD) + saider (Saider): instance from said in serder (SAD) + route (str): reply route + cigars (list): of Cigar instances that contain nontrans signing couple + signature in .raw and public key in .verfer + tsgs (list): tuples (quadruples) of form + (prefixer, seqner, diger, [sigers]) where: + prefixer is pre of trans endorser + seqner is sequence number of trans endorser's est evt for keys for sigs + diger is digest of trans endorser's est evt for keys for sigs + [sigers] is list of indexed sigs from trans endorser's keys from est evt + + Reply Message: + { + "v" : "KERI10JSON00011c_", + "t" : "rpy", + "d": "EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM", + "dt": "2020-08-22T17:50:12.988921+00:00", + "r" : "/watcher/BrHLayDN-mXKv62DAjFLX1_Y5yEUe0vA9YPe_ihiKYHE/add", + "a" : + { + "cid": "EyX-zd8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM" + "oid": "EM0-i05TNZJZAoH3UR2nmLaU6JwyvPzhzS6YAfSVbMC5" + "oobi": "http://example.com/oobi/EyX-zd8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM" + } + } + + """ + aid = kwargs["aid"] + action = kwargs["action"] + # reply specific logic + if not route.startswith("/watcher"): + raise ValidationError(f"Usupported route={route} in {Ilks.rpy} " + f"msg={serder.ked}.") + + # reply specific logic + if action == "add": + enabled = True + elif action == "cut": + enabled = False + else: # unsupported route + raise ValidationError(f"Usupported route={route} in {Ilks.rpy} " + f"msg={serder.ked}.") + route = f"/watcher/{aid}" # escrow based on route base + cigars = cigars if cigars is not None else [] + tsgs = tsgs if tsgs is not None else [] + + data = serder.ked["a"] + cid = data["cid"] + oid = data["oid"] + oobi = data["oobi"] if "oobi" in data else None + + keys = (cid, aid, oid) + + osaider = self.db.wwas.get(keys=keys) # get old said if any + + # BADA Logic + accepted = self.rvy.acceptReply(serder=serder, saider=saider, route=route, + aid=cid, osaider=osaider, cigars=cigars, + tsgs=tsgs) + if not accepted: + raise UnverifiedReplyError(f"Unverified watcher add reply. {serder.ked}") + + if oobi: + self.db.oobis.pin(keys=(oobi,), val=OobiRecord(date=help.nowIso8601())) + self.updateWatched(keys=keys, saider=saider, enabled=enabled) + + def updateWatched(self, keys, saider, enabled=None): + """ + Update loc auth database .lans and loc database .locs. + + Parameters: + keys (tuple): of key strs for databases (eid, scheme) + saider (Saider): instance from said in reply serder (SAD) + enabled (bool): True means add observed to watcher, False means remove (cut) + """ + self.db.wwas.pin(keys=keys, val=saider) # overwrite + if observed := self.db.obvs.get(keys=keys): # preexisting record + observed.enabled = enabled # update preexisting record + else: # no preexisting record + observed = basing.ObservedRecord(enabled=enabled, datetime=helping.nowIso8601()) # create new record + + self.db.obvs.pin(keys=keys, val=observed) # overwrite + def processQuery(self, serder, source=None, sigers=None, cigars=None): """ Process query mode replay message for collective or single element query. @@ -5175,6 +5263,14 @@ def processEscrowPartialSigs(self): raise ValidationError("Missing escrowed evt sigs at " "dig = {}.".format(bytes(edig))) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + if not wigs: # empty list + # wigs maybe empty while waiting for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) # seal source (delegator issuer if any) delseqner = delsaider = None @@ -5197,7 +5293,8 @@ def processEscrowPartialSigs(self): # process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - self.processEvent(serder=eserder, sigers=sigers, + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=delseqner, delsaider=delsaider, local=esr.local) # If process does NOT validate sigs or delegation seal (when delegated), diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 8d119a27b..3601529f0 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -198,7 +198,6 @@ def processReply(self, serder, cigars=None, tsgs=None): self.rtr.dispatch(serder=serder, saider=saider, cigars=cigars, tsgs=tsgs) - def acceptReply(self, serder, saider, route, aid, osaider=None, cigars=None, tsgs=None): """ Applies Best Available Data Acceptance policy to reply and signatures @@ -494,8 +493,6 @@ def processEscrowReply(self): # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) - else: - logger.error("Kevery unescrow attempt failed: %s", ex.args[0]) except Exception as ex: # other error so remove from reply escrow self.db.rpes.rem(keys=(route, ), val=saider) # remove escrow only diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index d911eeff2..f403a136a 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -347,7 +347,6 @@ class OobiRecord: urls: list = None - @dataclass class EndpointRecord: # baser.ends """ @@ -499,6 +498,56 @@ def __iter__(self): return iter(asdict(self)) +@dataclass +class ObservedRecord: # baser.obvs + """ + Watched Record with fields and keys to manage OIDs (Observed IDs) being watched by a watcher, keyed by + cid (controller ID), aid (watcher ID), and oid (observed ID). + + The namespace is a tree of branches with each leaf at a + specific (cid, aid, oid). Retrieval by branch returns groups of leaves as + appropriate for a cid braanch or cid.aid branch. + Database Keys are (cid, aid, oid) where cid is attributable controller identifier + (qb64 prefix). + + Attributes: + enabled (bool): AuthZ via expose message + True means oid is enabled as being observed + False means eid is disenabled being observed + None means eid is neither enabled or disenabled + name (str): user friendly name for eid in role + datetime (str): Date time this record was last observed + + + A watcher end reply message is required from which the field values + for this record are extracted. A routes of /watcher/{aid}/add /watcher/{aid}/cut + Uses add-cut model with allowed field + enabled==True oid is allowed (add) as being observed + enabled==False oid is disallowed (cut) as being observed + + { + "v" : "KERI10JSON00011c_", + "t" : "rpy", + "d": "EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM", + "dt": "2020-08-22T17:50:12.988921+00:00", + "r" : "/watcher/BrHLayDN-mXKv62DAjFLX1_Y5yEUe0vA9YPe_ihiKYHE/add", + "a" : + { + "cid": "EaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM", + "oid": "EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM", + "oobi": "http://example.com/oobi/EyX-zd8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM", + } + } + + """ + enabled: bool = None # True eid enabled (add), False eid disenabled (cut), None neither + name: str = "" # optional user friendly name of endpoint + datetime: str = None + + def __iter__(self): + return iter(asdict(self)) + + @dataclass class WellKnownAuthN: """ @@ -1015,6 +1064,11 @@ def reopen(self, **kwa): self.locs = koming.Komer(db=self, subkey='locs.', schema=LocationRecord, ) + # observed oids by watcher by cid.aid.oid (endpoint identifier) + # data extracted from reply loc + self.obvs = koming.Komer(db=self, + subkey='obvs.', + schema=ObservedRecord, ) # index of last retrieved message from witness mailbox # TODO: clean @@ -1095,6 +1149,11 @@ def reopen(self, **kwa): # TODO: clean self.knas = subing.CesrSuber(db=self, subkey='knas.', klas=coring.Saider) + # Watcher watched SAID database for successfully saved watched AIDs for a watcher + # maps key=(cid, aid, oid) to val=said of rpy message + # TODO: clean + self.wwas = subing.CesrSuber(db=self, subkey='wwas.', klas=coring.Saider) + # config loaded oobis to be processed asynchronously, keyed by oobi URL # TODO: clean self.oobis = koming.Komer(db=self, @@ -1182,6 +1241,9 @@ def reopen(self, **kwa): # delegated unanchored escrow self.dune = subing.SerderSuber(db=self, subkey='dune.') + # delegate publication escrow for sending delegator info to my witnesses + self.dpub = subing.SerderSuber(db=self, subkey='dpub.') + # completed group delegated AIDs # TODO: clean self.cdel = subing.CesrSuber(db=self, subkey='cdel.', @@ -1537,7 +1599,6 @@ def cloneEvtMsg(self, pre, fn, dig): msg.extend(atc) return msg - def cloneDelegation(self, kever): """ Recursively clone delegation chain from AID of Kever if one exits. @@ -1553,7 +1614,6 @@ def cloneDelegation(self, kever): for dmsg in self.clonePreIter(pre=kever.delpre, fn=0): yield dmsg - def findAnchoringSealEvent(self, pre, seal, sn=0): """ Search through a KEL for the event that contains a specific anchored diff --git a/tests/app/test_storing.py b/tests/app/test_storing.py index 82bf39352..d37eeae3c 100644 --- a/tests/app/test_storing.py +++ b/tests/app/test_storing.py @@ -1,6 +1,6 @@ # -*- encoding: utf-8 -*- """ -tests.peer.mailboxing +tests.app.storing """ import os diff --git a/tests/app/test_watching.py b/tests/app/test_watching.py new file mode 100644 index 000000000..4c7a49702 --- /dev/null +++ b/tests/app/test_watching.py @@ -0,0 +1,159 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.watching + +""" +from dataclasses import asdict + +import pytest + +from keri import core +from keri.app import watching, habbing +from keri.app.watching import DiffState +from keri.core import coring +from keri.db.basing import KeyStateRecord, ObservedRecord + + +def test_diffstate(): + d0 = {'vn': [1, 0], + 'i': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', + 's': '0', + 'p': 'ElsHFkbZQjRb7xHnuE-wyiarIZ9j-1CEQ89I0E3WevcE', + 'd': 'EBiIFxr_o1b4x1YR21PblAFpFG61qDghqFBDyVSOXYW0', + 'f': '0', + 'dt': '2021-06-09T17:35:54.169967+00:00', + 'et': '2021-06-09T17:35:54.169967+00:00', + 'kt': '1', + 'k': ["D-HwiqmaETxls3vAVSh0xpXYTs94NUJX6juupWj_EgsA"], + 'nt': '1', + 'n': ["ED6lKZwg-BWl_jlCrjosQkOEhqKD4BJnlqYqWmhqPhaU"], + 'bt': '0', + 'b': [], + 'c': [], + 'ee': { + 's': '0', + 'd': 'EBiIFxr_o1b4x1YR21PblAFpFG61qDghqFBDyVSOXYW0', + 'br': [], + 'ba': [] + }, + 'di': ''} + + ksr0 = KeyStateRecord(**d0) + d1 = {'vn': [1, 0], + 'i': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', + 's': '0', + 'p': 'ElsHFkbZQjRb7xHnuE-wyiarIZ9j-1CEQ89I0E3WevcE', + 'd': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc', + 'f': '0', + 'dt': '2021-06-09T17:35:54.169967+00:00', + 'et': '2021-06-09T17:35:54.169967+00:00', + 'kt': '1', + 'k': ["DxVTxls3vAwiqmaEXYTs94NUJX6juVSh0xpupEgsAWj_"], + 'nt': '1', + 'n': ["ED6lKZwg-BWl_jlCrjosQkOEhqKD4BJnlqYqWmhqPhaU"], + 'bt': '0', + 'b': [], + 'c': [], + 'ee': { + 's': '0', + 'd': 'EBiIFxr_o1b4x1YR21PblAFpFG61qDghqFBDyVSOXYW0', + 'br': [], + 'ba': [] + }, + 'di': ''} + ksr1 = KeyStateRecord(**d1) + + wat = "BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s" + diffstate = watching.diffState(wat, ksr0, ksr1) + + # Sequence numbers are the same, digest different == duplicitous + assert asdict(diffstate) == {'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + 'state': 'duplicitous', + 'sn': 0, 'dig': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc'} + + # Same state == event + diffstate = watching.diffState(wat, ksr0, ksr0) + assert asdict(diffstate) == {'dig': 'EBiIFxr_o1b4x1YR21PblAFpFG61qDghqFBDyVSOXYW0', + 'sn': 0, + 'state': 'even', + 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} + + ksr1.s = "2" + diffstate = watching.diffState(wat, ksr0, ksr1) + + # Sequence numbers are the same, digest different == duplicitous + assert asdict(diffstate) == {'dig': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc', + 'sn': 2, + 'state': 'ahead', + 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} + + ksr0.s = "3" + diffstate = watching.diffState(wat, ksr0, ksr1) + + # Sequence numbers are the same, digest different == duplicitous + assert asdict(diffstate) == {'dig': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc', + 'sn': 2, + 'state': 'behind', + 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} + + +def test_adjudicator(): + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 + with habbing.openHby(name="test", base="test", salt=default_salt) as hby: + hab = hby.makeHab("test") + assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" + wat = "BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s" + saider = coring.Saider(qb64b=b'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ') + + ksr = hab.kever.state() + ksr0 = KeyStateRecord(**asdict(ksr)) + + hab.db.knas.pin(keys=(hab.pre, wat), val=saider) + hab.db.ksns.pin(keys=(saider.qb64, ), val=ksr0) + hab.db.obvs.pin(keys=(hab.pre, wat, hab.pre), val=ObservedRecord(enabled=True)) + + adj = watching.Adjudicator(hby=hby, hab=hab) + + adj.adjudicate(hab.pre, 1) + assert len(adj.cues) == 1 + cue = adj.cues.pull() + + assert cue == {'cid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'kin': 'keyStateConsistent', + 'oid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'states': [DiffState(wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + state='even', + sn=0, + dig='EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3')], + 'wids': {'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'}} + + hab.rotate() + + adj.adjudicate(hab.pre, 1) + assert len(adj.cues) == 1 + cue = adj.cues.pull() + assert cue == {'behind': [DiffState(wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + state='behind', + sn=0, + dig='EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3')], + 'cid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'kin': 'keyStateLagging', + 'oid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'wids': {'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'}} + + ksr0.s = '1' + hab.db.ksns.pin(keys=(saider.qb64, ), val=ksr0) + adj.adjudicate(hab.pre, 1) + assert len(adj.cues) == 1 + cue = adj.cues.pull() + assert cue == {'cid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'dups': [DiffState(wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + state='duplicitous', + sn=1, + dig='EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3')], + 'kin': 'keyStateDuplicitous', + 'oid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'wids': {'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'}} + + with pytest.raises(ValueError): + adj.adjudicate(hab.pre, 2) diff --git a/tests/core/test_reply.py b/tests/core/test_reply.py index 4705065eb..383ab74ec 100644 --- a/tests/core/test_reply.py +++ b/tests/core/test_reply.py @@ -22,8 +22,7 @@ from keri.db import basing from keri.app import habbing, keeping - - +from keri.kering import Roles logger = help.ogler.getLogger() @@ -1283,5 +1282,165 @@ def test_reply(mockHelpingNowUTC): """Done Test""" +def test_watcher_add_cut(): + salt = core.Salter(raw=b'abcdef0123456789').qb64 + + with habbing.openHby(name="con", base="test", salt=salt) as conHby, \ + habbing.openHby(name="wat0", base="test", salt=salt) as wat0hby, \ + habbing.openHby(name="wat1", base="test", salt=salt) as wat1hby, \ + habbing.openHby(name="wat2", base="test", salt=salt) as wat2hby, \ + habbing.openHby(name="obv0", base="test", salt=salt) as obv0hby, \ + habbing.openHby(name="obv1", base="test", salt=salt) as obv1hby, \ + habbing.openHby(name="obv2", base="test", salt=salt) as obv2hby: + + conHab = conHby.makeHab(name="con", isith="1", icount=1, transferable=True) + assert conHab.kever.prefixer.transferable + conKvy = eventing.Kevery(db=conHab.db, lax=False, local=False) + + wat0hab = wat0hby.makeHab(name='wat0', isith="1", icount=1, transferable=False) + assert not wat0hab.kever.prefixer.transferable + # create non-local kevery for Wes to process nonlocal msgs + wat0kvy = eventing.Kevery(db=wat0hab.db, lax=False, local=False) + + wat1hab = wat1hby.makeHab(name='wat1', isith="1", icount=1, transferable=False) + assert not wat1hab.kever.prefixer.transferable + # create non-local kevery for Wes to process nonlocal msgs + wat1kvy = eventing.Kevery(db=wat1hab.db, lax=False, local=False) + + wat2hab = wat2hby.makeHab(name='wat2', isith="1", icount=1, transferable=False) + assert not wat2hab.kever.prefixer.transferable + # create non-local kevery for Wes to process nonlocal msgs + wat2kvy = eventing.Kevery(db=wat2hab.db, lax=False, local=False) + + obv0hab = obv0hby.makeHab(name='obv0', isith="1", icount=1, transferable=True) + assert obv0hab.kever.prefixer.transferable + # create non-local kevery for Wes to process nonlocal msgs + obv0kvy = eventing.Kevery(db=obv0hab.db, lax=False, local=False) + + obv1hab = obv1hby.makeHab(name='obv1', isith="1", icount=1, transferable=True) + assert obv1hab.kever.prefixer.transferable + # create non-local kevery for Wes to process nonlocal msgs + obv1kvy = eventing.Kevery(db=obv1hab.db, lax=False, local=False) + + obv2hab = obv2hby.makeHab(name='obv2', isith="1", icount=1, transferable=True) + assert obv2hab.kever.prefixer.transferable + # create non-local kevery for Wes to process nonlocal msgs + obv2kvy = eventing.Kevery(db=obv2hab.db, lax=False, local=False) + + for hab in [wat0hab, wat1hab, wat2hab, obv0hab, obv1hab, obv2hab]: + msg = hab.makeOwnInception() + parsing.Parser().parseOne(ims=msg, kvy=conKvy) + + conIcp = conHab.makeOwnInception() + for kvy in [wat0kvy, wat1kvy, wat2kvy, obv0kvy, obv1kvy, obv2kvy]: + parsing.Parser().parseOne(ims=bytes(conIcp), kvy=kvy) # make copy so we don't clobber it + + assert wat0hab.pre in conHab.kevers + assert wat1hab.pre in conHab.kevers + assert wat2hab.pre in conHab.kevers + assert obv0hab.pre in conHab.kevers + assert obv1hab.pre in conHab.kevers + assert obv2hab.pre in conHab.kevers + + data = dict(cid=conHab.pre, role=Roles.watcher, eid=wat0hab.pre) + rpy = conHab.reply(route="/end/role/add", data=data) + conHab.psr.parseOne(ims=bytes(rpy)) # make copy so we don't clobber it + wat0hab.psr.parseOne(ims=rpy) + + ender = conHab.db.ends.get(keys=(conHab.pre, Roles.watcher, wat0hab.pre)) + assert ender.allowed is True + ender = wat0hab.db.ends.get(keys=(conHab.pre, Roles.watcher, wat0hab.pre)) + assert ender.allowed is True + + for hab in [obv0hab, obv1hab, obv2hab]: + icp = hab.makeOwnInception() + conHab.psr.parseOne(ims=bytes(icp)) + wat0hab.psr.parseOne(ims=bytes(icp)) + wat1hab.psr.parseOne(ims=bytes(icp)) + wat2hab.psr.parseOne(ims=bytes(icp)) + + assert obv0hab.pre in conHab.kevers + assert obv1hab.pre in conHab.kevers + assert obv2hab.pre in conHab.kevers + assert obv2hab.pre in wat0hab.kevers + + route = f"/watcher/{wat0hab.pre}/add" + data = dict(cid=conHab.pre, oid=obv0hab.pre) + rpy = conHab.reply(route=route, data=data) + conHab.psr.parseOne(ims=bytes(rpy)) # make copy so we don't clobber it + wat0hab.psr.parseOne(ims=rpy) + + observed = conHab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv0hab.pre)) + assert observed.enabled is True + observed = wat0hab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv0hab.pre)) + assert observed.enabled is True + + route = f"/watcher/{wat0hab.pre}/add" + data = dict(cid=conHab.pre, oid=obv1hab.pre) + rpy = conHab.reply(route=route, data=data) + conHab.psr.parseOne(ims=bytes(rpy)) # make copy so we don't clobber it + wat0hab.psr.parseOne(ims=rpy) + + observed = conHab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv1hab.pre)) + assert observed.enabled is True + observed = wat0hab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv1hab.pre)) + assert observed.enabled is True + + route = f"/watcher/{wat0hab.pre}/add" + data = dict(cid=conHab.pre, oid=obv2hab.pre) + rpy = conHab.reply(route=route, data=data) + conHab.psr.parseOne(ims=bytes(rpy)) # make copy so we don't clobber it + wat0hab.psr.parseOne(ims=rpy) + + observed = conHab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv2hab.pre)) + assert observed.enabled is True + observed = wat0hab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv2hab.pre)) + assert observed.enabled is True + + route = f"/watcher/{wat0hab.pre}/cut" + data = dict(cid=conHab.pre, oid=obv1hab.pre) + rpy = conHab.reply(route=route, data=data) + conHab.psr.parseOne(ims=bytes(rpy)) # make copy so we don't clobber it + wat0hab.psr.parseOne(ims=rpy) + + observed = conHab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv1hab.pre)) + assert observed.enabled is False + observed = wat0hab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv1hab.pre)) + assert observed.enabled is False + + route = f"/watcher/{wat1hab.pre}/add" + data = dict(cid=conHab.pre, oid=obv0hab.pre) + rpy = conHab.reply(route=route, data=data) + conHab.psr.parseOne(ims=bytes(rpy)) # make copy so we don't clobber it + wat1hab.psr.parseOne(ims=rpy) + + observed = conHab.db.obvs.get(keys=(conHab.pre, wat1hab.pre, obv0hab.pre)) + assert observed.enabled is True + observed = wat1hab.db.obvs.get(keys=(conHab.pre, wat1hab.pre, obv0hab.pre)) + assert observed.enabled is True + + route = f"/watcher/{wat1hab.pre}/add" + data = dict(cid=conHab.pre, oid=obv2hab.pre, oobi=f"http://example.com/oobi/{obv2hab.pre}") + rpy = conHab.reply(route=route, data=data) + conHab.psr.parseOne(ims=bytes(rpy)) # make copy so we don't clobber it + wat1hab.psr.parseOne(ims=rpy) + + observed = conHab.db.obvs.get(keys=(conHab.pre, wat1hab.pre, obv0hab.pre)) + assert observed.enabled is True + assert conHab.db.oobis.get(keys=(f"http://example.com/oobi/{obv2hab.pre}",)) is not None + observed = wat1hab.db.obvs.get(keys=(conHab.pre, wat1hab.pre, obv2hab.pre)) + assert observed.enabled is True + observed = wat1hab.db.obvs.get(keys=(conHab.pre, wat1hab.pre, obv1hab.pre)) + assert observed is None + + # Make sure nothing was changed for Wat0 + observed = conHab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv0hab.pre)) + assert observed.enabled is True + observed = conHab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv1hab.pre)) + assert observed.enabled is False + observed = conHab.db.obvs.get(keys=(conHab.pre, wat0hab.pre, obv2hab.pre)) + assert observed.enabled is True + + if __name__ == "__main__": pytest.main(['-vv', 'test_reply.py::test_reply']) From 0ac693d105afd8fb47404cd62781839339754c26 Mon Sep 17 00:00:00 2001 From: SabbirIrfan <50948678+SabbirIrfan@users.noreply.github.com> Date: Thu, 27 Jun 2024 23:17:08 +0600 Subject: [PATCH 184/418] fix alias renaming (#798) * fix alias renaming * test added for renaming alias * test added for renaming alias * execution permission to the test script --- scripts/demo/basic/rename-alias.sh | 25 +++++++++++++++++++++++++ scripts/demo/test_scripts.sh | 6 ++++++ src/keri/app/cli/commands/rename.py | 9 ++++++--- 3 files changed, 37 insertions(+), 3 deletions(-) create mode 100755 scripts/demo/basic/rename-alias.sh diff --git a/scripts/demo/basic/rename-alias.sh b/scripts/demo/basic/rename-alias.sh new file mode 100755 index 000000000..5867aa174 --- /dev/null +++ b/scripts/demo/basic/rename-alias.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# CREATE DATABASE AND KEYSTORE +kli init --name rename-test --base "${KERI_TEMP_DIR}" --nopasscode + +# Incept with the initial alias "sabbir" +kli incept --name rename-test --base "${KERI_TEMP_DIR}" --alias sabbir --file ${KERI_DEMO_SCRIPT_DIR}/data/transferable-sample.json + +# Rename the alias from "sabbir" to "irfan" +kli rename --name rename-test --base "${KERI_TEMP_DIR}" --alias sabbir irfan + +# Extract alias from status +irfan_alias=$(kli status --name rename-test --base "${KERI_TEMP_DIR}" --alias irfan | grep -Eo 'Alias:\s+(.+)' | awk '{print $2}') + + + +# Check if the extracted alias is "irfan" +if [ "$irfan_alias" = "irfan" ]; then + echo "Alias successfully changed to 'irfan'." +else + echo "Alias did not change !" + +fi + +echo 'Test Complete' \ No newline at end of file diff --git a/scripts/demo/test_scripts.sh b/scripts/demo/test_scripts.sh index dd2c3cc88..6cce74d38 100755 --- a/scripts/demo/test_scripts.sh +++ b/scripts/demo/test_scripts.sh @@ -76,3 +76,9 @@ printf "Running multisig-join.sh" printf "\n************************************\n" "${script_dir}/basic/multisig-join.sh" isSuccess + +printf "\n************************************\n" +printf "Running rename.sh" +printf "\n************************************\n" +"${script_dir}/basic/rename-alias.sh" +isSuccess \ No newline at end of file diff --git a/src/keri/app/cli/commands/rename.py b/src/keri/app/cli/commands/rename.py index 10bcc54d8..b14b2f6a5 100644 --- a/src/keri/app/cli/commands/rename.py +++ b/src/keri/app/cli/commands/rename.py @@ -48,16 +48,19 @@ def rename(tymth, tock=0.0, **opts): if hby.habByName(newAlias) is not None: print(f"{newAlias} is already in use") - if (pre := hab.db.names.get(keys=("", name))) is not None: + if (pre := hab.db.names.get(keys=("", alias))) is not None: habord = hab.db.habs.get(keys=pre) - habord.name = name + habord.name = newAlias hab.db.habs.pin(keys=habord.hid, val=habord) - hab.db.names.pin(keys=("", name), val=pre) + hab.db.names.pin(keys=("", newAlias), val=pre) hab.db.names.rem(keys=("", alias)) print(f"Hab {alias} renamed to {newAlias}") + else: + raise ConfigurationError(f"No AID with name {alias} found") + except ConfigurationError as e: print(f"identifier prefix for {name} does not exist, incept must be run first", ) From a2c0566c45c7cbc9e9b06c2fd933e317641164ce Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Thu, 27 Jun 2024 10:39:53 -0700 Subject: [PATCH 185/418] Fixes to watcher add command to allow for multiple aliases that start with the same name. (#808) Added a feature to turn witnesses search off for MailboxDirector. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/watcher/add.py | 18 ++++++++++-------- src/keri/app/indirecting.py | 16 +++++++++------- src/keri/app/watching.py | 8 +++++++- src/keri/core/parsing.py | 4 ++-- src/keri/vdr/credentialing.py | 2 +- tests/app/test_watching.py | 18 +++++++++++++----- 6 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/keri/app/cli/commands/watcher/add.py b/src/keri/app/cli/commands/watcher/add.py index c993f8e90..546eec96c 100644 --- a/src/keri/app/cli/commands/watcher/add.py +++ b/src/keri/app/cli/commands/watcher/add.py @@ -54,24 +54,26 @@ def __init__(self, name, alias, base, bran, watcher, watched): self.hab = self.hby.habByName(alias) self.org = connecting.Organizer(hby=self.hby) + wat = None if watcher in self.hby.kevers: wat = watcher else: - wat = self.org.find("alias", watcher) - if len(wat) != 1: - raise ValueError(f"invalid recipient {watcher}") - wat = wat[0]['id'] + contacts = self.org.find("alias", watcher) + for contact in contacts: + if contact['alias'] == watcher: + wat = contact['id'] if not wat: raise ValueError(f"unknown watcher {watcher}") + watd = None if watched in self.hby.kevers: watd = watched else: - watd = self.org.find("alias", watched) - if len(watd) != 1: - raise ValueError(f"invalid recipient {watched}") - watd = watd[0]['id'] + contacts = self.org.find("alias", watched) + for contact in contacts: + if contact['alias'] == watched: + watd = contact['id'] if not watd: raise ValueError(f"unknown watched {watched}") diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 4dedb432a..80b3f8865 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -119,7 +119,7 @@ def createHttpServer(host, port, app, keypath=None, certpath=None, cafilepath=No Parameters: host(str) : host to bind to for this server, or None for default of '0.0.0.0', all ifaces port (int) : port to listen on for all HTTP(s) server instances - app (falcon.App) : application instance to pass to the http.Server instance + app (Any) : WSGI application instance to pass to the http.Server instance keypath (string) : the file path to the TLS private key certpath (string) : the file path to the TLS signed certificate (public key) cafilepath (string): the file path to the TLS CA certificate chain file @@ -506,7 +506,7 @@ class MailboxDirector(doing.DoDoer): """ def __init__(self, hby, topics, ims=None, verifier=None, kvy=None, exc=None, rep=None, cues=None, rvy=None, - tvy=None, **kwa): + tvy=None, witnesses=True, **kwa): """ Initialize instance. @@ -530,6 +530,7 @@ def __init__(self, hby, topics, ims=None, verifier=None, kvy=None, exc=None, rep self.pollers = list() self.prefixes = oset() self.cues = cues if cues is not None else decking.Deck() + self.witnesses = witnesses self.ims = ims if ims is not None else bytearray() @@ -622,11 +623,12 @@ def addPollers(self, hab): self.pollers.append(poller) self.extend([poller]) - wits = hab.kever.wits - for wit in wits: - poller = Poller(hab=hab, topics=self.topics, witness=wit) - self.pollers.append(poller) - self.extend([poller]) + if self.witnesses: + wits = hab.kever.wits + for wit in wits: + poller = Poller(hab=hab, topics=self.topics, witness=wit) + self.pollers.append(poller) + self.extend([poller]) self.prefixes.add(hab.pre) diff --git a/src/keri/app/watching.py b/src/keri/app/watching.py index 3a720d594..1016383a4 100644 --- a/src/keri/app/watching.py +++ b/src/keri/app/watching.py @@ -27,6 +27,7 @@ class DiffState: Uses Stateage to represent whether the remote KSR is even, ahead, behind or duplicitous """ + pre: str # The AID of the KSR wit: str # The entity reporting the KSR (non-local) state: Stateage # The state of the remote KSR relative to local sn: int # The sequence number of the remote KSR @@ -193,11 +194,16 @@ def diffState(wit, preksn, witksn): state (WitnessState): record indicating the differenced between the two provided KSN records """ + mypre = preksn.i + pre = witksn.i mysn = int(preksn.s, 16) mydig = preksn.d sn = int(witksn.s, 16) dig = witksn.d + if pre != mypre: + raise ValueError(f"can't compare key states from different AIDs {mypre}/{pre}") + # At the same sequence number, check the DIGs if mysn == sn: if mydig == dig: @@ -214,4 +220,4 @@ def diffState(wit, preksn, witksn): else: state = States.ahead - return DiffState(wit, state, sn, dig) + return DiffState(pre, wit, state, sn, dig) diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index 4faa7f36a..cd160c235 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -528,9 +528,9 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, # Non extraction errors happen after successfully extracted from stream # so we don't flush rest of stream just resume if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery msg non-extraction error: %s", ex.args[0]) + logger.exception("Kevery msg non-extraction error: %s", ex) else: - logger.exception("Kevery msg non-extraction error: %s", ex.args[0]) + logger.exception("Kevery msg non-extraction error: %s", ex) finally: done = True diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index 6c9d906fc..49f7e6357 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -473,7 +473,7 @@ def revoke(self, said, dt=None): raise kering.ValidationError("Invalid revoke of {} that has not been issued " "pre={}.".format(vci, self.regk)) ievt = self.reger.getTvt(dgKey(pre=vci, dig=vcser)) - iserder = serdering.serderACDC(raw=bytes(ievt)) # Serder(raw=bytes(ievt)) + iserder = serdering.SerderACDC(raw=bytes(ievt)) # Serder(raw=bytes(ievt)) if self.noBackers: serder = eventing.revoke(vcdig=vci, regk=self.regk, dig=iserder.said, dt=dt) diff --git a/tests/app/test_watching.py b/tests/app/test_watching.py index 4c7a49702..de6ca0da9 100644 --- a/tests/app/test_watching.py +++ b/tests/app/test_watching.py @@ -67,13 +67,16 @@ def test_diffstate(): diffstate = watching.diffState(wat, ksr0, ksr1) # Sequence numbers are the same, digest different == duplicitous - assert asdict(diffstate) == {'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + assert asdict(diffstate) == {'dig': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc', + 'pre': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', + 'sn': 0, 'state': 'duplicitous', - 'sn': 0, 'dig': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc'} + 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} # Same state == event diffstate = watching.diffState(wat, ksr0, ksr0) assert asdict(diffstate) == {'dig': 'EBiIFxr_o1b4x1YR21PblAFpFG61qDghqFBDyVSOXYW0', + 'pre': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', 'sn': 0, 'state': 'even', 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} @@ -83,6 +86,7 @@ def test_diffstate(): # Sequence numbers are the same, digest different == duplicitous assert asdict(diffstate) == {'dig': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc', + 'pre': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', 'sn': 2, 'state': 'ahead', 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} @@ -92,6 +96,7 @@ def test_diffstate(): # Sequence numbers are the same, digest different == duplicitous assert asdict(diffstate) == {'dig': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc', + 'pre': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', 'sn': 2, 'state': 'behind', 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} @@ -121,7 +126,8 @@ def test_adjudicator(): assert cue == {'cid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', 'kin': 'keyStateConsistent', 'oid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', - 'states': [DiffState(wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + 'states': [DiffState(pre="EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3", + wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', state='even', sn=0, dig='EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3')], @@ -132,7 +138,8 @@ def test_adjudicator(): adj.adjudicate(hab.pre, 1) assert len(adj.cues) == 1 cue = adj.cues.pull() - assert cue == {'behind': [DiffState(wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + assert cue == {'behind': [DiffState(pre="EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3", + wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', state='behind', sn=0, dig='EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3')], @@ -147,7 +154,8 @@ def test_adjudicator(): assert len(adj.cues) == 1 cue = adj.cues.pull() assert cue == {'cid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', - 'dups': [DiffState(wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + 'dups': [DiffState(pre="EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3", + wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', state='duplicitous', sn=1, dig='EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3')], From 566f5e311315934086dce31aca610da871b6955f Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Thu, 27 Jun 2024 10:57:27 -0700 Subject: [PATCH 186/418] Version bump to 1.2.0-dev7 Signed-off-by: pfeairheller --- Makefile | 4 ++-- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 72f319cf0..da71d85de 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev6 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev6-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev7 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev7-arm64 . .PHONY: build-witness-demo build-witness-demo: diff --git a/setup.py b/setup.py index b58a1b8c7..2ef9fa427 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev6', # also change in src/keri/__init__.py + version='1.2.0-dev7', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 8f8441c53..52515283c 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev6' # also change in setup.py +__version__ = '1.2.0-dev7' # also change in setup.py From e1bf1a3b9e3cb72c0fdaf47c82967b273b210b01 Mon Sep 17 00:00:00 2001 From: Kent Bull <65027257+kentbull@users.noreply.github.com> Date: Fri, 28 Jun 2024 10:13:53 -0600 Subject: [PATCH 187/418] fix: add smids group hab call (#809) --- src/keri/app/habbing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index ab9c2c9ee..d860f9bfc 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -342,7 +342,7 @@ def loadHabs(self): rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, name=habord.name, pre=habord.sid) elif habord.sid and habord.mid: - hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, + hab = SignifyGroupHab(smids=habord.smids, ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, name=habord.name, pre=pre) groups.append(habord) From d70d14828fc52f7555f2ff544a48637cba10d15d Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Fri, 28 Jun 2024 09:26:13 -0700 Subject: [PATCH 188/418] Externalize the size limit for LMDB map size for Baser to an environment variable. (#810) Signed-off-by: pfeairheller --- src/keri/db/basing.py | 6 ++++++ src/keri/db/dbing.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index f403a136a..8294cf551 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -593,6 +593,9 @@ def reopenDB(db, clear=False, **kwa): db.close(clear=clear) +KERIBaserMapSizeKey = "KERI_BASER_MAP_SIZE" + + class Baser(dbing.LMDBer): """ Baser sets up named sub databases with Keri Event Logs within main database @@ -935,6 +938,9 @@ def __init__(self, headDirPath=None, reopen=False, **kwa): self._kevers = dbdict() self._kevers.db = self # assign db for read through cache of kevers + if (mapSize := os.getenv(KERIBaserMapSizeKey)) is not None: + self.MapSize = int(mapSize) + super(Baser, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa) @property diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index ddb0ce4e0..1dc1208bc 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -303,6 +303,7 @@ class LMDBer(filing.Filer): TempSuffix = "_test" Perm = stat.S_ISVTX | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR # 0o1700==960 MaxNamedDBs = 96 + MapSize = 104857600 def __init__(self, readonly=False, **kwa): """ @@ -378,7 +379,7 @@ def reopen(self, readonly=False, **kwa): # open lmdb major database instance # creates files data.mdb and lock.mdb in .dbDirPath - self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=104857600, + self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=self.MapSize, mode=self.perm, readonly=self.readonly) self.opened = True if opened and self.env else False From 6b6970be92d851f94e58b7a9f4225a81582d2235 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Fri, 28 Jun 2024 12:41:09 -0400 Subject: [PATCH 189/418] v1.2.0-dev8 Signed-off-by: Kevin Griffin --- Makefile | 4 ++-- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index da71d85de..8db5a4274 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev7 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev7-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev8 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev8-arm64 . .PHONY: build-witness-demo build-witness-demo: diff --git a/setup.py b/setup.py index 2ef9fa427..cf891f233 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev7', # also change in src/keri/__init__.py + version='1.2.0-dev8', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 52515283c..f854dae39 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev7' # also change in setup.py +__version__ = '1.2.0-dev8' # also change in setup.py From 59a0c8a06a2839ae18d484ecfd9332222c8f4032 Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Mon, 8 Jul 2024 18:04:54 -0400 Subject: [PATCH 190/418] Fixed KERI_BASER_MAP_SIZE not integer bug. (#812) * Fixed KERI_BASER_MAP_SIZE not integer bug. Added log message and exception check for when KERI_BASER_MAP_SIZE not an integer. Signed-off-by: Charles Lanahan * Rethrow exception per PR request to fail fast. Updated log message to reflect the issue --------- Signed-off-by: Charles Lanahan --- src/keri/db/basing.py | 6 +++++- tests/db/test_basing.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 8294cf551..7426ae733 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -939,7 +939,11 @@ def __init__(self, headDirPath=None, reopen=False, **kwa): self._kevers.db = self # assign db for read through cache of kevers if (mapSize := os.getenv(KERIBaserMapSizeKey)) is not None: - self.MapSize = int(mapSize) + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error("KERI_BASER_MAP_SIZE must be an integer value >1!") + raise super(Baser, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa) diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 43d575007..89044c7d9 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -2338,6 +2338,22 @@ def test_group_members(): """End Test""" +def test_KERI_BASER_MAP_SIZE_handles_bad_values(caplog): + # Base case works because of above tests, they will all break if happy path + # is broken. We'll just test some unhappy values. + + # Pytest will fail if any exceptions raised here. + os.environ["KERI_BASER_MAP_SIZE"] = "foo" # Not an int + err_msg = "KERI_BASER_MAP_SIZE must be an integer value > 1!" + with pytest.raises(ValueError): + Baser(reopen=False, temp=True) + assert err_msg in caplog.messages + os.environ["KERI_BASER_MAP_SIZE"] = "1.0" # Not an int + with pytest.raises(ValueError): + Baser(reopen=False, temp=True) + assert err_msg in caplog.messages + os.environ.pop("KERI_BASER_MAP_SIZE") + if __name__ == "__main__": test_baser() From 89a2884adeb5ecd52bd69ec18d869d9159e459bf Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Mon, 8 Jul 2024 15:16:41 -0700 Subject: [PATCH 191/418] Update to latest Hio build. (#815) New command line command for adding location records for an AID Signed-off-by: pfeairheller --- setup.py | 2 +- .../app/cli/commands/location/__init__.py | 0 src/keri/app/cli/commands/location/add.py | 123 ++++++++++++++++++ 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 src/keri/app/cli/commands/location/__init__.py create mode 100644 src/keri/app/cli/commands/location/add.py diff --git a/setup.py b/setup.py index cf891f233..dea228df3 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ 'cbor2>=5.6.2', 'multidict>=6.0.5', 'ordered-set>=4.1.0', - 'hio>=0.6.12', + 'hio>=0.6.13', 'multicommand>=1.0.0', 'jsonschema>=4.21.1', 'falcon>=3.1.3', diff --git a/src/keri/app/cli/commands/location/__init__.py b/src/keri/app/cli/commands/location/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/keri/app/cli/commands/location/add.py b/src/keri/app/cli/commands/location/add.py new file mode 100644 index 000000000..d549c0ded --- /dev/null +++ b/src/keri/app/cli/commands/location/add.py @@ -0,0 +1,123 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse +from urllib.parse import urlparse + +from hio import help +from hio.base import doing + +from keri import kering +from keri.app import habbing, grouping, indirecting, forwarding +from keri.app.agenting import WitnessPublisher +from keri.app.cli.common import existing +from keri.app.notifying import Notifier +from keri.core import parsing +from keri.peer import exchanging + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Add new endpoint location record.') +parser.set_defaults(handler=lambda args: add_loc(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--url", "-u", help="Location URL", + required=True) +parser.add_argument("--eid", "-e", help="qualified base64 of AID to associate a location with, defaults to alias aid ", + required=False, default=None) +parser.add_argument("--time", help="timestamp for the end auth", required=False, default=None) + + +def add_loc(args): + """ Command line tool for adding location scheme records + + """ + ld = LocationDoer(name=args.name, + base=args.base, + alias=args.alias, + bran=args.bran, + url=args.url, + eid=args.eid, + timestamp=args.time) + return [ld] + + +class LocationDoer(doing.DoDoer): + + def __init__(self, name, base, alias, bran, url, eid, timestamp=None): + self.url = url + self.eid = eid + self.timestamp = timestamp + + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hab = self.hby.habByName(alias) + self.witpub = WitnessPublisher(hby=self.hby) + self.postman = forwarding.Poster(hby=self.hby) + notifier = Notifier(self.hby) + mux = grouping.Multiplexor(self.hby, notifier=notifier) + exc = exchanging.Exchanger(hby=self.hby, handlers=[]) + grouping.loadHandlers(exc, mux) + + mbx = indirecting.MailboxDirector(hby=self.hby, topics=["/receipt", "/multisig", "/replay"], exc=exc) + + if self.hab is None: + raise kering.ConfigurationError(f"unknown alias={alias}") + + self.toRemove = [self.witpub, self.postman, mbx] + + super(LocationDoer, self).__init__(doers=self.toRemove + [doing.doify(self.roleDo)]) + + def roleDo(self, tymth, tock=0.0): + """ Export any end reply messages previous saved for the provided AID + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + up = urlparse(self.url) + eid = self.eid if self.eid is not None else self.hab.pre + + msg = self.hab.makeLocScheme(url=self.url, eid=eid, scheme=up.scheme) + parsing.Parser().parse(ims=bytes(msg), kvy=self.hab.kvy, rvy=self.hab.rvy) + + if isinstance(self.hab, habbing.GroupHab): + smids = self.hab.db.signingMembers(pre=self.hab.pre) + smids.remove(self.hab.mhab.pre) + + for recp in smids: # this goes to other participants only as a signaling mechanism + exn, atc = grouping.multisigRpyExn(ghab=self.hab, rpy=msg) + self.postman.send(src=self.hab.mhab.pre, + dest=recp, + topic="multisig", + serder=exn, + attachment=atc) + + while not self.hab.loadLocScheme(scheme=up.scheme, eid=eid): + yield self.tock + + self.witpub.msgs.append(dict(pre=self.hab.pre, msg=bytes(msg))) + + while not self.witpub.cues: + yield self.tock + + print(f"Location {self.url} added for aid {eid} with scheme {up.scheme}") + + self.remove(self.toRemove) + return From 12a134e28df444426a828deb6b9eae274b91397f Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Mon, 8 Jul 2024 15:33:07 -0700 Subject: [PATCH 192/418] Version Bump to 1.2.0-dev9 --- Makefile | 7 ++++--- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 8db5a4274..5e84fafc0 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev8 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev8-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev9 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev9-arm64 . .PHONY: build-witness-demo build-witness-demo: @@ -11,7 +11,8 @@ build-witness-demo: .PHONY: publish-keri publish-keri: - @docker push weboftrust/keri --all-tags + @docker push weboftrust/keri:1.2.0-dev9 + @docker push weboftrust/keri:1.2.0-dev9-arm64 .PHONY: publish-keri-witness-demo publish-keri-witness-demo: diff --git a/setup.py b/setup.py index dea228df3..0db79a09b 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev8', # also change in src/keri/__init__.py + version='1.2.0-dev9', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index f854dae39..1afbef12d 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev8' # also change in setup.py +__version__ = '1.2.0-dev9' # also change in setup.py From d05049927fc7b90ca3b269110de482af7c6fda64 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Mon, 15 Jul 2024 14:19:20 -0700 Subject: [PATCH 193/418] - A new kli command to send an `/introduce` message with an AID and OOBI. (#817) - Bug fix to watching to check against the correct key state. - Logging additions, fixes and typo fixes. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/introduce.py | 137 ++++++++++++++++++ .../app/cli/commands/watcher/adjudicate.py | 5 +- src/keri/app/forwarding.py | 2 +- src/keri/app/oobiing.py | 2 +- src/keri/app/watching.py | 2 +- src/keri/core/eventing.py | 7 +- src/keri/core/parsing.py | 6 +- src/keri/core/routing.py | 4 + 8 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 src/keri/app/cli/commands/introduce.py diff --git a/src/keri/app/cli/commands/introduce.py b/src/keri/app/cli/commands/introduce.py new file mode 100644 index 000000000..f5f5a2ae7 --- /dev/null +++ b/src/keri/app/cli/commands/introduce.py @@ -0,0 +1,137 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio.base import doing + +from keri.core import serdering +from ..common import existing +from ... import habbing, connecting, forwarding + +parser = argparse.ArgumentParser(description='Send an rpy /introduce message to recipient with OOBI') +parser.set_defaults(handler=lambda args: introduce(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--recipient', help='alias or contact to send the introduction to', required=True) +parser.add_argument('--introducee', help='alias or contact of the OOBI to send', required=True) +parser.add_argument("--role", "-r", help="role of oobi to send", required=True) + + +def introduce(args): + """ + Performs a rotation of the identifier of the environment represented by the provided name parameter + + args (parseargs): Command line argument + + """ + name = args.name + alias = args.alias + base = args.base + bran = args.bran + introducee = args.introducee + recipient = args.recipient + role = args.role + + ixnDoer = IntroduceDoer(name=name, base=base, alias=alias, bran=bran, introducee=introducee, recipient=recipient, + role=role,) + + return [ixnDoer] + + +class IntroduceDoer(doing.DoDoer): + """ + + DoDoer that launches Doers needed to introduce one controller of an AID to another + + """ + + def __init__(self, name, base, bran, alias, introducee, recipient, role): + """ + Returns DoDoer with all registered Doers needed to perform an introduction. + + Parameters: + name is human readable str of identifier + """ + + self.alias = alias + self.introducee = introducee + self.recipient = recipient + self.role = role + + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer + self.org = connecting.Organizer(hby=self.hby) + self.postman = forwarding.Poster(hby=self.hby) + doers = [self.hbyDoer, self.postman, doing.doify(self.introduceDo)] + + super(IntroduceDoer, self).__init__(doers=doers) + + def introduceDo(self, tymth, tock=0.0, **opts): + """ + Returns: doifiable Doist compatible generator method + Usage: + add result of doify on this method to doers list + """ + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + hab = self.hby.habByName(name=self.alias) + + if self.recipient in self.hby.kevers: + recp = self.recipient + else: + recp = self.org.find("alias", self.recipient) + if len(recp) != 1: + raise ValueError(f"invalid recipient {self.recipient}") + recp = recp[0]['id'] + + if (ihab := self.hby.habByName(self.introducee)) is not None: + introducee = ihab.pre + elif self.introducee in self.hby.habs: + introducee = self.introducee + elif self.introducee in self.hby.kevers: + introducee = self.introducee + else: + introducee = None + results = self.org.find("alias", self.introducee) + for result in results: + if result["alias"] == self.introducee: + introducee = result['id'] + if not introducee: + raise ValueError(f"invalid introducee {self.introducee}") + + oobi = None + for (key,), obr in self.hby.db.roobi.getItemIter(): + if obr.cid == introducee and obr.role == self.role: + oobi = key + + if oobi is None: + raise ValueError(f"Unable to find OOBI with role {self.role} for {introducee}") + + msg = hab.reply(route="/introduce", data=dict( + cid=hab.pre, + oobi=oobi + )) + + serder = serdering.SerderKERI(raw=msg) + atc = msg[serder.size:] + self.postman.send(src=hab.pre, dest=recp, topic="credential", serder=serder, + attachment=atc) + + while not len(self.postman.cues) == 1: + yield self.tock + + print(f"Introduction with OOBI {oobi} sent for {self.introducee} sent to {self.recipient}") + + toRemove = [self.hbyDoer, self.postman] + self.remove(toRemove) + + return diff --git a/src/keri/app/cli/commands/watcher/adjudicate.py b/src/keri/app/cli/commands/watcher/adjudicate.py index 82e368e83..41c9efc9a 100644 --- a/src/keri/app/cli/commands/watcher/adjudicate.py +++ b/src/keri/app/cli/commands/watcher/adjudicate.py @@ -133,10 +133,11 @@ def adjudicate(self, tymth, tock=0.0, **opts): case "keyStateUpdate": ahds = cue["aheads"] - logger.info(f"Threshold ({self.toad}) satisfying number of watchers ({len(ahds)}) are ahead") + print(f"Threshold ({self.toad}) satisfying number of watchers ({len(ahds)}) are ahead") for state in ahds: - logger.info(f"\tWatcher {state.wit} at Seq No. {state.sn} with digest: {state.dig}") + print(f"\tWatcher {state.wit} at Seq No. {state.sn} with digest: {state.dig}") + print("Submitting query to update local copy of latest events.") state = random.choice(ahds) querier = querying.SeqNoQuerier(hby=self.hby, hab=hab, pre=self.watched, sn=state.sn, wits=[state.wit]) diff --git a/src/keri/app/forwarding.py b/src/keri/app/forwarding.py index 2356ebc19..db5818f09 100644 --- a/src/keri/app/forwarding.py +++ b/src/keri/app/forwarding.py @@ -85,7 +85,7 @@ def deliverDo(self, tymth=None, tock=0.0): elif Roles.witness in ends: yield from self.forwardToWitness(hab, ends[Roles.witness], recp=recp, serder=srdr, atc=atc, topic=tpc) else: - logger.info(f"No end roles for {recp} to send evt={recp}") + logger.info(f"No end roles for {recp} to send evt={srdr.said}") continue except kering.ConfigurationError as e: logger.error(f"Error sending to {recp} with ends={ends}. Err={e}") diff --git a/src/keri/app/oobiing.py b/src/keri/app/oobiing.py index 89f87a526..cb8b41d3a 100644 --- a/src/keri/app/oobiing.py +++ b/src/keri/app/oobiing.py @@ -382,7 +382,7 @@ def processReply(self, *, serder, saider, route, cigars=None, tsgs=None, **kwarg aid=aid, osaider=None, cigars=cigars, tsgs=tsgs) if not accepted: - raise UnverifiedReplyError(f"Unverified introduciton reply. {serder.ked}") + raise UnverifiedReplyError(f"Unverified introduction reply. {serder.ked}") obr = basing.OobiRecord(cid=cid, date=dt) self.hby.db.oobis.put(keys=(oobi,), val=obr) diff --git a/src/keri/app/watching.py b/src/keri/app/watching.py index 1016383a4..677ee63f5 100644 --- a/src/keri/app/watching.py +++ b/src/keri/app/watching.py @@ -101,7 +101,7 @@ def adjudicate(self, watched, toad=None): raise ValueError(f"Threshold of {toad} is greater than number watchers {len(watchers)}") states = [] - mystate = self.hab.kever.state() + mystate = self.hab.kevers[watched].state() for watcher in watchers: saider = self.hab.db.knas.get(keys=(watched, watcher)) if saider is None: diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 7b552b072..9580e8d9b 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2130,8 +2130,8 @@ def deriveBacks(self, serder): Parameters: serder (SerderKeri): instance of current event """ - if serder.ilk not in (Ilks.rot, Ilks.drt): # no changes - return (self.wits, self.cuts, self.adds) + if serder.ilk not in (Ilks.rot, Ilks.drt) or self.sn >= serder.sn: # no changes + return self.wits, self.cuts, self.adds witset = oset(self.wits) cuts = serder.cuts @@ -4170,7 +4170,8 @@ def processReplyEndRole(self, *, serder, saider, route, cigars=None, tsgs=None, aid=aid, osaider=osaider, cigars=cigars, tsgs=tsgs) if not accepted: - raise UnverifiedReplyError(f"Unverified end role reply. {serder.ked}") + logger.debug(f"Unverified end role reply ked={serder.ked}") + raise UnverifiedReplyError(f"Unverified end role reply. {serder.said}") self.updateEnd(keys=keys, saider=saider, allowed=allowed) # update .eans and .ends diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index cd160c235..1cd9b4cd4 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -445,7 +445,7 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, if logger.isEnabledFor(logging.DEBUG): logger.exception("Parser msg non-extraction error: %s", ex) else: - logger.exception("Parser msg non-extraction error: %s", ex) + logger.error("Parser msg non-extraction error: %s", ex) yield return True @@ -530,7 +530,7 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery msg non-extraction error: %s", ex) else: - logger.exception("Kevery msg non-extraction error: %s", ex) + logger.error("Kevery msg non-extraction error: %s", ex) finally: done = True @@ -620,7 +620,7 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, if logger.isEnabledFor(logging.DEBUG): logger.exception("Parser msg non-extraction error: %s", ex.args[0]) else: - logger.exception("Parser msg non-extraction error: %s", ex.args[0]) + logger.error("Parser msg non-extraction error: %s", ex.args[0]) yield return True # should never return diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 3601529f0..57d1346a6 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -261,6 +261,8 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, for cigar in cigars: # process each couple to verify sig and write to db if cigar.verfer.transferable: # ignore invalid transferable verfers + logger.info("Kevery process: skipped invalid transferable verfers" + " on reply said=", serder.said) continue # skip invalid transferable if not self.lax and cigar.verfer.qb64 in self.prefixes: # own cig @@ -338,6 +340,8 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, if sdig is None: # create cue here to request key state for sprefixer signer # signer's est event not yet in signer's KEL + logger.info("Kevery process: escrowing without key state for signer" + " on reply said=", serder.said) self.escrowReply(serder=serder, saider=saider, dater=dater, route=route, prefixer=prefixer, seqner=seqner, ssaider=ssaider, sigers=sigers) From 3febc54a463125a4cc3d86d694e57004134b2092 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Mon, 15 Jul 2024 14:57:40 -0700 Subject: [PATCH 194/418] Version Bump --- Makefile | 8 ++++---- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 5e84fafc0..e5c2c807e 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev9 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev9-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev10 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev10-arm64 . .PHONY: build-witness-demo build-witness-demo: @@ -11,8 +11,8 @@ build-witness-demo: .PHONY: publish-keri publish-keri: - @docker push weboftrust/keri:1.2.0-dev9 - @docker push weboftrust/keri:1.2.0-dev9-arm64 + @docker push weboftrust/keri:1.2.0-dev10 + @docker push weboftrust/keri:1.2.0-dev10-arm64 .PHONY: publish-keri-witness-demo publish-keri-witness-demo: diff --git a/setup.py b/setup.py index 0db79a09b..52bb137f8 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev9', # also change in src/keri/__init__.py + version='1.2.0-dev10', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 1afbef12d..cb048f4e4 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev9' # also change in setup.py +__version__ = '1.2.0-dev10' # also change in setup.py From 13c4a43f4f46471b6f108e0ec02fdfd6430d6f7c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 17 Jul 2024 16:57:24 -0600 Subject: [PATCH 195/418] added export of new Counter to core __init__ --- src/keri/core/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index e71721fc2..128c22537 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -14,4 +14,5 @@ from .coring import Tholder from .indexing import Indexer, Siger, IdrDex, IdxSigDex -from .signing import (Signer, Salter, Cipher, Encrypter, Decrypter, ) +from .signing import Signer, Salter, Cipher, Encrypter, Decrypter +from .counting import Counter From b3dd43b29bf6b9bbd9dbe4ff3c21a36a4e56e6a0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 21 Jul 2024 12:16:03 -0600 Subject: [PATCH 196/418] refactor change tags in counting to Codens simplified. Removed tag init parameter now code init parameter is polymorphic so can take code or code name (coden) started migrating coring.Counter to counting.Counter with gvrsn=Vrsn_1_0 --- src/keri/app/agenting.py | 6 +- src/keri/core/__init__.py | 2 +- src/keri/core/counting.py | 98 ++++--- src/keri/core/serdering.py | 12 +- src/keri/db/basing.py | 28 +- src/keri/vdr/credentialing.py | 4 +- tests/app/__init__.py | 5 +- tests/app/test_agenting.py | 8 +- tests/core/test_counting.py | 520 +++++++++++++++++++--------------- 9 files changed, 370 insertions(+), 313 deletions(-) diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 456e3f4bc..894a3cbd4 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -17,6 +17,7 @@ from . import httping, forwarding from .. import help from .. import kering +from .. import core from ..core import eventing, parsing, coring, serdering, indexing from ..core.coring import CtrDex from ..db import dbing @@ -104,7 +105,7 @@ def receipt(self, pre, sn=None, auths=None): del rct[:rserder.size] # pull off the count code - coring.Counter(qb64b=rct, strip=True) + core.Counter(qb64b=rct, strip=True, gvrsn=kering.Vrsn_1_0) rcts[wit] = rct else: print(f"invalid response {rep.status} from witnesses {wit}") @@ -124,7 +125,8 @@ def receipt(self, pre, sn=None, auths=None): sn=sn, said=ser.said) msg.extend(rserder.raw) - msg.extend(coring.Counter(code=CtrDex.NonTransReceiptCouples, count=len(wigs)).qb64b) + msg.extend(core.Counter(core.Codens.NonTransReceiptCouples, + count=len(wigs), gvrsn=kering.Vrsn_1_0).qb64b) for wig in wigs: msg.extend(wig) diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index 128c22537..734d8259c 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -15,4 +15,4 @@ from .coring import Tholder from .indexing import Indexer, Siger, IdrDex, IdxSigDex from .signing import Signer, Salter, Cipher, Encrypter, Decrypter -from .counting import Counter +from .counting import Counter, Codens diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 404b9b69b..255695518 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -4,6 +4,7 @@ Provides versioning support for Counter classes and codes """ +import copy from dataclasses import dataclass, astuple, asdict from collections import namedtuple @@ -147,18 +148,15 @@ def __iter__(self): CtrDex_2_0 = CounterCodex_2_0() -# keys and values as strings of keys -Codict1 = asdict(CtrDex_1_0) -Tagage_1_0 = namedtuple("Tagage_1_0", list(Codict1), defaults=list(Codict1)) -Tags_1_0 = Tagage_1_0() # uses defaults - -Codict2 = asdict(CtrDex_2_0) -Tagage_2_0 = namedtuple("Tagage_2_0", list(Codict2), defaults=list(Codict2)) -Tags_2_0 = Tagage_2_0() # uses defaults - -CodictAll = Codict2 | Codict1 -AllTagage = namedtuple("AllTagage", list(CodictAll), defaults=list(CodictAll)) -AllTags = AllTagage() # uses defaults +# CodeNames is tuple of codes names given by attributes of union of codices +CodeNames = tuple(asdict(CtrDex_2_0) | asdict(CtrDex_1_0)) +# Codens is namedtuple of CodeNames where its names are the code names +# Codens enables using the attributes of the named tuple to specify a code by +# name (indirection) so that changes in the code itself do not break the +# creation of a counter. Enables specifying a counter by the code name not the +# code itself. The code may change between versions but the code name does not. +Codenage = namedtuple("Codenage", CodeNames, defaults=CodeNames) +Codens = Codenage() @dataclass(frozen=True) @@ -200,8 +198,9 @@ class Counter: Includes the following attributes and properties: Class Attributes: - Codes (dict): of codexes keyed by version - Tags (dict): of tagages keyed by version + Codes (dict): nested of codexes keyed by major and minor version + Names (dict): nested of map of code names to codes keyed by + major and minor version Hards (dict): of hard code sizes keyed by text domain selector Bards (dict): of hard code sizes keyed by binary domain selector Sizes (dict): of size tables keyed by version. Size table is dict @@ -310,7 +309,15 @@ class Counter: }, } - Tags = {Vrsn_1_0: Tags_1_0, Vrsn_2_0: Tags_2_0} + + # invert dataclass codenames: codes to dict codes: codenames + Names = copy.deepcopy(Codes) # make deep nested copy so can invert nested values + for minor in Names.values(): + for key in minor: + minor[key] = {val: key for key, val in asdict(minor[key]).items()} + + + # Hards table maps from bytes Base64 first two code chars to int of # hard size, hs,(stable) of code. The soft size, ss, (unstable) for Counter @@ -415,17 +422,16 @@ class Counter: } - def __init__(self, tag=None, *, code = None, count=None, countB64=None, + def __init__(self, code=None, *, count=None, countB64=None, qb64b=None, qb64=None, qb2=None, strip=False, gvrsn=Vrsn_2_0): """ Validate as fully qualified Parameters: - tag (str | None): label of stable (hard) part of derivation code - to lookup in codex so it can depend on version. - takes precedence over code. - code (str | None): stable (hard) part of derivation code - if tag provided lookup code from tag - else if tag is None and code provided use code. + code (str | None): either stable (hard) part of derivation code or + code name. When code name then look up code from + ._codes. This allows versioning to change code + but keep stable code name. + count (int | None): count of framed material in quadlets/triplets for composition. Count does not include code. When both count and countB64 are None then count @@ -466,14 +472,15 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, self._version = gvrsn # provided version may be earlier than supported version - if tag: - if not hasattr(self._codes, tag): - raise kering.InvalidCodeError(f"Unsupported {tag=}.") - code = self._codes[tag] - - if code is not None: # code (hard) provided + if code: # code (hard) provided + # assumes ._sizes ._codes coherent if code not in self._sizes or len(code) < 2: - raise kering.InvalidCodeError(f"Unsupported {code=}.") + try: + code = self._codes[code] # code is code name so look up code + if code not in self._sizes or len(code) < 2: + raise kering.InvalidCodeError(f"Unsupported {code=}.") + except Exception as ex: + raise kering.InvalidCodeError(f"Unsupported {code=}.") from ex hs, ss, fs, ls = self._sizes[code] # get sizes for code cs = hs + ss # both hard + soft code size @@ -520,8 +527,7 @@ def __init__(self, tag=None, *, code = None, count=None, countB64=None, "(code and count) or qb64b or " "qb64 or qb2.") - codenames = { val: key for key, val in asdict(self.codes).items()} # map codes to code names - self._tag = codenames[self.code] + self._name = self.Names[gvrsn.major][latest][self.code] @property def version(self): @@ -547,13 +553,13 @@ def codes(self): """ return self._codes - @property - def tags(self): - """ - Returns tags for current .version - Makes .tags read only - """ - return self.Tags[self.version] # use own version + #@property + #def tags(self): + #""" + #Returns tags for current .version + #Makes .tags read only + #""" + #return self.Tags[self.version] # use own version @property def sizes(self): @@ -574,25 +580,17 @@ def code(self): """ return self._code - @property - def tag(self): - """ - Returns: - tag (str): code name for self.code - - Getter for ._tag. Makes .tag read only - """ - return self._tag - @property def name(self): """ Returns: - name (str): code name for self.code alias of .tag. Match interface + name (str): code name for self.code. Match interface for annotation for primitives like Matter + Getter for ._name. Makes .name read only + """ - return self.tag + return self._name @property diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 9c46826d0..bbfe64a97 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -38,7 +38,7 @@ from .coring import (Matter, Saider, Verfer, Diger, Number, Tholder, Tagger, Ilker, Traitor, Verser, ) -from .counting import GenDex, AllTags, Counter, SealDex_2_0 +from .counting import GenDex, Counter, Codens, SealDex_2_0 from .structing import Sealer, SClanDom @@ -1298,7 +1298,7 @@ def _dumps(self, sad=None, proto=None, vrsn=None): for e in v: # list frame.extend(e.encode("utf-8")) - val = bytearray(Counter(tag=AllTags.GenericListGroup, + val = bytearray(Counter(Codens.GenericListGroup, count=len(frame) // 4).qb64b) val.extend(frame) @@ -1307,7 +1307,7 @@ def _dumps(self, sad=None, proto=None, vrsn=None): for e in v: # list frame.extend(Traitor(trait=e).qb64b) - val = bytearray(Counter(tag=AllTags.GenericListGroup, + val = bytearray(Counter(Codens.GenericListGroup, count=len(frame) // 4).qb64b) val.extend(frame) @@ -1342,7 +1342,7 @@ def _dumps(self, sad=None, proto=None, vrsn=None): #generic seal no count type (v, Mapping): #for l, e in v.items(): #pass - #val = bytearray(Counter(tag=AllTags.GenericMapGroup, + #val = bytearray(Counter(tag=""GenericMapGroup"", # count=len(frame) // 4).qb64b) #val.extend(mapframe) @@ -1352,7 +1352,7 @@ def _dumps(self, sad=None, proto=None, vrsn=None): gframe = bytearray() gcode = None - val = bytearray(Counter(tag=AllTags.GenericListGroup, + val = bytearray(Counter(Codens.GenericListGroup, count=len(frame) // 4).qb64b) val.extend(frame) @@ -1378,7 +1378,7 @@ def _dumps(self, sad=None, proto=None, vrsn=None): # prepend count code for message if fixed: - raw = bytearray(Counter(tag=AllTags.FixedMessageBodyGroup, + raw = bytearray(Counter(Codens.FixedMessageBodyGroup, count=len(bdy) // 4).qb64b) raw.extend(bdy) else: diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 7426ae733..d50b8b12e 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1556,46 +1556,46 @@ def cloneEvtMsg(self, pre, fn, dig): # add indexed signatures to attachments if not (sigs := self.getSigs(key=dgkey)): raise kering.MissingEntryError("Missing sigs for dig={}.".format(dig)) - atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigs)).qb64b) + atc.extend(core.Counter(code=core.Codens.ControllerIdxSigs, + count=len(sigs), gvrsn=kering.Vrsn_1_0).qb64b) for sig in sigs: atc.extend(sig) # add indexed witness signatures to attachments if wigs := self.getWigs(key=dgkey): - atc.extend(coring.Counter(code=coring.CtrDex.WitnessIdxSigs, - count=len(wigs)).qb64b) + atc.extend(core.Counter(code=core.Codens.WitnessIdxSigs, + count=len(wigs), gvrsn=kering.Vrsn_1_0).qb64b) for wig in wigs: atc.extend(wig) # add authorizer (delegator/issuer) source seal event couple to attachments couple = self.getAes(dgkey) if couple is not None: - atc.extend(coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1).qb64b) + atc.extend(core.Counter(code=core.Codens.SealSourceCouples, + count=1, gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(couple) # add trans endorsement quadruples to attachments not controller # may have been originally key event attachments or receipted endorsements if quads := self.getVrcs(key=dgkey): - atc.extend(coring.Counter(code=coring.CtrDex.TransReceiptQuadruples, - count=len(quads)).qb64b) + atc.extend(core.Counter(code=core.Codens.TransReceiptQuadruples, + count=len(quads), gvrsn=kering.Vrsn_1_0).qb64b) for quad in quads: atc.extend(quad) # add nontrans endorsement couples to attachments not witnesses # may have been originally key event attachments or receipted endorsements if coups := self.getRcts(key=dgkey): - atc.extend(coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, - count=len(coups)).qb64b) + atc.extend(core.Counter(code=core.Codens.NonTransReceiptCouples, + count=len(coups), gvrsn=kering.Vrsn_1_0).qb64b) for coup in coups: atc.extend(coup) # add first seen replay couple to attachments if not (dts := self.getDts(key=dgkey)): raise kering.MissingEntryError("Missing datetime for dig={}.".format(dig)) - atc.extend(coring.Counter(code=coring.CtrDex.FirstSeenReplayCouples, - count=1).qb64b) + atc.extend(core.Counter(code=core.Codens.FirstSeenReplayCouples, + count=1, gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(core.Number(num=fn, code=core.NumDex.Huge).qb64b) # may not need to be Huge atc.extend(coring.Dater(dts=bytes(dts)).qb64b) @@ -1603,8 +1603,8 @@ def cloneEvtMsg(self, pre, fn, dig): if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - pcnt = coring.Counter(code=coring.CtrDex.AttachmentGroup, - count=(len(atc) // 4)).qb64b + pcnt = core.Counter(code=core.Codens.AttachmentGroup, + count=(len(atc) // 4), gvrsn=kering.Vrsn_1_0).qb64b msg.extend(pcnt) msg.extend(atc) return msg diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index 49f7e6357..416bb0af0 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -12,6 +12,7 @@ from .. import kering, help from ..app import agenting from ..app.habbing import GroupHab +from .. import kering, core from ..core import parsing, coring, scheming, serdering from ..core.coring import Seqner, MtrDex from ..core.eventing import TraitDex @@ -931,7 +932,8 @@ def sendCredential(hby, hab, reger, postman, creder, recp): postman.send(serder=source, attachment=atc) serder, prefixer, seqner, saider = reger.cloneCred(creder.said) - atc = bytearray(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + atc = bytearray(core.Counter(core.Codens.SealSourceTriples, + count=1, gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(prefixer.qb64b) atc.extend(seqner.qb64b) atc.extend(saider.qb64b) diff --git a/tests/app/__init__.py b/tests/app/__init__.py index 1120d113a..312d14603 100644 --- a/tests/app/__init__.py +++ b/tests/app/__init__.py @@ -5,6 +5,7 @@ from contextlib import contextmanager from keri.app import habbing +from keri import kering, core from keri.core import coring, eventing, parsing from keri.db import dbing @@ -54,8 +55,8 @@ def openMultiSig(prefix="test", salt=b'0123456789abcdef', temp=True, **kwa): sigs.extend(bytes(hab3.db.getSigs(dgkey)[0])) evt = bytearray(eraw) - evt.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=3).qb64b) # attach cnt + evt.extend(core.Counter(code=core.Codens.ControllerIdxSigs, + count=3, gvrsn=kering.Vrsn_1_0).qb64b) # attach cnt evt.extend(sigs) parsing.Parser().parse(ims=bytearray(evt), kvy=kev3, local=True) diff --git a/tests/app/test_agenting.py b/tests/app/test_agenting.py index 4b2022ddd..4f746168c 100644 --- a/tests/app/test_agenting.py +++ b/tests/app/test_agenting.py @@ -7,10 +7,7 @@ from hio.base import doing, tyming -from keri import kering - -from keri import core - +from keri import kering, core from keri.core import coring, serdering from keri.core.coring import Counter, CtrDex, Seqner from keri.help import nowIso8601 @@ -158,7 +155,8 @@ def testDo(self, tymth, tock=0.0): serder = eventing.issue(vcdig=regser.pre, regk="EbA1o_bItVC9i6YB3hr2C3I_Gtqvz02vCmavJNoBA3Jg") msg = bytearray(serder.raw) - msg.extend(Counter(CtrDex.SealSourceCouples, count=1).qb64b) + msg.extend(core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(Seqner(sn=self.palHab.kever.sn).qb64b) msg.extend(self.palHab.kever.serder.saidb) diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index e94d18e5e..443a28551 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -21,8 +21,8 @@ from keri.core import counting -from keri.core.counting import GenDex, Sizage, MapDom, Counter -from keri.core.counting import Versionage, Vrsn_1_0, Vrsn_2_0, AllTags +from keri.core.counting import GenDex, Sizage, MapDom, Counter, Codens +from keri.core.counting import Versionage, Vrsn_1_0, Vrsn_2_0 @@ -60,6 +60,8 @@ def test_codexes_tags(): """ Test supporting module attributes """ + + assert asdict(counting.CtrDex_1_0) == \ { 'ControllerIdxSigs': '-A', @@ -139,146 +141,122 @@ def test_codexes_tags(): 'KERIACDCGenusVersion': '--AAA' } - assert counting.Tags_1_0._asdict() == \ - { - 'ControllerIdxSigs': 'ControllerIdxSigs', - 'WitnessIdxSigs': 'WitnessIdxSigs', - 'NonTransReceiptCouples': 'NonTransReceiptCouples', - 'TransReceiptQuadruples': 'TransReceiptQuadruples', - 'FirstSeenReplayCouples': 'FirstSeenReplayCouples', - 'TransIdxSigGroups': 'TransIdxSigGroups', - 'SealSourceCouples': 'SealSourceCouples', - 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', - 'SealSourceTriples': 'SealSourceTriples', - 'SadPathSigGroups': 'SadPathSigGroups', - 'RootSadPathSigGroups': 'RootSadPathSigGroups', - 'PathedMaterialGroup': 'PathedMaterialGroup', - 'BigPathedMaterialGroup': 'BigPathedMaterialGroup', - 'AttachmentGroup': 'AttachmentGroup', - 'BigAttachmentGroup': 'BigAttachmentGroup', - 'ESSRPayloadGroup': 'ESSRPayloadGroup', - 'KERIACDCGenusVersion': 'KERIACDCGenusVersion' - } - - assert counting.Tags_1_0.ControllerIdxSigs == 'ControllerIdxSigs' + assert counting.CodeNames == ( + 'GenericGroup', + 'BigGenericGroup', + 'MessageGroup', + 'BigMessageGroup', + 'AttachmentGroup', + 'BigAttachmentGroup', + 'DatagramSegmentGroup', + 'BigDatagramSegmentGroup', + 'ESSRWrapperGroup', + 'BigESSRWrapperGroup', + 'FixedMessageBodyGroup', + 'BigFixedMessageBodyGroup', + 'MapMessageBodyGroup', + 'BigMapMessageBodyGroup', + 'GenericMapGroup', + 'BigGenericMapGroup', + 'GenericListGroup', + 'BigGenericListGroup', + 'ControllerIdxSigs', + 'BigControllerIdxSigs', + 'WitnessIdxSigs', + 'BigWitnessIdxSigs', + 'NonTransReceiptCouples', + 'BigNonTransReceiptCouples', + 'TransReceiptQuadruples', + 'BigTransReceiptQuadruples', + 'FirstSeenReplayCouples', + 'BigFirstSeenReplayCouples', + 'TransIdxSigGroups', + 'BigTransIdxSigGroups', + 'TransLastIdxSigGroups', + 'BigTransLastIdxSigGroups', + 'SealSourceCouples', + 'BigSealSourceCouples', + 'SealSourceTriples', + 'BigSealSourceTriples', + 'PathedMaterialGroup', + 'BigPathedMaterialGroup', + 'SadPathSigGroups', + 'BigSadPathSigGroups', + 'RootSadPathSigGroups', + 'BigRootSadPathSigGroups', + 'DigestSealSingles', + 'BigDigestSealSingles', + 'MerkleRootSealSingles', + 'BigMerkleRootSealSingles', + 'BackerRegistrarSealCouples', + 'BigBackerRegistrarSealCouples', + 'SealSourceLastSingles', + 'BigSealSourceLastSingles', + 'ESSRPayloadGroup', + 'BigESSRPayloadGroup', + 'KERIACDCGenusVersion' + ) + assert 'ControllerIdxSigs' in counting.CodeNames + + assert counting.Codens == counting.Codenage( + GenericGroup='GenericGroup', + BigGenericGroup='BigGenericGroup', + MessageGroup='MessageGroup', + BigMessageGroup='BigMessageGroup', + AttachmentGroup='AttachmentGroup', + BigAttachmentGroup='BigAttachmentGroup', + DatagramSegmentGroup='DatagramSegmentGroup', + BigDatagramSegmentGroup='BigDatagramSegmentGroup', + ESSRWrapperGroup='ESSRWrapperGroup', + BigESSRWrapperGroup='BigESSRWrapperGroup', + FixedMessageBodyGroup='FixedMessageBodyGroup', + BigFixedMessageBodyGroup='BigFixedMessageBodyGroup', + MapMessageBodyGroup='MapMessageBodyGroup', + BigMapMessageBodyGroup='BigMapMessageBodyGroup', + GenericMapGroup='GenericMapGroup', + BigGenericMapGroup='BigGenericMapGroup', + GenericListGroup='GenericListGroup', + BigGenericListGroup='BigGenericListGroup', + ControllerIdxSigs='ControllerIdxSigs', + BigControllerIdxSigs='BigControllerIdxSigs', + WitnessIdxSigs='WitnessIdxSigs', + BigWitnessIdxSigs='BigWitnessIdxSigs', + NonTransReceiptCouples='NonTransReceiptCouples', + BigNonTransReceiptCouples='BigNonTransReceiptCouples', + TransReceiptQuadruples='TransReceiptQuadruples', + BigTransReceiptQuadruples='BigTransReceiptQuadruples', + FirstSeenReplayCouples='FirstSeenReplayCouples', + BigFirstSeenReplayCouples='BigFirstSeenReplayCouples', + TransIdxSigGroups='TransIdxSigGroups', + BigTransIdxSigGroups='BigTransIdxSigGroups', + TransLastIdxSigGroups='TransLastIdxSigGroups', + BigTransLastIdxSigGroups='BigTransLastIdxSigGroups', + SealSourceCouples='SealSourceCouples', + BigSealSourceCouples='BigSealSourceCouples', + SealSourceTriples='SealSourceTriples', + BigSealSourceTriples='BigSealSourceTriples', + PathedMaterialGroup='PathedMaterialGroup', + BigPathedMaterialGroup='BigPathedMaterialGroup', + SadPathSigGroups='SadPathSigGroups', + BigSadPathSigGroups='BigSadPathSigGroups', + RootSadPathSigGroups='RootSadPathSigGroups', + BigRootSadPathSigGroups='BigRootSadPathSigGroups', + DigestSealSingles='DigestSealSingles', + BigDigestSealSingles='BigDigestSealSingles', + MerkleRootSealSingles='MerkleRootSealSingles', + BigMerkleRootSealSingles='BigMerkleRootSealSingles', + BackerRegistrarSealCouples='BackerRegistrarSealCouples', + BigBackerRegistrarSealCouples='BigBackerRegistrarSealCouples', + SealSourceLastSingles='SealSourceLastSingles', + BigSealSourceLastSingles='BigSealSourceLastSingles', + ESSRPayloadGroup='ESSRPayloadGroup', + BigESSRPayloadGroup='BigESSRPayloadGroup', + KERIACDCGenusVersion='KERIACDCGenusVersion' + ) + + assert counting.Codens.ControllerIdxSigs == 'ControllerIdxSigs' - assert counting.Tags_2_0._asdict() == \ - { - 'GenericGroup': 'GenericGroup', - 'BigGenericGroup': 'BigGenericGroup', - 'MessageGroup': 'MessageGroup', - 'BigMessageGroup': 'BigMessageGroup', - 'AttachmentGroup': 'AttachmentGroup', - 'BigAttachmentGroup': 'BigAttachmentGroup', - 'DatagramSegmentGroup': 'DatagramSegmentGroup', - 'BigDatagramSegmentGroup': 'BigDatagramSegmentGroup', - 'ESSRWrapperGroup': 'ESSRWrapperGroup', - 'BigESSRWrapperGroup': 'BigESSRWrapperGroup', - 'FixedMessageBodyGroup': 'FixedMessageBodyGroup', - 'BigFixedMessageBodyGroup': 'BigFixedMessageBodyGroup', - 'MapMessageBodyGroup': 'MapMessageBodyGroup', - 'BigMapMessageBodyGroup': 'BigMapMessageBodyGroup', - 'GenericMapGroup': 'GenericMapGroup', - 'BigGenericMapGroup': 'BigGenericMapGroup', - 'GenericListGroup': 'GenericListGroup', - 'BigGenericListGroup': 'BigGenericListGroup', - 'ControllerIdxSigs': 'ControllerIdxSigs', - 'BigControllerIdxSigs': 'BigControllerIdxSigs', - 'WitnessIdxSigs': 'WitnessIdxSigs', - 'BigWitnessIdxSigs': 'BigWitnessIdxSigs', - 'NonTransReceiptCouples': 'NonTransReceiptCouples', - 'BigNonTransReceiptCouples': 'BigNonTransReceiptCouples', - 'TransReceiptQuadruples': 'TransReceiptQuadruples', - 'BigTransReceiptQuadruples': 'BigTransReceiptQuadruples', - 'FirstSeenReplayCouples': 'FirstSeenReplayCouples', - 'BigFirstSeenReplayCouples': 'BigFirstSeenReplayCouples', - 'TransIdxSigGroups': 'TransIdxSigGroups', - 'BigTransIdxSigGroups': 'BigTransIdxSigGroups', - 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', - 'BigTransLastIdxSigGroups': 'BigTransLastIdxSigGroups', - 'SealSourceCouples': 'SealSourceCouples', - 'BigSealSourceCouples': 'BigSealSourceCouples', - 'SealSourceTriples': 'SealSourceTriples', - 'BigSealSourceTriples': 'BigSealSourceTriples', - 'PathedMaterialGroup': 'PathedMaterialGroup', - 'BigPathedMaterialGroup': 'BigPathedMaterialGroup', - 'SadPathSigGroups': 'SadPathSigGroups', - 'BigSadPathSigGroups': 'BigSadPathSigGroups', - 'RootSadPathSigGroups': 'RootSadPathSigGroups', - 'BigRootSadPathSigGroups': 'BigRootSadPathSigGroups', - 'DigestSealSingles': 'DigestSealSingles', - 'BigDigestSealSingles': 'BigDigestSealSingles', - 'MerkleRootSealSingles': 'MerkleRootSealSingles', - 'BigMerkleRootSealSingles': 'BigMerkleRootSealSingles', - 'BackerRegistrarSealCouples': 'BackerRegistrarSealCouples', - 'BigBackerRegistrarSealCouples': 'BigBackerRegistrarSealCouples', - 'SealSourceLastSingles': 'SealSourceLastSingles', - 'BigSealSourceLastSingles': 'BigSealSourceLastSingles', - 'ESSRPayloadGroup': 'ESSRPayloadGroup', - 'BigESSRPayloadGroup': 'BigESSRPayloadGroup', - 'KERIACDCGenusVersion': 'KERIACDCGenusVersion' - } - - assert counting.Tags_2_0.ControllerIdxSigs == 'ControllerIdxSigs' - - assert counting.AllTags._asdict() == \ - { - 'GenericGroup': 'GenericGroup', - 'BigGenericGroup': 'BigGenericGroup', - 'MessageGroup': 'MessageGroup', - 'BigMessageGroup': 'BigMessageGroup', - 'AttachmentGroup': 'AttachmentGroup', - 'BigAttachmentGroup': 'BigAttachmentGroup', - 'DatagramSegmentGroup': 'DatagramSegmentGroup', - 'BigDatagramSegmentGroup': 'BigDatagramSegmentGroup', - 'ESSRWrapperGroup': 'ESSRWrapperGroup', - 'BigESSRWrapperGroup': 'BigESSRWrapperGroup', - 'FixedMessageBodyGroup': 'FixedMessageBodyGroup', - 'BigFixedMessageBodyGroup': 'BigFixedMessageBodyGroup', - 'MapMessageBodyGroup': 'MapMessageBodyGroup', - 'BigMapMessageBodyGroup': 'BigMapMessageBodyGroup', - 'GenericMapGroup': 'GenericMapGroup', - 'BigGenericMapGroup': 'BigGenericMapGroup', - 'GenericListGroup': 'GenericListGroup', - 'BigGenericListGroup': 'BigGenericListGroup', - 'ControllerIdxSigs': 'ControllerIdxSigs', - 'BigControllerIdxSigs': 'BigControllerIdxSigs', - 'WitnessIdxSigs': 'WitnessIdxSigs', - 'BigWitnessIdxSigs': 'BigWitnessIdxSigs', - 'NonTransReceiptCouples': 'NonTransReceiptCouples', - 'BigNonTransReceiptCouples': 'BigNonTransReceiptCouples', - 'TransReceiptQuadruples': 'TransReceiptQuadruples', - 'BigTransReceiptQuadruples': 'BigTransReceiptQuadruples', - 'FirstSeenReplayCouples': 'FirstSeenReplayCouples', - 'BigFirstSeenReplayCouples': 'BigFirstSeenReplayCouples', - 'TransIdxSigGroups': 'TransIdxSigGroups', - 'BigTransIdxSigGroups': 'BigTransIdxSigGroups', - 'TransLastIdxSigGroups': 'TransLastIdxSigGroups', - 'BigTransLastIdxSigGroups': 'BigTransLastIdxSigGroups', - 'SealSourceCouples': 'SealSourceCouples', - 'BigSealSourceCouples': 'BigSealSourceCouples', - 'SealSourceTriples': 'SealSourceTriples', - 'BigSealSourceTriples': 'BigSealSourceTriples', - 'PathedMaterialGroup': 'PathedMaterialGroup', - 'BigPathedMaterialGroup': 'BigPathedMaterialGroup', - 'SadPathSigGroups': 'SadPathSigGroups', - 'BigSadPathSigGroups': 'BigSadPathSigGroups', - 'RootSadPathSigGroups': 'RootSadPathSigGroups', - 'BigRootSadPathSigGroups': 'BigRootSadPathSigGroups', - 'DigestSealSingles': 'DigestSealSingles', - 'BigDigestSealSingles': 'BigDigestSealSingles', - 'MerkleRootSealSingles': 'MerkleRootSealSingles', - 'BigMerkleRootSealSingles': 'BigMerkleRootSealSingles', - 'BackerRegistrarSealCouples': 'BackerRegistrarSealCouples', - 'BigBackerRegistrarSealCouples': 'BigBackerRegistrarSealCouples', - 'SealSourceLastSingles': 'SealSourceLastSingles', - 'BigSealSourceLastSingles': 'BigSealSourceLastSingles', - 'ESSRPayloadGroup': 'ESSRPayloadGroup', - 'BigESSRPayloadGroup': 'BigESSRPayloadGroup', - 'KERIACDCGenusVersion': 'KERIACDCGenusVersion' - } - assert counting.AllTags.ControllerIdxSigs == 'ControllerIdxSigs' assert asdict(counting.SealDex_2_0) == \ { @@ -319,25 +297,90 @@ def test_counter_class(): }, } - assert Counter.Tags == \ - { - counting.Vrsn_1_0: counting.Tags_1_0, - counting.Vrsn_2_0: counting.Tags_2_0, - } + assert Counter.Names == \ + {1: + {0: + { + '-A': 'ControllerIdxSigs', + '-B': 'WitnessIdxSigs', + '-C': 'NonTransReceiptCouples', + '-D': 'TransReceiptQuadruples', + '-E': 'FirstSeenReplayCouples', + '-F': 'TransIdxSigGroups', + '-G': 'SealSourceCouples', + '-H': 'TransLastIdxSigGroups', + '-I': 'SealSourceTriples', + '-J': 'SadPathSigGroups', + '-K': 'RootSadPathSigGroups', + '-L': 'PathedMaterialGroup', + '-0L': 'BigPathedMaterialGroup', + '-V': 'AttachmentGroup', + '-0V': 'BigAttachmentGroup', + '-Z': 'ESSRPayloadGroup', + '--AAA': 'KERIACDCGenusVersion' + } + }, + 2: + {0: + { + '-A': 'GenericGroup', + '-0A': 'BigGenericGroup', + '-B': 'MessageGroup', + '-0B': 'BigMessageGroup', + '-C': 'AttachmentGroup', + '-0C': 'BigAttachmentGroup', + '-D': 'DatagramSegmentGroup', + '-0D': 'BigDatagramSegmentGroup', + '-E': 'ESSRWrapperGroup', + '-0E': 'BigESSRWrapperGroup', + '-F': 'FixedMessageBodyGroup', + '-0F': 'BigFixedMessageBodyGroup', + '-G': 'MapMessageBodyGroup', + '-0G': 'BigMapMessageBodyGroup', + '-H': 'GenericMapGroup', + '-0H': 'BigGenericMapGroup', + '-I': 'GenericListGroup', + '-0I': 'BigGenericListGroup', + '-J': 'ControllerIdxSigs', + '-0J': 'BigControllerIdxSigs', + '-K': 'WitnessIdxSigs', + '-0K': 'BigWitnessIdxSigs', + '-L': 'NonTransReceiptCouples', + '-0L': 'BigNonTransReceiptCouples', + '-M': 'TransReceiptQuadruples', + '-0M': 'BigTransReceiptQuadruples', + '-N': 'FirstSeenReplayCouples', + '-0N': 'BigFirstSeenReplayCouples', + '-O': 'TransIdxSigGroups', + '-0O': 'BigTransIdxSigGroups', + '-P': 'TransLastIdxSigGroups', + '-0P': 'BigTransLastIdxSigGroups', + '-Q': 'SealSourceCouples', + '-0Q': 'BigSealSourceCouples', + '-R': 'SealSourceTriples', + '-0R': 'BigSealSourceTriples', + '-S': 'PathedMaterialGroup', + '-0S': 'BigPathedMaterialGroup', + '-T': 'SadPathSigGroups', + '-0T': 'BigSadPathSigGroups', + '-U': 'RootSadPathSigGroups', + '-0U': 'BigRootSadPathSigGroups', + '-V': 'DigestSealSingles', + '-0V': 'BigDigestSealSingles', + '-W': 'MerkleRootSealSingles', + '-0W': 'BigMerkleRootSealSingles', + '-X': 'BackerRegistrarSealCouples', + '-0X': 'BigBackerRegistrarSealCouples', + '-Y': 'SealSourceLastSingles', + '-0Y': 'BigSealSourceLastSingles', + '-Z': 'ESSRPayloadGroup', + '-0Z': 'BigESSRPayloadGroup', + '--AAA': 'KERIACDCGenusVersion' + } + } + } - # first character of code with hard size of code - assert Counter.Hards == \ - { - '-A': 2, '-B': 2, '-C': 2, '-D': 2, '-E': 2, '-F': 2, '-G': 2, '-H': 2, '-I': 2, - '-J': 2, '-K': 2, '-L': 2, '-M': 2, '-N': 2, '-O': 2, '-P': 2, '-Q': 2, '-R': 2, - '-S': 2, '-T': 2, '-U': 2, '-V': 2, '-W': 2, '-X': 2, '-Y': 2, '-Z': 2, - '-a': 2, '-b': 2, '-c': 2, '-d': 2, '-e': 2, '-f': 2, '-g': 2, '-h': 2, '-i': 2, - '-j': 2, '-k': 2, '-l': 2, '-m': 2, '-n': 2, '-o': 2, '-p': 2, '-q': 2, '-r': 2, - '-s': 2, '-t': 2, '-u': 2, '-v': 2, '-w': 2, '-x': 2, '-y': 2, '-z': 2, - '-0': 3, '--': 5, - } - # Codes table with sizes of code (hard) and full primitive material assert Counter.Sizes == \ { @@ -435,6 +478,20 @@ def test_counter_class(): assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].ls == 0 # lead size + # first character of code with hard size of code + assert Counter.Hards == \ + { + '-A': 2, '-B': 2, '-C': 2, '-D': 2, '-E': 2, '-F': 2, '-G': 2, '-H': 2, '-I': 2, + '-J': 2, '-K': 2, '-L': 2, '-M': 2, '-N': 2, '-O': 2, '-P': 2, '-Q': 2, '-R': 2, + '-S': 2, '-T': 2, '-U': 2, '-V': 2, '-W': 2, '-X': 2, '-Y': 2, '-Z': 2, + '-a': 2, '-b': 2, '-c': 2, '-d': 2, '-e': 2, '-f': 2, '-g': 2, '-h': 2, '-i': 2, + '-j': 2, '-k': 2, '-l': 2, '-m': 2, '-n': 2, '-o': 2, '-p': 2, '-q': 2, '-r': 2, + '-s': 2, '-t': 2, '-u': 2, '-v': 2, '-w': 2, '-x': 2, '-y': 2, '-z': 2, + '-0': 3, '--': 5, + } + + + # verify first hs Sizes matches hs in Codes for same first char for vmajor in Counter.Sizes.values(): for vminor in vmajor.values(): @@ -531,9 +588,10 @@ def test_counter_v1(): qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(tag="ControllerIdxSigs", count=count, gvrsn=Vrsn_1_0) + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs == counter.hard - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -543,25 +601,24 @@ def test_counter_v1(): assert counter.soft =='AB' assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 assert counter.codes == counting.CtrDex_1_0 - assert counter.tags == counting.Tags_1_0 + #assert counter.tags == counting.Tags_1_0 assert counter.sizes == Counter.Sizes[1][0] - counter = Counter(tag=AllTags.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_1_0 - # test tag takes precedence - counter = Counter(tag=AllTags.ControllerIdxSigs, - code=CtrDex.WitnessIdxSigs, + # test keyword buth with code name + counter = Counter(code=Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -570,7 +627,7 @@ def test_counter_v1(): counter = Counter(code=CtrDex.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default count = 1 assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -579,7 +636,7 @@ def test_counter_v1(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -588,7 +645,7 @@ def test_counter_v1(): counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -597,7 +654,7 @@ def test_counter_v1(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -634,7 +691,7 @@ def test_counter_v1(): counter = Counter(code=CtrDex.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -643,7 +700,7 @@ def test_counter_v1(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -652,7 +709,7 @@ def test_counter_v1(): counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -661,7 +718,7 @@ def test_counter_v1(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -677,7 +734,7 @@ def test_counter_v1(): counter = Counter(code=CtrDex.BigAttachmentGroup, count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.BigAttachmentGroup - assert counter.tag == AllTags.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -686,7 +743,7 @@ def test_counter_v1(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.BigAttachmentGroup - assert counter.tag == AllTags.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -695,7 +752,7 @@ def test_counter_v1(): counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.BigAttachmentGroup - assert counter.tag == AllTags.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -704,7 +761,7 @@ def test_counter_v1(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.BigAttachmentGroup - assert counter.tag == AllTags.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -718,7 +775,7 @@ def test_counter_v1(): qb2 = counter.qb2 counter._bexfil(qb2) assert counter.code == code - assert counter.tag == AllTags.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" assert counter.count == count assert counter.qb64 == qsc assert counter.qb2 == qb2 @@ -737,7 +794,7 @@ def test_counter_v1(): counter = Counter(code=CtrDex.BigPathedMaterialGroup, count=count, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.BigPathedMaterialGroup - assert counter.tag == AllTags.BigPathedMaterialGroup + assert counter.name == "BigPathedMaterialGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -746,7 +803,7 @@ def test_counter_v1(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.BigPathedMaterialGroup - assert counter.tag == AllTags.BigPathedMaterialGroup + assert counter.name == "BigPathedMaterialGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -755,7 +812,7 @@ def test_counter_v1(): counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.BigPathedMaterialGroup - assert counter.tag == AllTags.BigPathedMaterialGroup + assert counter.name == "BigPathedMaterialGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -764,7 +821,7 @@ def test_counter_v1(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.BigPathedMaterialGroup - assert counter.tag == AllTags.BigPathedMaterialGroup + assert counter.name == "BigPathedMaterialGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -778,7 +835,7 @@ def test_counter_v1(): qb2 = counter.qb2 counter._bexfil(qb2) assert counter.code == code - assert counter.tag == AllTags.BigPathedMaterialGroup + assert counter.name == "BigPathedMaterialGroup" assert counter.count == count assert counter.qb64 == qsc assert counter.qb2 == qb2 @@ -799,7 +856,7 @@ def test_counter_v1(): # strip ignored if qb64 counter = Counter(qb64=qsc, strip=True, gvrsn=Vrsn_1_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -810,7 +867,7 @@ def test_counter_v1(): counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_1_0) # strip assert not ims # deleted assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -821,7 +878,7 @@ def test_counter_v1(): counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_1_0) assert not ims # deleted assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -864,7 +921,7 @@ def test_counter_v1(): ims = bytearray(qscb) counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_1_0) # test with bytes not str assert counter.code == CtrDex.BigAttachmentGroup - assert counter.tag == AllTags.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -876,7 +933,7 @@ def test_counter_v1(): ims = bytearray(qscb2) counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_1_0) # test with qb2 assert counter.code == CtrDex.BigAttachmentGroup - assert counter.tag == AllTags.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -900,7 +957,7 @@ def test_counter_v1(): count=genverint, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.KERIACDCGenusVersion - assert counter.tag == AllTags.KERIACDCGenusVersion + assert counter.name == "KERIACDCGenusVersion" assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length @@ -909,7 +966,7 @@ def test_counter_v1(): assert counter.qb2 == qscb2 assert counter.version == Vrsn_1_0 assert counter.codes == counting.CtrDex_1_0 - assert counter.tags == counting.Tags_1_0 + #assert counter.tags == counting.Tags_1_0 assert counter.sizes == Counter.Sizes[1][0] @@ -917,7 +974,7 @@ def test_counter_v1(): countB64=genver, gvrsn=Vrsn_1_0) assert counter.code == CtrDex.KERIACDCGenusVersion - assert counter.tag == AllTags.KERIACDCGenusVersion + assert counter.name == "KERIACDCGenusVersion" assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length @@ -949,7 +1006,7 @@ def test_counter_v2(): # default version and default count = 1 counter = Counter(code=CtrDex.ControllerIdxSigs) assert counter.code == CtrDex.ControllerIdxSigs == counter.hard - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -959,7 +1016,7 @@ def test_counter_v2(): assert counter.soft =='AB' assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 assert counter.codes == counting.CtrDex_2_0 - assert counter.tags == counting.Tags_2_0 + #assert counter.tags == counting.Tags_2_0 assert counter.sizes == Counter.Sizes[2][0] @@ -967,7 +1024,7 @@ def test_counter_v2(): # default count = 1 counter = Counter(code=CtrDex.ControllerIdxSigs, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -976,7 +1033,7 @@ def test_counter_v2(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -985,7 +1042,7 @@ def test_counter_v2(): counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -994,7 +1051,7 @@ def test_counter_v2(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1029,33 +1086,32 @@ def test_counter_v2(): qscb = qsc.encode("utf-8") qscb2 = decodeB64(qscb) - counter = Counter(tag="ControllerIdxSigs", count=count, gvrsn=Vrsn_2_0) + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_2_0 - counter = Counter(tag=AllTags.ControllerIdxSigs, + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc assert counter.qb2 == qscb2 assert counter.version == Vrsn_2_0 - # test tag takes precedence - counter = Counter(tag=AllTags.ControllerIdxSigs, - code=CtrDex.WitnessIdxSigs, + # test keyword with code name + counter = Counter(code=Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1064,7 +1120,7 @@ def test_counter_v2(): counter = Counter(code=CtrDex.ControllerIdxSigs, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1073,7 +1129,7 @@ def test_counter_v2(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1082,7 +1138,7 @@ def test_counter_v2(): counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1091,7 +1147,7 @@ def test_counter_v2(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1105,7 +1161,7 @@ def test_counter_v2(): qb2 = counter.qb2 counter._bexfil(qb2) assert counter.code == code - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64 == qsc assert counter.qb2 == qb2 @@ -1126,7 +1182,7 @@ def test_counter_v2(): # strip ignored if qb64 counter = Counter(qb64=qsc, strip=True, gvrsn=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1137,7 +1193,7 @@ def test_counter_v2(): counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) # strip assert not ims # deleted assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1148,7 +1204,7 @@ def test_counter_v2(): counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) assert not ims # deleted assert counter.code == CtrDex.ControllerIdxSigs - assert counter.tag == AllTags.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1190,7 +1246,7 @@ def test_counter_v2(): counter = Counter(code=CtrDex.BigGenericGroup, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1199,7 +1255,7 @@ def test_counter_v2(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1208,7 +1264,7 @@ def test_counter_v2(): counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1217,7 +1273,7 @@ def test_counter_v2(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1234,7 +1290,7 @@ def test_counter_v2(): ims = bytearray(qscb) counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1245,7 +1301,7 @@ def test_counter_v2(): ims = bytearray(qscb2) counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1262,7 +1318,7 @@ def test_counter_v2(): counter = Counter(code=CtrDex.BigGenericGroup, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.BigGenericGroup == counter.hard - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1274,7 +1330,7 @@ def test_counter_v2(): counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1283,7 +1339,7 @@ def test_counter_v2(): counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1292,7 +1348,7 @@ def test_counter_v2(): counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1309,7 +1365,7 @@ def test_counter_v2(): ims = bytearray(qscb) counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) # test with bytes not str assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1321,7 +1377,7 @@ def test_counter_v2(): ims = bytearray(qscb2) counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) # test with qb2 assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1338,7 +1394,7 @@ def test_counter_v2(): counter = Counter(code=CtrDex.GenericGroup, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.BigGenericGroup == counter.hard - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1347,9 +1403,9 @@ def test_counter_v2(): assert counter.soft =='AACAB' assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 - counter = Counter(tag=AllTags.GenericGroup, count=count, gvrsn=Vrsn_2_0) + counter = Counter(Codens.GenericGroup, count=count, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.BigGenericGroup - assert counter.tag == AllTags.BigGenericGroup + assert counter.name == "BigGenericGroup" assert counter.count == count assert counter.qb64b == qscb assert counter.qb64 == qsc @@ -1370,7 +1426,7 @@ def test_counter_v2(): count=genverint, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.KERIACDCGenusVersion - assert counter.tag == AllTags.KERIACDCGenusVersion + assert counter.name == "KERIACDCGenusVersion" assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length @@ -1380,14 +1436,14 @@ def test_counter_v2(): assert counter.version == Vrsn_2_0 assert counter.fullSize == 8 assert counter.codes == counting.CtrDex_2_0 - assert counter.tags == counting.Tags_2_0 + #assert counter.tags == counting.Tags_2_0 assert counter.sizes == Counter.Sizes[2][0] counter = Counter(code=CtrDex.KERIACDCGenusVersion, countB64=genver, gvrsn=Vrsn_2_0) assert counter.code == CtrDex.KERIACDCGenusVersion - assert counter.tag == AllTags.KERIACDCGenusVersion + assert counter.name == "KERIACDCGenusVersion" assert counter.count == genverint assert counter.countToB64(l=3) == genver assert counter.countToB64() == genver # default length From 606b10679bb7e37f3e373db91074826d2fe6d792 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 23 Jul 2024 14:12:44 -0600 Subject: [PATCH 197/418] more porting to use new versioned Counter in core.counting --- src/keri/app/grouping.py | 10 +++---- src/keri/app/signing.py | 8 ++++-- src/keri/core/coring.py | 2 +- src/keri/core/counting.py | 4 ++- src/keri/core/eventing.py | 53 ++++++++++++++++++++++------------- src/keri/core/indexing.py | 2 +- src/keri/core/parsing.py | 37 +++++++++++++++++------- src/keri/peer/exchanging.py | 29 +++++++++++-------- tests/app/test_agenting.py | 2 +- tests/core/test_counting.py | 3 ++ tests/peer/test_exchanging.py | 9 ++++-- 11 files changed, 105 insertions(+), 54 deletions(-) diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index af7faab22..41f8249c4 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -9,7 +9,7 @@ from hio.base import doing from hio.help import decking -from .. import kering +from .. import kering, core from .. import help from ..app import delegating, agenting from ..core import coring, routing, eventing, parsing, serdering, indexing @@ -518,14 +518,14 @@ def getEscrowedEvent(db, pre, sn): msg = bytearray() msg.extend(serder.raw) - msg.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigs)).qb64b) # attach cnt + msg.extend(core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigs), gvrsn=kering.Vrsn_1_0).qb64b) # attach cnt for sig in sigs: msg.extend(sig.qb64b) # attach sig if couple is not None: - msg.extend(coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1).qb64b) + msg.extend(core.Counter(core.Codens.SealSourceCouples, + count=1, gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(couple) return msg diff --git a/src/keri/app/signing.py b/src/keri/app/signing.py index f8aed4456..3c99c50f5 100644 --- a/src/keri/app/signing.py +++ b/src/keri/app/signing.py @@ -4,13 +4,17 @@ keri.app.signing module """ +from ..kering import Vrsn_1_0, Vrsn_2_0 from ..app.habbing import GroupHab -from ..core import coring, eventing +from .. import core +from ..core import coring, eventing, counting + def serialize(creder, prefixer, seqner, saider): craw = bytearray(creder.raw) - craw.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + craw.extend(core.Counter(core.Codens.SealSourceTriples, count=1, + gvrsn=Vrsn_1_0).qb64b) craw.extend(prefixer.qb64b) craw.extend(seqner.qb64b) craw.extend(saider.qb64b) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index fc2f51009..22a1d4a0b 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -909,7 +909,7 @@ class Matter: def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, - qb64b=None, qb64=None, qb2=None, strip=False): + qb64b=None, qb64=None, qb2=None, strip=False, **kwa): """ Validate as fully qualified Parameters: diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 255695518..0a5ca96f3 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -357,6 +357,7 @@ class Counter: '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), '-V': Sizage(hs=2, ss=2, fs=4, ls=0), '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), }, }, @@ -423,7 +424,8 @@ class Counter: def __init__(self, code=None, *, count=None, countB64=None, - qb64b=None, qb64=None, qb2=None, strip=False, gvrsn=Vrsn_2_0): + qb64b=None, qb64=None, qb2=None, strip=False, + gvrsn=Vrsn_2_0, **kwa): """ Validate as fully qualified Parameters: diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 9580e8d9b..5e0f9e770 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -14,7 +14,7 @@ from hio.help import decking -from .. import kering +from .. import kering, core from ..kering import (MissingEntryError, ValidationError, MissingSignatureError, MissingWitnessSignatureError, UnverifiedReplyError, @@ -30,10 +30,12 @@ from . import coring from .coring import (versify, Serials, Ilks, PreDex, DigDex, - NonTransDex, CtrDex, Counter, + NonTransDex, Number, Seqner, Cigar, Dater, Verfer, Diger, Prefixer, Tholder, Saider) +from .counting import Counter, Codens + from . import indexing from .indexing import Siger @@ -1304,21 +1306,25 @@ def messagize(serder, *, sigers=None, seal=None, wigers=None, cigars=None, if sigers: if isinstance(seal, SealEvent): - atc.extend(Counter(CtrDex.TransIdxSigGroups, count=1).qb64b) + atc.extend(Counter(Codens.TransIdxSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(seal.i.encode("utf-8")) atc.extend(Seqner(snh=seal.s).qb64b) atc.extend(seal.d.encode("utf-8")) elif isinstance(seal, SealLast): - atc.extend(Counter(CtrDex.TransLastIdxSigGroups, count=1).qb64b) + atc.extend(Counter(Codens.TransLastIdxSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(seal.i.encode("utf-8")) - atc.extend(Counter(code=CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) + atc.extend(Counter(Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0).qb64b) for siger in sigers: atc.extend(siger.qb64b) if wigers: - atc.extend(Counter(code=CtrDex.WitnessIdxSigs, count=len(wigers)).qb64b) + atc.extend(Counter(Codens.WitnessIdxSigs, count=len(wigers), + gvrsn=kering.Vrsn_1_0).qb64b) for wiger in wigers: if wiger.verfer and wiger.verfer.code not in NonTransDex: raise ValueError("Attempt to use tranferable prefix={} for " @@ -1326,7 +1332,8 @@ def messagize(serder, *, sigers=None, seal=None, wigers=None, cigars=None, atc.extend(wiger.qb64b) if cigars: - atc.extend(Counter(code=CtrDex.NonTransReceiptCouples, count=len(cigars)).qb64b) + atc.extend(Counter(Codens.NonTransReceiptCouples, count=len(cigars), + gvrsn=kering.Vrsn_1_0).qb64b) for cigar in cigars: if cigar.verfer.code not in NonTransDex: raise ValueError("Attempt to use tranferable prefix={} for " @@ -1338,8 +1345,8 @@ def messagize(serder, *, sigers=None, seal=None, wigers=None, cigars=None, if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - msg.extend(Counter(code=CtrDex.AttachmentGroup, - count=(len(atc) // 4)).qb64b) + msg.extend(Counter(Codens.AttachmentGroup, + count=(len(atc) // 4), gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(atc) return msg @@ -1370,33 +1377,40 @@ def proofize(sadtsgs=None, *, sadsigers=None, sadcigars=None, pipelined=False): count = 0 for (pather, sigers) in sadsigers: count += 1 - atc.extend(coring.Counter(coring.CtrDex.SadPathSigGroups, count=1).qb64b) + atc.extend(Counter(Codens.SadPathSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(pather.qb64b) - atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) + atc.extend(Counter(Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0).qb64b) for siger in sigers: atc.extend(siger.qb64b) for (pather, prefixer, seqner, saider, sigers) in sadtsgs: count += 1 - atc.extend(coring.Counter(coring.CtrDex.SadPathSigGroups, count=1).qb64b) + atc.extend(Counter(Codens.SadPathSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(pather.qb64b) - atc.extend(coring.Counter(coring.CtrDex.TransIdxSigGroups, count=1).qb64b) + atc.extend(Counter(Codens.TransIdxSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(prefixer.qb64b) atc.extend(seqner.qb64b) atc.extend(saider.qb64b) - atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) + atc.extend(Counter(Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0).qb64b) for siger in sigers: atc.extend(siger.qb64b) for (pather, cigars) in sadcigars: count += 1 - atc.extend(coring.Counter(coring.CtrDex.SadPathSigGroups, count=1).qb64b) + atc.extend(Counter(Codens.SadPathSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(pather.qb64b) - atc.extend(coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=len(sadcigars)).qb64b) + atc.extend(Counter(Codens.NonTransReceiptCouples, + count=len(sadcigars), gvrsn=kering.Vrsn_1_0).qb64b) for cigar in cigars: if cigar.verfer.code not in coring.NonTransDex: raise ValueError("Attempt to use tranferable prefix={} for " @@ -1410,12 +1424,13 @@ def proofize(sadtsgs=None, *, sadsigers=None, sadcigars=None, pipelined=False): if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - msg.extend(coring.Counter(code=coring.CtrDex.AttachmentGroup, - count=(len(atc) // 4)).qb64b) + msg.extend(Counter(Codens.AttachmentGroup, count=(len(atc) // 4), + gvrsn=kering.Vrsn_1_0).qb64b) if count > 1: root = coring.Pather(bext="-") - msg.extend(coring.Counter(code=coring.CtrDex.RootSadPathSigGroups, count=count).qb64b) + msg.extend(Counter(Codens.RootSadPathSigGroups, count=count, + gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(root.qb64b) msg.extend(atc) diff --git a/src/keri/core/indexing.py b/src/keri/core/indexing.py index 7022fbc1a..65c933b5a 100644 --- a/src/keri/core/indexing.py +++ b/src/keri/core/indexing.py @@ -233,7 +233,7 @@ class Indexer: def __init__(self, raw=None, code=IdrDex.Ed25519_Sig, index=0, ondex=None, - qb64b=None, qb64=None, qb2=None, strip=False): + qb64b=None, qb64=None, qb2=None, strip=False, **kwa): """ Validate as fully qualified Parameters: diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index 1cd9b4cd4..95798e7e3 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -7,13 +7,15 @@ import logging -from .coring import (Ilks, CtrDex, Counter, Seqner, Cigar, +from ..kering import Vrsn_1_0, Vrsn_2_0 +from .coring import (Ilks, CtrDex, Seqner, Cigar, Dater, Verfer, Prefixer, Saider, Pather, Texter) +from .counting import Counter, Codens from .indexing import (Siger, ) from . import serdering from .. import help from .. import kering -from ..kering import Colds, sniff +from ..kering import Colds, sniff, Vrsn_1_0, Vrsn_2_0 logger = help.ogler.getLogger() @@ -94,7 +96,7 @@ def extract(ims, klas, cold=Colds.txt): @staticmethod - def _extractor(ims, klas, cold=Colds.txt, abort=False): + def _extractor(ims, klas, cold=Colds.txt, abort=False, gvrsn=Vrsn_1_0): """ Returns generator to extract and return instance of klas from input message stream, ims, given stream state, cold, is txt or bny. @@ -103,6 +105,14 @@ def _extractor(ims, klas, cold=Colds.txt, abort=False): Inits klas from ims using qb64b or qb2 parameter based on cold. Yields if not enough bytes in ims to fill out klas instance. + Parameters: + ims (bytearray): input message stream (must be strippable) + klas (Serder | Counter | Matter | Indexer): subclass that is parsable + cold (Coldage): instance str value + abort (bool): True means abort if bad pipelined frame Shortage + False means do not abort if Shortage just wait for more + gvrsn (Versionage): instance of genera version of CESR code tables + Usage: instance = self._extractor @@ -110,9 +120,9 @@ def _extractor(ims, klas, cold=Colds.txt, abort=False): while True: try: if cold == Colds.txt: - return klas(qb64b=ims, strip=True) + return klas(qb64b=ims, strip=True, gvrsn=gvrsn) elif cold == Colds.bny: - return klas(qb2=ims, strip=True) + return klas(qb2=ims, strip=True, gvrsn=gvrsn) else: raise kering.ColdStartError("Invalid stream state cold={}.".format(cold)) except kering.ShortageError as ex: @@ -263,7 +273,7 @@ def _nonTransReceiptCouples(self, ctr, ims, cold=Colds.txt, pipelined=False): def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, - exc=None, rvy=None, vry=None, local=None): + exc=None, rvy=None, vry=None, local=None, gvrsn=Vrsn_1_0): """ Processes all messages from incoming message stream, ims, when provided. Otherwise process messages from .ims @@ -289,6 +299,7 @@ def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, local (bool): True means event source is local (protected) for validation False means event source is remote (unprotected) for validation None means use default .local + gvrsn (Versionage): instance of genera version of CESR code tables New Logic: Attachments must all have counters so know if txt or bny format for @@ -306,7 +317,8 @@ def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=exc, rvy=rvy, vry=vry, - local=local) + local=local, + gvrsn=gvrsn) while True: try: @@ -366,7 +378,8 @@ def parseOne(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, - tvy=None, exc=None, rvy=None, vry=None, local=None): + tvy=None, exc=None, rvy=None, vry=None, local=None, + gvrsn=Vrsn_1_0): """ Returns generator to parse all messages from incoming message stream, ims until ims is exhausted (empty) then returns. @@ -392,6 +405,7 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, local (bool): True means event source is local (protected) for validation False means event source is remote (unprotected) for validation None means use default .local + gvrsn (Versionage): instance of genera version of CESR code tables New Logic: Attachments must all have counters so know if txt or bny format for @@ -423,7 +437,8 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, exc=exc, rvy=rvy, vry=vry, - local=local) + local=local, + gvrsn=gvrsn) except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream @@ -627,7 +642,8 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, def msgParsator(self, ims=None, framed=True, pipeline=False, - kvy=None, tvy=None, exc=None, rvy=None, vry=None, local=None): + kvy=None, tvy=None, exc=None, rvy=None, vry=None, + local=None, gvrsn=Vrsn_1_0): """ Returns generator that upon each iteration extracts and parses msg with attached crypto material (signature etc) from incoming message @@ -657,6 +673,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, local (bool): True means event source is local (protected) for validation False means event source is remote (unprotected) for validation None means use default .local + gvrsn (Versionage): instance of genera version of CESR code tables Logic: Currently only support couters on attachments not on combined or diff --git a/src/keri/peer/exchanging.py b/src/keri/peer/exchanging.py index ea96b7d41..d8d5a777b 100644 --- a/src/keri/peer/exchanging.py +++ b/src/keri/peer/exchanging.py @@ -8,7 +8,7 @@ from hio.help import decking -from .. import help, kering +from .. import help, kering, core from ..app import habbing from ..core import eventing, coring, serdering from ..help import helping @@ -350,11 +350,13 @@ def exchange(route, pathed.extend(pather.qb64b) pathed.extend(atc) if len(pathed) // 4 < 4096: - end.extend(coring.Counter(code=coring.CtrDex.PathedMaterialGroup, - count=(len(pathed) // 4)).qb64b) + end.extend(core.Counter(core.Codens.PathedMaterialGroup, + count=(len(pathed) // 4), + gvrsn=kering.Vrsn_1_0).qb64b) else: - end.extend(coring.Counter(code=coring.CtrDex.BigPathedMaterialGroup, - count=(len(pathed) // 4)).qb64b) + end.extend(core.Counter(core.Codens.BigPathedMaterialGroup, + count=(len(pathed) // 4), + gvrsn=kering.Vrsn_1_0).qb64b) end.extend(pathed) if e: @@ -434,17 +436,20 @@ def serializeMessage(hby, said, pipelined=False): if len(tsgs) > 0: for (prefixer, seqner, saider, sigers) in tsgs: - atc.extend(coring.Counter(coring.CtrDex.TransIdxSigGroups, count=1).qb64b) + atc.extend(core.Counter(core.Codens.TransIdxSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(prefixer.qb64b) atc.extend(seqner.qb64b) atc.extend(saider.qb64b) - atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) + atc.extend(core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0).qb64b) for siger in sigers: atc.extend(siger.qb64b) if len(cigars) > 0: - atc.extend(coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=len(cigars)).qb64b) + atc.extend(core.Counter(core.Codens.CtrDex.NonTransReceiptCouples, + count=len(cigars), gvrsn=kering.Vrsn_1_0).qb64b) for cigar in cigars: if cigar.verfer.code not in coring.NonTransDex: raise ValueError("Attempt to use tranferable prefix={} for " @@ -454,8 +459,8 @@ def serializeMessage(hby, said, pipelined=False): # Smash the pathed components on the end for p in hby.db.epath.get(keys=(exn.said,)): - atc.extend(coring.Counter(code=coring.CtrDex.PathedMaterialGroup, - count=(len(p) // 4)).qb64b) + atc.extend(core.Counter(core.Codens.PathedMaterialGroup, + count=(len(p) // 4), gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(p.encode("utf-8")) msg = bytearray() @@ -464,8 +469,8 @@ def serializeMessage(hby, said, pipelined=False): if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - msg.extend(coring.Counter(code=coring.CtrDex.AttachmentGroup, - count=(len(atc) // 4)).qb64b) + msg.extend(core.Counter(core.Codens.AttachmentGroup, + count=(len(atc) // 4), gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(atc) return msg diff --git a/tests/app/test_agenting.py b/tests/app/test_agenting.py index 4f746168c..6f2470369 100644 --- a/tests/app/test_agenting.py +++ b/tests/app/test_agenting.py @@ -9,7 +9,7 @@ from keri import kering, core from keri.core import coring, serdering -from keri.core.coring import Counter, CtrDex, Seqner +from keri.core.coring import Seqner from keri.help import nowIso8601 from keri.app import habbing, indirecting, agenting, directing from keri.db import dbing diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 443a28551..53d6f210e 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -403,6 +403,7 @@ def test_counter_class(): '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), '-V': Sizage(hs=2, ss=2, fs=4, ls=0), '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), + '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) }, }, @@ -467,6 +468,8 @@ def test_counter_class(): }, } + # add test that ensures there is an entry in Sizes for each entry in Codes + assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].hs == 2 # hard size assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].ss == 2 # soft size assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].fs == 4 # full size diff --git a/tests/peer/test_exchanging.py b/tests/peer/test_exchanging.py index 5e7e96949..0f19944f9 100644 --- a/tests/peer/test_exchanging.py +++ b/tests/peer/test_exchanging.py @@ -7,6 +7,7 @@ from base64 import urlsafe_b64encode as encodeB64 from base64 import urlsafe_b64decode as decodeB64 +from keri import kering from keri import core from keri.core import coring, serdering, MtrDex, parsing @@ -68,11 +69,15 @@ def test_essrs(): essr, _ = exchanging.exchange(route='/essr/req', sender=hab.pre, diger=diger, modifiers=dict(src=hab.pre, dest=recHab.pre)) ims = hab.endorse(serder=essr, pipelined=False) - ims.extend(coring.Counter(code=CtrDex.ESSRPayloadGroup, count=1).qb64b) + ims.extend(core.Counter(core.Codens.ESSRPayloadGroup, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) ims.extend(texter.qb64b) exc = exchanging.Exchanger(hby=recHby, handlers=[]) - parsing.Parser().parse(ims=ims, kvy=recHby.kvy, exc=exc) + parsing.Parser().parse(ims=ims, + kvy=recHby.kvy, + exc=exc, + gvrsn=kering.Vrsn_1_0) # parser does not support version2 count codes # Pull the logged exn and verify the attributes digest matches the attachment serder = recHby.db.exns.get(keys=(essr.said,)) From 0d36aba1b9a058a69462385f65bda08748da0354 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 23 Jul 2024 19:22:23 -0600 Subject: [PATCH 198/418] Added test to ensure Counter.Codes and Counter.Sizes have same keys space --- tests/core/test_counting.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 53d6f210e..3433127a6 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -468,7 +468,15 @@ def test_counter_class(): }, } - # add test that ensures there is an entry in Sizes for each entry in Codes + # Ensure there is an entry in Sizes for each entry in Codes + assert Counter.Codes.keys() == Counter.Sizes.keys() + for majorc, majors in zip(Counter.Codes.items(), Counter.Sizes.items(), strict=True): + assert majorc[0] == majors[0] # major version, keys match + for minorc, minors in zip(majorc[1].items(), majors[1].items(), strict=True): + assert minorc[0] == minors[0] # minor version keys match + for code, size in zip(asdict(minorc[1]).items(), minors[1].items(), strict=True): + code[0] == size[0] # code and size keys match + assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].hs == 2 # hard size assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].ss == 2 # soft size From f8420a3b6abff20b16dc9243a68e2700dc8c6d54 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 24 Jul 2024 08:19:43 -0700 Subject: [PATCH 199/418] more porting to new versioned Counter --- src/keri/app/habbing.py | 17 ++++----- src/keri/app/indirecting.py | 7 ++-- tests/core/test_delegating.py | 65 ++++++++++++++++------------------- 3 files changed, 43 insertions(+), 46 deletions(-) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index d860f9bfc..44516caa8 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -17,7 +17,8 @@ from .. import help from .. import kering from .. import core -from ..core import coring, eventing, parsing, routing, serdering, indexing +from ..core import (coring, eventing, parsing, routing, serdering, indexing, + Counter, Codens) from ..db import dbing, basing from ..kering import MissingSignatureError, Roles @@ -1567,8 +1568,8 @@ def makeOtherEvent(self, pre, sn): dig = bytes(dig) key = dbing.dgKey(pre, dig) # digest key msg.extend(self.db.getEvt(key)) - msg.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=self.db.cntSigs(key)).qb64b) # attach cnt + msg.extend(Counter(Codens.ControllerIdxSigs, count=self.db.cntSigs(key), + gvrsn=kering.Vrsn_1_0).qb64b) # attach cnt for sig in self.db.getSigsIter(key): msg.extend(sig) # attach sig return msg @@ -2035,14 +2036,14 @@ def makeOwnEvent(self, sn, allowPartiallySigned=False): serder, sigs, couple = self.getOwnEvent(sn=sn, allowPartiallySigned=allowPartiallySigned) msg.extend(serder.raw) - msg.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigs)).qb64b) # attach cnt + msg.extend(Counter(Codens.ControllerIdxSigs, count=len(sigs), + gvrsn=kering.Vrsn_1_0).qb64b) # attach cnt for sig in sigs: msg.extend(sig.qb64b) # attach sig if couple is not None: - msg.extend(coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1).qb64b) + msg.extend(Counter(Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(couple) return msg @@ -2727,7 +2728,7 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, rmids=self.rmids) self.save(habord) self.prefixes.add(self.pre) - + # during delegation initialization of a habitat we ignore the MissingDelegationError and # MissingSignatureError try: diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 80b3f8865..8fd4730c4 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -22,7 +22,8 @@ from . import directing, storing, httping, forwarding, agenting, oobiing from .habbing import GroupHab from .. import help, kering -from ..core import eventing, parsing, routing, coring, serdering +from ..core import (eventing, parsing, routing, coring, serdering, + Counter, Codens) from ..core.coring import Ilks from ..db import basing, dbing from ..end import ending @@ -1142,8 +1143,8 @@ def on_get(self, req, rep): said=said.decode("utf-8")) rct = bytearray(rserder.raw) if wigs := self.hab.db.getWigs(key=dgkey): - rct.extend(coring.Counter(code=coring.CtrDex.WitnessIdxSigs, - count=len(wigs)).qb64b) + rct.extend(Counter(Codens.CtrDex.WitnessIdxSigs, count=len(wigs), + gvrsn=kering.Vrsn_1_0).qb64b) for wig in wigs: rct.extend(wig) diff --git a/tests/core/test_delegating.py b/tests/core/test_delegating.py index 0c05f27c8..2f183f3e2 100644 --- a/tests/core/test_delegating.py +++ b/tests/core/test_delegating.py @@ -7,7 +7,7 @@ from keri import help -from keri import core +from keri import kering, core from keri.core import coring, eventing, parsing from keri.app import keeping, habbing @@ -55,7 +55,8 @@ def test_delegation(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -107,8 +108,8 @@ def test_delegation(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -135,13 +136,13 @@ def test_delegation(): sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) @@ -201,8 +202,8 @@ def test_delegation(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -229,13 +230,13 @@ def test_delegation(): sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) @@ -462,14 +463,7 @@ def test_load_event(mockHelpingNowUTC): count = botHab.db.cntWigs(dbing.dgKey(botHab.pre, serder.said)) assert count >= 1 - - - """End Test""" - - - - - + # This needs to be fixedup to actually test delegating superseding recovery with (basing.openDB(name="bob") as bobDB, keeping.openKS(name="bob") as bobKS, basing.openDB(name="del") as delDB, @@ -497,7 +491,8 @@ def test_load_event(mockHelpingNowUTC): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -549,8 +544,8 @@ def test_load_event(mockHelpingNowUTC): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -577,13 +572,13 @@ def test_load_event(mockHelpingNowUTC): sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) @@ -643,8 +638,8 @@ def test_load_event(mockHelpingNowUTC): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -671,13 +666,13 @@ def test_load_event(mockHelpingNowUTC): sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) From 4a0abf6ea9b09549973d43cf54f7b6f7021fb7a2 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Thu, 25 Jul 2024 11:07:43 -0400 Subject: [PATCH 200/418] updates Hio version (#822) Signed-off-by: Kevin Griffin --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 52bb137f8..cfc98f821 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ 'cbor2>=5.6.2', 'multidict>=6.0.5', 'ordered-set>=4.1.0', - 'hio>=0.6.13', + 'hio>=0.6.14', 'multicommand>=1.0.0', 'jsonschema>=4.21.1', 'falcon>=3.1.3', From f378939c883cf001858b79d415b9214aaaa0ea25 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 25 Jul 2024 12:09:50 -0600 Subject: [PATCH 201/418] more porting to new Counter --- src/keri/vdr/viring.py | 31 +- tests/core/test_coring.py | 410 +------------------------- tests/core/test_weighted_threshold.py | 22 +- tests/vc/test_walleting.py | 5 +- 4 files changed, 32 insertions(+), 436 deletions(-) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index a65c62d57..f90f7dad1 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -13,7 +13,7 @@ from ..db import koming, subing, escrowing -from .. import kering +from .. import kering, core from ..app import signing from ..core import coring, serdering, indexing from ..db import dbing, basing @@ -439,9 +439,9 @@ def cloneCreds(self, saids, db): ) ) - ctr = coring.Counter(qb64b=iss, strip=True) + ctr = core.Counter(qb64b=iss, strip=True, gvrsn=kering.Vrsn_1_0) if ctr.code == coring.CtrDex.AttachmentGroup: - ctr = coring.Counter(qb64b=iss, strip=True) + ctr = core.Counter(qb64b=iss, strip=True, gvrsn=kering.Vrsn_1_0) if ctr.code == coring.CtrDex.SealSourceCouples: coring.Seqner(qb64b=iss, strip=True) @@ -525,24 +525,24 @@ def cloneTvt(self, pre, dig): # add indexed backer signatures to attachments if tibs := self.getTibs(key=dgkey): - atc.extend(coring.Counter(code=coring.CtrDex.WitnessIdxSigs, - count=len(tibs)).qb64b) + atc.extend(core.Counter(core.Codens.WitnessIdxSigs, count=len(tibs), + gvrsn=kering.Vrsn_1_0).qb64b) for tib in tibs: atc.extend(tib) # add authorizer (delegator/issure) source seal event couple to attachments couple = self.getAnc(dgkey) if couple is not None: - atc.extend(coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1).qb64b) + atc.extend(core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(couple) # prepend pipelining counter to attachments if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - pcnt = coring.Counter(code=coring.CtrDex.AttachmentGroup, - count=(len(atc) // 4)).qb64b + pcnt = core.Counter(core.Codens.AttachmentGroup, count=(len(atc) // 4), + gvrsn=kering.Vrsn_1_0).qb64b msg.extend(pcnt) msg.extend(atc) return msg @@ -573,7 +573,8 @@ def sources(self, db, creder): for said in saids: screder, prefixer, seqner, saider = self.cloneCred(said=said) - atc = bytearray(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + atc = bytearray(core.Counter(core.Codens.SealSourceTriples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(prefixer.qb64b) atc.extend(seqner.qb64b) atc.extend(saider.qb64b) @@ -981,12 +982,14 @@ def buildProof(prefixer, seqner, diger, sigers): """ prf = bytearray() - prf.extend(coring.Counter(coring.CtrDex.TransIdxSigGroups, count=1).qb64b) + prf.extend(core.Counter(core.Codens.TransIdxSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) prf.extend(prefixer.qb64b) prf.extend(seqner.qb64b) prf.extend(diger.qb64b) - prf.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) + prf.extend(core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0).qb64b) for siger in sigers: prf.extend(siger.qb64b) @@ -1009,8 +1012,8 @@ def messagize(creder, proof): if len(proof) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(proof))) - craw.extend(coring.Counter(code=coring.CtrDex.AttachmentGroup, - count=(len(proof) // 4)).qb64b) + craw.extend(core.Counter(core.Codens.AttachmentGroup, count=(len(proof) // 4), + gvrsn=kering.Vrsn_1_0).qb64b) craw.extend(proof) return craw diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index e8eee29b4..83e0f1193 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -43,7 +43,7 @@ TagDex, PadTagDex, Tagger, Ilker, Traitor, Verser, Versage, ) from keri.core.coring import Serialage, Serials -from keri.core.coring import (Sizage, MtrDex, Matter, CtrDex, Counter) +from keri.core.coring import (Sizage, MtrDex, Matter) from keri.core.coring import (Verfer, Cigar, Saider, DigDex, Diger, Prefixer,) from keri.core.coring import versify, deversify, Rever, MAXVERFULLSPAN @@ -2162,414 +2162,6 @@ def test_matter_special(): """ Done Test """ -def test_counter(): - """ - Test Counter class - """ - assert asdict(CtrDex) == { - 'ControllerIdxSigs': '-A', - 'WitnessIdxSigs': '-B', - 'NonTransReceiptCouples': '-C', - 'TransReceiptQuadruples': '-D', - 'FirstSeenReplayCouples': '-E', - 'TransIdxSigGroups': '-F', - 'SealSourceCouples': '-G', - 'TransLastIdxSigGroups': '-H', - 'SealSourceTriples': '-I', - 'SadPathSigGroups': '-J', - 'RootSadPathSigGroups': '-K', - 'PathedMaterialGroup': '-L', - 'BigPathedMaterialGroup': '-0L', - 'AttachmentGroup': '-V', - 'ESSRPayloadGroup': '-Z', - 'BigAttachmentGroup': '-0V', - 'KERIACDCGenusVersion': '--AAA', - } - - - assert CtrDex.ControllerIdxSigs == '-A' - assert CtrDex.WitnessIdxSigs == '-B' - - assert Counter.Codex == CtrDex - - # first character of code with hard size of code - assert Counter.Hards == { - '-A': 2, '-B': 2, '-C': 2, '-D': 2, '-E': 2, '-F': 2, '-G': 2, '-H': 2, '-I': 2, - '-J': 2, '-K': 2, '-L': 2, '-M': 2, '-N': 2, '-O': 2, '-P': 2, '-Q': 2, '-R': 2, - '-S': 2, '-T': 2, '-U': 2, '-V': 2, '-W': 2, '-X': 2, '-Y': 2, '-Z': 2, - '-a': 2, '-b': 2, '-c': 2, '-d': 2, '-e': 2, '-f': 2, '-g': 2, '-h': 2, '-i': 2, - '-j': 2, '-k': 2, '-l': 2, '-m': 2, '-n': 2, '-o': 2, '-p': 2, '-q': 2, '-r': 2, - '-s': 2, '-t': 2, '-u': 2, '-v': 2, '-w': 2, '-x': 2, '-y': 2, '-z': 2, - '-0': 3, '--': 5, - } - - # Codes table with sizes of code (hard) and full primitive material - assert Counter.Sizes == { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) - } - - assert Counter.Sizes['-A'].hs == 2 # hard size - assert Counter.Sizes['-A'].ss == 2 # soft size - assert Counter.Sizes['-A'].fs == 4 # full size - assert Counter.Sizes['-A'].ls == 0 # lead size - - # verify first hs Sizes matches hs in Codes for same first char - for ckey in Counter.Sizes.keys(): - assert Counter.Hards[ckey[:2]] == Counter.Sizes[ckey].hs - - # verify all Codes have hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 - for val in Counter.Sizes.values(): - assert val.hs > 0 and val.ss > 0 and val.hs + val.ss == val.fs and not val.fs % 4 - - # Bizes maps bytes of sextet of decoded first character of code with hard size of code - # verify equivalents of items for Sizes and Bizes - for skey, sval in Counter.Hards.items(): - ckey = codeB64ToB2(skey) - assert Counter.Bards[ckey] == sval - - with pytest.raises(EmptyMaterialError): - counter = Counter() - - # create code manually - count = 1 - qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) - assert qsc == '-AAB' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.ControllerIdxSigs) # default count = 1 - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64b=qscb) # test with bytes not str - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64=qsc) # test with str not bytes - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb2=qscb2) # test with qb2 - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - # test truncates extra bytes from qb64 parameter - longqsc64 = qsc + "ABCD" - counter = Counter(qb64=longqsc64) - assert len(counter.qb64) == Counter.Sizes[counter.code].fs - - # test raises ShortageError if not enough bytes in qb64 parameter - shortqsc64 = qsc[:-1] # too short - with pytest.raises(ShortageError): - counter = Counter(qb64=shortqsc64) - - # test truncates extra bytes from qb2 parameter - longqscb2 = qscb2 + bytearray([1, 2, 3, 4, 5]) # extra bytes in size - counter = Counter(qb2=longqscb2) - assert counter.qb2 == qscb2 - assert len(counter.qb64) == Counter.Sizes[counter.code].fs - - # test raises ShortageError if not enough bytes in qb2 parameter - shortqscb2 = qscb2[:-4] # too few bytes in size - with pytest.raises(ShortageError): - counter = Counter(qb2=shortqscb2) - - # test with non-zero count=5 - count = 5 - qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) - assert qsc == '-AAF' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.ControllerIdxSigs, count=count) - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64b=qscb) # test with bytes not str - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64=qsc) # test with str not bytes - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb2=qscb2) # test with qb2 - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - # test with big codes index=1024000 - count = 1024000 - qsc = CtrDex.BigAttachmentGroup + intToB64(count, l=5) - assert qsc == '-0VAD6AA' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.BigAttachmentGroup, count=count) - assert counter.code == CtrDex.BigAttachmentGroup - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64b=qscb) # test with bytes not str - assert counter.code == CtrDex.BigAttachmentGroup - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64=qsc) # test with str not bytes - assert counter.code == CtrDex.BigAttachmentGroup - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb2=qscb2) # test with qb2 - assert counter.code == CtrDex.BigAttachmentGroup - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - # Test ._bexfil - counter = Counter(qb64=qsc) # - code = counter.code - count = counter.count - qb2 = counter.qb2 - counter._bexfil(qb2) - assert counter.code == code - assert counter.count == count - assert counter.qb64 == qsc - assert counter.qb2 == qb2 - - # Test ._binfil - test = counter._binfil() - assert test == qb2 - - # Test limits of PathedMaterialGroup - count = 255 - qsc = CtrDex.PathedMaterialGroup + intToB64(count, l=2) - assert qsc == '-LD_' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.PathedMaterialGroup, count=count) - assert counter.code == CtrDex.PathedMaterialGroup - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64='-L__') - assert counter.count == 4095 - - with pytest.raises(kering.InvalidVarIndexError): - Counter(code=CtrDex.PathedMaterialGroup, count=4096) # Too big - - # Test BigPathedMaterialGroup - # test with big codes index=1024000 - count = 1024000 - qsc = CtrDex.BigPathedMaterialGroup + intToB64(count, l=5) - assert qsc == '-0LAD6AA' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.BigPathedMaterialGroup, count=count) - assert counter.code == CtrDex.BigPathedMaterialGroup - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - - # Test with strip - # create code manually - count = 1 - qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) - assert qsc == '-AAB' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - # strip ignored if qb64 - counter = Counter(qb64=qsc, strip=True) # test with str not bytes - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - ims = bytearray(qscb) # test with qb64b - counter = Counter(qb64b=ims, strip=True) # strip - assert not ims # deleted - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - ims = bytearray(qscb2) # test with qb2 - counter = Counter(qb2=ims, strip=True) - assert not ims # deleted - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - # test with longer ims for qb64b - extra = b"ABCD" - ims = bytearray(qscb + b"ABCD") - counter = Counter(qb64b=ims, strip=True) - assert counter.qb64b == qscb - assert len(counter.qb64b) == Counter.Sizes[counter.code].fs - assert ims == extra - - # test with longer ims for qb2 - extra = bytearray([1, 2, 3, 4, 5]) - ims = bytearray(qscb2) + extra - counter = Counter(qb2=ims, strip=True) - assert counter.qb2 == qscb2 - assert len(counter.qb2) == Counter.Sizes[counter.code].fs * 3 // 4 - assert ims == extra - - # raises error if not bytearray - - ims = bytes(qscb) # test with qb64b - with pytest.raises(TypeError): - counter = Counter(qb64b=ims, strip=True) # strip - - ims = bytes(qscb2) # test with qb2 - with pytest.raises(TypeError): - counter = Counter(qb2=ims, strip=True) - - # test with big codes index=1024 - count = 1024 - qsc = CtrDex.BigAttachmentGroup + intToB64(count, l=5) - assert qsc == '-0VAAAQA' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - ims = bytearray(qscb) - counter = Counter(qb64b=ims, strip=True) # test with bytes not str - assert counter.code == CtrDex.BigAttachmentGroup - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - assert not ims - - ims = bytearray(qscb2) - counter = Counter(qb2=ims, strip=True) # test with qb2 - assert counter.code == CtrDex.BigAttachmentGroup - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - assert not ims - - # test protocol genus with CESR version - # test with big codes index=1024 - verint = 0 - version = intToB64(verint, l=3) - assert version == 'AAA' - assert verint == b64ToInt(version) - qsc = CtrDex.KERIACDCGenusVersion + version - assert qsc == '--AAAAAA' # keri Cesr version 0.0.0 - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.KERIACDCGenusVersion, count=verint) - assert counter.code == CtrDex.KERIACDCGenusVersion - assert counter.count == verint - assert counter.countToB64(l=3) == version - assert counter.countToB64() == version # default length - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(code=CtrDex.KERIACDCGenusVersion, countB64=version) - assert counter.code == CtrDex.KERIACDCGenusVersion - assert counter.count == verint - assert counter.countToB64(l=3) == version - assert counter.countToB64() == version # default length - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - assert Counter.semVerToB64("1.2.3") == "BCD" - assert Counter.semVerToB64() == "AAA" - assert Counter.semVerToB64(major=1) == "BAA" - assert Counter.semVerToB64(minor=1) == "ABA" - assert Counter.semVerToB64(patch=1) == "AAB" - assert Counter.semVerToB64(major=3, minor=4, patch=5) == "DEF" - - # test defaults for missing parts in string version - assert Counter.semVerToB64(version="1.1") == "BBA" - assert Counter.semVerToB64(version="1.") == "BAA" - assert Counter.semVerToB64(version="1") == "BAA" - assert Counter.semVerToB64(version="1.2.") == "BCA" - assert Counter.semVerToB64(version="..") == "AAA" - assert Counter.semVerToB64(version="1..3") == "BAD" - assert Counter.semVerToB64(version="4", major=1, minor=2, patch=3) == "ECD" - - with pytest.raises(ValueError): - Counter.semVerToB64(version="64.0.1") - with pytest.raises(ValueError): - Counter.semVerToB64(version="-1.0.1") - with pytest.raises(ValueError): - Counter.semVerToB64(version="0.0.64") - with pytest.raises(ValueError): - Counter.semVerToB64(major=64) - with pytest.raises(ValueError): - Counter.semVerToB64(minor=-1) - with pytest.raises(ValueError): - Counter.semVerToB64(patch=-1) - - """ Done Test """ - - - def test_seqner(): """ diff --git a/tests/core/test_weighted_threshold.py b/tests/core/test_weighted_threshold.py index 3f65bc88d..8342f49d3 100644 --- a/tests/core/test_weighted_threshold.py +++ b/tests/core/test_weighted_threshold.py @@ -9,7 +9,7 @@ from keri import help -from keri import core +from keri import core, kering from keri.core import coring, eventing, parsing from keri.db import basing @@ -53,8 +53,8 @@ def test_weighted(): sigers = wesMgr.sign(ser=wesSrdr.raw, verfers=verfers) msg = bytearray(wesSrdr.raw) - counter = coring.Counter(coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -89,8 +89,8 @@ def test_weighted(): sigers = wesMgr.sign(ser=wesSrdr.raw, verfers=wesK.verfers) msg = bytearray(wesSrdr.raw) - counter = coring.Counter(coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -126,8 +126,8 @@ def test_weighted(): sigers = wesMgr.sign(ser=wesSrdr.raw, verfers=verfers) msg = bytearray(wesSrdr.raw) - counter = coring.Counter(coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -172,8 +172,8 @@ def test_weighted(): sigers = wesMgr.sign(ser=wesSrdr.raw, verfers=verfers) msg = bytearray(wesSrdr.raw) - counter = coring.Counter(coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -218,8 +218,8 @@ def test_weighted(): sigers = wesMgr.sign(ser=wesSrdr.raw, verfers=verfers) msg = bytearray(wesSrdr.raw) - counter = coring.Counter(coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) diff --git a/tests/vc/test_walleting.py b/tests/vc/test_walleting.py index 7049477fa..ddbbf4e35 100644 --- a/tests/vc/test_walleting.py +++ b/tests/vc/test_walleting.py @@ -3,7 +3,7 @@ tests.vc.walleting module """ -from keri import core +from keri import core, kering from keri.core import coring, parsing from keri.core.eventing import SealEvent @@ -56,7 +56,8 @@ def test_wallet(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): sidReg.processEscrows() msg = bytearray(creder.raw) - msg.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + msg.extend(core.Counter(core.Codens.SealSourceTriples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(coring.Prefixer(qb64=iss.pre).qb64b) msg.extend(coring.Seqner(sn=0).qb64b) msg.extend(iss.saidb) From a953ba915aadbd42c3253479ecf86d7492b1b60d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 25 Jul 2024 12:47:27 -0600 Subject: [PATCH 202/418] ported test_escrow to new Counter --- tests/core/test_escrow.py | 112 ++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 53 deletions(-) diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index a100cf03a..d3f419fa6 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -10,7 +10,7 @@ from keri import help from keri.help import helping -from keri import core +from keri import core, kering from keri.core import coring, eventing, parsing from keri.db import dbing, basing @@ -55,7 +55,8 @@ def test_partial_signed_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[0].qb64b) @@ -85,7 +86,8 @@ def test_partial_signed_escrow(): # Send message again but with signature from other siger msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[2].qb64b) # apply msg to Kevery to process @@ -119,8 +121,8 @@ def test_partial_signed_escrow(): # send duplicate message with all three sigs msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -148,7 +150,7 @@ def test_partial_signed_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=kvr.verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[1].qb64b) @@ -162,7 +164,7 @@ def test_partial_signed_escrow(): # add another sig msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[0].qb64b) @@ -190,7 +192,7 @@ def test_partial_signed_escrow(): # resend events to load escrow msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[1].qb64b) @@ -204,7 +206,7 @@ def test_partial_signed_escrow(): # add another sig msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[0].qb64b) @@ -233,7 +235,7 @@ def test_partial_signed_escrow(): # send duplicate message but add last sig msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[2].qb64b) psr.parse(ims=bytearray(msg), kvy=kvy) @@ -270,8 +272,8 @@ def test_partial_signed_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -300,7 +302,8 @@ def test_partial_signed_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=2) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=2, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[0].qb64b) msg.extend(sigers[3].qb64b) @@ -315,7 +318,7 @@ def test_partial_signed_escrow(): assert kvr.serder.said != srdr.said # key state not updated msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[1].qb64b) @@ -383,8 +386,8 @@ def test_missing_delegator_escrow(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -425,8 +428,8 @@ def test_missing_delegator_escrow(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -441,13 +444,13 @@ def test_missing_delegator_escrow(): sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = core.Counter(core.Codens.SealSourceCouples, + count=1, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) @@ -533,8 +536,8 @@ def test_missing_delegator_escrow(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -553,13 +556,13 @@ def test_missing_delegator_escrow(): sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = core.Counter(core.Codens.SealSourceCouples, + count=1, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) @@ -640,8 +643,8 @@ def test_out_of_order_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -654,8 +657,8 @@ def test_out_of_order_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -683,8 +686,8 @@ def test_out_of_order_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -850,8 +853,8 @@ def test_unverified_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -864,7 +867,8 @@ def test_unverified_receipt_escrow(): wit0Cigar = mgr.sign(ser=srdr.raw, verfers=[wit0Verfer], indexed=False)[0] # returns Cigar unindexed wit1Cigar = mgr.sign(ser=srdr.raw, verfers=[wit1Verfer], indexed=False)[0] # returns Cigar unindexed - recnt = coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=2) + recnt = core.Counter(core.Codens.NonTransReceiptCouples, count=2, + gvrsn=kering.Vrsn_1_0) msg = bytearray() msg.extend(reserder.raw) @@ -897,8 +901,8 @@ def test_unverified_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -911,7 +915,8 @@ def test_unverified_receipt_escrow(): wit0Cigar = mgr.sign(ser=srdr.raw, verfers=[wit0Verfer], indexed=False)[0] # returns Cigar unindexed wit1Cigar = mgr.sign(ser=srdr.raw, verfers=[wit1Verfer], indexed=False)[0] # returns Cigar unindexed - recnt = coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=2) + recnt = core.Counter(core.Codens.NonTransReceiptCouples, count=2, + gvrsn=kering.Vrsn_1_0) msg = bytearray() msg.extend(reserder.raw) @@ -959,8 +964,8 @@ def test_unverified_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -973,7 +978,8 @@ def test_unverified_receipt_escrow(): wit0Cigar = mgr.sign(ser=srdr.raw, verfers=[wit0Verfer], indexed=False)[0] # returns Cigar unindexed wit1Cigar = mgr.sign(ser=srdr.raw, verfers=[wit1Verfer], indexed=False)[0] # returns Cigar unindexed - recnt = coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=2) + recnt = core.Counter(core.Codens.NonTransReceiptCouples, count=2, + gvrsn=kering.Vrsn_1_0) msg = bytearray() msg.extend(reserder.raw) @@ -1122,8 +1128,8 @@ def test_unverified_trans_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -1149,8 +1155,8 @@ def test_unverified_trans_receipt_escrow(): rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) msg = bytearray(rsrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(rsigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(rsigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in rsigers: msg.extend(siger.qb64b) @@ -1189,8 +1195,8 @@ def test_unverified_trans_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -1215,8 +1221,8 @@ def test_unverified_trans_receipt_escrow(): rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) msg = bytearray(rsrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(rsigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(rsigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in rsigers: msg.extend(siger.qb64b) @@ -1269,8 +1275,8 @@ def test_unverified_trans_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) From f91d8940296fce11ca3b6957efc13b460f4c2fc9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 25 Jul 2024 14:09:57 -0600 Subject: [PATCH 203/418] ported test_eventing to new Counter --- tests/core/test_eventing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index 942a20cac..22a080a65 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -13,10 +13,10 @@ from keri.app import habbing, keeping from keri.app.keeping import openKS, Manager from keri import core -from keri.core import Signer +from keri.core import Signer, Counter, Codens from keri.core import coring, eventing, parsing, serdering from keri.core.coring import (Diger, MtrDex, Matter, - CtrDex, Counter, Cigar, + CtrDex, Cigar, Seqner, Verfer, Prefixer, DigDex) from keri.core.indexing import (IdrDex, IdxSigDex, Indexer, Siger) from keri.core.eventing import Kever, Kevery From f1b397a56ef1d654fc3d10e21d34563109187bf1 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 25 Jul 2024 14:24:12 -0600 Subject: [PATCH 204/418] fixed port of test_eventing to new Counter --- tests/core/test_eventing.py | 79 +++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index 22a080a65..8751c02c7 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -10,13 +10,14 @@ import pytest from keri import kering +from keri.kering import Vrsn_1_0 from keri.app import habbing, keeping from keri.app.keeping import openKS, Manager from keri import core from keri.core import Signer, Counter, Codens from keri.core import coring, eventing, parsing, serdering from keri.core.coring import (Diger, MtrDex, Matter, - CtrDex, Cigar, + Cigar, Seqner, Verfer, Prefixer, DigDex) from keri.core.indexing import (IdrDex, IdxSigDex, Indexer, Siger) from keri.core.eventing import Kever, Kevery @@ -2851,7 +2852,7 @@ def test_multisig_digprefix(): # create sig counter count = len(keys) - counter = Counter(CtrDex.ControllerIdxSigs, count=count) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization sigers = [signers[i].sign(serder.raw, index=i) for i in range(count)] # create key event verifier state @@ -2887,7 +2888,7 @@ def test_multisig_digprefix(): sn=1) # create sig counter count = len(keys) - counter = Counter(CtrDex.ControllerIdxSigs, count=count) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization sigers = [signers[i].sign(serder.raw, index=i - count) for i in range(count, count + count)] # update key event verifier state @@ -2903,7 +2904,7 @@ def test_multisig_digprefix(): dig=kever.serder.said, sn=2) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs, count=count) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization sigers = [signers[i].sign(serder.raw, index=i - count) for i in range(count, count + count)] # update key event verifier state @@ -2919,7 +2920,7 @@ def test_multisig_digprefix(): dig=kever.serder.said, sn=3) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs, count=count) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization sigers = [signers[i].sign(serder.raw, index=i - count) for i in range(count, count + count)] # update key event verifier state @@ -2939,7 +2940,7 @@ def test_multisig_digprefix(): dig=kever.serder.said, sn=4) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs, count=count) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization sigers = [signers[i].sign(serder.raw, index=i - 5) for i in range(5, 8)] # update key event verifier state @@ -2991,7 +2992,7 @@ def test_recovery(): event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # return siger # create key event verifier state @@ -3013,7 +3014,7 @@ def test_recovery(): event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # returns siger # update key event verifier state @@ -3032,7 +3033,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3054,7 +3055,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3073,7 +3074,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3092,7 +3093,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3111,7 +3112,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3134,7 +3135,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3153,7 +3154,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3244,7 +3245,7 @@ def test_receipt(): event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) # return Siger if index @@ -3273,7 +3274,7 @@ def test_receipt(): valCigar = valSigner.sign(ser=serder.raw) # returns Cigar cause no index assert valCigar.qb64 == ('0BADE2aOlwLi6OCF-jzRWSPuaOo916ADjwhA92hBQ1km' 'LSSYdzDiZIpJNFf0uislNR8uhCbB6x2Y1I6rqbNeBXwF') - recnt = Counter(code=CtrDex.NonTransReceiptCouples, count=1) + recnt = Counter(code=Codens.NonTransReceiptCouples, count=1, gvrsn=Vrsn_1_0) assert recnt.qb64 == '-CAB' res.extend(reserder.raw) @@ -3302,7 +3303,7 @@ def test_receipt(): said=fake) # sign event not receipt valCigar = valSigner.sign(ser=serder.raw) # returns Cigar cause no index - recnt = Counter(code=CtrDex.NonTransReceiptCouples, count=1) + recnt = Counter(code=Codens.NonTransReceiptCouples, count=1, gvrsn=Vrsn_1_0) # attach to receipt msg stream res.extend(reserder.raw) res.extend(recnt.qb64b) @@ -3324,7 +3325,7 @@ def test_receipt(): said=fake) # sign event not receipt valCigar = valSigner.sign(ser=serder.raw) # returns Cigar cause no index - recnt = Counter(code=CtrDex.NonTransReceiptCouples, count=1) + recnt = Counter(code=Codens.NonTransReceiptCouples, count=1, gvrsn=Vrsn_1_0) # attach to receipt msg stream res.extend(reserder.raw) res.extend(recnt.qb64b) @@ -3354,7 +3355,7 @@ def test_receipt(): event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) # returns siger # extend key event stream @@ -3375,7 +3376,7 @@ def test_receipt(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) @@ -3400,7 +3401,7 @@ def test_receipt(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) @@ -3422,7 +3423,7 @@ def test_receipt(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) @@ -3444,7 +3445,7 @@ def test_receipt(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) @@ -3466,7 +3467,7 @@ def test_receipt(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) @@ -3536,7 +3537,7 @@ def test_direct_mode(): coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) # return Siger if index @@ -3569,7 +3570,7 @@ def test_direct_mode(): val_event_digs.append(valSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = valSigners[vesn].sign(valSerder.raw, index=0) # return Siger if index @@ -3750,7 +3751,7 @@ def test_direct_mode(): sn=csn) coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) # returns siger @@ -3844,7 +3845,7 @@ def test_direct_mode(): sn=csn) coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) @@ -4002,7 +4003,7 @@ def test_direct_mode_cbor_mgpk(): coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) # return Siger if index @@ -4034,7 +4035,7 @@ def test_direct_mode_cbor_mgpk(): val_event_digs.append(valSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = valSigners[vesn].sign(valSerder.raw, index=0) # return Siger if index @@ -4213,7 +4214,7 @@ def test_direct_mode_cbor_mgpk(): kind=Serials.cbor) coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) # returns siger @@ -4307,7 +4308,7 @@ def test_direct_mode_cbor_mgpk(): kind=Serials.cbor) coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) @@ -4472,7 +4473,7 @@ def test_process_nontransferable(): assert skp0.verfer.verify(tsig0.raw, tser0.raw) # create attached sig counter - cnt0 = Counter(CtrDex.ControllerIdxSigs) + cnt0 = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # create packet msgb0 = bytearray(tser0.raw + cnt0.qb64b + tsig0.qb64b) @@ -4483,7 +4484,7 @@ def test_process_nontransferable(): del msgb0[:rser0.size] # strip off event from front # extract sig counter - rcnt0 = Counter(qb64=msgb0) + rcnt0 = Counter(qb64=msgb0, gvrsn=Vrsn_1_0) nrsigs = rcnt0.count assert nrsigs == 1 del msgb0[:len(rcnt0.qb64)] @@ -4565,7 +4566,7 @@ def test_process_transferable(): assert skp0.verfer.verify(tsig0.raw, tser0.raw) # create attached sig counter - cnt0 = Counter(CtrDex.ControllerIdxSigs) + cnt0 = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # create packet msgb0 = bytearray(tser0.raw + cnt0.qb64b + tsig0.qb64b) @@ -4576,7 +4577,7 @@ def test_process_transferable(): del msgb0[:rser0.size] # strip off event from front # extract sig counter - rcnt0 = Counter(qb64=msgb0) + rcnt0 = Counter(qb64=msgb0, gvrsn=Vrsn_1_0) nrsigs = rcnt0.count assert nrsigs == 1 del msgb0[:len(rcnt0.qb64)] @@ -4891,8 +4892,8 @@ def test_load_event(mockHelpingNowUTC): 'witnesses': []} # Add seal source couple to Tee's inception before sending to Wan - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = Counter(Codens.SealSourceCouples, + count=1, gvrsn=Vrsn_1_0) teeIcp.extend(counter.qb64b) seqner = coring.Seqner(sn=torHab.kever.sn) teeIcp.extend(seqner.qb64b) From def7b7efebb68ba3501a72cadff4e905b13810b2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 25 Jul 2024 19:04:23 -0600 Subject: [PATCH 205/418] port to new Counter --- src/keri/app/agenting.py | 1 - src/keri/app/indirecting.py | 2 +- src/keri/core/parsing.py | 54 +++++++++++++++++------------------ src/keri/peer/exchanging.py | 2 +- src/keri/vdr/viring.py | 6 ++-- tests/app/test_grouping.py | 6 ++-- tests/app/test_signing.py | 5 ++-- tests/core/test_kevery.py | 26 ++++++++--------- tests/core/test_parsing.py | 26 ++++++++--------- tests/core/test_replay.py | 24 ++++++++-------- tests/peer/test_exchanging.py | 4 +-- tests/vc/test_protocoling.py | 4 +-- tests/vc/test_proving.py | 19 ++++++------ 13 files changed, 90 insertions(+), 89 deletions(-) diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 894a3cbd4..a6023c3d6 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -19,7 +19,6 @@ from .. import kering from .. import core from ..core import eventing, parsing, coring, serdering, indexing -from ..core.coring import CtrDex from ..db import dbing from ..kering import Roles diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 8fd4730c4..16b9a0374 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -1143,7 +1143,7 @@ def on_get(self, req, rep): said=said.decode("utf-8")) rct = bytearray(rserder.raw) if wigs := self.hab.db.getWigs(key=dgkey): - rct.extend(Counter(Codens.CtrDex.WitnessIdxSigs, count=len(wigs), + rct.extend(Counter(Codens.WitnessIdxSigs, count=len(wigs), gvrsn=kering.Vrsn_1_0).qb64b) for wig in wigs: rct.extend(wig) diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index 95798e7e3..db0a86fa5 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -8,9 +8,9 @@ import logging from ..kering import Vrsn_1_0, Vrsn_2_0 -from .coring import (Ilks, CtrDex, Seqner, Cigar, +from .coring import (Ilks, Seqner, Cigar, Dater, Verfer, Prefixer, Saider, Pather, Texter) -from .counting import Counter, Codens +from .counting import Counter, Codens, CtrDex_1_0 from .indexing import (Siger, ) from . import serdering from .. import help @@ -147,10 +147,10 @@ def _sadPathSigGroup(self, ctr, ims, root=None, cold=Colds.txt, pipelined=False) Returns: """ - if ctr.code != CtrDex.SadPathSigGroups: + if ctr.code != CtrDex_1_0.SadPathSigGroups: raise kering.UnexpectedCountCodeError("Wrong " "count code={}.Expected code={}." - "".format(ctr.code, CtrDex.ControllerIdxSigs)) + "".format(ctr.code, CtrDex_1_0.ControllerIdxSigs)) subpath = yield from self._extractor(ims, klas=Pather, @@ -163,10 +163,10 @@ def _sadPathSigGroup(self, ctr, ims, root=None, cold=Colds.txt, pipelined=False) klas=Counter, cold=cold, abort=pipelined) - if sctr.code == CtrDex.TransIdxSigGroups: + if sctr.code == CtrDex_1_0.TransIdxSigGroups: for prefixer, seqner, saider, isigers in self._transIdxSigGroups(sctr, ims, cold=cold, pipelined=pipelined): yield sctr.code, (subpath, prefixer, seqner, saider, isigers) - elif sctr.code == CtrDex.ControllerIdxSigs: + elif sctr.code == CtrDex_1_0.ControllerIdxSigs: isigers = [] for i in range(sctr.count): # extract each attached signature isiger = yield from self._extractor(ims=ims, @@ -175,13 +175,13 @@ def _sadPathSigGroup(self, ctr, ims, root=None, cold=Colds.txt, pipelined=False) abort=pipelined) isigers.append(isiger) yield sctr.code, (subpath, isigers) - elif sctr.code == CtrDex.NonTransReceiptCouples: + elif sctr.code == CtrDex_1_0.NonTransReceiptCouples: for cigar in self._nonTransReceiptCouples(ctr=sctr, ims=ims, cold=cold, pipelined=pipelined): yield sctr.code, (subpath, cigar) else: raise kering.UnexpectedCountCodeError("Wrong " "count code={}.Expected code={}." - "".format(ctr.code, CtrDex.ControllerIdxSigs)) + "".format(ctr.code, CtrDex_1_0.ControllerIdxSigs)) def _transIdxSigGroups(self, ctr, ims, cold=Colds.txt, pipelined=False): @@ -224,10 +224,10 @@ def _transIdxSigGroups(self, ctr, ims, cold=Colds.txt, pipelined=False): klas=Counter, cold=cold, abort=pipelined) - if ictr.code != CtrDex.ControllerIdxSigs: + if ictr.code != CtrDex_1_0.ControllerIdxSigs: raise kering.UnexpectedCountCodeError("Wrong " "count code={}.Expected code={}." - "".format(ictr.code, CtrDex.ControllerIdxSigs)) + "".format(ictr.code, CtrDex_1_0.ControllerIdxSigs)) isigers = [] for i in range(ictr.count): # extract each attached signature isiger = yield from self._extractor(ims=ims, @@ -748,7 +748,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, cold = sniff(ims) # expect counter at front of attachments if cold != Colds.msg: # not new message so process attachments ctr = yield from self._extractor(ims=ims, klas=Counter, cold=cold) - if ctr.code == CtrDex.AttachmentGroup: # pipeline ctr? + if ctr.code == CtrDex_1_0.AttachmentGroup: # pipeline ctr? pipelined = True # compute pipelined attached group size based on txt or bny pags = ctr.count * 4 if cold == Colds.txt else ctr.count * 3 @@ -770,7 +770,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, # iteratively process attachment counters (all non pipelined) while True: # do while already extracted first counter is ctr - if ctr.code == CtrDex.ControllerIdxSigs: + if ctr.code == CtrDex_1_0.ControllerIdxSigs: for i in range(ctr.count): # extract each attached signature siger = yield from self._extractor(ims=ims, klas=Siger, @@ -778,7 +778,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) sigers.append(siger) - elif ctr.code == CtrDex.WitnessIdxSigs: + elif ctr.code == CtrDex_1_0.WitnessIdxSigs: for i in range(ctr.count): # extract each attached signature wiger = yield from self._extractor(ims=ims, klas=Siger, @@ -786,7 +786,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) wigers.append(wiger) - elif ctr.code == CtrDex.NonTransReceiptCouples: + elif ctr.code == CtrDex_1_0.NonTransReceiptCouples: # extract attached rct couplets into list of sigvers # verfer property of cigar is the identifier prefix # cigar itself has the attached signature @@ -796,7 +796,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, pipelined=pipelined): cigars.append(cigar) - elif ctr.code == CtrDex.TransReceiptQuadruples: + elif ctr.code == CtrDex_1_0.TransReceiptQuadruples: # extract attaced trans receipt vrc quadruple # spre+ssnu+sdig+sig # spre is pre of signer of vrc @@ -823,7 +823,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) trqs.append((prefixer, seqner, saider, siger)) - elif ctr.code == CtrDex.TransIdxSigGroups: + elif ctr.code == CtrDex_1_0.TransIdxSigGroups: # extract attaced trans indexed sig groups each made of # triple pre+snu+dig plus indexed sig group # pre is pre of signer (endorser) of msg @@ -836,7 +836,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, pipelined=pipelined): tsgs.append((prefixer, seqner, saider, isigers)) - elif ctr.code == CtrDex.TransLastIdxSigGroups: + elif ctr.code == CtrDex_1_0.TransLastIdxSigGroups: # extract attaced signer seal indexed sig groups each made of # identifier pre plus indexed sig group # pre is pre of signer (endorser) of msg @@ -851,10 +851,10 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, klas=Counter, cold=cold, abort=pipelined) - if ictr.code != CtrDex.ControllerIdxSigs: + if ictr.code != CtrDex_1_0.ControllerIdxSigs: raise kering.UnexpectedCountCodeError("Wrong " "count code={}.Expected code={}." - "".format(ictr.code, CtrDex.ControllerIdxSigs)) + "".format(ictr.code, CtrDex_1_0.ControllerIdxSigs)) isigers = [] for i in range(ictr.count): # extract each attached signature isiger = yield from self._extractor(ims=ims, @@ -864,7 +864,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, isigers.append(isiger) ssgs.append((prefixer, isigers)) - elif ctr.code == CtrDex.FirstSeenReplayCouples: + elif ctr.code == CtrDex_1_0.FirstSeenReplayCouples: # extract attached first seen replay couples # snu+dtm # snu is fn (first seen ordinal) of event @@ -880,7 +880,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) frcs.append((firner, dater)) - elif ctr.code == CtrDex.SealSourceCouples: + elif ctr.code == CtrDex_1_0.SealSourceCouples: # extract attached first seen replay couples # snu+dig # snu is sequence number of event @@ -896,7 +896,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) sscs.append((seqner, saider)) - elif ctr.code == CtrDex.SealSourceTriples: + elif ctr.code == CtrDex_1_0.SealSourceTriples: # extract attached anchoring source event information # pre+snu+dig # pre is prefix of event @@ -917,7 +917,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) ssts.append((prefixer, seqner, saider)) - elif ctr.code == CtrDex.SadPathSigGroups: + elif ctr.code == CtrDex_1_0.SadPathSigGroups: path = yield from self._extractor(ims, klas=Pather, cold=cold, @@ -932,12 +932,12 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, root=path, cold=cold, pipelined=pipelined): - if code == CtrDex.TransIdxSigGroups: + if code == CtrDex_1_0.TransIdxSigGroups: sadtsgs.append(sigs) else: sadcigs.append(sigs) - elif ctr.code == CtrDex.PathedMaterialGroup: # pathed ctr? + elif ctr.code == CtrDex_1_0.PathedMaterialGroup: # pathed ctr? # compute pipelined attached group size based on txt or bny pags = ctr.count * 4 if cold == Colds.txt else ctr.count * 3 while len(ims) < pags: # wait until rx full pipelned group @@ -947,7 +947,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, del ims[:pags] # strip off from ims pathed.append(pims) - elif ctr.code == CtrDex.BigPathedMaterialGroup: # pathed ctr? + elif ctr.code == CtrDex_1_0.BigPathedMaterialGroup: # pathed ctr? # compute pipelined attached group size based on txt or bny pags = ctr.count * 4 if cold == Colds.txt else ctr.count * 3 while len(ims) < pags: # wait until rx full pipelned group @@ -957,7 +957,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, del ims[:pags] # strip off from ims pathed.append(pims) - elif ctr.code == CtrDex.ESSRPayloadGroup: + elif ctr.code == CtrDex_1_0.ESSRPayloadGroup: for i in range(ctr.count): texter = yield from self._extractor(ims, klas=Texter, diff --git a/src/keri/peer/exchanging.py b/src/keri/peer/exchanging.py index d8d5a777b..c6c6b8c23 100644 --- a/src/keri/peer/exchanging.py +++ b/src/keri/peer/exchanging.py @@ -448,7 +448,7 @@ def serializeMessage(hby, said, pipelined=False): atc.extend(siger.qb64b) if len(cigars) > 0: - atc.extend(core.Counter(core.Codens.CtrDex.NonTransReceiptCouples, + atc.extend(core.Counter(core.Codens.NonTransReceiptCouples, count=len(cigars), gvrsn=kering.Vrsn_1_0).qb64b) for cigar in cigars: if cigar.verfer.code not in coring.NonTransDex: diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index f90f7dad1..434a2d16e 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -15,7 +15,7 @@ from .. import kering, core from ..app import signing -from ..core import coring, serdering, indexing +from ..core import coring, serdering, indexing, counting from ..db import dbing, basing from ..db.dbing import snKey from ..help import helping @@ -440,10 +440,10 @@ def cloneCreds(self, saids, db): ) ctr = core.Counter(qb64b=iss, strip=True, gvrsn=kering.Vrsn_1_0) - if ctr.code == coring.CtrDex.AttachmentGroup: + if ctr.code == counting.CtrDex_1_0.AttachmentGroup: ctr = core.Counter(qb64b=iss, strip=True, gvrsn=kering.Vrsn_1_0) - if ctr.code == coring.CtrDex.SealSourceCouples: + if ctr.code == counting.CtrDex_1_0.SealSourceCouples: coring.Seqner(qb64b=iss, strip=True) saider = coring.Saider(qb64b=iss) diff --git a/tests/app/test_grouping.py b/tests/app/test_grouping.py index 5fd0090a7..c50d9402d 100644 --- a/tests/app/test_grouping.py +++ b/tests/app/test_grouping.py @@ -5,7 +5,7 @@ """ from contextlib import contextmanager - +from keri import kering, core from keri.app import habbing, grouping, notifying from keri.core import coring, eventing, parsing, serdering from keri.vdr import eventing as veventing @@ -616,8 +616,8 @@ def openMultiSig(prefix="test", salt=b'0123456789abcdef', temp=True, **kwa): sigs.extend(bytes(hab3.db.getSigs(dgkey)[0])) evt = bytearray(eraw) - evt.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=3).qb64b) # attach cnt + evt.extend(core.Counter(core.Codens.ControllerIdxSigs, + count=3, gvrsn=kering.Vrsn_1_0).qb64b) # attach cnt evt.extend(sigs) parsing.Parser().parse(ims=bytearray(evt), kvy=kev3, local=True) diff --git a/tests/app/test_signing.py b/tests/app/test_signing.py index 0b297d17e..9a132990a 100644 --- a/tests/app/test_signing.py +++ b/tests/app/test_signing.py @@ -3,7 +3,7 @@ tests.app.signing module """ -from keri import core +from keri import kering, core from keri.core import coring, parsing, eventing from keri.core.eventing import SealEvent @@ -212,7 +212,8 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs # Sign with non-transferable identifier, defaults to single signature on entire SAD sig0 = bytearray(cred.raw) - sig0.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + sig0.extend(core.Counter(core.Codens.SealSourceTriples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) sig0.extend(coring.Prefixer(qb64=issuer.regk).qb64b) sig0.extend(seqner.qb64b) sig0.extend(coring.Saider(qb64=issuer.regd).qb64b) diff --git a/tests/core/test_kevery.py b/tests/core/test_kevery.py index 183b073a2..97b6dd7d5 100644 --- a/tests/core/test_kevery.py +++ b/tests/core/test_kevery.py @@ -2,13 +2,13 @@ import pytest -from keri.kering import (ValidationError) +from keri.kering import (ValidationError, Vrsn_1_0) from keri import help from keri import core -from keri.core import parsing, eventing, coring, serdering -from keri.core.coring import CtrDex, Counter +from keri.core import parsing, eventing, coring, serdering, Counter, Codens + from keri.core.eventing import Kever, Kevery from keri.core.eventing import (incept, rotate, interact) @@ -44,7 +44,7 @@ def test_kevery(): ndigs=[coring.Diger(ser=signers[1].verfer.qb64b).qb64]) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[0].sign(serder.raw, index=0) # return siger # create key event verifier state @@ -70,7 +70,7 @@ def test_kevery(): sn=1) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[1].sign(serder.raw, index=0) # returns siger # update key event verifier state @@ -88,7 +88,7 @@ def test_kevery(): sn=2) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[2].sign(serder.raw, index=0) # update key event verifier state @@ -104,7 +104,7 @@ def test_kevery(): sn=3) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[2].sign(serder.raw, index=0) # update key event verifier state @@ -120,7 +120,7 @@ def test_kevery(): sn=4) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[2].sign(serder.raw, index=0) # update key event verifier state @@ -138,7 +138,7 @@ def test_kevery(): sn=5) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[3].sign(serder.raw, index=0) # update key event verifier state @@ -154,7 +154,7 @@ def test_kevery(): sn=6) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[3].sign(serder.raw, index=0) # update key event verifier state @@ -172,7 +172,7 @@ def test_kevery(): sn=7) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[4].sign(serder.raw, index=0) # update key event verifier state @@ -187,7 +187,7 @@ def test_kevery(): dig=kever.serder.said, sn=8) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[4].sign(serder.raw, index=0) # update key event verifier state @@ -205,7 +205,7 @@ def test_kevery(): ndigs=[coring.Diger(ser=signers[5].verfer.qb64b).qb64], sn=8) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[4].sign(serder.raw, index=0) # update key event verifier state diff --git a/tests/core/test_parsing.py b/tests/core/test_parsing.py index 95ebc7e9e..a6438ea01 100644 --- a/tests/core/test_parsing.py +++ b/tests/core/test_parsing.py @@ -9,13 +9,13 @@ from hio.help import decking -from keri.kering import ValidationError +from keri.kering import ValidationError, Vrsn_1_0 from keri import help from keri import core -from keri.core import parsing, coring -from keri.core.coring import (CtrDex, Counter,) +from keri.core import parsing, coring, Counter, Codens + from keri.core.eventing import (Kever, Kevery, incept, rotate, interact) from keri.db.basing import openDB @@ -46,7 +46,7 @@ def test_parser(): ndigs=[coring.Diger(ser=signers[1].verfer.qb64b).qb64]) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[0].sign(serder.raw, index=0) # return siger # create key event verifier state @@ -72,7 +72,7 @@ def test_parser(): sn=1) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[1].sign(serder.raw, index=0) # returns siger # update key event verifier state @@ -90,7 +90,7 @@ def test_parser(): sn=2) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[2].sign(serder.raw, index=0) # update key event verifier state @@ -106,7 +106,7 @@ def test_parser(): sn=3) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[2].sign(serder.raw, index=0) # update key event verifier state @@ -122,7 +122,7 @@ def test_parser(): sn=4) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[2].sign(serder.raw, index=0) # update key event verifier state @@ -140,7 +140,7 @@ def test_parser(): sn=5) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[3].sign(serder.raw, index=0) # update key event verifier state @@ -156,7 +156,7 @@ def test_parser(): sn=6) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[3].sign(serder.raw, index=0) # update key event verifier state @@ -174,7 +174,7 @@ def test_parser(): sn=7) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[4].sign(serder.raw, index=0) # update key event verifier state @@ -189,7 +189,7 @@ def test_parser(): dig=kever.serder.said, sn=8) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[4].sign(serder.raw, index=0) # update key event verifier state @@ -207,7 +207,7 @@ def test_parser(): ndigs=[coring.Diger(ser=signers[5].verfer.qb64b).qb64], sn=8) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[4].sign(serder.raw, index=0) # update key event verifier state diff --git a/tests/core/test_replay.py b/tests/core/test_replay.py index 29547a88c..70b5ecf02 100644 --- a/tests/core/test_replay.py +++ b/tests/core/test_replay.py @@ -9,8 +9,8 @@ from keri import help from keri.help import helping -from keri import core -from keri.core import coring, eventing, parsing, serdering, indexing +from keri import core, kering +from keri.core import coring, eventing, parsing, serdering, indexing, counting from keri.app import habbing @@ -340,14 +340,14 @@ def test_replay(): del msg[:len(serder.raw)] assert len(msg) == 1076 - counter = coring.Counter(qb64b=msg) # attachment length quadlets counter - assert counter.code == coring.CtrDex.AttachmentGroup + counter = core.Counter(qb64b=msg, gvrsn=kering.Vrsn_1_0) # attachment length quadlets counter + assert counter.code == counting.CtrDex_1_0.AttachmentGroup assert counter.count == (len(msg) - len(counter.qb64b)) // 4 == 268 del msg[:len(counter.qb64b)] assert len(msg) == 1072 == 268 * 4 - counter = coring.Counter(qb64b=msg) # indexed signatures counter - assert counter.code == coring.CtrDex.ControllerIdxSigs + counter = core.Counter(qb64b=msg, gvrsn=kering.Vrsn_1_0) # indexed signatures counter + assert counter.code == counting.CtrDex_1_0.ControllerIdxSigs assert counter.count == 3 # multisig deb del msg[:len(counter.qb64b)] assert len(msg) == 1068 @@ -357,8 +357,8 @@ def test_replay(): del msg[:len(siger.qb64b)] assert len(msg) == 1068 - 3 * len(siger.qb64b) == 804 - counter = coring.Counter(qb64b=msg) # trans receipt (vrc) counter - assert counter.code == coring.CtrDex.TransReceiptQuadruples + counter = core.Counter(qb64b=msg, gvrsn=kering.Vrsn_1_0) # trans receipt (vrc) counter + assert counter.code == counting.CtrDex_1_0.TransReceiptQuadruples assert counter.count == 3 # multisig cam del msg[:len(counter.qb64b)] assert len(msg) == 800 @@ -368,8 +368,8 @@ def test_replay(): assert len(msg) == 800 - 3 * (len(prefixer.qb64b) + len(seqner.qb64b) + len(diger.qb64b) + len(siger.qb64b)) == 200 - counter = coring.Counter(qb64b=msg) # nontrans receipt (rct) counter - assert counter.code == coring.CtrDex.NonTransReceiptCouples + counter = core.Counter(qb64b=msg, gvrsn=kering.Vrsn_1_0) # nontrans receipt (rct) counter + assert counter.code == counting.CtrDex_1_0.NonTransReceiptCouples assert counter.count == 1 # single sig bev del msg[:len(counter.qb64b)] assert len(msg) == 196 @@ -378,8 +378,8 @@ def test_replay(): prefixer, cigar = eventing.deReceiptCouple(msg, strip=True) assert len(msg) == 196 - 1 * (len(prefixer.qb64b) + len(cigar.qb64b)) == 64 - counter = coring.Counter(qb64b=msg) # first seen replay couple counter - assert counter.code == coring.CtrDex.FirstSeenReplayCouples + counter = core.Counter(qb64b=msg, gvrsn=kering.Vrsn_1_0) # first seen replay couple counter + assert counter.code == counting.CtrDex_1_0.FirstSeenReplayCouples assert counter.count == 1 del msg[:len(counter.qb64b)] assert len(msg) == 60 diff --git a/tests/peer/test_exchanging.py b/tests/peer/test_exchanging.py index 0f19944f9..54115ae60 100644 --- a/tests/peer/test_exchanging.py +++ b/tests/peer/test_exchanging.py @@ -12,7 +12,6 @@ from keri.core import coring, serdering, MtrDex, parsing from keri.app import habbing, forwarding, storing, signing -from keri.core.coring import CtrDex from keri.peer import exchanging from keri.vdr.eventing import incept @@ -93,7 +92,8 @@ def test_essrs(): essr, _ = exchanging.exchange(route='/essr/req', sender=hab.pre, diger=diger, modifiers=dict(src=hab.pre, dest=recHab.pre)) ims = hab.endorse(serder=essr, pipelined=False) - ims.extend(coring.Counter(code=CtrDex.ESSRPayloadGroup, count=1).qb64b) + ims.extend(core.Counter(core.Codens.ESSRPayloadGroup, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) ims.extend(texter[0].qb64b) parsing.Parser().parse(ims=ims, kvy=recHby.kvy, exc=exc) diff --git a/tests/vc/test_protocoling.py b/tests/vc/test_protocoling.py index f6dde10bd..69a5b4853 100644 --- a/tests/vc/test_protocoling.py +++ b/tests/vc/test_protocoling.py @@ -4,7 +4,7 @@ """ -from keri import core +from keri import core, kering from keri.core import coring, scheming, parsing from keri.core.eventing import SealEvent @@ -96,7 +96,7 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN b'I":"254900OPPU84GM83MG36"}}') atc = bytearray(msg) - atc.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + atc.extend(core.Counter(core.Codens.SealSourceTriples, count=1, gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(coring.Prefixer(qb64=iss.pre).qb64b) atc.extend(coring.Seqner(sn=0).qb64b) atc.extend(iss.saidb) diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index 9860ca072..c2ab1cbac 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -6,11 +6,12 @@ import pytest from keri import kering -from keri.kering import Versionage +from keri.kering import Versionage, Vrsn_1_0 from keri import core from keri.core import coring, scheming, parsing, serdering -from keri.core.coring import Serials, Counter, CtrDex, Prefixer, Seqner, Diger +from keri.core import counting, Counter, Codens +from keri.core.coring import Serials, Prefixer, Seqner, Diger from keri.core.indexing import Siger from keri.core.scheming import CacheResolver @@ -74,15 +75,15 @@ def test_proving(mockHelpingNowIso8601): creder = serdering.SerderACDC(raw=msg) # Creder(raw=msg) proof = msg[creder.size:] - ctr = Counter(qb64b=proof, strip=True) - assert ctr.code == CtrDex.AttachmentGroup + ctr = Counter(qb64b=proof, strip=True, gvrsn=Vrsn_1_0) + assert ctr.code == counting.CtrDex_1_0.AttachmentGroup assert ctr.count == 52 pags = ctr.count * 4 assert len(proof) == pags - ctr = Counter(qb64b=proof, strip=True) - assert ctr.code == CtrDex.TransIdxSigGroups + ctr = Counter(qb64b=proof, strip=True, gvrsn=Vrsn_1_0) + assert ctr.code == counting.CtrDex_1_0.TransIdxSigGroups assert ctr.count == 1 prefixer = Prefixer(qb64b=proof, strip=True) @@ -94,8 +95,8 @@ def test_proving(mockHelpingNowIso8601): diger = Diger(qb64b=proof, strip=True) assert diger.qb64 == sidHab.kever.serder.said - ictr = Counter(qb64b=proof, strip=True) - assert ictr.code == CtrDex.ControllerIdxSigs + ictr = Counter(qb64b=proof, strip=True, gvrsn=Vrsn_1_0) + assert ictr.code == counting.CtrDex_1_0.ControllerIdxSigs isigers = [] for i in range(ictr.count): @@ -292,7 +293,7 @@ def test_credential_parsator(): status=issuer.regk) msg = bytearray(creder.raw) - msg.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + msg.extend(Counter(Codens.SealSourceTriples, count=1, gvrsn=Vrsn_1_0).qb64b) msg.extend(hab.kever.prefixer.qb64b) msg.extend(coring.Seqner(sn=hab.kever.sn).qb64b) msg.extend(hab.kever.serder.said.encode("utf-8")) From 6fae00dff1c68586aacd7c9c6f7004eb0738f985 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 26 Jul 2024 09:37:55 -0600 Subject: [PATCH 206/418] finished initial port to counting.Counter. coring.Counter has been removed. Now all Counter instances are versionable. --- src/keri/core/coring.py | 402 ---------------------------------------- 1 file changed, 402 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 22a1d4a0b..1699ab426 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -3906,408 +3906,6 @@ def verify(self, sad, *, prefixed=False, versioned=True, code=None, return True -@dataclass(frozen=True) -class CounterCodex: - """ - CounterCodex is codex hard (stable) part of all counter derivation codes. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - - ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. - WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. - NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. - TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. - FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. - TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. - SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegator/issuer/transaction event - TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. - SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event - SadPathSigGroups: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content - RootSadPathSigGroups: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples - PathedMaterialGroup: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) - BigPathedMaterialGroup: str = '-0L' # Composed Grouped Pathed Material Quadlet (4 char each) - AttachmentGroup: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) - BigAttachmentGroup: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) - ESSRPayloadGroup: str = '-Z' # ESSR Payload Group, dig of content+Texter group - KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Protocol Stack CESR Version - - def __iter__(self): - return iter(astuple(self)) # enables inclusion test with "in" - -CtrDex = CounterCodex() - - -class Counter: - """ - Counter is fully qualified cryptographic material primitive base class for - counter primitives (framing composition grouping count codes). - - Sub classes are derivation code and key event element context specific. - - Includes the following attributes and properties: - - Class Attributes: - - Attributes: - - Properties: - .code is str derivation code to indicate cypher suite - .raw is bytes crypto material only without code - .pad is int number of pad chars given raw - .count is int count of grouped following material (not part of counter) - .qb64 is str in Base64 fully qualified with derivation code + crypto mat - .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat - .qb2 is bytes in binary with derivation code + crypto material - - Hidden: - ._code is str value for .code property - ._raw is bytes value for .raw property - ._pad is method to compute .pad property - ._count is int value for .count property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 - - """ - # Hards table maps from bytes Base64 first two code chars to int of - # hard size, hs,(stable) of code. The soft size, ss, (unstable) for Counter - # is always > 0 and hs + ss = fs always - Hards = ({('-' + chr(c)): 2 for c in range(65, 65 + 26)}) - Hards.update({('-' + chr(c)): 2 for c in range(97, 97 + 26)}) - Hards.update([('-0', 3)]) - Hards.update([('--', 5)]) - - # Bards table maps to hard size, hs, of code from bytes holding sextets - # converted from first two code char. Used for ._bexfil. - Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) - - # Sizes table maps hs chars of code to Sizage namedtuple of (hs, ss, fs) - # where hs is hard size, ss is soft size, and fs is full size - # soft size, ss, should always be > 0 and hs+ss=fs for Counter - Sizes = { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), - } - - Codex = CtrDex - - - def __init__(self, code=None, count=None, countB64=None, - qb64b=None, qb64=None, qb2=None, strip=False): - """ - Validate as fully qualified - Parameters: - code (str | None): stable (hard) part of derivation code - count (int | None): count for composition. - Count may represent quadlets/triplet, groups, primitives or - other numericy - When both count and countB64 are None then count defaults to 1 - countB64 (str | None): count for composition as Base64 - countB64 may represent quadlets/triplet, groups, primitives or - other numericy - qb64b (bytes | bytearray | None): fully qualified crypto material text domain - qb64 (str | None) fully qualified crypto material text domain - qb2 (bytes | bytearray | None) fully qualified crypto material binary domain - strip (bool): True means strip counter contents from input stream - bytearray after parsing qb64b or qb2. False means do not strip. - default False - - - Needs either code or qb64b or qb64 or qb2 - Otherwise raises EmptyMaterialError - When code and count provided then validate that code and count are correct - Else when qb64b or qb64 or qb2 provided extract and assign - .code and .count - - """ - if code is not None: # code provided - if code not in self.Sizes: - raise InvalidCodeError("Unsupported code={}.".format(code)) - - hs, ss, fs, ls = self.Sizes[code] # get sizes for code - cs = hs + ss # both hard + soft code size - if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes - raise InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) - - if count is None: - count = 1 if countB64 is None else b64ToInt(countB64) - - if count < 0 or count > (64 ** ss - 1): - raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) - - self._code = code - self._count = count - - elif qb64b is not None: - self._exfil(qb64b) - if strip: # assumes bytearray - del qb64b[:self.Sizes[self.code].fs] - - elif qb64 is not None: - self._exfil(qb64) - - elif qb2 is not None: # rewrite to use direct binary exfiltration - self._bexfil(qb2) - if strip: # assumes bytearray - del qb2[:self.Sizes[self.code].fs * 3 // 4] - - else: - raise EmptyMaterialError("Improper initialization need either " - "(code and count) or qb64b or " - "qb64 or qb2.") - - @property - def code(self): - """ - Returns ._code - Makes .code read only - """ - return self._code - - - @property - def count(self): - """ - Returns ._count - Makes ._count read only - """ - return self._count - - - @property - def qb64b(self): - """ - Property qb64b: - Returns Fully Qualified Base64 Version encoded as bytes - Assumes self.raw and self.code are correctly populated - """ - return self._infil() - - - @property - def qb64(self): - """ - Property qb64: - Returns Fully Qualified Base64 Version - Assumes self.raw and self.code are correctly populated - """ - return self.qb64b.decode("utf-8") - - - @property - def qb2(self): - """ - Property qb2: - Returns Fully Qualified Binary Version Bytes - """ - return self._binfil() - - - def countToB64(self, l=None): - """ Returns count as Base64 left padded with "A"s - Parameters: - l (int | None): minimum number characters including left padding - When not provided use the softsize of .code - - """ - if l is None: - _, ss, _, _ = self.Sizes[self.code] - l = ss - return (intToB64(self.count, l=l)) - - - @staticmethod - def semVerToB64(version="", major=0, minor=0, patch=0): - """ Converts semantic version to Base64 representation of countB64 - suitable for CESR protocol genus and version - - Returns: - countB64 (str): suitable for input to Counter - example: Counter(countB64=semVerToB64(version = "1.0.0")) - - Parameters: - version (str | None): dot separated semantic version string of format - "major.minor.patch" - major (int): When version is None or empty then use major,minor, patch - minor (int): When version is None or empty then use major,minor, patch - patch (int): When version is None or empty then use major,minor, patch - - each of major, minor, patch must be in range [0,63] for represenation as - three Base64 characters - - """ - parts = [major, minor, patch] - if version: - splits = version.split(".", maxsplit=3) - splits = [(int(s) if s else 0) for s in splits] - for i in range(3-len(splits),0, -1): - splits.append(parts[-i]) - parts = splits - - for p in parts: - if p < 0 or p > 63: - raise ValueError(f"Out of bounds semantic version. " - f"Part={p} is < 0 or > 63.") - return ("".join(intToB64(p, l=1) for p in parts)) - - - def _infil(self): - """ - Returns fully qualified attached sig base64 bytes computed from - self.code and self.count. - """ - code = self.code # codex value chars hard code - count = self.count # index value int used for soft - - hs, ss, fs, ls = self.Sizes[code] - cs = hs + ss # both hard + soft size - if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes - raise InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) - if count < 0 or count > (64 ** ss - 1): - raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) - - # both is hard code + converted count - both = "{}{}".format(code, intToB64(count, l=ss)) - - # check valid pad size for whole code size - if len(both) % 4: # no pad - raise InvalidCodeSizeError("Invalid size = {} of {} not a multiple of 4." - .format(len(both), both)) - # prepending full derivation code with index and strip off trailing pad characters - return (both.encode("utf-8")) - - - def _binfil(self): - """ - Returns bytes of fully qualified base2 bytes, that is .qb2 - self.code converted to Base2 left shifted with pad bits - equivalent of Base64 decode of .qb64 into .qb2 - """ - code = self.code # codex chars hard code - count = self.count # index value int used for soft - - hs, ss, fs, ls = self.Sizes[code] - cs = hs + ss - if fs != cs or cs % 4: # fs must be cs and multiple of 4 for count codes - raise InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) - - if count < 0 or count > (64 ** ss - 1): - raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) - - # both is hard code + converted count - both = "{}{}".format(code, intToB64(count, l=ss)) - if len(both) != cs: - raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." - .format(cs, len(both))) - - return (codeB64ToB2(both)) # convert to b2 left shift if any - - - def _exfil(self, qb64b): - """ - Extracts self.code and self.count from qualified base64 bytes qb64b - """ - if not qb64b: # empty need more bytes - raise ShortageError("Empty material, Need more characters.") - - first = qb64b[:2] # extract first two char code selector - if hasattr(first, "decode"): - first = first.decode("utf-8") - if first not in self.Hards: - if first[0] == '_': - raise UnexpectedOpCodeError("Unexpected op code start" - "while extracing Counter.") - else: - raise UnexpectedCodeError("Unsupported code start ={}.".format(first)) - - hs = self.Hards[first] # get hard code size - if len(qb64b) < hs: # need more bytes - raise ShortageError("Need {} more characters.".format(hs - len(qb64b))) - - hard = qb64b[:hs] # get hard code - if hasattr(hard, "decode"): - hard = hard.decode("utf-8") # decode converts bytearray/bytes to str - if hard not in self.Sizes: # Sizes needs str not bytes - raise UnexpectedCodeError("Unsupported code ={}.".format(hard)) - - hs, ss, fs, ls = self.Sizes[hard] # assumes hs consistent in both tables - cs = hs + ss # both hard + soft code size - - # assumes that unit tests on Counter and CounterCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 - - if len(qb64b) < cs: # need more bytes - raise ShortageError("Need {} more characters.".format(cs - len(qb64b))) - - count = qb64b[hs:hs + ss] # extract count chars - if hasattr(count, "decode"): - count = count.decode("utf-8") - count = b64ToInt(count) # compute int count - - self._code = hard - self._count = count - - - def _bexfil(self, qb2): - """ - Extracts self.code and self.count from qualified base2 bytes qb2 - """ - if not qb2: # empty need more bytes - raise ShortageError("Empty material, Need more bytes.") - - first = nabSextets(qb2, 2) # extract first two sextets as code selector - if first not in self.Bards: - if first[0] == b'\xfc': # b64ToB2('_') - raise UnexpectedOpCodeError("Unexpected op code start" - "while extracing Matter.") - else: - raise UnexpectedCodeError("Unsupported code start sextet={}.".format(first)) - - hs = self.Bards[first] # get code hard size equvalent sextets - bhs = sceil(hs * 3 / 4) # bhs is min bytes to hold hs sextets - if len(qb2) < bhs: # need more bytes - raise ShortageError("Need {} more bytes.".format(bhs - len(qb2))) - - hard = codeB2ToB64(qb2, hs) # extract and convert hard part of code - if hard not in self.Sizes: - raise UnexpectedCodeError("Unsupported code ={}.".format(hard)) - - hs, ss, fs, ls = self.Sizes[hard] - cs = hs + ss # both hs and ss - # assumes that unit tests on Counter and CounterCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 - - bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets - if len(qb2) < bcs: # need more bytes - raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) - - both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code - count = b64ToInt(both[hs:hs + ss]) # get count - - self._code = hard - self._count = count - - class Tholder: """ From 46dfb35b658b8eaf4b56ae6cf99c9bbbe2a8f1fa Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 26 Jul 2024 10:08:27 -0600 Subject: [PATCH 207/418] Refactor Serials to Kinds. This has bugged me for some time since the variable nanme we use is kind for serialization kind but the namedtuple was called Serials. This realigns the naming. --- src/keri/app/forwarding.py | 2 +- src/keri/app/indirecting.py | 4 +- src/keri/core/coring.py | 22 ++--- src/keri/core/eventing.py | 20 ++--- src/keri/core/scheming.py | 12 +-- src/keri/core/serdering.py | 32 ++++---- src/keri/db/koming.py | 16 ++-- src/keri/kering.py | 12 +-- src/keri/peer/exchanging.py | 2 +- src/keri/vc/proving.py | 4 +- src/keri/vdr/eventing.py | 18 ++-- tests/core/test_coring.py | 8 +- tests/core/test_eventing.py | 28 +++---- tests/core/test_serdering.py | 154 +++++++++++++++++------------------ tests/core/test_streaming.py | 2 +- tests/db/test_basing.py | 6 +- tests/db/test_escrowing.py | 4 +- tests/db/test_koming.py | 14 ++-- tests/end/test_ending.py | 12 +-- tests/test_kering.py | 86 +++++++++---------- tests/vc/test_proving.py | 14 ++-- tests/vdr/test_eventing.py | 4 +- tests/vdr/test_viring.py | 8 +- 23 files changed, 242 insertions(+), 242 deletions(-) diff --git a/src/keri/app/forwarding.py b/src/keri/app/forwarding.py index db5818f09..b44c67a3a 100644 --- a/src/keri/app/forwarding.py +++ b/src/keri/app/forwarding.py @@ -451,7 +451,7 @@ def handle(self, serder, attachments=None): pevt = bytearray() for pather, atc in attachments: ked = pather.resolve(embeds) - sadder = coring.Sadder(ked=ked, kind=eventing.Serials.json) + sadder = coring.Sadder(ked=ked, kind=eventing.Kinds.json) pevt.extend(sadder.raw) pevt.extend(atc) diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 16b9a0374..482d79124 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -894,7 +894,7 @@ def on_post(self, req, rep): rep.set_header('connection', "close") cr = httping.parseCesrHttpRequest(req=req) - sadder = coring.Sadder(ked=cr.payload, kind=eventing.Serials.json) + sadder = coring.Sadder(ked=cr.payload, kind=eventing.Kinds.json) msg = bytearray(sadder.raw) msg.extend(cr.attachments.encode("utf-8")) @@ -1064,7 +1064,7 @@ def on_post(self, req, rep): rep.set_header('connection', "close") cr = httping.parseCesrHttpRequest(req=req) - serder = serdering.SerderKERI(sad=cr.payload, kind=eventing.Serials.json) + serder = serdering.SerderKERI(sad=cr.payload, kind=eventing.Kinds.json) pre = serder.ked["i"] if self.aids is not None and pre not in self.aids: diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 1699ab426..c7e662f97 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -39,7 +39,7 @@ from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, VERRAWSIZE, VERFMT, MAXVERFULLSPAN, versify, deversify, Rever, smell) -from ..kering import (Serials, Serialage, Protocols, Protocolage, Ilkage, Ilks, +from ..kering import (Kinds, Kindage, Protocols, Protocolage, Ilkage, Ilks, TraitDex, ) from ..help import helping @@ -94,7 +94,7 @@ def sizeify(ked, kind=None, version=Version): if not kind: kind = knd - if kind not in Serials: + if kind not in Kinds: raise ValueError("Invalid serialization kind = {}".format(kind)) raw = dumps(ked, kind) @@ -118,7 +118,7 @@ def sizeify(ked, kind=None, version=Version): -def dumps(ked, kind=Serials.json): +def dumps(ked, kind=Kinds.json): """ utility function to handle serialization by kind @@ -129,13 +129,13 @@ def dumps(ked, kind=Serials.json): ked (Optional(dict, list)): key event dict or message dict to serialize kind (str): serialization kind (JSON, MGPK, CBOR) """ - if kind == Serials.json: + if kind == Kinds.json: raw = json.dumps(ked, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - elif kind == Serials.mgpk: + elif kind == Kinds.mgpk: raw = msgpack.dumps(ked) - elif kind == Serials.cbor: + elif kind == Kinds.cbor: raw = cbor.dumps(ked) else: raise ValueError("Invalid serialization kind = {}".format(kind)) @@ -143,7 +143,7 @@ def dumps(ked, kind=Serials.json): return raw -def loads(raw, size=None, kind=Serials.json): +def loads(raw, size=None, kind=Kinds.json): """ utility function to handle deserialization by kind @@ -156,21 +156,21 @@ def loads(raw, size=None, kind=Serials.json): then consume all bytes kind (str): serialization kind (JSON, MGPK, CBOR) """ - if kind == Serials.json: + if kind == Kinds.json: try: ked = json.loads(raw[:size].decode("utf-8")) except Exception as ex: raise DeserializeError("Error deserializing JSON: {}" "".format(raw[:size].decode("utf-8"))) - elif kind == Serials.mgpk: + elif kind == Kinds.mgpk: try: ked = msgpack.loads(raw[:size]) except Exception as ex: raise DeserializeError("Error deserializing MGPK: {}" "".format(raw[:size])) - elif kind == Serials.cbor: + elif kind == Kinds.cbor: try: ked = cbor.loads(raw[:size]) except Exception as ex: @@ -3747,7 +3747,7 @@ def _serialize(clas, sad: dict, kind: str = None): otherwise default is Serials.json """ - knd = Serials.json + knd = Kinds.json if 'v' in sad: # versioned sad _, _, knd, _, _ = deversify(sad['v']) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 5e0f9e770..bef5475d3 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -29,7 +29,7 @@ from ..help import helping from . import coring -from .coring import (versify, Serials, Ilks, PreDex, DigDex, +from .coring import (versify, Kinds, Ilks, PreDex, DigDex, NonTransDex, Number, Seqner, Cigar, Dater, Verfer, Diger, Prefixer, Tholder, Saider) @@ -506,7 +506,7 @@ def state(pre, cnfg=None, # default to [] dpre=None, version=Version, - kind=Serials.json, + kind=Kinds.json, intive = False, ): """ @@ -657,7 +657,7 @@ def incept(keys, cnfg=None, data=None, version=Version, - kind=Serials.json, + kind=Kinds.json, code=None, intive=False, delpre=None, @@ -806,7 +806,7 @@ def rotate(pre, adds=None, data=None, version=Version, - kind=Serials.json, + kind=Kinds.json, intive = False, ): """ @@ -974,7 +974,7 @@ def interact(pre, sn=1, data=None, version=Version, - kind=Serials.json, + kind=Kinds.json, ): """ Returns serder of interaction event message. @@ -1020,7 +1020,7 @@ def receipt(pre, said, *, version=Version, - kind=Serials.json + kind=Kinds.json ): """ Returns serder of event receipt message. Used for both non-trans and trans @@ -1059,7 +1059,7 @@ def query(route="", query=None, stamp=None, version=Version, - kind=Serials.json): + kind=Kinds.json): """ Returns serder of query 'qry' message. Utility function to automate creation of query messages. @@ -1117,7 +1117,7 @@ def reply(route="", data=None, stamp=None, version=Version, - kind=Serials.json): + kind=Kinds.json): """ Returns serder of reply 'rpy' message. Utility function to automate creation of reply messages. @@ -1171,7 +1171,7 @@ def prod(route="", query=None, stamp=None, version=Version, - kind=Serials.json): + kind=Kinds.json): """ Returns serder of prod, 'pro', msg to request disclosure via bare, 'bar' msg of data anchored via seal(s) on KEL for identifier prefix, pre, when given @@ -1215,7 +1215,7 @@ def bare(route="", data=None, stamp=None, version=Version, - kind=Serials.json): + kind=Kinds.json): """ Returns serder of bare 'bar' message. Utility function to automate creation of unhiding (bareing) messages for diff --git a/src/keri/core/scheming.py b/src/keri/core/scheming.py index 5c87667a2..919d8e035 100644 --- a/src/keri/core/scheming.py +++ b/src/keri/core/scheming.py @@ -12,7 +12,7 @@ import msgpack from . import coring -from .coring import MtrDex, Serials, Saider, Saids +from .coring import MtrDex, Kinds, Saider, Saids from .. import help, kering from ..kering import ValidationError, DeserializeError @@ -109,7 +109,7 @@ def resolve(self, uri): return self.resolver.resolve(uri) - def load(self, raw, kind=Serials.json): + def load(self, raw, kind=Kinds.json): """ Schema loader Loads schema based on kind by performing deserialization on raw bytes of schema @@ -122,21 +122,21 @@ def load(self, raw, kind=Serials.json): tuple: (dict, Serials, Saider) of schema """ - if kind == Serials.json: + if kind == Kinds.json: try: sed = json.loads(raw.decode("utf-8")) except Exception as ex: raise DeserializeError("Error deserializing JSON: {} {}" "".format(raw.decode("utf-8"), ex)) - elif kind == Serials.mgpk: + elif kind == Kinds.mgpk: try: sed = msgpack.loads(raw) except Exception as ex: raise DeserializeError("Error deserializing MGPK: {} {}" "".format(raw, ex)) - elif kind == Serials.cbor: + elif kind == Kinds.cbor: try: sed = cbor.loads(raw) except Exception as ex: @@ -158,7 +158,7 @@ def load(self, raw, kind=Serials.json): return sed, kind, saider @staticmethod - def dump(sed, kind=Serials.json): + def dump(sed, kind=Kinds.json): """ Serailize schema based on kind Parameters: diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index bbfe64a97..afe6686d7 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -27,7 +27,7 @@ MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN) from ..kering import SMELLSIZE, Smellage, smell -from ..kering import Protocols, Serials, versify, deversify, Ilks +from ..kering import Protocols, Kinds, versify, deversify, Ilks from .. import help from ..help import helping @@ -379,7 +379,7 @@ class Serder: Protocol = None # class based message protocol, None means any in Protocols is ok Proto = Protocols.keri # default message protocol type for makify on base Serder Vrsn = Vrsn_1_0 # default protocol version for protocol type - Kind = Serials.json # default serialization kind + Kind = Kinds.json # default serialization kind CVrsn = Vrsn_2_0 # default CESR code table version @@ -809,7 +809,7 @@ def _verify(self): if self.kind != kind: raise ValidationError(f"Inconsistent kind={self.kind} in {sad}.") - if self.kind in (Serials.json, Serials.cbor, Serials.mgpk): + if self.kind in (Kinds.json, Kinds.cbor, Kinds.mgpk): if size != self.size != len(raw): raise ValidationError(f"Inconsistent size={self.size} in {sad}.") else: # size is not set in version string when kind is CESR @@ -898,7 +898,7 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, ilk = (silk if silk is not None else list(self.Fields[proto][vrsn])[0]) # list(dict) gives list of keys - if kind not in Serials: + if kind not in Kinds: raise SerializeError(f"Invalid serialization kind = {kind}") if ilk not in self.Fields[proto][vrsn]: @@ -986,7 +986,7 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, raise SerializeError(f"Missing requires version string field 'v'" f" in sad = {sad}.") - if kind in (Serials.json, Serials.cbor, Serials.mgpk): + if kind in (Kinds.json, Kinds.cbor, Kinds.mgpk): # this size of sad needs to be computed based on actual version string span # since not same for all versions sad['v'] = self.Dummy * self.Spans[vrsn] # ensure span of vs is dummied MAXVERFULLSPAN @@ -1015,7 +1015,7 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, sad[label] = dig raw = self.dumps(sad, kind=kind, proto=proto, vrsn=vrsn) # compute final raw - if kind == Serials.cesr:# cesr kind version string does not set size + if kind == Kinds.cesr:# cesr kind version string does not set size size = len(raw) # size of whole message self._raw = raw @@ -1074,7 +1074,7 @@ def _inhale(self, raw, *, smellage=None): - def loads(self, raw, size=None, kind=Serials.json): + def loads(self, raw, size=None, kind=Kinds.json): """method to handle deserialization by kind assumes already sniffed and smelled to determine serialization size and kind @@ -1093,21 +1093,21 @@ def loads(self, raw, size=None, kind=Serials.json): Notes: loads of json uses str whereas loads of cbor and msgpack use bytes """ - if kind == Serials.json: + if kind == Kinds.json: try: sad = json.loads(raw[:size].decode("utf-8")) except Exception as ex: raise DeserializeError(f"Error deserializing JSON: " f"{raw[:size].decode('utf-8')}") from ex - elif kind == Serials.mgpk: + elif kind == Kinds.mgpk: try: sad = msgpack.loads(raw[:size]) except Exception as ex: raise DeserializeError(f"Error deserializing MGPK: " f"{raw[:size].decode('utf-8')}") from ex - elif kind == Serials.cbor: + elif kind == Kinds.cbor: try: sad = cbor.loads(raw[:size]) except Exception as ex: @@ -1152,7 +1152,7 @@ def _exhale(self, sad): raw = self.dumps(sad, kind) - if kind in (Serials.cesr): # cesr kind version string does not set size + if kind in (Kinds.cesr): # cesr kind version string does not set size size = len(raw) # size of whole message # must call .verify to ensure these are compatible @@ -1164,7 +1164,7 @@ def _exhale(self, sad): self._size = size - def dumps(self, sad=None, kind=Serials.json, proto=None, vrsn=None): + def dumps(self, sad=None, kind=Kinds.json, proto=None, vrsn=None): """Method to handle serialization by kind Assumes sad fields are properly filled out for serialization kind. @@ -1187,17 +1187,17 @@ def dumps(self, sad=None, kind=Serials.json, proto=None, vrsn=None): """ sad = sad if sad is not None else self.sad - if kind == Serials.json: + if kind == Kinds.json: raw = json.dumps(sad, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - elif kind == Serials.mgpk: + elif kind == Kinds.mgpk: raw = msgpack.dumps(sad) - elif kind == Serials.cbor: + elif kind == Kinds.cbor: raw = cbor.dumps(sad) - elif kind == Serials.cesr: # does not support list only dict + elif kind == Kinds.cesr: # does not support list only dict raw = self._dumps(sad, proto=proto, vrsn=vrsn) else: diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index a4987563a..f69fd8b1f 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -47,7 +47,7 @@ class KomerBase: def __init__(self, db: dbing.LMDBer, *, subkey: str = 'docs.', schema: Type[dataclass], # class not instance - kind: str = coring.Serials.json, + kind: str = coring.Kinds.json, dupsort: bool = False, sep: str = None, **kwa): @@ -133,9 +133,9 @@ def _serializer(self, kind): Parameters: kind (str): serialization """ - if kind == coring.Serials.mgpk: + if kind == coring.Kinds.mgpk: return self.__serializeMGPK - elif kind == coring.Serials.cbor: + elif kind == coring.Kinds.cbor: return self.__serializeCBOR else: return self.__serializeJSON @@ -146,9 +146,9 @@ def _deserializer(self, kind): Parameters: kind (str): deserialization """ - if kind == coring.Serials.mgpk: + if kind == coring.Kinds.mgpk: return self.__deserializeMGPK - elif kind == coring.Serials.cbor: + elif kind == coring.Kinds.cbor: return self.__deserializeCBOR else: return self.__deserializeJSON @@ -218,7 +218,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str = 'docs.', schema: Type[dataclass], # class not instance - kind: str = coring.Serials.json, + kind: str = coring.Kinds.json, **kwa): """ Parameters: @@ -378,7 +378,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str = 'recs.', schema: Type[dataclass], # class not instance - kind: str = coring.Serials.json, + kind: str = coring.Kinds.json, **kwa): """ Parameters: @@ -665,7 +665,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str = 'recs.', schema: Type[dataclass], # class not instance - kind: str = coring.Serials.json, + kind: str = coring.Kinds.json, **kwa): """ Parameters: diff --git a/src/keri/kering.py b/src/keri/kering.py index 2f277ffd8..725e3b961 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -15,8 +15,8 @@ # Serialization Kinds -Serialage = namedtuple("Serialage", 'json mgpk cbor cesr') -Serials = Serialage(json='JSON', mgpk='MGPK', cbor='CBOR', cesr='CESR') +Kindage = namedtuple("Kindage", 'json mgpk cbor cesr') +Kinds = Kindage(json='JSON', mgpk='MGPK', cbor='CBOR', cesr='CESR') # Protocol Types Protocolage = namedtuple("Protocolage", "keri acdc") @@ -98,7 +98,7 @@ def rematch(match): raise VersionError(f"Incompatible {vrsn=} with version string.") kind = kind.decode("utf-8") - if kind not in Serials: + if kind not in Kinds: raise KindError(f"Invalid serialization kind = {kind}.") size = b64ToInt(size) @@ -116,7 +116,7 @@ def rematch(match): raise VersionError(f"Incompatible {vrsn=} with version string.") kind = kind.decode("utf-8") - if kind not in Serials: + if kind not in Kinds: raise KindError(f"Invalid serialization kind = {kind}.") size = int(size, 16) @@ -126,7 +126,7 @@ def rematch(match): return Smellage(proto=proto, vrsn=vrsn, kind=kind, size=size) -def versify(protocol=Protocols.keri, version=Version, kind=Serials.json, size=0): +def versify(protocol=Protocols.keri, version=Version, kind=Kinds.json, size=0): """ Returns: vs (str): version string @@ -139,7 +139,7 @@ def versify(protocol=Protocols.keri, version=Version, kind=Serials.json, size=0) """ if protocol not in Protocols: raise ProtocolError("Invalid message identifier = {}".format(protocol)) - if kind not in Serials: + if kind not in Kinds: raise KindError("Invalid serialization kind = {}".format(kind)) if version.major < 2: # version1 version string diff --git a/src/keri/peer/exchanging.py b/src/keri/peer/exchanging.py index c6c6b8c23..589fb5a6a 100644 --- a/src/keri/peer/exchanging.py +++ b/src/keri/peer/exchanging.py @@ -311,7 +311,7 @@ def exchange(route, modifiers=None, embeds=None, version=coring.Version, - kind=coring.Serials.json): + kind=coring.Kinds.json): """ Create an `exn` message with the specified route and payload Parameters: diff --git a/src/keri/vc/proving.py b/src/keri/vc/proving.py index bc2fdbb28..bd0a3e015 100644 --- a/src/keri/vc/proving.py +++ b/src/keri/vc/proving.py @@ -11,7 +11,7 @@ from .. import core from ..core import coring, serdering -from ..core.coring import (Serials, versify) +from ..core.coring import (Kinds, versify) from ..db import subing from ..kering import Version from ..help import helping @@ -31,7 +31,7 @@ def credential(schema, source=None, rules=None, version=Version, - kind=Serials.json): + kind=Kinds.json): """Utility function to create an ACDC. Creates dict SAD for credential from parameters and Saidifyies it before creation. diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 9deb9a544..bee2c9dcd 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -18,7 +18,7 @@ from .. import core from .. import help from ..core import serdering, coring, indexing -from ..core.coring import (MtrDex, Serials, versify, Prefixer, +from ..core.coring import (MtrDex, Kinds, versify, Prefixer, Ilks, Seqner, Verfer, Number) from ..core.signing import (Salter,) from ..core.eventing import SealEvent, ample, TraitDex, verifySigs @@ -39,7 +39,7 @@ def incept( nonce=None, cnfg=None, version=Version, - kind=Serials.json, + kind=Kinds.json, code=MtrDex.Blake3_256, ): """ Returns serder of credential registry inception (vcp) message event @@ -118,7 +118,7 @@ def rotate( cuts=None, adds=None, version=Version, - kind=Serials.json, + kind=Kinds.json, ): """ Returns serder of registry rotation (brt) message event @@ -214,7 +214,7 @@ def issue( vcdig, regk, version=Version, - kind=Serials.json, + kind=Kinds.json, dt=None ): """ Returns serder of issuance (iss) message event @@ -256,7 +256,7 @@ def revoke( regk, dig, version=Version, - kind=Serials.json, + kind=Kinds.json, dt=None ): """ Returns serder of backerless credential revocation (rev) message event @@ -307,7 +307,7 @@ def backerIssue( regsn, regd, version=Version, - kind=Serials.json, + kind=Kinds.json, dt=None, ): """ Returns serder of backer issuance (bis) message event @@ -361,7 +361,7 @@ def backerRevoke( regd, dig, version=Version, - kind=Serials.json, + kind=Kinds.json, dt=None ): """ Returns serder of backer credential revocation (brv) message event @@ -521,7 +521,7 @@ def vcstate(vcpre, ra=None, dts=None, # default current datetime version=Version, - kind=Serials.json, + kind=Kinds.json, ): """ Returns the credential transaction state notification @@ -591,7 +591,7 @@ def query(regk, dtb=None, stamp=None, version=Version, - kind=Serials.json + kind=Kinds.json ): """ Returns serder of credentialquery (qry) event message. diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 83e0f1193..952a75b0d 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -42,7 +42,7 @@ Dater, Bexter, Texter, TagDex, PadTagDex, Tagger, Ilker, Traitor, Verser, Versage, ) -from keri.core.coring import Serialage, Serials +from keri.core.coring import Kindage, Kinds from keri.core.coring import (Sizage, MtrDex, Matter) from keri.core.coring import (Verfer, Cigar, Saider, DigDex, Diger, Prefixer,) @@ -4732,7 +4732,7 @@ def test_prefixer(): prefixer = Prefixer(ked=badked, code=MtrDex.Ed25519) # Test digest derivation from inception ked - vs = versify(version=Version, kind=Serials.json, size=0) + vs = versify(version=Version, kind=Kinds.json, size=0) sn = 0 ilk = Ilks.icp sith = "1" @@ -5012,7 +5012,7 @@ def test_saider(): assert set(Saider.Digests.keys()) == set(code for code in DigDex) code = MtrDex.Blake3_256 - kind = Serials.json + kind = Kinds.json label = Saids.dollar # Test with valid said qb64 @@ -5214,7 +5214,7 @@ def test_saider(): assert saider.verify(sad8, prefixed=True) # verify gets kind from version string if provided when loading from dict - vs = versify(version=Version, kind=Serials.mgpk, size=0) # vaccuous size == 0 + vs = versify(version=Version, kind=Kinds.mgpk, size=0) # vaccuous size == 0 assert vs == 'KERI10MGPK000000_' sad9 = dict(sad4) sad9['v'] = vs diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index 8751c02c7..7384bbf0f 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -23,7 +23,7 @@ from keri.core.eventing import Kever, Kevery from keri.core.eventing import (SealDigest, SealRoot, SealBacker, SealEvent, SealLast, StateEvent, StateEstEvent) -from keri.core.eventing import (TraitDex, LastEstLoc, Serials, versify, +from keri.core.eventing import (TraitDex, LastEstLoc, Kinds, versify, simple, ample) from keri.core.eventing import (deWitnessCouple, deReceiptCouple, deSourceCouple, deReceiptTriple, @@ -3996,7 +3996,7 @@ def test_direct_mode_cbor_mgpk(): coeSerder = incept(keys=[coeSigners[cesn].verfer.qb64], ndigs=[coring.Diger(ser=coeSigners[cesn + 1].verfer.qb64b).qb64], code=MtrDex.Blake3_256, - kind=Serials.cbor) + kind=Kinds.cbor) assert csn == int(coeSerder.ked["s"], 16) == 0 coepre = coeSerder.ked["i"] @@ -4028,7 +4028,7 @@ def test_direct_mode_cbor_mgpk(): valSerder = incept(keys=[valSigners[vesn].verfer.qb64], ndigs=[coring.Diger(ser=valSigners[vesn + 1].verfer.qb64b).qb64], code=MtrDex.Blake3_256, - kind=Serials.mgpk) + kind=Kinds.mgpk) assert vsn == int(valSerder.ked["s"], 16) == 0 valpre = valSerder.ked["i"] @@ -4072,7 +4072,7 @@ def test_direct_mode_cbor_mgpk(): reserder = receipt(pre=coeK.prefixer.qb64, sn=coeK.sn, said=coeK.serder.said, - kind=Serials.mgpk) + kind=Kinds.mgpk) # sign coe's event not receipt # look up event to sign from val's kever for coe coeIcpDig = bytes(valKevery.db.getKeLast(key=snKey(pre=coepre, sn=csn))) @@ -4123,7 +4123,7 @@ def test_direct_mode_cbor_mgpk(): reserder = receipt(pre=coeK.prefixer.qb64, sn=10, said=fake, - kind=Serials.mgpk) + kind=Kinds.mgpk) # sign event not receipt siger = valSigners[vesn].sign(ser=coeIcpRaw, index=0) # return Siger if index @@ -4158,7 +4158,7 @@ def test_direct_mode_cbor_mgpk(): reserder = receipt(pre=valK.prefixer.qb64, sn=valK.sn, said=valK.serder.said, - kind=Serials.cbor) + kind=Kinds.cbor) # sign vals's event not receipt # look up event to sign from coe's kever for val valIcpDig = bytes(coeKevery.db.getKeLast(key=snKey(pre=valpre, sn=vsn))) @@ -4211,7 +4211,7 @@ def test_direct_mode_cbor_mgpk(): dig=coeKever.serder.said, ndigs=[coring.Diger(ser=coeSigners[cesn + 1].verfer.qb64b).qb64], sn=csn, - kind=Serials.cbor) + kind=Kinds.cbor) coe_event_digs.append(coeSerder.said) # create sig counter counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 @@ -4253,7 +4253,7 @@ def test_direct_mode_cbor_mgpk(): reserder = receipt(pre=coeK.prefixer.qb64, sn=coeK.sn, said=coeK.serder.said, - kind=Serials.mgpk) + kind=Kinds.mgpk) # sign coe's event not receipt # look up event to sign from val's kever for coe coeRotDig = bytes(valKevery.db.getKeLast(key=snKey(pre=coepre, sn=csn))) @@ -4305,7 +4305,7 @@ def test_direct_mode_cbor_mgpk(): coeSerder = interact(pre=coeKever.prefixer.qb64, dig=coeKever.serder.said, sn=csn, - kind=Serials.cbor) + kind=Kinds.cbor) coe_event_digs.append(coeSerder.said) # create sig counter counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 @@ -4345,7 +4345,7 @@ def test_direct_mode_cbor_mgpk(): reserder = receipt(pre=coeK.prefixer.qb64, sn=coeK.sn, said=coeK.serder.said, - kind=Serials.mgpk) + kind=Kinds.mgpk) # sign coe's event not receipt # look up event to sign from val's kever for coe coeIxnDig = bytes(valKevery.db.getKeLast(key=snKey(pre=coepre, sn=csn))) @@ -4444,7 +4444,7 @@ def test_process_nontransferable(): nsigs = 1 # one attached signature unspecified index #["v", "t", "d", "i", "s", "kt", "k", "nt", "n","bt", "b", "c", "a"] - ked0 = dict(v=versify(kind=Serials.json, size=0), + ked0 = dict(v=versify(kind=Kinds.json, size=0), t=Ilks.icp, d="", i=aid0.qb64, # qual base 64 prefix @@ -4532,7 +4532,7 @@ def test_process_transferable(): nsigs = 1 # one attached signature unspecified index - ked0 = dict(v=versify(kind=Serials.json, size=0), # version string + ked0 = dict(v=versify(kind=Kinds.json, size=0), # version string t=Ilks.icp, d="", # SAID i="", # qb64 prefix @@ -4659,7 +4659,7 @@ def test_process_manual(): index = 0 # create key event dict - ked0 = dict(v=versify(kind=Serials.json, size=0), + ked0 = dict(v=versify(kind=Kinds.json, size=0), t=Ilks.icp, d="", i=aidmat.qb64, # qual base 64 prefix @@ -4675,7 +4675,7 @@ def test_process_manual(): ) _, ked0 = coring.Saider.saidify(sad=ked0) - txsrdr = serdering.SerderKERI(sad=ked0, kind=Serials.json) + txsrdr = serdering.SerderKERI(sad=ked0, kind=Kinds.json) assert txsrdr.raw == (b'{"v":"KERI10JSON00012b_","t":"icp","d":"EKYHED-wvkYDZv4tNUF9qiC1kgnnGLS9YUU8' b'PCWig_n4","i":"DK-WsHD7MKfQpBjJ3B2GwjqY9z90G94uzMs7irCiT-dL","s":"0","kt":"1' b'","k":["DK-WsHD7MKfQpBjJ3B2GwjqY9z90G94uzMs7irCiT-dL"],"nt":"1","n":["EDcWJG' diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index ef7cbd07e..bbeebd0e4 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -159,7 +159,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -180,7 +180,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -194,7 +194,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -208,7 +208,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -220,7 +220,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -247,7 +247,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == None @@ -261,7 +261,7 @@ def test_serder(): #Test makify bootstrap for ACDC with CBOR - serder = Serder(makify=True, proto=Protocols.acdc, kind=kering.Serials.cbor) + serder = Serder(makify=True, proto=Protocols.acdc, kind=kering.Kinds.cbor) assert serder.sad == {'v': 'ACDC10CBOR00004b_', 'd': 'EGahYhEMb_Sz0L1UwhrUvbyxyzoi_G85-pD9jRjhnqgU', 'i': '', @@ -281,7 +281,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.cbor + assert serder.kind == kering.Kinds.cbor assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -302,7 +302,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.cbor + assert serder.kind == kering.Kinds.cbor assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -316,7 +316,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.cbor + assert serder.kind == kering.Kinds.cbor assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -324,7 +324,7 @@ def test_serder(): #Test makify bootstrap for ACDC with MGPK - serder = Serder(makify=True, proto=Protocols.acdc, kind=kering.Serials.mgpk) + serder = Serder(makify=True, proto=Protocols.acdc, kind=kering.Kinds.mgpk) assert serder.sad == {'v': 'ACDC10MGPK00004b_', 'd': 'EGV5wdF1nRbSXatBgZDpAxlGL6BuATjpUYBuk0AQW7GC', 'i': '', @@ -344,7 +344,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.mgpk + assert serder.kind == kering.Kinds.mgpk assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -365,7 +365,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.mgpk + assert serder.kind == kering.Kinds.mgpk assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -379,7 +379,7 @@ def test_serder(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.mgpk + assert serder.kind == kering.Kinds.mgpk assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -417,7 +417,7 @@ def test_serder(): assert serder.sad == sad assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk == kering.Ilks.icp @@ -426,7 +426,7 @@ def test_serder(): assert serder.sad == sad assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk == kering.Ilks.icp @@ -468,7 +468,7 @@ def test_serder(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -478,7 +478,7 @@ def test_serder(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -552,7 +552,7 @@ def test_serder(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -562,7 +562,7 @@ def test_serder(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -604,7 +604,7 @@ def test_serder(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -614,7 +614,7 @@ def test_serder(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -755,7 +755,7 @@ def test_serderkeri(): assert serder.sad == sad assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.pre == pre assert serder.ilk == kering.Ilks.icp @@ -788,7 +788,7 @@ def test_serderkeri(): assert serder.sad == sad assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == kering.Ilks.icp @@ -857,7 +857,7 @@ def test_serderkeri_icp(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -893,7 +893,7 @@ def test_serderkeri_icp(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -993,7 +993,7 @@ def test_serderkeri_icp(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1029,7 +1029,7 @@ def test_serderkeri_icp(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1115,7 +1115,7 @@ def test_serderkeri_rot(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1146,7 +1146,7 @@ def test_serderkeri_rot(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1223,7 +1223,7 @@ def test_serderkeri_ixn(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1257,7 +1257,7 @@ def test_serderkeri_ixn(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1357,7 +1357,7 @@ def test_serderkeri_dip(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1392,7 +1392,7 @@ def test_serderkeri_dip(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1526,7 +1526,7 @@ def test_serderkeri_dip(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1561,7 +1561,7 @@ def test_serderkeri_dip(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1654,7 +1654,7 @@ def test_serderkeri_drt(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1687,7 +1687,7 @@ def test_serderkeri_drt(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1756,7 +1756,7 @@ def test_serderkeri_rct(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1786,7 +1786,7 @@ def test_serderkeri_rct(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1856,7 +1856,7 @@ def test_serderkeri_qry(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1887,7 +1887,7 @@ def test_serderkeri_qry(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1959,7 +1959,7 @@ def test_serderkeri_rpy(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -1989,7 +1989,7 @@ def test_serderkeri_rpy(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -2060,7 +2060,7 @@ def test_serderkeri_pro(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -2091,7 +2091,7 @@ def test_serderkeri_pro(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -2162,7 +2162,7 @@ def test_serderkeri_bar(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -2193,7 +2193,7 @@ def test_serderkeri_bar(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -2272,7 +2272,7 @@ def test_serderkeri_exn(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -2305,7 +2305,7 @@ def test_serderkeri_exn(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -2373,7 +2373,7 @@ def test_serderkeri_vcp(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -2405,7 +2405,7 @@ def test_serderkeri_vcp(): assert serder.proto == Protocols.keri assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == ilk @@ -2452,7 +2452,7 @@ def test_serderacdc(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == 90 - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == 'EMk7BvrqO_2sYjpI_-BmSELOFNie-muw4XTi3iYCz6pT' assert serder.ilk == None @@ -2484,7 +2484,7 @@ def test_serderacdc(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == None assert serder.issuer == isr @@ -2496,7 +2496,7 @@ def test_serderacdc(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_1_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.ilk == None assert serder.issuer == isr @@ -2549,7 +2549,7 @@ def test_serder_v2(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_2_0 == serder.version assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -2570,7 +2570,7 @@ def test_serder_v2(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_2_0 == serder.version assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -2584,7 +2584,7 @@ def test_serder_v2(): assert serder.proto == Protocols.acdc assert serder.vrsn == kering.Vrsn_2_0 assert serder.size == size - assert serder.kind == kering.Serials.json + assert serder.kind == kering.Kinds.json assert serder.said == said assert serder.saidb == said.encode("utf-8") assert serder.ilk == None @@ -2614,7 +2614,7 @@ def test_serder_v2(): assert serder.verify() assert serder.proto == Protocols.keri == Serder.Proto # default assert serder.vrsn == kering.Vrsn_2_0 - assert serder.kind == kering.Serials.json == Serder.Kind # default + assert serder.kind == kering.Kinds.json == Serder.Kind # default assert serder.ilk == kering.Ilks.icp # default first one @@ -2692,7 +2692,7 @@ def test_cesr_native_dumps(): keys = [csigners[0].verfer.qb64] assert keys == ['DG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ'] - serder = incept(keys, version=Vrsn_2_0, kind=kering.Serials.cesr) + serder = incept(keys, version=Vrsn_2_0, kind=kering.Kinds.cesr) assert serder.sad == \ { @@ -2731,10 +2731,10 @@ def test_cesr_native_dumps(): rawjson = serder.dumps(serder.sad) assert len(rawjson) == 252 - rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + rawcbor = serder.dumps(serder.sad, kind=kering.Kinds.cbor) assert len(rawcbor) == 202 - rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + rawmgpk = serder.dumps(serder.sad, kind=kering.Kinds.mgpk) assert len(rawmgpk) == 202 raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] @@ -2777,7 +2777,7 @@ def test_cesr_native_dumps(): data=data, code=core.MtrDex.Blake3_256, version=Vrsn_2_0, - kind=kering.Serials.cesr) + kind=kering.Kinds.cesr) pre = serder.pre assert pre == 'EKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi' @@ -2878,10 +2878,10 @@ def test_cesr_native_dumps(): rawjson = serder.dumps(serder.sad) assert len(rawjson) == 915 - rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + rawcbor = serder.dumps(serder.sad, kind=kering.Kinds.cbor) assert len(rawcbor) == 829 - rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + rawmgpk = serder.dumps(serder.sad, kind=kering.Kinds.mgpk) assert len(rawmgpk) == 829 raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] @@ -2930,7 +2930,7 @@ def test_cesr_native_dumps(): sn=1, data=data, version=Vrsn_2_0, - kind=kering.Serials.cesr) + kind=kering.Kinds.cesr) said = serder.said assert said == 'EHXLwMJsZLyG643VW8Do1cqqiMxD_E65Mc3Z1we6vTaR' @@ -2992,10 +2992,10 @@ def test_cesr_native_dumps(): rawjson = serder.dumps(serder.sad) assert len(rawjson) == 601 - rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + rawcbor = serder.dumps(serder.sad, kind=kering.Kinds.cbor) assert len(rawcbor) == 536 - rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + rawmgpk = serder.dumps(serder.sad, kind=kering.Kinds.mgpk) assert len(rawmgpk) == 536 raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] @@ -3024,7 +3024,7 @@ def test_cesr_native_dumps(): adds=adds, data=data, version=Vrsn_2_0, - kind=kering.Serials.cesr) + kind=kering.Kinds.cesr) said = serder.said assert said == 'EDHlTlOcSXZInbTE4iXzb1iFjZcxJZn3C3UXhckb3uQm' @@ -3085,10 +3085,10 @@ def test_cesr_native_dumps(): rawjson = serder.dumps(serder.sad) assert len(rawjson) == 638 - rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + rawcbor = serder.dumps(serder.sad, kind=kering.Kinds.cbor) assert len(rawcbor) == 577 - rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + rawmgpk = serder.dumps(serder.sad, kind=kering.Kinds.mgpk) assert len(rawmgpk) == 577 raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] @@ -3140,7 +3140,7 @@ def test_cesr_native_dumps(): delpre=delpre, code=core.MtrDex.Blake3_256, version=Vrsn_2_0, - kind=kering.Serials.cesr) + kind=kering.Kinds.cesr) pre = serder.pre assert pre == 'EKCFMk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjC' @@ -3233,10 +3233,10 @@ def test_cesr_native_dumps(): rawjson = serder.dumps(serder.sad) assert len(rawjson) == 1059 - rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + rawcbor = serder.dumps(serder.sad, kind=kering.Kinds.cbor) assert len(rawcbor) == 953 - rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + rawmgpk = serder.dumps(serder.sad, kind=kering.Kinds.mgpk) assert len(rawmgpk) == 953 raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] @@ -3266,7 +3266,7 @@ def test_cesr_native_dumps(): adds=adds, data=data, version=Vrsn_2_0, - kind=kering.Serials.cesr) + kind=kering.Kinds.cesr) said = serder.said assert said == 'EKfRY6YrpqUU0HyKWMGvNtzuZCaeMcIBrdKzHAqpmtTA' @@ -3314,10 +3314,10 @@ def test_cesr_native_dumps(): rawjson = serder.dumps(serder.sad) assert len(rawjson) == 450 - rawcbor = serder.dumps(serder.sad, kind=kering.Serials.cbor) + rawcbor = serder.dumps(serder.sad, kind=kering.Kinds.cbor) assert len(rawcbor) == 393 - rawmgpk = serder.dumps(serder.sad, kind=kering.Serials.mgpk) + rawmgpk = serder.dumps(serder.sad, kind=kering.Kinds.mgpk) assert len(rawmgpk) == 393 raws = [rawqb2, rawqb64, rawcbor, rawmgpk, rawjson] diff --git a/tests/core/test_streaming.py b/tests/core/test_streaming.py index 60b3ecc47..86eefc399 100644 --- a/tests/core/test_streaming.py +++ b/tests/core/test_streaming.py @@ -162,7 +162,7 @@ def test_annot(): "kid" : "FdFYFzERwC2uCBB46pZQi4GG85LujR8obt-KWRBICVQ" } - jwk = dumps(djwk, kering.Serials.json) + jwk = dumps(djwk, kering.Kinds.json) assert jwk == (b'{"kty":"OKP","crv":"Ed25519","x":"11qYAYKxCrfVS_7TyWQHOg7hcvPapiMlrwIaaPcHUR' b'o","d":"nWGxne_9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A","use":"sig","kid":"FdFY' b'FzERwC2uCBB46pZQi4GG85LujR8obt-KWRBICVQ"}') diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 89044c7d9..16a0819eb 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -16,7 +16,7 @@ from keri import core from keri.core import coring, eventing, serdering -from keri.core.coring import Serials, versify +from keri.core.coring import Kinds, versify from keri.core.eventing import incept, rotate, interact, Kever @@ -124,7 +124,7 @@ def test_baser(): preb = 'DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc'.encode("utf-8") digb = 'EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4'.encode("utf-8") sn = 3 - vs = versify(kind=Serials.json, size=20) + vs = versify(kind=Kinds.json, size=20) assert vs == 'KERI10JSON000014_' ked = dict(vs=vs, pre=preb.decode("utf-8"), @@ -1838,7 +1838,7 @@ def test_fetchkeldel(): preb = 'BWzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc'.encode("utf-8") digb = 'EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4'.encode("utf-8") sn = 3 - vs = versify(kind=Serials.json, size=20) + vs = versify(kind=Kinds.json, size=20) assert vs == 'KERI10JSON000014_' ked = dict(vs=vs, pre=preb.decode("utf-8"), diff --git a/tests/db/test_escrowing.py b/tests/db/test_escrowing.py index 87d12d9bc..6a7e5af24 100644 --- a/tests/db/test_escrowing.py +++ b/tests/db/test_escrowing.py @@ -66,7 +66,7 @@ def test_broker_nontrans(): rrsr = viring.RegStateRecord._fromdict(ked["a"]) # reply RegStateRecord #tserder = serdering.SerderKERI(sad=ked["a"]) - saider, _ = coring.Saider.saidify(sad=ked, kind=coring.Serials.json, label=coring.Saids.d) + saider, _ = coring.Saider.saidify(sad=ked, kind=coring.Kinds.json, label=coring.Saids.d) dater = coring.Dater(dts=dts) cigars = wesHab.sign(ser=serder.raw, @@ -132,7 +132,7 @@ def test_broker_trans(): serder = serdering.SerderKERI(sad=ked) rrsr = viring.RegStateRecord._fromdict(ked["a"]) # reply RegStateRecord #tserder = serdering.SerderKERI(sad=ked["a"]) - saider, _ = coring.Saider.saidify(sad=ked, kind=coring.Serials.json, label=coring.Saids.d) + saider, _ = coring.Saider.saidify(sad=ked, kind=coring.Kinds.json, label=coring.Saids.d) dater = coring.Dater(dts=dts) sigers = bobHab.sign(ser=serder.raw, diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index dfafa5327..ae1975e36 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -10,7 +10,7 @@ import pytest -from keri.core.coring import Serials +from keri.core.coring import Kinds from keri.db import dbing, koming from keri.help import helping @@ -299,16 +299,16 @@ class Record: with dbing.openLMDB() as db: k = koming.Komer(db=db, schema=Record, subkey='records.') - srl = k._serializer(Serials.mgpk) + srl = k._serializer(Kinds.mgpk) expected = b'\x86\xa5first\xa3Jim\xa4last\xa5Black\xa6street\xaf100 Main Street\xa4city\xa8Riverton\xa5state\xa2UT\xa3zip\xce\x00\x01HZ' assert srl(jim) == expected - srl = k._serializer(Serials.cbor) + srl = k._serializer(Kinds.cbor) expected = b'\xa6efirstcJimdlasteBlackfstreeto100 Main StreetdcityhRivertonestatebUTczip\x1a\x00\x01HZ' assert srl(jim) == expected - srl = k._serializer(Serials.json) + srl = k._serializer(Kinds.json) expected = b'{"first":"Jim","last":"Black","street":"100 Main Street","city":"Riverton","state":"UT","zip":84058}' assert srl(jim) == expected @@ -389,7 +389,7 @@ class Record: with dbing.openLMDB() as db: k = koming.Komer(db=db, schema=Record, subkey='records.') - desrl = k._deserializer(Serials.mgpk) + desrl = k._deserializer(Kinds.mgpk) actual = helping.datify(Record, desrl(msgp)) assert actual.first == "Jim" assert actual.last == "Black" @@ -398,7 +398,7 @@ class Record: assert actual.state == "UT" assert actual.zip == 84058 - desrl = k._deserializer(Serials.json) + desrl = k._deserializer(Kinds.json) actual = helping.datify(Record, desrl(json)) assert actual.first == "Jim" assert actual.last == "Black" @@ -407,7 +407,7 @@ class Record: assert actual.state == "UT" assert actual.zip == 84058 - desrl = k._deserializer(Serials.cbor) + desrl = k._deserializer(Kinds.cbor) actual = helping.datify(Record, desrl(cbor)) assert actual.first == "Jim" assert actual.last == "Black" diff --git a/tests/end/test_ending.py b/tests/end/test_ending.py index b27a9378b..4e90bad64 100644 --- a/tests/end/test_ending.py +++ b/tests/end/test_ending.py @@ -39,13 +39,13 @@ def test_mimes(): assert ending.KeriMimes.cesr == 'application/keri+cesr' # Usage: to get Mime from serialization kind - assert getattr(ending.Mimes, coring.Serials.json.lower()) == ending.Mimes.json - assert getattr(ending.Mimes, coring.Serials.mgpk.lower()) == ending.Mimes.mgpk - assert getattr(ending.Mimes, coring.Serials.cbor.lower()) == ending.Mimes.cbor + assert getattr(ending.Mimes, coring.Kinds.json.lower()) == ending.Mimes.json + assert getattr(ending.Mimes, coring.Kinds.mgpk.lower()) == ending.Mimes.mgpk + assert getattr(ending.Mimes, coring.Kinds.cbor.lower()) == ending.Mimes.cbor - assert getattr(ending.KeriMimes, coring.Serials.json.lower()) == ending.KeriMimes.json - assert getattr(ending.KeriMimes, coring.Serials.mgpk.lower()) == ending.KeriMimes.mgpk - assert getattr(ending.KeriMimes, coring.Serials.cbor.lower()) == ending.KeriMimes.cbor + assert getattr(ending.KeriMimes, coring.Kinds.json.lower()) == ending.KeriMimes.json + assert getattr(ending.KeriMimes, coring.Kinds.mgpk.lower()) == ending.KeriMimes.mgpk + assert getattr(ending.KeriMimes, coring.Kinds.cbor.lower()) == ending.KeriMimes.cbor """Done Test""" diff --git a/tests/test_kering.py b/tests/test_kering.py index aeb86e717..a8f2941ff 100644 --- a/tests/test_kering.py +++ b/tests/test_kering.py @@ -14,7 +14,7 @@ from keri import kering from keri.kering import Protocolage, Protocols -from keri.kering import Serialage, Serials +from keri.kering import Kindage, Kinds from keri.kering import Ilkage, Ilks from keri.kering import ColdCodex, ColdDex, TraitCodex, TraitDex from keri.kering import (Versionage, Version, MAXVERFULLSPAN, @@ -251,7 +251,7 @@ def snatch(match, size=0): if gvrsn.major < 2: # version2 vs but major < 2 raise VersionError(f"Incompatible {gvrsn=} with CESR native version" f"field.") - kind = Serials.cesr + kind = Kinds.cesr size = size else: raise VersionError(f"Bad snatch.") @@ -456,22 +456,22 @@ def test_serials(): assert Version == Versionage(major=1, minor=0) - assert isinstance(Serials, Serialage) + assert isinstance(Kinds, Kindage) - assert Serials.json == 'JSON' - assert Serials.mgpk == 'MGPK' - assert Serials.cbor == 'CBOR' - assert Serials.cesr == 'CESR' + assert Kinds.json == 'JSON' + assert Kinds.mgpk == 'MGPK' + assert Kinds.cbor == 'CBOR' + assert Kinds.cesr == 'CESR' - assert 'JSON' in Serials - assert 'MGPK' in Serials - assert 'CBOR' in Serials - assert 'CESR' in Serials + assert 'JSON' in Kinds + assert 'MGPK' in Kinds + assert 'CBOR' in Kinds + assert 'CESR' in Kinds - Vstrings = Serialage(json=versify(kind=Serials.json, size=0), - mgpk=versify(kind=Serials.mgpk, size=0), - cbor=versify(kind=Serials.cbor, size=0), - cesr=versify(kind=Serials.cesr, size=0)) + Vstrings = Kindage(json=versify(kind=Kinds.json, size=0), + mgpk=versify(kind=Kinds.mgpk, size=0), + cbor=versify(kind=Kinds.cbor, size=0), + cesr=versify(kind=Kinds.cesr, size=0)) assert Vstrings.json == 'KERI10JSON000000_' @@ -589,61 +589,61 @@ def test_versify_v1(): assert len(vs) == VER1FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.json + assert kind == Kinds.json assert vrsn == Version assert size == 0 - vs = versify(kind=Serials.json, size=65) + vs = versify(kind=Kinds.json, size=65) assert vs == "KERI10JSON000041_" assert len(vs) == VER1FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.json + assert kind == Kinds.json assert vrsn == Version assert size == 65 - vs = versify(protocol=Protocols.acdc, kind=Serials.json, size=86) + vs = versify(protocol=Protocols.acdc, kind=Kinds.json, size=86) assert vs == "ACDC10JSON000056_" assert len(vs) == VER1FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.acdc - assert kind == Serials.json + assert kind == Kinds.json assert vrsn == Version assert size == 86 - vs = versify(kind=Serials.mgpk, size=0) + vs = versify(kind=Kinds.mgpk, size=0) assert vs == "KERI10MGPK000000_" assert len(vs) == VER1FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.mgpk + assert kind == Kinds.mgpk assert vrsn == Version assert size == 0 - vs = versify(kind=Serials.mgpk, size=65) + vs = versify(kind=Kinds.mgpk, size=65) assert vs == "KERI10MGPK000041_" assert len(vs) == VER1FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.mgpk + assert kind == Kinds.mgpk assert vrsn == Version assert size == 65 - vs = versify(kind=Serials.cbor, size=0) + vs = versify(kind=Kinds.cbor, size=0) assert vs == "KERI10CBOR000000_" assert len(vs) == VER1FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.cbor + assert kind == Kinds.cbor assert vrsn == Version assert size == 0 - vs = versify(kind=Serials.cbor, size=65) + vs = versify(kind=Kinds.cbor, size=65) assert vs == "KERI10CBOR000041_" assert len(vs) == VER1FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.cbor + assert kind == Kinds.cbor assert vrsn == Version assert size == 65 @@ -652,7 +652,7 @@ def test_versify_v1(): assert len(vs) == VER1FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.json + assert kind == Kinds.json assert vrsn == (1, 1) assert size == 0 @@ -685,61 +685,61 @@ def test_versify_v2(): assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.json + assert kind == Kinds.json assert vrsn == version assert size == 0 - vs = versify(version=version, kind=Serials.json, size=65) + vs = versify(version=version, kind=Kinds.json, size=65) assert vs == "KERICAAJSONAABB." assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.json + assert kind == Kinds.json assert vrsn == version assert size == 65 - vs = versify(protocol=Protocols.acdc, version=version, kind=Serials.json, size=86) + vs = versify(protocol=Protocols.acdc, version=version, kind=Kinds.json, size=86) assert vs == "ACDCCAAJSONAABW." assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.acdc - assert kind == Serials.json + assert kind == Kinds.json assert version == version assert size == 86 - vs = versify(version=version, kind=Serials.mgpk, size=0) + vs = versify(version=version, kind=Kinds.mgpk, size=0) assert vs == 'KERICAAMGPKAAAA.' assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.mgpk + assert kind == Kinds.mgpk assert vrsn == version assert size == 0 - vs = versify(version=version, kind=Serials.mgpk, size=65) + vs = versify(version=version, kind=Kinds.mgpk, size=65) assert vs == 'KERICAAMGPKAABB.' assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.mgpk + assert kind == Kinds.mgpk assert vrsn == version assert size == 65 - vs = versify(version=version, kind=Serials.cbor, size=0) + vs = versify(version=version, kind=Kinds.cbor, size=0) assert vs == 'KERICAACBORAAAA.' assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.cbor + assert kind == Kinds.cbor assert vrsn == version assert size == 0 - vs = versify(version=version, kind=Serials.cbor, size=65) + vs = versify(version=version, kind=Kinds.cbor, size=65) assert vs == 'KERICAACBORAABB.' assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.cbor + assert kind == Kinds.cbor assert vrsn == version assert size == 65 @@ -748,7 +748,7 @@ def test_versify_v2(): assert len(vs) == VER2FULLSPAN proto, vrsn, kind, size, opt = deversify(vs) assert proto == Protocols.keri - assert kind == Serials.json + assert kind == Kinds.json assert vrsn == (2, 1) assert size == 0 diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index c2ab1cbac..798478be3 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -11,7 +11,7 @@ from keri import core from keri.core import coring, scheming, parsing, serdering from keri.core import counting, Counter, Codens -from keri.core.coring import Serials, Prefixer, Seqner, Diger +from keri.core.coring import Kinds, Prefixer, Seqner, Diger from keri.core.indexing import Siger from keri.core.scheming import CacheResolver @@ -119,7 +119,7 @@ def test_credentialer(): sub = dict(a=123, b="abc", issuanceDate="2021-06-27T21:26:21.233257+00:00") d = dict( - v=coring.versify(protocol=coring.Protocols.acdc, kind=Serials.json, size=0), + v=coring.versify(protocol=coring.Protocols.acdc, kind=Kinds.json, size=0), d="", i="EF6maPM_d5ZN7U3NRFC1-6TM7k_E00_a8AG9YyLA4uWi", s="abc", @@ -131,7 +131,7 @@ def test_credentialer(): creder = serdering.SerderACDC(sad=d) # Creder(ked=d) assert creder.said == said - assert creder.kind == Serials.json + assert creder.kind == Kinds.json assert creder.issuer == "EF6maPM_d5ZN7U3NRFC1-6TM7k_E00_a8AG9YyLA4uWi" assert creder.schema == "abc" assert creder.attrib == sub @@ -147,19 +147,19 @@ def test_credentialer(): knd1 = creder.kind sad1 = creder.sad - assert knd1 == Serials.json + assert knd1 == Kinds.json assert sad1 == d assert ver1 == Versionage(major=1, minor=0) creder = serdering.SerderACDC(raw=raw1) # Creder(raw=raw1) - assert creder.kind == Serials.json + assert creder.kind == Kinds.json assert creder.issuer == "EF6maPM_d5ZN7U3NRFC1-6TM7k_E00_a8AG9YyLA4uWi" assert creder.sad == d assert creder.size == 211 d2 = dict(d) d2['d'] = "" - d2["v"] = coring.versify(protocol=coring.Protocols.acdc, kind=Serials.cbor, size=0) + d2["v"] = coring.versify(protocol=coring.Protocols.acdc, kind=Kinds.cbor, size=0) _, d2 = coring.Saider.saidify(sad=d2) creder = serdering.SerderACDC(sad=d2) # Creder(ked=d2) @@ -185,7 +185,7 @@ def test_credentialer(): assert creder.sad == d2 d3 = dict(d) - d3["v"] = coring.versify(protocol=coring.Protocols.acdc, kind=Serials.mgpk, size=0) + d3["v"] = coring.versify(protocol=coring.Protocols.acdc, kind=Kinds.mgpk, size=0) _, d3 = coring.Saider.saidify(sad=d3) creder = serdering.SerderACDC(sad=d3) # Creder(ked=d3) diff --git a/tests/vdr/test_eventing.py b/tests/vdr/test_eventing.py index 124030cb3..6cfdf6196 100644 --- a/tests/vdr/test_eventing.py +++ b/tests/vdr/test_eventing.py @@ -9,7 +9,7 @@ from keri.core import Signer from keri.core import coring, serdering from keri.core import eventing as keventing -from keri.core.coring import versify, Serials, Ilks, MtrDex, Prefixer, Seqner, Saider +from keri.core.coring import versify, Kinds, Ilks, MtrDex, Prefixer, Seqner, Saider from keri.db import basing from keri.db.dbing import snKey, dgKey from keri.kering import Version, EmptyMaterialError, DerivationError, MissingAnchorError, ValidationError, \ @@ -323,7 +323,7 @@ def test_backer_issue_revoke(mockHelpingNowUTC): def test_prefixer(): pre = "DAtNTPnDFBnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM" - vs = versify(version=Version, kind=Serials.json, size=0) + vs = versify(version=Version, kind=Kinds.json, size=0) with pytest.raises(EmptyMaterialError): prefixer = Prefixer() diff --git a/tests/vdr/test_viring.py b/tests/vdr/test_viring.py index dd84c1448..8ec90e476 100644 --- a/tests/vdr/test_viring.py +++ b/tests/vdr/test_viring.py @@ -9,7 +9,7 @@ import lmdb -from keri.core.coring import Diger, versify, Serials +from keri.core.coring import Diger, versify, Kinds from keri.db.dbing import openLMDB, dgKey, snKey from keri.vdr.viring import Reger @@ -77,7 +77,7 @@ def test_issuer(): # test with registry inception (vcp) event regk = regb sn = 0 - vs = versify(kind=Serials.json, size=20) + vs = versify(kind=Kinds.json, size=20) vcp = dict(v=vs, i=regk.decode("utf-8"), s="{:x}".format(sn), b=[rarb.decode("utf-8")], @@ -184,7 +184,7 @@ def test_issuer(): # test with verifiable credential issuance (iss) event vcdig = b'EAvR3p8V95W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc' sn = 0 - vs = versify(kind=Serials.json, size=20) + vs = versify(kind=Kinds.json, size=20) vcp = dict(v=vs, i=vcdig.decode("utf-8"), s="{:x}".format(sn), @@ -267,7 +267,7 @@ def test_clone(): # test with registry inception (vcp) event sn = 0 - vs = versify(kind=Serials.json, size=20) + vs = versify(kind=Kinds.json, size=20) vcp = dict(v=vs, i=regk.decode("utf-8"), s="{:x}".format(sn), b=[rarb.decode("utf-8")], From 97098670cc19fe11e99b66a86a86300d4f4a30bc Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Fri, 26 Jul 2024 12:48:30 -0400 Subject: [PATCH 208/418] Version bump Signed-off-by: Kevin Griffin --- Makefile | 8 ++++---- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index e5c2c807e..68d73ccb7 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev10 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev10-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev11 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev11-arm64 . .PHONY: build-witness-demo build-witness-demo: @@ -11,8 +11,8 @@ build-witness-demo: .PHONY: publish-keri publish-keri: - @docker push weboftrust/keri:1.2.0-dev10 - @docker push weboftrust/keri:1.2.0-dev10-arm64 + @docker push weboftrust/keri:1.2.0-dev11 + @docker push weboftrust/keri:1.2.0-dev11-arm64 .PHONY: publish-keri-witness-demo publish-keri-witness-demo: diff --git a/setup.py b/setup.py index cfc98f821..a0240086a 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev10', # also change in src/keri/__init__.py + version='1.2.0-dev11', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index cb048f4e4..0e48f5c9a 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev10' # also change in setup.py +__version__ = '1.2.0-dev11' # also change in setup.py From 120ad5c42f11ccf7e7730fa0e6f6f5c1db5805e7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 26 Jul 2024 11:05:17 -0600 Subject: [PATCH 209/418] decouple sizes from Matter use new Cizage named tuple instead of Sizage. This way can add entries to Matter sizes table without have to change Counter . --- src/keri/core/counting.py | 174 ++++++++++++++++++------------------ tests/core/test_counting.py | 165 +++++++++++++++++----------------- 2 files changed, 166 insertions(+), 173 deletions(-) diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 0a5ca96f3..33325e5f8 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -42,9 +42,6 @@ def __iter__(self): GenDex = GenusCodex() # Make instance - - - @dataclass(frozen=True) class CounterCodex_1_0(MapDom): """ @@ -187,6 +184,12 @@ def __iter__(self): SealDex_2_0 = SealCodex_2_0() +# namedtuple for size entries in Counter derivation code tables +# hs is the hard size int number of chars in hard (stable) part of code +# ss is the soft size int number of chars in soft (unstable) part of code +# fs is the full size int number of chars in code +Cizage = namedtuple("Cizage", "hs ss fs") + class Counter: """ @@ -333,7 +336,7 @@ class Counter: # Sizes table indexes size tables first by major version and then by # lastest minor version - # Each size table maps hs chars of code to Sizage namedtuple of (hs, ss, fs) + # Each size table maps hs chars of code to Cizage namedtuple of (hs, ss, fs) # where hs is hard size, ss is soft size, and fs is full size # soft size, ss, should always be > 0 and hs+ss=fs for Counter Sizes = \ @@ -342,82 +345,82 @@ class Counter: { Vrsn_1_0.minor: \ { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + '-A': Cizage(hs=2, ss=2, fs=4), + '-B': Cizage(hs=2, ss=2, fs=4), + '-C': Cizage(hs=2, ss=2, fs=4), + '-D': Cizage(hs=2, ss=2, fs=4), + '-E': Cizage(hs=2, ss=2, fs=4), + '-F': Cizage(hs=2, ss=2, fs=4), + '-G': Cizage(hs=2, ss=2, fs=4), + '-H': Cizage(hs=2, ss=2, fs=4), + '-I': Cizage(hs=2, ss=2, fs=4), + '-J': Cizage(hs=2, ss=2, fs=4), + '-K': Cizage(hs=2, ss=2, fs=4), + '-L': Cizage(hs=2, ss=2, fs=4), + '-0L': Cizage(hs=3, ss=5, fs=8), + '-V': Cizage(hs=2, ss=2, fs=4), + '-0V': Cizage(hs=3, ss=5, fs=8), + '-Z': Cizage(hs=2, ss=2, fs=4), + '--AAA': Cizage(hs=5, ss=3, fs=8), }, }, Vrsn_2_0.major: \ { Vrsn_2_0.minor: \ { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0A': Sizage(hs=3, ss=5, fs=8, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0B': Sizage(hs=3, ss=5, fs=8, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0C': Sizage(hs=3, ss=5, fs=8, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0D': Sizage(hs=3, ss=5, fs=8, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0E': Sizage(hs=3, ss=5, fs=8, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0F': Sizage(hs=3, ss=5, fs=8, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0G': Sizage(hs=3, ss=5, fs=8, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0H': Sizage(hs=3, ss=5, fs=8, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0I': Sizage(hs=3, ss=5, fs=8, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0J': Sizage(hs=3, ss=5, fs=8, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0K': Sizage(hs=3, ss=5, fs=8, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), - '-M': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0M': Sizage(hs=3, ss=5, fs=8, ls=0), - '-N': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0N': Sizage(hs=3, ss=5, fs=8, ls=0), - '-O': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0O': Sizage(hs=3, ss=5, fs=8, ls=0), - '-P': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0P': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Q': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0Q': Sizage(hs=3, ss=5, fs=8, ls=0), - '-R': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0R': Sizage(hs=3, ss=5, fs=8, ls=0), - '-S': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0S': Sizage(hs=3, ss=5, fs=8, ls=0), - '-T': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0T': Sizage(hs=3, ss=5, fs=8, ls=0), - '-U': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0U': Sizage(hs=3, ss=5, fs=8, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '-W': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0W': Sizage(hs=3, ss=5, fs=8, ls=0), - '-X': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0X': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Y': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0Y': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0Z': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), + '-A': Cizage(hs=2, ss=2, fs=4), + '-0A': Cizage(hs=3, ss=5, fs=8), + '-B': Cizage(hs=2, ss=2, fs=4), + '-0B': Cizage(hs=3, ss=5, fs=8), + '-C': Cizage(hs=2, ss=2, fs=4), + '-0C': Cizage(hs=3, ss=5, fs=8), + '-D': Cizage(hs=2, ss=2, fs=4), + '-0D': Cizage(hs=3, ss=5, fs=8), + '-E': Cizage(hs=2, ss=2, fs=4), + '-0E': Cizage(hs=3, ss=5, fs=8), + '-F': Cizage(hs=2, ss=2, fs=4), + '-0F': Cizage(hs=3, ss=5, fs=8), + '-G': Cizage(hs=2, ss=2, fs=4), + '-0G': Cizage(hs=3, ss=5, fs=8,), + '-H': Cizage(hs=2, ss=2, fs=4), + '-0H': Cizage(hs=3, ss=5, fs=8), + '-I': Cizage(hs=2, ss=2, fs=4), + '-0I': Cizage(hs=3, ss=5, fs=8), + '-J': Cizage(hs=2, ss=2, fs=4,), + '-0J': Cizage(hs=3, ss=5, fs=8), + '-K': Cizage(hs=2, ss=2, fs=4), + '-0K': Cizage(hs=3, ss=5, fs=8), + '-L': Cizage(hs=2, ss=2, fs=4), + '-0L': Cizage(hs=3, ss=5, fs=8), + '-M': Cizage(hs=2, ss=2, fs=4), + '-0M': Cizage(hs=3, ss=5, fs=8), + '-N': Cizage(hs=2, ss=2, fs=4), + '-0N': Cizage(hs=3, ss=5, fs=8), + '-O': Cizage(hs=2, ss=2, fs=4), + '-0O': Cizage(hs=3, ss=5, fs=8), + '-P': Cizage(hs=2, ss=2, fs=4), + '-0P': Cizage(hs=3, ss=5, fs=8), + '-Q': Cizage(hs=2, ss=2, fs=4), + '-0Q': Cizage(hs=3, ss=5, fs=8), + '-R': Cizage(hs=2, ss=2, fs=4), + '-0R': Cizage(hs=3, ss=5, fs=8), + '-S': Cizage(hs=2, ss=2, fs=4), + '-0S': Cizage(hs=3, ss=5, fs=8), + '-T': Cizage(hs=2, ss=2, fs=4), + '-0T': Cizage(hs=3, ss=5, fs=8), + '-U': Cizage(hs=2, ss=2, fs=4), + '-0U': Cizage(hs=3, ss=5, fs=8), + '-V': Cizage(hs=2, ss=2, fs=4), + '-0V': Cizage(hs=3, ss=5, fs=8), + '-W': Cizage(hs=2, ss=2, fs=4), + '-0W': Cizage(hs=3, ss=5, fs=8), + '-X': Cizage(hs=2, ss=2, fs=4), + '-0X': Cizage(hs=3, ss=5, fs=8), + '-Y': Cizage(hs=2, ss=2, fs=4), + '-0Y': Cizage(hs=3, ss=5, fs=8), + '-Z': Cizage(hs=2, ss=2, fs=4), + '-0Z': Cizage(hs=3, ss=5, fs=8), + '--AAA': Cizage(hs=5, ss=3, fs=8), }, }, } @@ -484,7 +487,7 @@ def __init__(self, code=None, *, count=None, countB64=None, except Exception as ex: raise kering.InvalidCodeError(f"Unsupported {code=}.") from ex - hs, ss, fs, ls = self._sizes[code] # get sizes for code + hs, ss, fs = self._sizes[code] # get sizes for code cs = hs + ss # both hard + soft code size if hs < 2 or fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes raise kering.InvalidCodeSizeError(f"Whole code size not full " @@ -555,13 +558,6 @@ def codes(self): """ return self._codes - #@property - #def tags(self): - #""" - #Returns tags for current .version - #Makes .tags read only - #""" - #return self.Tags[self.version] # use own version @property def sizes(self): @@ -624,7 +620,7 @@ def soft(self): quadlets/triples chars/bytes of material framed by counter. Converts .count to b64 """ - _, ss, _, _ = self.sizes[self.code] + _, ss, _ = self.sizes[self.code] return intToB64(self._count, l=ss) @@ -643,7 +639,7 @@ def fullSize(self): Returns full size of counter in bytes """ - _, _, fs, _ = self.sizes[self.code] # get from sizes table + _, _, fs = self.sizes[self.code] # get from sizes table return fs @@ -685,7 +681,7 @@ def countToB64(self, l=None): """ if l is None: - _, ss, _, _ = self._sizes[self.code] + _, ss, _ = self._sizes[self.code] l = ss return (intToB64(self.count, l=l)) @@ -766,7 +762,7 @@ def _infil(self): code = self.code # codex value chars hard code count = self.count # index value int used for soft - hs, ss, fs, ls = self._sizes[code] + hs, ss, fs = self._sizes[code] # assumes fs = hs + ss # both hard + soft size # assumes unit tests ensure ._sizes table entries are consistent # hs >= 2, ss > 0 fs == hs + ss, not (fs % 4) @@ -794,7 +790,7 @@ def _binfil(self): code = self.code # codex chars hard code count = self.count # index value int used for soft - hs, ss, fs, ls = self._sizes[code] + hs, ss, fs = self._sizes[code] # assumes fs = hs + ss # assumes unit tests ensure ._sizes table entries are consistent # hs >= 2, ss>0 fs == hs + ss, not (fs % 4) @@ -838,7 +834,7 @@ def _exfil(self, qb64b): if hard not in self._sizes: # Sizes needs str not bytes raise kering.UnexpectedCodeError("Unsupported code ={}.".format(hard)) - hs, ss, fs, ls = self._sizes[hard] # assumes hs consistent in both tables + hs, ss, fs = self._sizes[hard] # assumes hs consistent in both tables # assumes fs = hs + ss # both hard + soft code size # assumes that unit tests on Counter and CounterCodex ensure that # .Codes and .Sizes are well formed. @@ -880,7 +876,7 @@ def _bexfil(self, qb2): if hard not in self._sizes: raise kering.UnexpectedCodeError("Unsupported code ={}.".format(hard)) - hs, ss, fs, ls = self._sizes[hard] + hs, ss, fs = self._sizes[hard] # assumes fs = hs + ss # both hs and ss # assumes that unit tests on Counter and CounterCodex ensure that # .Codes and .Sizes are well formed. diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py index 3433127a6..7ec6cfbea 100644 --- a/tests/core/test_counting.py +++ b/tests/core/test_counting.py @@ -21,7 +21,7 @@ from keri.core import counting -from keri.core.counting import GenDex, Sizage, MapDom, Counter, Codens +from keri.core.counting import GenDex, Cizage, Counter, Codens from keri.core.counting import Versionage, Vrsn_1_0, Vrsn_2_0 @@ -383,90 +383,90 @@ def test_counter_class(): # Codes table with sizes of code (hard) and full primitive material assert Counter.Sizes == \ - { - Vrsn_1_0.major: \ { - Vrsn_1_0.minor: \ + 1: { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) + 0: + { + '-A': Cizage(hs=2, ss=2, fs=4), + '-B': Cizage(hs=2, ss=2, fs=4), + '-C': Cizage(hs=2, ss=2, fs=4), + '-D': Cizage(hs=2, ss=2, fs=4), + '-E': Cizage(hs=2, ss=2, fs=4), + '-F': Cizage(hs=2, ss=2, fs=4), + '-G': Cizage(hs=2, ss=2, fs=4), + '-H': Cizage(hs=2, ss=2, fs=4), + '-I': Cizage(hs=2, ss=2, fs=4), + '-J': Cizage(hs=2, ss=2, fs=4), + '-K': Cizage(hs=2, ss=2, fs=4), + '-L': Cizage(hs=2, ss=2, fs=4), + '-0L': Cizage(hs=3, ss=5, fs=8), + '-V': Cizage(hs=2, ss=2, fs=4), + '-0V': Cizage(hs=3, ss=5, fs=8), + '-Z': Cizage(hs=2, ss=2, fs=4), + '--AAA': Cizage(hs=5, ss=3, fs=8) + } }, - }, - Vrsn_2_0.major: \ - { - Vrsn_2_0.minor: \ + 2: { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0A': Sizage(hs=3, ss=5, fs=8, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0B': Sizage(hs=3, ss=5, fs=8, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0C': Sizage(hs=3, ss=5, fs=8, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0D': Sizage(hs=3, ss=5, fs=8, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0E': Sizage(hs=3, ss=5, fs=8, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0F': Sizage(hs=3, ss=5, fs=8, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0G': Sizage(hs=3, ss=5, fs=8, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0H': Sizage(hs=3, ss=5, fs=8, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0I': Sizage(hs=3, ss=5, fs=8, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0J': Sizage(hs=3, ss=5, fs=8, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0K': Sizage(hs=3, ss=5, fs=8, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0L': Sizage(hs=3, ss=5, fs=8, ls=0), - '-M': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0M': Sizage(hs=3, ss=5, fs=8, ls=0), - '-N': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0N': Sizage(hs=3, ss=5, fs=8, ls=0), - '-O': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0O': Sizage(hs=3, ss=5, fs=8, ls=0), - '-P': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0P': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Q': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0Q': Sizage(hs=3, ss=5, fs=8, ls=0), - '-R': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0R': Sizage(hs=3, ss=5, fs=8, ls=0), - '-S': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0S': Sizage(hs=3, ss=5, fs=8, ls=0), - '-T': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0T': Sizage(hs=3, ss=5, fs=8, ls=0), - '-U': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0U': Sizage(hs=3, ss=5, fs=8, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '-W': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0W': Sizage(hs=3, ss=5, fs=8, ls=0), - '-X': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0X': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Y': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0Y': Sizage(hs=3, ss=5, fs=8, ls=0), - '-Z': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0Z': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0) - }, - }, - } + 0: + { + '-A': Cizage(hs=2, ss=2, fs=4), + '-0A': Cizage(hs=3, ss=5, fs=8), + '-B': Cizage(hs=2, ss=2, fs=4), + '-0B': Cizage(hs=3, ss=5, fs=8), + '-C': Cizage(hs=2, ss=2, fs=4), + '-0C': Cizage(hs=3, ss=5, fs=8), + '-D': Cizage(hs=2, ss=2, fs=4), + '-0D': Cizage(hs=3, ss=5, fs=8), + '-E': Cizage(hs=2, ss=2, fs=4), + '-0E': Cizage(hs=3, ss=5, fs=8), + '-F': Cizage(hs=2, ss=2, fs=4), + '-0F': Cizage(hs=3, ss=5, fs=8), + '-G': Cizage(hs=2, ss=2, fs=4), + '-0G': Cizage(hs=3, ss=5, fs=8), + '-H': Cizage(hs=2, ss=2, fs=4), + '-0H': Cizage(hs=3, ss=5, fs=8), + '-I': Cizage(hs=2, ss=2, fs=4), + '-0I': Cizage(hs=3, ss=5, fs=8), + '-J': Cizage(hs=2, ss=2, fs=4), + '-0J': Cizage(hs=3, ss=5, fs=8), + '-K': Cizage(hs=2, ss=2, fs=4), + '-0K': Cizage(hs=3, ss=5, fs=8), + '-L': Cizage(hs=2, ss=2, fs=4), + '-0L': Cizage(hs=3, ss=5, fs=8), + '-M': Cizage(hs=2, ss=2, fs=4), + '-0M': Cizage(hs=3, ss=5, fs=8), + '-N': Cizage(hs=2, ss=2, fs=4), + '-0N': Cizage(hs=3, ss=5, fs=8), + '-O': Cizage(hs=2, ss=2, fs=4), + '-0O': Cizage(hs=3, ss=5, fs=8), + '-P': Cizage(hs=2, ss=2, fs=4), + '-0P': Cizage(hs=3, ss=5, fs=8), + '-Q': Cizage(hs=2, ss=2, fs=4), + '-0Q': Cizage(hs=3, ss=5, fs=8), + '-R': Cizage(hs=2, ss=2, fs=4), + '-0R': Cizage(hs=3, ss=5, fs=8), + '-S': Cizage(hs=2, ss=2, fs=4), + '-0S': Cizage(hs=3, ss=5, fs=8), + '-T': Cizage(hs=2, ss=2, fs=4), + '-0T': Cizage(hs=3, ss=5, fs=8), + '-U': Cizage(hs=2, ss=2, fs=4), + '-0U': Cizage(hs=3, ss=5, fs=8), + '-V': Cizage(hs=2, ss=2, fs=4), + '-0V': Cizage(hs=3, ss=5, fs=8), + '-W': Cizage(hs=2, ss=2, fs=4), + '-0W': Cizage(hs=3, ss=5, fs=8), + '-X': Cizage(hs=2, ss=2, fs=4), + '-0X': Cizage(hs=3, ss=5, fs=8), + '-Y': Cizage(hs=2, ss=2, fs=4), + '-0Y': Cizage(hs=3, ss=5, fs=8), + '-Z': Cizage(hs=2, ss=2, fs=4), + '-0Z': Cizage(hs=3, ss=5, fs=8), + '--AAA': Cizage(hs=5, ss=3, fs=8) + } + } + } # Ensure there is an entry in Sizes for each entry in Codes assert Counter.Codes.keys() == Counter.Sizes.keys() @@ -481,13 +481,10 @@ def test_counter_class(): assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].hs == 2 # hard size assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].ss == 2 # soft size assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].fs == 4 # full size - assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].ls == 0 # lead size assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].hs == 3 # hard size assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].ss == 5 # soft size assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].fs == 8 # full size - assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].ls == 0 # lead size - # first character of code with hard size of code assert Counter.Hards == \ From 668550dbac0866e26e7242d21097d6cda7ea14f8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 26 Jul 2024 11:48:18 -0600 Subject: [PATCH 210/418] added xs field to Sizage prepatory to making soft prepad part of matter not bolt on in Tagger --- src/keri/core/coring.py | 207 +++++++++++++++++++------------------- src/keri/core/counting.py | 2 +- tests/core/test_coring.py | 187 +++++++++++++++++----------------- 3 files changed, 199 insertions(+), 197 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index c7e662f97..cc18ac22b 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -727,9 +727,10 @@ def __iter__(self): # namedtuple for size entries in Matter and Counter derivation code tables # hs is the hard size int number of chars in hard (stable) part of code # ss is the soft size int number of chars in soft (unstable) part of code +# xs is the xtra size into number of xtra (pre-pad) chars as part of soft # fs is the full size int number of chars in code plus appended material if any # ls is the lead size int number of bytes to pre-pad pre-converted raw binary -Sizage = namedtuple("Sizage", "hs ss fs ls") +Sizage = namedtuple("Sizage", "hs ss xs fs ls") class Matter: @@ -810,97 +811,97 @@ class Matter: # soft size, ss, should always be 0 for Matter unless fs is None which allows # for variable size multiple of 4, i.e. not (hs + ss) % 4. Sizes = { - 'A': Sizage(hs=1, ss=0, fs=44, ls=0), - 'B': Sizage(hs=1, ss=0, fs=44, ls=0), - 'C': Sizage(hs=1, ss=0, fs=44, ls=0), - 'D': Sizage(hs=1, ss=0, fs=44, ls=0), - 'E': Sizage(hs=1, ss=0, fs=44, ls=0), - 'F': Sizage(hs=1, ss=0, fs=44, ls=0), - 'G': Sizage(hs=1, ss=0, fs=44, ls=0), - 'H': Sizage(hs=1, ss=0, fs=44, ls=0), - 'I': Sizage(hs=1, ss=0, fs=44, ls=0), - 'J': Sizage(hs=1, ss=0, fs=44, ls=0), - 'K': Sizage(hs=1, ss=0, fs=76, ls=0), - 'L': Sizage(hs=1, ss=0, fs=76, ls=0), - 'M': Sizage(hs=1, ss=0, fs=4, ls=0), - 'N': Sizage(hs=1, ss=0, fs=12, ls=0), - 'O': Sizage(hs=1, ss=0, fs=44, ls=0), - 'P': Sizage(hs=1, ss=0, fs=124, ls=0), - 'Q': Sizage(hs=1, ss=0, fs=44, ls=0), - 'R': Sizage(hs=1, ss=0, fs=8, ls=0), - 'S': Sizage(hs=1, ss=0, fs=16, ls=0), - 'T': Sizage(hs=1, ss=0, fs=20, ls=0), - 'U': Sizage(hs=1, ss=0, fs=24, ls=0), - 'V': Sizage(hs=1, ss=0, fs=4, ls=1), - 'W': Sizage(hs=1, ss=0, fs=4, ls=0), - 'X': Sizage(hs=1, ss=3, fs=4, ls=0), - 'Y': Sizage(hs=1, ss=7, fs=8, ls=0), - 'Z': Sizage(hs=1, ss=0, fs=44, ls=0), - '0A': Sizage(hs=2, ss=0, fs=24, ls=0), - '0B': Sizage(hs=2, ss=0, fs=88, ls=0), - '0C': Sizage(hs=2, ss=0, fs=88, ls=0), - '0D': Sizage(hs=2, ss=0, fs=88, ls=0), - '0E': Sizage(hs=2, ss=0, fs=88, ls=0), - '0F': Sizage(hs=2, ss=0, fs=88, ls=0), - '0G': Sizage(hs=2, ss=0, fs=88, ls=0), - '0H': Sizage(hs=2, ss=0, fs=8, ls=0), - '0I': Sizage(hs=2, ss=0, fs=88, ls=0), - '0J': Sizage(hs=2, ss=2, fs=4, ls=0), - '0K': Sizage(hs=2, ss=2, fs=4, ls=0), - '0L': Sizage(hs=2, ss=6, fs=8, ls=0), - '0M': Sizage(hs=2, ss=6, fs=8, ls=0), - '0N': Sizage(hs=2, ss=10, fs=12, ls=0), - '0O': Sizage(hs=2, ss=10, fs=12, ls=0), - '1AAA': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAB': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), - '1AAD': Sizage(hs=4, ss=0, fs=80, ls=0), - '1AAE': Sizage(hs=4, ss=0, fs=56, ls=0), - '1AAF': Sizage(hs=4, ss=4, fs=8, ls=0), - '1AAG': Sizage(hs=4, ss=0, fs=36, ls=0), - '1AAH': Sizage(hs=4, ss=0, fs=100, ls=0), - '1AAI': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAJ': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAN': Sizage(hs=4, ss=8, fs=12, ls=0), - '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), - '1___': Sizage(hs=4, ss=0, fs=8, ls=0), - '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), - '2___': Sizage(hs=4, ss=0, fs=8, ls=1), - '3__-': Sizage(hs=4, ss=2, fs=12, ls=2), - '3___': Sizage(hs=4, ss=0, fs=8, ls=2), - '4A': Sizage(hs=2, ss=2, fs=None, ls=0), - '5A': Sizage(hs=2, ss=2, fs=None, ls=1), - '6A': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAA': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAA': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAA': Sizage(hs=4, ss=4, fs=None, ls=2), - '4B': Sizage(hs=2, ss=2, fs=None, ls=0), - '5B': Sizage(hs=2, ss=2, fs=None, ls=1), - '6B': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAB': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAB': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAB': Sizage(hs=4, ss=4, fs=None, ls=2), - '4C': Sizage(hs=2, ss=2, fs=None, ls=0), - '5C': Sizage(hs=2, ss=2, fs=None, ls=1), - '6C': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAC': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAC': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAC': Sizage(hs=4, ss=4, fs=None, ls=2), - '4D': Sizage(hs=2, ss=2, fs=None, ls=0), - '5D': Sizage(hs=2, ss=2, fs=None, ls=1), - '6D': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAD': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAD': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAD': Sizage(hs=4, ss=4, fs=None, ls=2), - '4E': Sizage(hs=2, ss=2, fs=None, ls=0), - '5E': Sizage(hs=2, ss=2, fs=None, ls=1), - '6E': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAE': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAE': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAE': Sizage(hs=4, ss=4, fs=None, ls=2), + 'A': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'B': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'C': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'D': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'E': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'F': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'G': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'H': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'I': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'J': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'K': Sizage(hs=1, ss=0, xs=0, fs=76, ls=0), + 'L': Sizage(hs=1, ss=0, xs=0, fs=76, ls=0), + 'M': Sizage(hs=1, ss=0, xs=0, fs=4, ls=0), + 'N': Sizage(hs=1, ss=0, xs=0, fs=12, ls=0), + 'O': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'P': Sizage(hs=1, ss=0, xs=0, fs=124, ls=0), + 'Q': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'R': Sizage(hs=1, ss=0, xs=0, fs=8, ls=0), + 'S': Sizage(hs=1, ss=0, xs=0, fs=16, ls=0), + 'T': Sizage(hs=1, ss=0, xs=0, fs=20, ls=0), + 'U': Sizage(hs=1, ss=0, xs=0, fs=24, ls=0), + 'V': Sizage(hs=1, ss=0, xs=0, fs=4, ls=1), + 'W': Sizage(hs=1, ss=0, xs=0, fs=4, ls=0), + 'X': Sizage(hs=1, ss=3, xs=0, fs=4, ls=0), + 'Y': Sizage(hs=1, ss=7, xs=0, fs=8, ls=0), + 'Z': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + '0A': Sizage(hs=2, ss=0, xs=0, fs=24, ls=0), + '0B': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0C': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0D': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0E': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0F': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0G': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0H': Sizage(hs=2, ss=0, xs=0, fs=8, ls=0), + '0I': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0J': Sizage(hs=2, ss=2, xs=0, fs=4, ls=0), + '0K': Sizage(hs=2, ss=2, xs=0, fs=4, ls=0), + '0L': Sizage(hs=2, ss=6, xs=0, fs=8, ls=0), + '0M': Sizage(hs=2, ss=6, xs=0, fs=8, ls=0), + '0N': Sizage(hs=2, ss=10, xs=0, fs=12, ls=0), + '0O': Sizage(hs=2, ss=10, xs=0, fs=12, ls=0), + '1AAA': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAB': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAC': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), + '1AAD': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), + '1AAE': Sizage(hs=4, ss=0, xs=0, fs=56, ls=0), + '1AAF': Sizage(hs=4, ss=4, xs=0, fs=8, ls=0), + '1AAG': Sizage(hs=4, ss=0, xs=0, fs=36, ls=0), + '1AAH': Sizage(hs=4, ss=0, xs=0, fs=100, ls=0), + '1AAI': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAJ': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAK': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAL': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAM': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAN': Sizage(hs=4, ss=8, xs=0, fs=12, ls=0), + '1__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=0), + '1___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=0), + '2__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=1), + '2___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=1), + '3__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=2), + '3___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=2), + '4A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), } Codes = asdict(MtrDex) # map code name to code @@ -953,7 +954,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, if code not in self.Sizes: raise InvalidCodeError(f"Unsupported {code=}.") - hs, ss, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes + hs, ss, xs, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes if fs is None: # variable sized assumes code[0] in SmallVrzDex or LargeVrzDex if rize: # use rsize to determine length of raw to extract @@ -1021,7 +1022,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, self._raw = bytes(raw) # crypto ops require bytes not bytearray elif soft and code: # fixed size and special when raw None - hs, ss, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes + hs, ss, xs, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes if not fs: # variable sized code so can't be soft raise InvalidSoftError(f"Unsupported variable sized {code=} " f" with {fs=} for special {soft=}.") @@ -1069,7 +1070,7 @@ def _rawSize(cls, code): Parameters: code (str): derivation code Base64 """ - hs, ss, fs, ls = cls.Sizes[code] # get sizes + hs, ss, xs, fs, ls = cls.Sizes[code] # get sizes cs = hs + ss # both hard + soft code size if fs is None: raise InvalidCodeSizeError(f"Non-fixed raw size code {code}.") @@ -1084,7 +1085,7 @@ def _leadSize(cls, code): Parameters: code (str): derivation code Base64 """ - _, _, _, ls = cls.Sizes[code] # get lead size from .Sizes table + _, _, _, _, ls = cls.Sizes[code] # get lead size from .Sizes table return ls @@ -1097,7 +1098,7 @@ def _special(cls, code): False otherwise """ - hs, ss, fs, ls = cls.Sizes[code] + hs, ss, xs, fs, ls = cls.Sizes[code] return (fs is not None and ss > 0) @@ -1188,7 +1189,7 @@ def fullSize(self): Fixed size codes returns fs from .Sizes Variable size codes where fs==None computes fs from .size and sizes """ - hs, ss, fs, _ = self.Sizes[self.code] # get sizes + hs, ss, _, fs, _ = self.Sizes[self.code] # get sizes if fs is None: # compute fs from ss characters in code fs = hs + ss + (self.size * 4) @@ -1295,7 +1296,7 @@ def _infil(self): both = self.both # code + soft, soft may be empty raw = self.raw # bytes or bytearray, raw may be empty rs = len(raw) # raw size - hs, ss, fs, ls = self.Sizes[code] + hs, ss, xs, fs, ls = self.Sizes[code] cs = hs + ss # assumes unit tests on Matter.Sizes ensure valid size entries @@ -1357,7 +1358,7 @@ def _binfil(self): both = self.both # code + soft, soft may be empty raw = self.raw # bytes or bytearray may be empty - hs, ss, fs, ls = self.Sizes[code] + hs, ss, xs, fs, ls = self.Sizes[code] cs = hs + ss # assumes unit tests on Matter.Sizes ensure valid size entries n = sceil(cs * 3 / 4) # number of b2 bytes to hold b64 code @@ -1416,7 +1417,7 @@ def _exfil(self, qb64b): if hard not in self.Sizes: raise UnexpectedCodeError(f"Unsupported code ={hard}.") - hs, ss, fs, ls = self.Sizes[hard] # assumes hs in both tables match + hs, ss, xs, fs, ls = self.Sizes[hard] # assumes hs in both tables match cs = hs + ss # both hs and ss # assumes that unit tests on Matter .Sizes .Hards and .Bards ensure that # these are well formed. @@ -1493,7 +1494,7 @@ def _bexfil(self, qb2): if hard not in self.Sizes: raise UnexpectedCodeError(f"Unsupported code ={hard}.") - hs, ss, fs, ls = self.Sizes[hard] + hs, ss, xs, fs, ls = self.Sizes[hard] cs = hs + ss # both hs and ss # assumes that unit tests on Matter .Sizes .Hards and .Bards ensure that # these are well formed. @@ -2735,7 +2736,7 @@ def bext(self): Property bext: Base64 text value portion of qualified b64 str Returns the value portion of .qb64 with text code and leader removed """ - _, _, _, ls = self.Sizes[self.code] + _, _, _, _, ls = self.Sizes[self.code] bext = encodeB64(bytes([0] * ls) + self.raw) ws = 0 if ls == 0 and bext: diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py index 33325e5f8..4fc8774df 100644 --- a/src/keri/core/counting.py +++ b/src/keri/core/counting.py @@ -17,7 +17,7 @@ from .. import kering from ..kering import (Versionage, Vrsn_1_0, Vrsn_2_0) -from ..core.coring import Sizage, MapDom +from ..core.coring import MapDom diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 952a75b0d..78e51ba8d 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -411,102 +411,103 @@ def test_matter_class(): # Codes table with sizes of code (hard) and full primitive material assert Matter.Sizes == \ { - 'A': Sizage(hs=1, ss=0, fs=44, ls=0), - 'B': Sizage(hs=1, ss=0, fs=44, ls=0), - 'C': Sizage(hs=1, ss=0, fs=44, ls=0), - 'D': Sizage(hs=1, ss=0, fs=44, ls=0), - 'E': Sizage(hs=1, ss=0, fs=44, ls=0), - 'F': Sizage(hs=1, ss=0, fs=44, ls=0), - 'G': Sizage(hs=1, ss=0, fs=44, ls=0), - 'H': Sizage(hs=1, ss=0, fs=44, ls=0), - 'I': Sizage(hs=1, ss=0, fs=44, ls=0), - 'J': Sizage(hs=1, ss=0, fs=44, ls=0), - 'K': Sizage(hs=1, ss=0, fs=76, ls=0), - 'L': Sizage(hs=1, ss=0, fs=76, ls=0), - 'M': Sizage(hs=1, ss=0, fs=4, ls=0), - 'N': Sizage(hs=1, ss=0, fs=12, ls=0), - 'O': Sizage(hs=1, ss=0, fs=44, ls=0), - 'P': Sizage(hs=1, ss=0, fs=124, ls=0), - 'Q': Sizage(hs=1, ss=0, fs=44, ls=0), - 'R': Sizage(hs=1, ss=0, fs=8, ls=0), - 'S': Sizage(hs=1, ss=0, fs=16, ls=0), - 'T': Sizage(hs=1, ss=0, fs=20, ls=0), - 'U': Sizage(hs=1, ss=0, fs=24, ls=0), - 'V': Sizage(hs=1, ss=0, fs=4, ls=1), - 'W': Sizage(hs=1, ss=0, fs=4, ls=0), - 'X': Sizage(hs=1, ss=3, fs=4, ls=0), - 'Y': Sizage(hs=1, ss=7, fs=8, ls=0), - 'Z': Sizage(hs=1, ss=0, fs=44, ls=0), - '0A': Sizage(hs=2, ss=0, fs=24, ls=0), - '0B': Sizage(hs=2, ss=0, fs=88, ls=0), - '0C': Sizage(hs=2, ss=0, fs=88, ls=0), - '0D': Sizage(hs=2, ss=0, fs=88, ls=0), - '0E': Sizage(hs=2, ss=0, fs=88, ls=0), - '0F': Sizage(hs=2, ss=0, fs=88, ls=0), - '0G': Sizage(hs=2, ss=0, fs=88, ls=0), - '0H': Sizage(hs=2, ss=0, fs=8, ls=0), - '0I': Sizage(hs=2, ss=0, fs=88, ls=0), - '0J': Sizage(hs=2, ss=2, fs=4, ls=0), - '0K': Sizage(hs=2, ss=2, fs=4, ls=0), - '0L': Sizage(hs=2, ss=6, fs=8, ls=0), - '0M': Sizage(hs=2, ss=6, fs=8, ls=0), - '0N': Sizage(hs=2, ss=10, fs=12, ls=0), - '0O': Sizage(hs=2, ss=10, fs=12, ls=0), - '1AAA': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAB': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), - '1AAD': Sizage(hs=4, ss=0, fs=80, ls=0), - '1AAE': Sizage(hs=4, ss=0, fs=56, ls=0), - '1AAF': Sizage(hs=4, ss=4, fs=8, ls=0), - '1AAG': Sizage(hs=4, ss=0, fs=36, ls=0), - '1AAH': Sizage(hs=4, ss=0, fs=100, ls=0), - '1AAI': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAJ': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAN': Sizage(hs=4, ss=8, fs=12, ls=0), - '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), - '1___': Sizage(hs=4, ss=0, fs=8, ls=0), - '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), - '2___': Sizage(hs=4, ss=0, fs=8, ls=1), - '3__-': Sizage(hs=4, ss=2, fs=12, ls=2), - '3___': Sizage(hs=4, ss=0, fs=8, ls=2), - '4A': Sizage(hs=2, ss=2, fs=None, ls=0), - '5A': Sizage(hs=2, ss=2, fs=None, ls=1), - '6A': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAA': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAA': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAA': Sizage(hs=4, ss=4, fs=None, ls=2), - '4B': Sizage(hs=2, ss=2, fs=None, ls=0), - '5B': Sizage(hs=2, ss=2, fs=None, ls=1), - '6B': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAB': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAB': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAB': Sizage(hs=4, ss=4, fs=None, ls=2), - '4C': Sizage(hs=2, ss=2, fs=None, ls=0), - '5C': Sizage(hs=2, ss=2, fs=None, ls=1), - '6C': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAC': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAC': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAC': Sizage(hs=4, ss=4, fs=None, ls=2), - '4D': Sizage(hs=2, ss=2, fs=None, ls=0), - '5D': Sizage(hs=2, ss=2, fs=None, ls=1), - '6D': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAD': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAD': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAD': Sizage(hs=4, ss=4, fs=None, ls=2), - '4E': Sizage(hs=2, ss=2, fs=None, ls=0), - '5E': Sizage(hs=2, ss=2, fs=None, ls=1), - '6E': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAE': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAE': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAE': Sizage(hs=4, ss=4, fs=None, ls=2) + 'A': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'B': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'C': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'D': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'E': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'F': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'G': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'H': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'I': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'J': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'K': Sizage(hs=1, ss=0, xs=0, fs=76, ls=0), + 'L': Sizage(hs=1, ss=0, xs=0, fs=76, ls=0), + 'M': Sizage(hs=1, ss=0, xs=0, fs=4, ls=0), + 'N': Sizage(hs=1, ss=0, xs=0, fs=12, ls=0), + 'O': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'P': Sizage(hs=1, ss=0, xs=0, fs=124, ls=0), + 'Q': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'R': Sizage(hs=1, ss=0, xs=0, fs=8, ls=0), + 'S': Sizage(hs=1, ss=0, xs=0, fs=16, ls=0), + 'T': Sizage(hs=1, ss=0, xs=0, fs=20, ls=0), + 'U': Sizage(hs=1, ss=0, xs=0, fs=24, ls=0), + 'V': Sizage(hs=1, ss=0, xs=0, fs=4, ls=1), + 'W': Sizage(hs=1, ss=0, xs=0, fs=4, ls=0), + 'X': Sizage(hs=1, ss=3, xs=0, fs=4, ls=0), + 'Y': Sizage(hs=1, ss=7, xs=0, fs=8, ls=0), + 'Z': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + '0A': Sizage(hs=2, ss=0, xs=0, fs=24, ls=0), + '0B': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0C': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0D': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0E': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0F': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0G': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0H': Sizage(hs=2, ss=0, xs=0, fs=8, ls=0), + '0I': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0J': Sizage(hs=2, ss=2, xs=0, fs=4, ls=0), + '0K': Sizage(hs=2, ss=2, xs=0, fs=4, ls=0), + '0L': Sizage(hs=2, ss=6, xs=0, fs=8, ls=0), + '0M': Sizage(hs=2, ss=6, xs=0, fs=8, ls=0), + '0N': Sizage(hs=2, ss=10, xs=0, fs=12, ls=0), + '0O': Sizage(hs=2, ss=10, xs=0, fs=12, ls=0), + '1AAA': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAB': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAC': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), + '1AAD': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), + '1AAE': Sizage(hs=4, ss=0, xs=0, fs=56, ls=0), + '1AAF': Sizage(hs=4, ss=4, xs=0, fs=8, ls=0), + '1AAG': Sizage(hs=4, ss=0, xs=0, fs=36, ls=0), + '1AAH': Sizage(hs=4, ss=0, xs=0, fs=100, ls=0), + '1AAI': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAJ': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAK': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAL': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAM': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAN': Sizage(hs=4, ss=8, xs=0, fs=12, ls=0), + '1__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=0), + '1___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=0), + '2__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=1), + '2___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=1), + '3__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=2), + '3___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=2), + '4A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2) } assert Matter.Sizes['A'].hs == 1 # hard size assert Matter.Sizes['A'].ss == 0 # soft size + assert Matter.Sizes['A'].xs == 0 # xtra size assert Matter.Sizes['A'].fs == 44 # full size assert Matter.Sizes['A'].ls == 0 # lead size @@ -515,13 +516,14 @@ def test_matter_class(): for code, val in Matter.Sizes.items(): # hard code hs = val.hs ss = val.ss + xs = val.xs fs = val.fs ls = val.ls cs = hs + ss assert (isinstance(hs, int) and isinstance(ss, int) and isinstance(ls, int)) - assert hs > 0 and ss >= 0 and ls in (0, 1, 2) + assert hs > 0 and ss >= 0 and xs in (0, 1, 2) and ls in (0, 1, 2) assert len(code) == hs if fs is None: # variable sized @@ -5784,7 +5786,6 @@ def test_tholder(): test_traitor() test_verser() #test_texter() - #test_counter() #test_prodex() #test_indexer() test_number() From bc9c0f982a210f8c1b4119694d7fa597c7808466 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 27 Jul 2024 10:52:53 -0600 Subject: [PATCH 211/418] added xs and comments to Matter Sizes --- src/keri/core/coring.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index cc18ac22b..9d4ddcecc 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -655,15 +655,15 @@ class TagCodex: Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ Tag1: str = '0J' # 1 B64 char tag with 1 pre pad - Tag2: str = '0K' # 1 B64 char tag - Tag3: str = 'X' # 1 B64 char tag - Tag4: str = '1AAF' # 1 B64 char tag - Tag5: str = '0L' # 1 B64 char tag with 1 pre pad - Tag6: str = '0M' # 1 B64 char tag - Tag7: str = 'Y' # 1 B64 char tag - Tag8: str = '1AAN' # 1 B64 char tag - Tag9: str = '0N' # 1 B64 char tag with 1 pre pad - Tag10: str = '0O' # 1 B64 char tag + Tag2: str = '0K' # 2 B64 char tag + Tag3: str = 'X' # 3 B64 char tag + Tag4: str = '1AAF' # 4 B64 char tag + Tag5: str = '0L' # 5 B64 char tag with 1 pre pad + Tag6: str = '0M' # 6 B64 char tag + Tag7: str = 'Y' # 7 B64 char tag + Tag8: str = '1AAN' # 8 B64 char tag + Tag9: str = '0N' # 9 B64 char tag with 1 pre pad + Tag10: str = '0O' # 10 B64 char tag def __iter__(self): return iter(astuple(self)) @@ -806,10 +806,8 @@ class Matter: Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) # Sizes table maps from value of hs chars of code to Sizage namedtuple of - # (hs, ss, fs, ls) where hs is hard size, ss is soft size, and fs is full size - # and ls is lead size - # soft size, ss, should always be 0 for Matter unless fs is None which allows - # for variable size multiple of 4, i.e. not (hs + ss) % 4. + # (hs, ss, xs, fs, ls) where hs is hard size, ss is soft size, + # xs is extra size of soft, fs is full size, and ls is lead size of raw. Sizes = { 'A': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), 'B': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), @@ -2121,11 +2119,12 @@ def __init__(self, tag='', soft='', code=None, **kwa): tag = tag.decode("utf-8") if not Reb64.match(tag.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {tag=}.") + # TagDex tags appear in order of size 1 to 10, at indices 0 to 9 codes = astuple(TagDex) l = len(tag) # soft not empty so l > 0 if l > len(codes): raise InvalidSoftError("Oversized tag={soft}.") - code = codes[l-1] # get code for size of soft + code = codes[l-1] # get code for for tag of len where (index = len - 1) if code in PadTagDex: soft = self.Pad + tag # pre pad for those that need it else: From db1ec092dc398ec6bf133dc7377f8a4b61eb6caf Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 27 Jul 2024 13:08:56 -0600 Subject: [PATCH 212/418] added xs xtra prepad support to Matter --- src/keri/core/coring.py | 75 +++++++++++++++++++++++++++++---------- tests/core/test_coring.py | 8 ++++- 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 9d4ddcecc..161c06e8c 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -747,6 +747,9 @@ class Matter: Hards (dict): hard sizes keyed by qb64 selector Bards (dict): hard size keyed by qb2 selector Sizes (dict): sizes tables for codes + Codes (dict): maps code name to code + Names (dict): maps code to code name + Pad (str): B64 pad char for xtra size pre-padded soft values Calss Methods: @@ -756,7 +759,7 @@ class Matter: Properties: code (str): hard part of derivation code to indicate cypher suite hard (str): hard part of derivation code. alias for code - soft (str | bytes): soft part of derivation code fs any. + soft (str | bytes): soft part of full code exclusive of xs xtra prepad. Empty when ss = 0. both (str): hard + soft parts of full text code size (int | None): Number of quadlets/triplets of chars/bytes including @@ -791,6 +794,8 @@ class Matter: Special soft values are indicated when fn in table is None and ss > 0. """ + Codex = MtrDex # class variable holding MatterDex reference + # Hards table maps from bytes Base64 first code char to int of hard size, hs, # (stable) of code. The soft size, ss, (unstable) is always 0 for Matter # unless fs is None which allows for variable size multiple of 4, i.e. @@ -904,7 +909,7 @@ class Matter: Codes = asdict(MtrDex) # map code name to code Names = {val : key for key, val in Codes.items()} # invert map code to code name - + Pad = '_' # B64 pad char for special codes with xtra size pre-padded soft values def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, @@ -915,7 +920,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, raw (bytes | bytearray | None): unqualified crypto material usable for crypto operations. code (str): stable (hard) part of derivation code - soft (str | bytes): soft part for special codes + soft (str | bytes): soft part exclusive of prepad for special codes rize (int | None): raw size in bytes when variable sized material not including lead bytes if any Otherwise None @@ -955,6 +960,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, hs, ss, xs, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes if fs is None: # variable sized assumes code[0] in SmallVrzDex or LargeVrzDex + # assumes xs must be 0 when variable sized if rize: # use rsize to determine length of raw to extract if rize < 0: raise InvalidVarRawSizeError(f"Missing var raw size for " @@ -997,12 +1003,13 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, else: # fixed size but raw may be empty and/or special soft rize = Matter._rawSize(code) # get raw size from Sizes for code + # if raw then ls may be nonzero if ss > 0: # special soft size, so soft must be provided - soft = soft[:ss] - if len(soft) != ss: + soft = soft[:ss-xs] # + if len(soft) != ss - xs: raise SoftMaterialError(f"Not enough chars in {soft=} " - f"with {ss=} for {code=}.") + f"with {ss=} {xs=} for {code=}.") if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") @@ -1015,13 +1022,13 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, raise RawMaterialError(f"Not enougth raw bytes for code={code}" f"expected {rize=} got {len(raw)}.") - self._code = code # str hard part of code - self._soft = soft # str soft part of code, empty when ss=0 + self._code = code # str hard part of full code + self._soft = soft # str soft part of full code exclusive of xs prepad, empty when ss=0 self._raw = bytes(raw) # crypto ops require bytes not bytearray - elif soft and code: # fixed size and special when raw None + elif soft and code: # raw None so ls == 0 with fixed size and special hs, ss, xs, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes - if not fs: # variable sized code so can't be soft + if not fs: # variable sized code so can't be special soft raise InvalidSoftError(f"Unsupported variable sized {code=} " f" with {fs=} for special {soft=}.") @@ -1029,17 +1036,17 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, raise InvalidSoftError("Invalid soft size={ss} or lead={ls} " f" or {code=} {fs=} when special soft.") - soft = soft[:ss] - if len(soft) != ss: + soft = soft[:ss-xs] + if len(soft) != ss - xs: raise SoftMaterialError(f"Not enough chars in {soft=} " - f"with {ss=} for {code=}.") + f"with {ss=} {xs=} for {code=}.") if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") self._code = code # str hard part of code self._soft = soft # str soft part of code, empty when ss=0 - self._raw = b'' # make raw empty when None and when special soft + self._raw = b'' # force raw empty when None given and special soft elif qb64b is not None: self._exfil(qb64b) @@ -1086,6 +1093,16 @@ def _leadSize(cls, code): _, _, _, _, ls = cls.Sizes[code] # get lead size from .Sizes table return ls + @classmethod + def _xtraSize(cls, code): + """ + Returns xtra size in bytes for a given code + Parameters: + code (str): derivation code Base64 + """ + _, _, xs, _, _ = cls.Sizes[code] # get lead size from .Sizes table + return xs + @classmethod def _special(cls, code): @@ -1177,7 +1194,9 @@ def both(self): #else: #return (f"{self.code}{self.soft}") - return (f"{self.code}{self.soft}") + _, _, xs, _, _ = self.Sizes[self.code] + + return (f"{self.code}{self.Pad * xs}{self.soft}") @property @@ -1287,6 +1306,8 @@ def composable(self): def _infil(self): """ + Create text domain representation + Returns: primitive (bytes): fully qualified base64 characters. """ @@ -1341,13 +1362,15 @@ def _infil(self): if (len(full) % 4) or (fs and len(full) != fs): raise InvalidCodeSizeError(f"Invalid full size given code{both=} " f" with raw size={rs}, {cs=}, {hs=}, " - f"{ss=}, {fs=}, and {ls=}.") + f"{ss=}, {xs=} {fs=}, and {ls=}.") return full def _binfil(self): """ + Create binary domain representation + Returns bytes of fully qualified base2 bytes, that is .qb2 self.code converted to Base2 + self.raw left shifted with pad bits equivalent of Base64 decode of .qb64 into .qb2 @@ -1421,7 +1444,17 @@ def _exfil(self, qb64b): # these are well formed. # when fs is None then ss > 0 otherwise fs > hs + ss when ss > 0 - soft = qb64b[hs:hs + ss] # extract soft chars, empty when ss==0 + xtra = qb64b[hs:hs+xs] # extract xtra prepad chars of soft, empty when xs==0 + if isinstance(xtra, memoryview): + xtra = bytes(xtra) + if hasattr(xtra, "decode"): + xtra = xtra.decode() # converts bytes/bytearray to str + if xtra != f"{self.Pad * xs}": + raise UnexpectedCodeError(f"Invalid prepad xtra ={xtra}.") + + # extract soft chars excluding xtra, empty when ss==0 and xs == 0 + # assumes that when ss == 0 then xs must be 0 + soft = qb64b[hs+xs:hs+ss] if isinstance(soft, memoryview): soft = bytes(soft) if hasattr(soft, "decode"): @@ -1503,7 +1536,13 @@ def _bexfil(self, qb2): raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code - soft = both[hs:hs + ss] # get soft may be empty + xtra = both[hs:hs+xs] # extract xtra prepad chars of soft, empty when xs==0 + if xtra != f"{self.Pad * xs}": + raise UnexpectedCodeError(f"Invalid prepad xtra ={xtra}.") + + # extract soft chars excluding xtra, empty when ss==0 and xs == 0 + # assumes that when ss == 0 then xs must be 0 + soft = both[hs+xs:hs+ss] # get soft may be empty if not fs: # compute fs from size chars in ss part of code if len(qb2) < bcs: # need more bytes diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 78e51ba8d..5206a2d79 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -204,6 +204,9 @@ def test_matter_class(): """ Test Matter class attributes """ + assert Matter.Codex == MtrDex + + assert Matter.Pad == '_' assert Matter.Codes == \ { @@ -527,7 +530,7 @@ def test_matter_class(): assert len(code) == hs if fs is None: # variable sized - assert ss > 0 and not (cs % 4) # full code is 24 bit aligned + assert ss > 0 and xs == 0 and not (cs % 4) # full code is 24 bit aligned # assumes that Matter methods also ensure (ls + rs) % 3 == 0 i.e. # variable raw with lead is 24 bit aligned, where rs is raw size. assert code[0] in coring.SmallVrzDex or code[0] in coring.LargeVrzDex @@ -563,10 +566,12 @@ def test_matter_class(): assert not (code[0] in coring.SmallVrzDex or code[0] in coring.LargeVrzDex) assert isinstance(fs, int) and fs > 0 and not fs % 4 assert fs >= cs + assert xs <= ss # xs must be zero if ss is assert cs % 4 != 3 # prevent ambiguous conversion if ss > 0 and fs == cs: # special soft value with raw empty assert ls == 0 # no lead assert Matter._rawSize(code) == 0 + assert xs < ss # soft must not be empty, not all prepad # verify correct sizes given raw size. Assumes properties above rs = ((fs - cs) * 3 // 4) - ls # raw size bytes sans lead @@ -601,6 +606,7 @@ def test_matter_class(): assert Matter._rawSize(MtrDex.Ed25519) == 32 assert Matter._leadSize(MtrDex.Ed25519) == 0 + assert Matter._xtraSize(MtrDex.Ed25519) == 0 assert not Matter._special(MtrDex.Ed25519) assert Matter._special(MtrDex.Tag3) From de5c84a47efb96e55e50e04f6e0ed698e5667607 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 27 Jul 2024 13:37:33 -0600 Subject: [PATCH 213/418] added test vector for xs --- src/keri/core/coring.py | 27 ++++++++++++++------------- tests/core/test_coring.py | 31 +++++++++++++++---------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 161c06e8c..6bbc3c10a 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -849,11 +849,11 @@ class Matter: '0G': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), '0H': Sizage(hs=2, ss=0, xs=0, fs=8, ls=0), '0I': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), - '0J': Sizage(hs=2, ss=2, xs=0, fs=4, ls=0), + '0J': Sizage(hs=2, ss=2, xs=1, fs=4, ls=0), '0K': Sizage(hs=2, ss=2, xs=0, fs=4, ls=0), - '0L': Sizage(hs=2, ss=6, xs=0, fs=8, ls=0), + '0L': Sizage(hs=2, ss=6, xs=1, fs=8, ls=0), '0M': Sizage(hs=2, ss=6, xs=0, fs=8, ls=0), - '0N': Sizage(hs=2, ss=10, xs=0, fs=12, ls=0), + '0N': Sizage(hs=2, ss=10, xs=1, fs=12, ls=0), '0O': Sizage(hs=2, ss=10, xs=0, fs=12, ls=0), '1AAA': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), '1AAB': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), @@ -871,7 +871,7 @@ class Matter: '1AAN': Sizage(hs=4, ss=8, xs=0, fs=12, ls=0), '1__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=0), '1___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=0), - '2__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=1), + '2__-': Sizage(hs=4, ss=2, xs=1, fs=12, ls=1), '2___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=1), '3__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=2), '3___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=2), @@ -2164,10 +2164,11 @@ def __init__(self, tag='', soft='', code=None, **kwa): if l > len(codes): raise InvalidSoftError("Oversized tag={soft}.") code = codes[l-1] # get code for for tag of len where (index = len - 1) - if code in PadTagDex: - soft = self.Pad + tag # pre pad for those that need it - else: - soft = tag + #if code in PadTagDex: + #soft = self.Pad + tag # pre pad for those that need it + #else: + #soft = tag + soft = tag super(Tagger, self).__init__(soft=soft, code=code, **kwa) @@ -2182,11 +2183,11 @@ def tag(self): """ tag = self.soft - if self.code in PadTagDex: - pad = self.soft[0] - tag = self.soft[1:] - if pad != self.Pad: - raise InvalidSoftError("Invaid pre {pad=} for {tag=}.") + #if self.code in PadTagDex: + #pad = self.soft[0] + #tag = self.soft[1:] + #if pad != self.Pad: + #raise InvalidSoftError("Invaid pre {pad=} for {tag=}.") return tag diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 5206a2d79..6eca06a47 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -449,11 +449,11 @@ def test_matter_class(): '0G': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), '0H': Sizage(hs=2, ss=0, xs=0, fs=8, ls=0), '0I': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), - '0J': Sizage(hs=2, ss=2, xs=0, fs=4, ls=0), + '0J': Sizage(hs=2, ss=2, xs=1, fs=4, ls=0), '0K': Sizage(hs=2, ss=2, xs=0, fs=4, ls=0), - '0L': Sizage(hs=2, ss=6, xs=0, fs=8, ls=0), + '0L': Sizage(hs=2, ss=6, xs=1, fs=8, ls=0), '0M': Sizage(hs=2, ss=6, xs=0, fs=8, ls=0), - '0N': Sizage(hs=2, ss=10, xs=0, fs=12, ls=0), + '0N': Sizage(hs=2, ss=10, xs=1, fs=12, ls=0), '0O': Sizage(hs=2, ss=10, xs=0, fs=12, ls=0), '1AAA': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), '1AAB': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), @@ -471,7 +471,7 @@ def test_matter_class(): '1AAN': Sizage(hs=4, ss=8, xs=0, fs=12, ls=0), '1__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=0), '1___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=0), - '2__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=1), + '2__-': Sizage(hs=4, ss=2, xs=1, fs=12, ls=1), '2___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=1), '3__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=2), '3___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=2), @@ -1985,7 +1985,7 @@ def test_matter_special(): # Test TBD0S '1__-' # soft special but valid non-empty raw as part of primitive - code = MtrDex.TBD0S # sizes '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), + code = MtrDex.TBD0S # sizes '1__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=0), rs = Matter._rawSize(code) # raw size soft = 'TG' qb64 = '1__-TGB1dnd4' @@ -2046,11 +2046,11 @@ def test_matter_special(): # Test TBD1S '2__-' # soft special but valid non-empty raw as part of primitive - code = MtrDex.TBD1S # sizes '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), + code = MtrDex.TBD1S # sizes '2__-': Sizage(hs=4, ss=2, xs=1, fs=12, ls=1), rs = Matter._rawSize(code) # raw size - soft = 'TG' - qb64 = '2__-TGAAdXZ3' # see lead byte - qb2 = b'\xdb\xff\xfeL`\x00uvw' + soft = 'T' + qb64 = '2__-_TAAdXZ3' # see prepad and see lead byte + qb2 = b'\xdb\xff\xfe\xfd0\x00uvw' raw = b'uvw' assert rs == 3 @@ -2087,8 +2087,8 @@ def test_matter_special(): # Same as above but raw all zeros - qb64 = '2__-TGAAAAAA' - qb2 = b'\xdb\xff\xfeL`\x00\x00\x00\x00' + qb64 = '2__-_TAAAAAA' + qb2 = b'\xdb\xff\xfe\xfd0\x00\x00\x00\x00' raw = b'\x00\x00\x00' assert rs == 3 @@ -3367,7 +3367,6 @@ def test_tagger(): # Tag1 tag = 'v' code = MtrDex.Tag1 - soft = '_v' qb64 = '0J_v' qb64b = qb64.encode("utf-8") qb2 = decodeB64(qb64b) @@ -3375,7 +3374,7 @@ def test_tagger(): tagger = Tagger(tag=tag) # defaults assert tagger.code == tagger.hard == code - assert tagger.soft == soft + assert tagger.soft == tag assert tagger.raw == raw assert tagger.qb64 == qb64 assert tagger.qb2 == qb2 @@ -3385,7 +3384,7 @@ def test_tagger(): tagger = Tagger(qb2=qb2) assert tagger.code == tagger.hard == code - assert tagger.soft == soft + assert tagger.soft == tag assert tagger.raw == raw assert tagger.qb64 == qb64 assert tagger.qb2 == qb2 @@ -3395,7 +3394,7 @@ def test_tagger(): tagger = Tagger(qb64=qb64) assert tagger.code == tagger.hard == code - assert tagger.soft == soft + assert tagger.soft == tag assert tagger.raw == raw assert tagger.qb64 == qb64 assert tagger.qb2 == qb2 @@ -3405,7 +3404,7 @@ def test_tagger(): tagger = Tagger(qb64b=qb64b) assert tagger.code == tagger.hard == code - assert tagger.soft == soft + assert tagger.soft == tag assert tagger.raw == raw assert tagger.qb64 == qb64 assert tagger.qb2 == qb2 From f5840bc78794171ea832c368d4b1e2b0cce32fd2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 27 Jul 2024 13:51:17 -0600 Subject: [PATCH 214/418] fixed up comments removed dead code. Now Tagger does not have to fix up prepad after the fact but is natively supported by Matter with xtra xs code table size field. --- src/keri/core/coring.py | 41 ++++----------------------------------- tests/core/test_coring.py | 2 +- 2 files changed, 5 insertions(+), 38 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 6bbc3c10a..552fffca9 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -672,26 +672,6 @@ def __iter__(self): TagDex = TagCodex() # Make instance -@dataclass(frozen=True) -class PadTagCodex: - """ - TagCodex is codex of Base64 derivation codes for compactly representing - various small Base64 tag values as prepadded special code soft part values. - Prepad is 1 B64 char. - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Tag1: str = '0J' # 1 B64 char tag with 1 pre pad - Tag5: str = '0L' # 1 B64 char tag with 1 pre pad - Tag9: str = '0N' # 1 B64 char tag with 1 pre pad - - def __iter__(self): - return iter(astuple(self)) - - -PadTagDex = PadTagCodex() # Make instance - @dataclass(frozen=True) class PreCodex: @@ -2106,7 +2086,7 @@ class Tagger(Matter): composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip Properties: - tag (str): B64 primitive without prepad (strips prepad from soft) + tag (str): B64 primitive without prepad (alias of .soft) Inherited Hidden: (See Matter) @@ -2127,8 +2107,6 @@ class Tagger(Matter): Methods: - def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, - qb64b=None, qb64=None, qb2=None, strip=False): """ Pad = '_' # B64 pad char for tag codes with pre-padded soft values @@ -2150,7 +2128,7 @@ def __init__(self, tag='', soft='', code=None, **kwa): bytearray after parsing qb64b or qb2. False means do not strip Parameters: - tag (str | bytes): Base64 plain. Prepad is added as needed. + tag (str | bytes): Base64 automatic sets code given size of tag """ if tag: @@ -2164,10 +2142,6 @@ def __init__(self, tag='', soft='', code=None, **kwa): if l > len(codes): raise InvalidSoftError("Oversized tag={soft}.") code = codes[l-1] # get code for for tag of len where (index = len - 1) - #if code in PadTagDex: - #soft = self.Pad + tag # pre pad for those that need it - #else: - #soft = tag soft = tag @@ -2179,17 +2153,10 @@ def __init__(self, tag='', soft='', code=None, **kwa): @property def tag(self): """Returns: - tag (str): B64 primitive without prepad (strips prepad from soft) + tag (str): B64 primitive without prepad (alias of self.soft) """ - tag = self.soft - #if self.code in PadTagDex: - #pad = self.soft[0] - #tag = self.soft[1:] - #if pad != self.Pad: - #raise InvalidSoftError("Invaid pre {pad=} for {tag=}.") - - return tag + return self.soft class Ilker(Tagger): diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 6eca06a47..2dc1344e7 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -40,7 +40,7 @@ from keri.core import coring from keri.core.coring import (Saids, Sadder, Tholder, Seqner, NumDex, Number, Dater, Bexter, Texter, - TagDex, PadTagDex, Tagger, Ilker, Traitor, + TagDex, Tagger, Ilker, Traitor, Verser, Versage, ) from keri.core.coring import Kindage, Kinds from keri.core.coring import (Sizage, MtrDex, Matter) From b63a9f9629d7bf5541f699ab98c622d295e2d6c8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 27 Jul 2024 13:53:37 -0600 Subject: [PATCH 215/418] removed spurious Pad, now in super class --- src/keri/core/coring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 552fffca9..84bd0893c 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -2109,7 +2109,7 @@ class Tagger(Matter): """ - Pad = '_' # B64 pad char for tag codes with pre-padded soft values + def __init__(self, tag='', soft='', code=None, **kwa): """ From af2e280567550aa2f284e1f426172331cf70eaa1 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 28 Jul 2024 22:00:49 -0600 Subject: [PATCH 216/418] refactor streamlined xtra xs field support in Matter --- src/keri/core/coring.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 84bd0893c..2c47104e1 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -1424,21 +1424,18 @@ def _exfil(self, qb64b): # these are well formed. # when fs is None then ss > 0 otherwise fs > hs + ss when ss > 0 - xtra = qb64b[hs:hs+xs] # extract xtra prepad chars of soft, empty when xs==0 - if isinstance(xtra, memoryview): - xtra = bytes(xtra) - if hasattr(xtra, "decode"): - xtra = xtra.decode() # converts bytes/bytearray to str - if xtra != f"{self.Pad * xs}": - raise UnexpectedCodeError(f"Invalid prepad xtra ={xtra}.") - # extract soft chars excluding xtra, empty when ss==0 and xs == 0 + # extract soft chars including xtra, empty when ss==0 and xs == 0 # assumes that when ss == 0 then xs must be 0 - soft = qb64b[hs+xs:hs+ss] + soft = qb64b[hs:hs+ss] if isinstance(soft, memoryview): soft = bytes(soft) if hasattr(soft, "decode"): soft = soft.decode() # converts bytes/bytearray to str + xtra = soft[:xs] # extract xtra if any from front of soft + soft = soft[xs:] # strip xtra from soft + if xtra != f"{self.Pad * xs}": + raise UnexpectedCodeError(f"Invalid prepad xtra ={xtra}.") if not fs: # compute fs from soft from ss part which provides size B64 # compute variable size as int may have value 0 @@ -1516,13 +1513,14 @@ def _bexfil(self, qb2): raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code - xtra = both[hs:hs+xs] # extract xtra prepad chars of soft, empty when xs==0 - if xtra != f"{self.Pad * xs}": - raise UnexpectedCodeError(f"Invalid prepad xtra ={xtra}.") - # extract soft chars excluding xtra, empty when ss==0 and xs == 0 + # extract soft chars including xtra, empty when ss==0 and xs == 0 # assumes that when ss == 0 then xs must be 0 - soft = both[hs+xs:hs+ss] # get soft may be empty + soft = both[hs:hs+ss] # get soft may be empty + xtra = soft[:xs] # extract xtra if any from front of soft + soft = soft[xs:] # strip xtra from soft + if xtra != f"{self.Pad * xs}": + raise UnexpectedCodeError(f"Invalid prepad xtra ={xtra}.") if not fs: # compute fs from size chars in ss part of code if len(qb2) < bcs: # need more bytes From 77010bd02a7d6849db246d75ba1081ca3e7b996e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 29 Jul 2024 10:09:26 -0600 Subject: [PATCH 217/418] move Cipher related Codices to core.signing.py so same place as Cipher --- src/keri/core/coring.py | 109 ------------------------------------ src/keri/core/signing.py | 112 +++++++++++++++++++++++++++++++++++++ tests/core/test_signing.py | 57 +++++++++---------- 3 files changed, 141 insertions(+), 137 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 2c47104e1..ebbab6309 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -465,115 +465,6 @@ def __iter__(self): TexDex = TextCodex() # Make instance -@dataclass(frozen=True) -class CipherX25519VarCodex: - """ - CipherX25519VarCodex is codex all variable sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is B2. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_L0: str = '4D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 - X25519_Cipher_L1: str = '5D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 - X25519_Cipher_L2: str = '6D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 - - def __iter__(self): - return iter(astuple(self)) - - -CiXVarDex = CipherX25519VarCodex() # Make instance - - -@dataclass(frozen=True) -class CipherX25519FixQB64Codex: - """ - CipherX25519FixQB64Codex is codex all fixed sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is B64. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed - X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt - - def __iter__(self): - return iter(astuple(self)) - - -CiXFixQB64Dex = CipherX25519FixQB64Codex() # Make instance - - -@dataclass(frozen=True) -class CipherX25519VarQB64Codex: - """ - CipherX25519VarQB64Codex is codex all variable sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is QB64. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 - X25519_Cipher_QB64_L1: str = '5E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 - X25519_Cipher_QB64_L2: str = '6E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 - X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 - X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 - X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 - - def __iter__(self): - return iter(astuple(self)) - - -CiXVarQB64Dex = CipherX25519VarQB64Codex() # Make instance - - -@dataclass(frozen=True) -class CipherX25519AllQB64Codex: - """ - CipherX25519AllQB64Codex is codex all both fixed and variable sized cipher bytes - derivation codes for sealed box encryped ciphertext. Plaintext is B64. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed - X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt - X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 - X25519_Cipher_QB64_L1: str = '5E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 - X25519_Cipher_QB64_L2: str = '6E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 - X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 - X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 - X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 - - def __iter__(self): - return iter(astuple(self)) - - -CiXAllQB64Dex = CipherX25519AllQB64Codex() # Make instance - - -@dataclass(frozen=True) -class CipherX25519QB2VarCodex: - """ - CipherX25519QB2VarCodex is codex all variable sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is B2. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 - X25519_Cipher_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 - X25519_Cipher_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 - - def __iter__(self): - return iter(astuple(self)) - - -CiXVarQB2Dex = CipherX25519QB2VarCodex() # Make instance - - - @dataclass(frozen=True) class NonTransCodex: diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index 875ce179b..404c77062 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -4,6 +4,7 @@ Provides support Signer class """ +from dataclasses import dataclass, astuple, asdict import pysodium @@ -25,6 +26,117 @@ ECDSA_256k1_SEEDBYTES = 32 +@dataclass(frozen=True) +class CipherX25519VarCodex: + """ + CipherX25519VarCodex is codex all variable sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is B2. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_L0: str = '4D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 + X25519_Cipher_L1: str = '5D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 + X25519_Cipher_L2: str = '6D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + + +CiXVarDex = CipherX25519VarCodex() # Make instance + + +@dataclass(frozen=True) +class CipherX25519FixQB64Codex: + """ + CipherX25519FixQB64Codex is codex all fixed sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is B64. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed + X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt + + def __iter__(self): + return iter(astuple(self)) + + +CiXFixQB64Dex = CipherX25519FixQB64Codex() # Make instance + + +@dataclass(frozen=True) +class CipherX25519VarQB64Codex: + """ + CipherX25519VarQB64Codex is codex all variable sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is QB64. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 + X25519_Cipher_QB64_L1: str = '5E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 + X25519_Cipher_QB64_L2: str = '6E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 + X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 + X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 + X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + + +CiXVarQB64Dex = CipherX25519VarQB64Codex() # Make instance + + +@dataclass(frozen=True) +class CipherX25519AllQB64Codex: + """ + CipherX25519AllQB64Codex is codex all both fixed and variable sized cipher bytes + derivation codes for sealed box encryped ciphertext. Plaintext is B64. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed + X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt + X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 + X25519_Cipher_QB64_L1: str = '5E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 + X25519_Cipher_QB64_L2: str = '6E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 + X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 + X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 + X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + + +CiXAllQB64Dex = CipherX25519AllQB64Codex() # Make instance + + +@dataclass(frozen=True) +class CipherX25519QB2VarCodex: + """ + CipherX25519QB2VarCodex is codex all variable sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is B2. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 + X25519_Cipher_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 + X25519_Cipher_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + + +CiXVarQB2Dex = CipherX25519QB2VarCodex() # Make instance + + + + + class Signer(Matter): """ Signer is Matter subclass with method to create signature of serialization diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index f0e519db2..b38477183 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -375,33 +375,6 @@ def test_signer(): """ Done Test """ -def test_generatesigners(): - """ - Test the support function genSigners - - """ - signers = Salter().signers(count=2, temp=True, transferable=False) - assert len(signers) == 2 - for signer in signers: - assert signer.verfer.code == MtrDex.Ed25519N - - # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) # raw salt - raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - assert len(raw) == 16 - signers = Salter(raw=raw).signers(count=4) - assert len(signers) == 4 - for signer in signers: - assert signer.code == MtrDex.Ed25519_Seed - assert signer.verfer.code == MtrDex.Ed25519 - - sigkeys = [signer.qb64 for signer in signers] - assert sigkeys == ['AK8F6AAiYDpXlWdj2O5F5-6wNCCNJh2A4XOlqwR_HwwH', - 'AOs8-zNPPh0EhavdrCfCiTk9nGeO8e6VxUCzwdKXJAd0', - 'AHMBU5PsIJN2U9m7j0SGyvs8YD8fkym2noELzxIrzfdG', - 'AJZ7ZLd7unQ4IkMUwE69NXcvDO9rrmmRH_Xk3TPu9BpP'] - - """ End Test """ - def test_salter(): """ @@ -452,6 +425,34 @@ def test_salter(): """ Done Test """ +def test_gensignerswithsalter(): + """ + Test generating a set of signers using Salter + + """ + signers = Salter().signers(count=2, temp=True, transferable=False) + assert len(signers) == 2 + for signer in signers: + assert signer.verfer.code == MtrDex.Ed25519N + + # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) # raw salt + raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' + assert len(raw) == 16 + signers = Salter(raw=raw).signers(count=4) + assert len(signers) == 4 + for signer in signers: + assert signer.code == MtrDex.Ed25519_Seed + assert signer.verfer.code == MtrDex.Ed25519 + + sigkeys = [signer.qb64 for signer in signers] + assert sigkeys == ['AK8F6AAiYDpXlWdj2O5F5-6wNCCNJh2A4XOlqwR_HwwH', + 'AOs8-zNPPh0EhavdrCfCiTk9nGeO8e6VxUCzwdKXJAd0', + 'AHMBU5PsIJN2U9m7j0SGyvs8YD8fkym2noELzxIrzfdG', + 'AJZ7ZLd7unQ4IkMUwE69NXcvDO9rrmmRH_Xk3TPu9BpP'] + + """ End Test """ + + def test_cipher(): """ Test Cipher subclass of Matter @@ -702,8 +703,8 @@ def test_decrypter(): if __name__ == "__main__": test_signer() - test_generatesigners() test_salter() + test_gensignerswithsalter() test_cipher() test_encrypter() test_decrypter() From b8d56b842a8bb238e447296609a8f4dfbb1fed48 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 29 Jul 2024 11:56:27 -0600 Subject: [PATCH 218/418] Started adding support for variable sized ciphers to Cipher --- src/keri/core/__init__.py | 2 +- src/keri/core/signing.py | 307 +++++++++++++++++++++++-------------- tests/core/test_signing.py | 51 +++++- 3 files changed, 240 insertions(+), 120 deletions(-) diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index 734d8259c..1042deb62 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -14,5 +14,5 @@ from .coring import Tholder from .indexing import Indexer, Siger, IdrDex, IdxSigDex -from .signing import Signer, Salter, Cipher, Encrypter, Decrypter +from .signing import Signer, Salter, Cipher, CiXDex, Encrypter, Decrypter from .counting import Counter, Codens diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index 404c77062..298e5b78f 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -12,7 +12,7 @@ from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from cryptography.hazmat.primitives.asymmetric import ec, utils -from ..kering import (EmptyMaterialError,) +from ..kering import (EmptyMaterialError, InvalidSizeError, InvalidVarRawSizeError) from ..help import helping @@ -26,115 +26,6 @@ ECDSA_256k1_SEEDBYTES = 32 -@dataclass(frozen=True) -class CipherX25519VarCodex: - """ - CipherX25519VarCodex is codex all variable sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is B2. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_L0: str = '4D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 - X25519_Cipher_L1: str = '5D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 - X25519_Cipher_L2: str = '6D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 - - def __iter__(self): - return iter(astuple(self)) - - -CiXVarDex = CipherX25519VarCodex() # Make instance - - -@dataclass(frozen=True) -class CipherX25519FixQB64Codex: - """ - CipherX25519FixQB64Codex is codex all fixed sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is B64. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed - X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt - - def __iter__(self): - return iter(astuple(self)) - - -CiXFixQB64Dex = CipherX25519FixQB64Codex() # Make instance - - -@dataclass(frozen=True) -class CipherX25519VarQB64Codex: - """ - CipherX25519VarQB64Codex is codex all variable sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is QB64. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 - X25519_Cipher_QB64_L1: str = '5E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 - X25519_Cipher_QB64_L2: str = '6E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 - X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 - X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 - X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 - - def __iter__(self): - return iter(astuple(self)) - - -CiXVarQB64Dex = CipherX25519VarQB64Codex() # Make instance - - -@dataclass(frozen=True) -class CipherX25519AllQB64Codex: - """ - CipherX25519AllQB64Codex is codex all both fixed and variable sized cipher bytes - derivation codes for sealed box encryped ciphertext. Plaintext is B64. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed - X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt - X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 - X25519_Cipher_QB64_L1: str = '5E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 - X25519_Cipher_QB64_L2: str = '6E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 - X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 - X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 - X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 - - def __iter__(self): - return iter(astuple(self)) - - -CiXAllQB64Dex = CipherX25519AllQB64Codex() # Make instance - - -@dataclass(frozen=True) -class CipherX25519QB2VarCodex: - """ - CipherX25519QB2VarCodex is codex all variable sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is B2. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 - X25519_Cipher_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 - X25519_Cipher_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 - - def __iter__(self): - return iter(astuple(self)) - - -CiXVarQB2Dex = CipherX25519QB2VarCodex() # Make instance - - - class Signer(Matter): @@ -588,6 +479,153 @@ def signers(self, count=1, start=0, path="", **kwa): return [self.signer(path=f"{path}{i + start:x}", **kwa) for i in range(count)] + + +# Codes for for ciphers of variable sized sniffable QB2 or QB64 plain text +@dataclass(frozen=True) +class CipherX25519VarSnifCodex: + """ + CipherX25519VarCodex is codex all variable sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is Sniffable QB2 or QB64. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXVarSnifDex = CipherX25519VarSnifCodex() # Make instance + + +# Codes for for ciphers of variable sized QB64 plain text +@dataclass(frozen=True) +class CipherX25519VarQB64Codex: + """ + CipherX25519VarQB64Codex is codex all variable sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is QB64. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 + X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 + X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 + X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 + X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 + X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXVarQB64Dex = CipherX25519VarQB64Codex() # Make instance + +# Codes for for ciphers of fixed sized QB64 plain text +@dataclass(frozen=True) +class CipherX25519FixQB64Codex: + """ + CipherX25519FixQB64Codex is codex all fixed sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is B64. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed + X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt + + def __iter__(self): + return iter(astuple(self)) + +CiXFixQB64Dex = CipherX25519FixQB64Codex() # Make instance + +# Codes for for ciphers of all sizes fixed and variable of QB64 plain text +@dataclass(frozen=True) +class CipherX25519AllQB64Codex: + """ + CipherX25519AllQB64Codex is codex all both fixed and variable sized cipher bytes + derivation codes for sealed box encryped ciphertext. Plaintext is B64. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed + X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt + X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 + X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 + X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 + X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 + X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 + X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXAllQB64Dex = CipherX25519AllQB64Codex() # Make instance + + + +# Codes for for ciphers of variable sized QB2 plain text +@dataclass(frozen=True) +class CipherX25519QB2VarCodex: + """ + CipherX25519QB2VarCodex is codex all variable sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is B2. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_QB2_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 + X25519_Cipher_QB2_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 + X25519_Cipher_QB2_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 + X25519_Cipher_QB2_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 + X25519_Cipher_QB2_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 + X25519_Cipher_QB2_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXVarQB2Dex = CipherX25519QB2VarCodex() # Make instance + + +# Codes for for ciphers of all sizes and all types of plain text +@dataclass(frozen=True) +class CipherX25519AllCodex: + """ + CipherX25519AllCodex is codex all codes and types of cipher bytes + for sealed box encryped ciphertext. Plaintext maybe sniffable or qb64 or qb2. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 + X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed + X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt + X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 + X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 + X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 + X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 + X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 + X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + X25519_Cipher_QB2_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 + X25519_Cipher_QB2_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 + X25519_Cipher_QB2_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 + X25519_Cipher_QB2_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 + X25519_Cipher_QB2_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 + X25519_Cipher_QB2_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXDex = CipherX25519AllCodex() # Make instance + + + + class Cipher(Matter): """ Cipher is Matter subclass holding a cipher text of a secret that may be @@ -605,18 +643,53 @@ class Cipher(Matter): See Matter for inherited attributes and properties """ + Codex = CiXDex + Codes = asdict(CiXDex) # map code name to code - def __init__(self, raw=None, code=None, **kwa): + def __init__(self, raw=None, code=None, fix=False, **kwa): """ Parmeters: - raw (Union[bytes, str]): cipher text + raw (bytes | str): cipher text code (str): cipher suite + """ - if raw is not None and code is None: - if len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Salt): - code = MtrDex.X25519_Cipher_Salt - elif len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Seed): - code = MtrDex.X25519_Cipher_Seed + if raw is not None: + if code is None or code in CiXFixQB64Dex: + if len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Salt): + code = MtrDex.X25519_Cipher_Salt + elif len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Seed): + code = MtrDex.X25519_Cipher_Seed + else: + raise InvalidSizeError(f"Unsupported fixed raw size" + f" {len(raw)} for {code=}.") + + #if code[0] in SmallVrzDex: # compute code with sizes + #if size <= (64 ** 2 - 1): # ss = 2 + #hs = 2 + #s = astuple(SmallVrzDex)[ls] + #code = f"{s}{code[1:hs]}" + #ss = 2 + #elif size <= (64 ** 4 - 1): # ss = 4 make big version of code + #hs = 4 + #s = astuple(LargeVrzDex)[ls] + #code = f"{s}{'A' * (hs - 2)}{code[1]}" + #soft = intToB64(size, 4) + #ss = 4 + #else: + #raise InvalidVarRawSizeError(f"Unsupported raw size for " + #f"{code=}.") + #elif code[0] in LargeVrzDex: # compute code with sizes + #if size <= (64 ** 4 - 1): # ss = 4 + #hs = 4 + #s = astuple(LargeVrzDex)[ls] + #code = f"{s}{code[1:hs]}" + #ss = 4 + #else: + #raise InvalidVarRawSizeError(f"Unsupported raw size for large " + #f"{code=}. {size} <= {64 ** 4 - 1}") + #else: + #raise InvalidVarRawSizeError(f"Unsupported variable raw size " + #f"{code=}.") if hasattr(raw, "encode"): raw = raw.encode("utf-8") # ensure bytes not str diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index b38477183..a9f4d98d4 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -11,8 +11,7 @@ from keri import kering from keri.core import (Indexer, IdrDex, ) from keri.core import (Matter, MtrDex, Cigar, Verfer, Prefixer) -from keri.core import (Signer, Salter, - Cipher, Encrypter, Decrypter, ) +from keri.core import (Signer, Salter, Cipher, CiXDex, Encrypter, Decrypter, ) from keri.core import (Tiers, ) @@ -452,6 +451,38 @@ def test_gensignerswithsalter(): """ End Test """ +def test_cipher_closs(): + """ + Test class attributes of Cipher + """ + assert Cipher.Codex == CiXDex + + assert Cipher.Codes == \ + { + 'X25519_Cipher_L0': '4C', + 'X25519_Cipher_L1': '5C', + 'X25519_Cipher_L2': '6C', + 'X25519_Cipher_Big_L0': '7AAC', + 'X25519_Cipher_Big_L1': '8AAC', + 'X25519_Cipher_Big_L2': '9AAC', + 'X25519_Cipher_Seed': 'P', + 'X25519_Cipher_Salt': '1AAH', + 'X25519_Cipher_QB64_L0': '4D', + 'X25519_Cipher_QB64_L1': '5D', + 'X25519_Cipher_QB64_L2': '6D', + 'X25519_Cipher_QB64_Big_L0': '7AAD', + 'X25519_Cipher_QB64_Big_L1': '8AAD', + 'X25519_Cipher_QB64_Big_L2': '9AAD', + 'X25519_Cipher_QB2_L0': '4E', + 'X25519_Cipher_QB2_L1': '5E', + 'X25519_Cipher_QB2_L2': '6E', + 'X25519_Cipher_QB2_Big_L0': '7AAE', + 'X25519_Cipher_QB2_Big_L1': '8AAE', + 'X25519_Cipher_QB2_Big_L2': '9AAE' + } + + """End Test""" + def test_cipher(): """ @@ -512,6 +543,21 @@ def test_cipher(): with pytest.raises(ValueError): # bad code cipher = Cipher(raw=raw, code=MtrDex.Ed25519N) + + # Test bad raw size + raw = pysodium.crypto_box_seal(seedqb64b, pubkey) # uses nonce so different everytime + with pytest.raises(kering.InvalidSizeError): + cipher = Cipher(raw=raw[:len(raw)-1]) # make raw too small + with pytest.raises(kering.InvalidSizeError): + cipher = Cipher(raw=raw + b'_') # make raw too big + + raw = pysodium.crypto_box_seal(saltqb64b, pubkey) # uses nonce so different everytime + with pytest.raises(kering.InvalidSizeError): + cipher = Cipher(raw=raw[:len(raw)-1]) # make raw too small + with pytest.raises(kering.InvalidSizeError): + cipher = Cipher(raw=raw + b'_') # make raw too big + + """ Done Test """ @@ -705,6 +751,7 @@ def test_decrypter(): test_signer() test_salter() test_gensignerswithsalter() + test_cipher_closs() test_cipher() test_encrypter() test_decrypter() From 486a5f33138242fcbc45d32df7a285080caec783 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 29 Jul 2024 12:32:09 -0600 Subject: [PATCH 219/418] some refactoring --- src/keri/core/signing.py | 81 +++++++++++++++++++++----------------- tests/core/test_signing.py | 2 +- 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index 298e5b78f..716e796ad 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -12,12 +12,13 @@ from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from cryptography.hazmat.primitives.asymmetric import ec, utils -from ..kering import (EmptyMaterialError, InvalidSizeError, InvalidVarRawSizeError) +from ..kering import (EmptyMaterialError, InvalidCodeError, InvalidSizeError, + InvalidVarRawSizeError) from ..help import helping from .coring import (Tiers, ) -from .coring import (Matter, MtrDex, Verfer, Cigar) +from .coring import (SmallVrzDex, LargeVrzDex, Matter, MtrDex, Verfer, Cigar) from .indexing import IdrDex, Siger @@ -587,6 +588,39 @@ def __iter__(self): CiXVarQB2Dex = CipherX25519QB2VarCodex() # Make instance +# Codes for for ciphers of all varibale sizes and all types of plain text +@dataclass(frozen=True) +class CipherX25519AllVarCodex: + """ + CipherX25519AllVarCodex is codex all variable size codes of cipher bytes + for sealed box encryped ciphertext. Plaintext maybe sniffable or qb64 or qb2. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 + X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 + X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 + X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 + X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 + X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 + X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + X25519_Cipher_QB2_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 + X25519_Cipher_QB2_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 + X25519_Cipher_QB2_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 + X25519_Cipher_QB2_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 + X25519_Cipher_QB2_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 + X25519_Cipher_QB2_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXVarDex = CipherX25519AllVarCodex() # Make instance + # Codes for for ciphers of all sizes and all types of plain text @dataclass(frozen=True) @@ -653,51 +687,26 @@ def __init__(self, raw=None, code=None, fix=False, **kwa): code (str): cipher suite """ - if raw is not None: - if code is None or code in CiXFixQB64Dex: + # default when raw is not None and code is None is to use fixed size + # code given by raw size. Otherwise provided code fixed or variable size + # is handled by Matter superclass. + if raw is not None and code is None: if len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Salt): code = MtrDex.X25519_Cipher_Salt elif len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Seed): code = MtrDex.X25519_Cipher_Seed else: raise InvalidSizeError(f"Unsupported fixed raw size" - f" {len(raw)} for {code=}.") - - #if code[0] in SmallVrzDex: # compute code with sizes - #if size <= (64 ** 2 - 1): # ss = 2 - #hs = 2 - #s = astuple(SmallVrzDex)[ls] - #code = f"{s}{code[1:hs]}" - #ss = 2 - #elif size <= (64 ** 4 - 1): # ss = 4 make big version of code - #hs = 4 - #s = astuple(LargeVrzDex)[ls] - #code = f"{s}{'A' * (hs - 2)}{code[1]}" - #soft = intToB64(size, 4) - #ss = 4 - #else: - #raise InvalidVarRawSizeError(f"Unsupported raw size for " - #f"{code=}.") - #elif code[0] in LargeVrzDex: # compute code with sizes - #if size <= (64 ** 4 - 1): # ss = 4 - #hs = 4 - #s = astuple(LargeVrzDex)[ls] - #code = f"{s}{code[1:hs]}" - #ss = 4 - #else: - #raise InvalidVarRawSizeError(f"Unsupported raw size for large " - #f"{code=}. {size} <= {64 ** 4 - 1}") - #else: - #raise InvalidVarRawSizeError(f"Unsupported variable raw size " - #f"{code=}.") + f" {len(raw)} for {code=}.") if hasattr(raw, "encode"): raw = raw.encode("utf-8") # ensure bytes not str super(Cipher, self).__init__(raw=raw, code=code, **kwa) - if self.code not in (MtrDex.X25519_Cipher_Salt, MtrDex.X25519_Cipher_Seed): - raise ValueError("Unsupported cipher code = {}.".format(self.code)) + if self.code not in CiXDex: + raise InvalidCodeError(f"Unsupported cipher code = {self.code}.") + def decrypt(self, prikey=None, seed=None): """ @@ -706,7 +715,7 @@ def decrypt(self, prikey=None, seed=None): qualified (qb64) so derivaton code of plain text preserved through encryption/decryption round trip. - Uses either decryption key given by prikey or derives prikey from + Decrypter uses either decryption key given by prikey or derives prikey from signing key derived from private seed. Parameters: diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index a9f4d98d4..1061c9a9f 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -541,7 +541,7 @@ def test_cipher(): cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b assert cipher.decrypt(seed=cryptseedqb64).qb64b == saltqb64b - with pytest.raises(ValueError): # bad code + with pytest.raises(kering.InvalidCodeError): # bad code cipher = Cipher(raw=raw, code=MtrDex.Ed25519N) # Test bad raw size From ad49eee6fcb27ea0ea4936a0f705808567b18c38 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 29 Jul 2024 12:37:17 -0600 Subject: [PATCH 220/418] updated Usage comment --- src/keri/help/helping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index 838f69bf4..5333887e5 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -266,7 +266,7 @@ def fromIso8601(dts): B64REX = rb'^[0-9A-Za-z_-]*\Z' # [A-Za-z0-9\-\_] Reb64 = re.compile(B64REX) # compile is faster -# to use if Reb64.match(bext): +# Usage: if Reb64.match(bext): or if not Reb64.match(bext): def intToB64(i, l=1): From e6274a7b4f5d8c346b2486c119093996fbbba52c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 29 Jul 2024 12:51:14 -0600 Subject: [PATCH 221/418] some refactoring --- src/keri/core/signing.py | 8 ++++---- src/keri/kering.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index 716e796ad..79e08e238 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -13,7 +13,7 @@ from cryptography.hazmat.primitives.asymmetric import ec, utils from ..kering import (EmptyMaterialError, InvalidCodeError, InvalidSizeError, - InvalidVarRawSizeError) + RawMaterialError) from ..help import helping @@ -670,7 +670,7 @@ class Cipher(Matter): private key for decryption. The default is to use X25519 sealed box encryption. The Cipher instances .raw is the raw binary encrypted cipher text and its - .code indicates what type of secret has been encrypted. The cipher suite used + .code indicates what type of plain text has been encrypted. The cipher suite used for the encryption/decryption is implied by the context where the cipher is used. @@ -680,10 +680,10 @@ class Cipher(Matter): Codex = CiXDex Codes = asdict(CiXDex) # map code name to code - def __init__(self, raw=None, code=None, fix=False, **kwa): + def __init__(self, raw=None, code=None, **kwa): """ Parmeters: - raw (bytes | str): cipher text + raw (bytes | str): cipher text (not plain text) code (str): cipher suite """ diff --git a/src/keri/kering.py b/src/keri/kering.py index 725e3b961..2177b1be8 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -461,7 +461,7 @@ class MaterialError(KeriError): class RawMaterialError(MaterialError): """ - Not Enough bytes in buffer bytearray for raw material + Invalid raw material Usage: raise RawMaterialError("error message") """ @@ -469,7 +469,7 @@ class RawMaterialError(MaterialError): class SoftMaterialError(MaterialError): """ - Not Enough chars in soft for soft material + Invalid soft material Usage: raise SoftMaterialError("error message") """ From c7c547f306eb5d1ee6c555a9ad05abb2cc46976b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 29 Jul 2024 12:51:56 -0600 Subject: [PATCH 222/418] removed unused import --- src/keri/core/signing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index 79e08e238..dcfaec05a 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -12,8 +12,7 @@ from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from cryptography.hazmat.primitives.asymmetric import ec, utils -from ..kering import (EmptyMaterialError, InvalidCodeError, InvalidSizeError, - RawMaterialError) +from ..kering import (EmptyMaterialError, InvalidCodeError, InvalidSizeError) from ..help import helping From 4678826605cec4de46fa042514ccab23752fe663 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 29 Jul 2024 15:41:16 -0600 Subject: [PATCH 223/418] Finished support in Cipher for variable sized raw of all the types Sniffable, QB64, and QB2 with unit tests. --- src/keri/core/__init__.py | 2 +- src/keri/core/coring.py | 2 +- tests/core/test_signing.py | 185 ++++++++++++++++++++++++++++++++++++- 3 files changed, 185 insertions(+), 4 deletions(-) diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index 1042deb62..ec83d8bbb 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -15,4 +15,4 @@ from .coring import Tholder from .indexing import Indexer, Siger, IdrDex, IdxSigDex from .signing import Signer, Salter, Cipher, CiXDex, Encrypter, Decrypter -from .counting import Counter, Codens +from .counting import Counter, Codens, CtrDex_2_0 diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index ebbab6309..5896ffe09 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -622,7 +622,7 @@ class Matter: Names (dict): maps code to code name Pad (str): B64 pad char for xtra size pre-padded soft values - Calss Methods: + Class Methods: Attributes: diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index 1061c9a9f..9b8ec4d16 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -3,12 +3,12 @@ tests.core.test_indexing module """ - +from base64 import urlsafe_b64decode as decodeB64 import pysodium import pytest -from keri import kering +from keri import kering, core from keri.core import (Indexer, IdrDex, ) from keri.core import (Matter, MtrDex, Cigar, Verfer, Prefixer) from keri.core import (Signer, Salter, Cipher, CiXDex, Encrypter, Decrypter, ) @@ -521,6 +521,11 @@ def test_cipher(): uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) assert uncb == seedqb64b + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_Seed) + assert cipher.code == MtrDex.X25519_Cipher_Seed + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == seedqb64b + # test .decrypt method needs qb64 prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b assert cipher.decrypt(prikey=prikeyqb64).qb64b == seedqb64b @@ -528,12 +533,27 @@ def test_cipher(): cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b assert cipher.decrypt(seed=cryptseedqb64).qb64b == seedqb64b + # wrong but shorter code so instance creation succeeds + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_Salt) + assert cipher.code == MtrDex.X25519_Cipher_Salt + with pytest.raises(ValueError): # but decryption fails + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + + raw = pysodium.crypto_box_seal(saltqb64b, pubkey) # uses nonce so different everytime cipher = Cipher(raw=raw) assert cipher.code == MtrDex.X25519_Cipher_Salt uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) assert uncb == saltqb64b + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_Salt) + assert cipher.code == MtrDex.X25519_Cipher_Salt + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == saltqb64b + + with pytest.raises(kering.RawMaterialError): # wrong code to big for raw bytes + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_Seed) + # test .decrypt method needs qb64 prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b assert cipher.decrypt(prikey=prikeyqb64).qb64b == saltqb64b @@ -557,6 +577,167 @@ def test_cipher(): with pytest.raises(kering.InvalidSizeError): cipher = Cipher(raw=raw + b'_') # make raw too big + # variable sized ciphers + # Always init with L0 code and let Matter auto correct the code for the + # actual lead size of raw + + # qb64 lead 0 + plain = "The quick brown fox jumps over the lazy " + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 108 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L0 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == texter.qb64b + texter = core.Texter(qb64b=uncb) + assert texter.text == plain + + # qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dogcats" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 116 + assert (3 - (len(raw) % 3)) % 3 == 1 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L1 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == texter.qb64b + texter = core.Texter(qb64b=uncb) + assert texter.text == plain + + # qb64 lead 2 + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 112 + assert (3 - (len(raw) % 3)) % 3 == 2 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L2 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == texter.qb64b + texter = core.Texter(qb64b=uncb) + assert texter.text == plain + + # qb2 lead 0 (always lead 0 when qb2 from texter) + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb2, pubkey) # uses nonce so different everytime + assert len(raw) == 96 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB2_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB2_L0 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == texter.qb2 + texter = core.Texter(qb2=uncb) + assert texter.text == plain + + # sniffable qb64 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + peb = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(peb, pubkey) # uses nonce so different everytime + assert len(raw) == 108 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == peb + peb = bytearray(peb) + counter = core.Counter(qb64b=peb, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=peb, strip=True) + assert texter.text == plain + + # sniffable qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + peb = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(peb, pubkey) # uses nonce so different everytime + assert len(raw) == 116 + assert (3 - (len(raw) % 3)) % 3 == 1 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L1 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == peb + peb = bytearray(peb) + counter = core.Counter(qb64b=peb, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=peb, strip=True) + assert texter.text == plain + + # sniffable qb64 lead 2 + plain = "The quick brown fox jumps over the lazy " + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + peb = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(peb, pubkey) # uses nonce so different everytime + assert len(raw) == 112 + assert (3 - (len(raw) % 3)) % 3 == 2 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L2 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == peb + peb = bytearray(peb) + counter = core.Counter(qb64b=peb, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=peb, strip=True) + assert texter.text == plain + + # sniffable qb2 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + assert counter.code == core.CtrDex_2_0.GenericGroup + peb = counter.qb64b + texter.qb64b + pebqb2 = decodeB64(peb) + raw = pysodium.crypto_box_seal(pebqb2, pubkey) # uses nonce so different everytime + assert len(raw) == 93 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == pebqb2 + pebqb2 = bytearray(pebqb2) + counter = core.Counter(qb2=pebqb2, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb2=pebqb2, strip=True) + assert texter.text == plain + + + # sniffable qb2 lead 0 Big + + plain = "The quick brown fox jumps over the lazy" * 324 + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + assert counter.code == core.CtrDex_2_0.BigGenericGroup + peb = counter.qb64b + texter.qb64b + pebqb2 = decodeB64(peb) + raw = pysodium.crypto_box_seal(pebqb2, pubkey) # uses nonce so different everytime + assert len(raw) == 12696 + assert (3 - (len(raw) % 3)) % 3 == 0 + assert (len(raw) // 3 ) > (64 ** 2 - 1) # triplets + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_Big_L0 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == pebqb2 + pebqb2 = bytearray(pebqb2) + counter = core.Counter(qb2=pebqb2, strip=True) + assert counter.code == core.CtrDex_2_0.BigGenericGroup + texter = core.Texter(qb2=pebqb2, strip=True) + assert texter.text == plain """ Done Test """ From 3909bd8dae30453b723885104ca80fc2d050511e Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Wed, 31 Jul 2024 09:32:59 -0400 Subject: [PATCH 224/418] updates dev tag Signed-off-by: Kevin Griffin --- Makefile | 8 ++++---- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 68d73ccb7..4ef478940 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ .PHONY: build-keri build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev11 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev11-arm64 . + @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev12 . + @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev12-arm64 . .PHONY: build-witness-demo build-witness-demo: @@ -11,8 +11,8 @@ build-witness-demo: .PHONY: publish-keri publish-keri: - @docker push weboftrust/keri:1.2.0-dev11 - @docker push weboftrust/keri:1.2.0-dev11-arm64 + @docker push weboftrust/keri:1.2.0-dev12 + @docker push weboftrust/keri:1.2.0-dev12-arm64 .PHONY: publish-keri-witness-demo publish-keri-witness-demo: diff --git a/setup.py b/setup.py index a0240086a..34112fa0d 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev11', # also change in src/keri/__init__.py + version='1.2.0-dev12', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 0e48f5c9a..e4125ad57 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev11' # also change in setup.py +__version__ = '1.2.0-dev12' # also change in setup.py From 2918f86cb465f7f4d3d08252dce69c758bccd273 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 3 Aug 2024 13:07:52 -0600 Subject: [PATCH 225/418] fix doc strings comments --- src/keri/app/habbing.py | 6 +++--- src/keri/app/keeping.py | 7 +++--- src/keri/core/streaming.py | 44 ++++++++++++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 44516caa8..d32a858f5 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -1321,13 +1321,13 @@ def sign(self, ser, verfers=None, indexed=True, indices=None, ondices=None, **kw ondices=ondices) def decrypt(self, ser, verfers=None, **kwa): - """Sign given serialization ser using appropriate keys. + """Decrypt given serialization ser using appropriate keys. Use provided verfers or .kever.verfers to lookup keys to sign. Parameters: - ser (bytes): serialization to sign + ser (bytes): serialization to decrypt verfers (list[Verfer] | None): Verfer instances to get pub verifier - keys to lookup private siging keys. + keys to lookup and convert to private decryption keys. verfers None means use .kever.verfers. Assumes that when group and verfers is not None then provided verfers must be .kever.verfers diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index 3a626a622..0e29fff6f 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -1394,10 +1394,10 @@ def sign(self, ser, pubs=None, verfers=None, indexed=True, cigars.append(signer.sign(ser)) # assigns .verfer to cigar return cigars + def decrypt(self, ser, pubs=None, verfers=None): """ - Returns list of signatures of ser if indexed as Sigers else as Cigars with - .verfer assigned. + Returns plain text of decrypted serialization Parameters: ser (bytes): serialization to sign @@ -1405,7 +1405,8 @@ def decrypt(self, ser, pubs=None, verfers=None): one of pubs or verfers is required. If both then verfers is ignored. verfers (list[Verfer] | None): Verfer instances of public keys one of pubs or verfers is required. If both then verfers is ignored. - If not pubs then gets public key from verfer.qb64 + If not pubs then gets public key from verfer.qb64 used to lookup + private keys Returns: bytes: decrypted data diff --git a/src/keri/core/streaming.py b/src/keri/core/streaming.py index 617a74d21..4ad253ecc 100644 --- a/src/keri/core/streaming.py +++ b/src/keri/core/streaming.py @@ -48,7 +48,7 @@ def annot(ims): if not isinstance(ims, bytearray): # going to strip - ims = bytearray(ims) # so make bytearray copy + ims = bytearray(ims) # so make bytearray copy, converts str to bytearray while ims: # right now just for KERI event messages cold = sniff(ims) # check for spurious counters at front of stream @@ -342,14 +342,22 @@ class Streamer: def __init__(self, stream): """Initialize instance + Holds sniffable CESR stream as byte like string + either (bytes, bytearray, or memoryview) Parameters: - stream (bytes | bytearray): sniffable CESR stream + stream (str | bytes | bytearray | memoryview): sniffable CESR stream + """ - self._stream = bytes(stream) + if hasattr(stream, "encode"): + stream = bytearray(stream.encode()) # convert str to bytearray + if not isinstance(stream, (bytes, bytearray, memoryview)): + raise kering.InvalidTypeError(f"Invalid stream type, not byteable.") + + self._stream = stream @property @@ -360,7 +368,9 @@ def stream(self): @property def text(self): - """expanded stream as qb64 text + """expanded stream where all primitives and groups in stream are + individually expanded to qb64. + Requires parsing full depth to ensure expanded consistently. Returns: stream (bytes): expanded text qb64 version of stream @@ -369,7 +379,9 @@ def text(self): @property def binary(self): - """compacted stream as qb2 binary + """compacted stream where all primitives and groups in stream are + individually compacted to qb2. + Requires parsing full depth to ensure compacted consistently Returns: stream (bytes): compacted binary qb2 version of stream @@ -378,13 +390,33 @@ def binary(self): @property def texter(self): - """expanded stream as Texter instance + """stream as Texter instance. + Texter(text=self.stream) Returns: texter (Texter): Texter primitive of stream suitable wrapping """ return self._stream + @property + def bexter(self): + """stream as Bexter instance. + Bexter of expanded text version of stream. + First expand to text which requires parsing then create bexter + Bexter(bext=self.text) + Because sniffable stream MUST NOT start with 'A' then there is no + length ambiguity. The only tritet collison of 'A' is with '-' but the + remaining 5 bits are guaranteed to always be different. So bexter must + check not just the starting tritet but the full starting byte to ensure + not 'A' as first byte. + + Requires parsing to ensure qb64 + Returns: + bexter (Bexter): Bexter primitive of stream suitable wrapping + + """ + return self._stream + From edbdcee4809eb0313cc5c9e0044fd7471dcbafe4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 3 Aug 2024 13:37:59 -0600 Subject: [PATCH 226/418] start refactoring Encrypter Fix comments doc strings --- src/keri/app/keeping.py | 2 +- src/keri/core/signing.py | 23 +++++++++++++---------- src/keri/db/subing.py | 4 ++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index 0e29fff6f..6949d748d 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -805,7 +805,7 @@ def updateAeid(self, aeid, seed): for keys, data in self.ks.prms.getItemIter(): # keys is tuple of pre qb64 if data.salt: salter = self.decrypter.decrypt(ser=data.salt) - data.salt = (self.encrypter.encrypt(matter=salter).qb64 + data.salt = (self.encrypter.encrypt(prim=salter).qb64 if self.encrypter else salter.qb64) self.ks.prms.pin(keys, val=data) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index dcfaec05a..e448bcbc0 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -794,33 +794,36 @@ def verifySeed(self, seed): pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) return (pubkey == self.raw) - def encrypt(self, ser=None, matter=None): + def encrypt(self, ser=None, prim=None, code=None): """ Returns: Cipher instance of cipher text encryption of plain text serialization provided by either ser or Matter instance when provided. Parameters: + ser (Union[bytes,str]): qb64b or qb64 serialization of plain text - matter (Matter): plain text as Matter instance of seed or salt to - be encrypted + prim (Matter | Indexer): CESR primitive instance whose serialization + qb64 or qb2 is to be encrypted based on code + code (str): code of plain text type for resultant encrypted cipher """ - if not (ser or matter): - raise EmptyMaterialError("Neither ser or plain are provided.") + if not (ser or prim): + raise EmptyMaterialError(f"Neither bar serialization or primitive " + f"are provided.") if ser: - matter = Matter(qb64b=ser) + prim = Matter(qb64b=ser) - if matter.code == MtrDex.Salt_128: # future other salt codes + if prim.code == MtrDex.Salt_128: # future other salt codes code = MtrDex.X25519_Cipher_Salt - elif matter.code == MtrDex.Ed25519_Seed: # future other seed codes + elif prim.code == MtrDex.Ed25519_Seed: # future other seed codes code = MtrDex.X25519_Cipher_Seed else: - raise ValueError("Unsupported plain text code = {}.".format(matter.code)) + raise ValueError("Unsupported plain text code = {}.".format(prim.code)) # encrypting fully qualified qb64 version of plain text ensures its # derivation code round trips through eventual decryption - return (self._encrypt(ser=matter.qb64b, pubkey=self.raw, code=code)) + return (self._encrypt(ser=prim.qb64b, pubkey=self.raw, code=code)) @staticmethod def _x25519(ser, pubkey, code): diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index f007a28aa..5d092b53c 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -949,7 +949,7 @@ def put(self, keys: Union[str, Iterable], val: coring.Matter, already in database. """ if encrypter: - val = encrypter.encrypt(matter=val) # returns Cipher instance + val = encrypter.encrypt(prim=val) # returns Cipher instance return (self.db.putVal(db=self.sdb, key=self._tokey(keys), val=val.qb64b)) @@ -970,7 +970,7 @@ def pin(self, keys: Union[str, Iterable], val: coring.Matter, result (bool): True If successful. False otherwise. """ if encrypter: - val = encrypter.encrypt(matter=val) # returns Cipher instance + val = encrypter.encrypt(prim=val) # returns Cipher instance return (self.db.setVal(db=self.sdb, key=self._tokey(keys), val=val.qb64b)) From 108a27434762fa29979da25aba4005c4a241b38e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 3 Aug 2024 14:13:20 -0600 Subject: [PATCH 227/418] Refactored Encrypter to support all cipher codes. Need units tests --- src/keri/app/keeping.py | 8 +++-- src/keri/core/signing.py | 65 ++++++++++++++++++++++++++++---------- tests/core/test_signing.py | 8 ++--- 3 files changed, 58 insertions(+), 23 deletions(-) diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index 6949d748d..b1d2f1089 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -902,7 +902,7 @@ def salt(self, salt): may be plain text or cipher text handled by updateAeid """ if self.encrypter: - salt = self.encrypter.encrypt(ser=salt).qb64 + salt = self.encrypter.encrypt(ser=salt, code=core.MtrDex.X25519_Cipher_Salt).qb64 self.ks.gbls.pin('salt', salt) @@ -1020,7 +1020,8 @@ def incept(self, icodes=None, icount=1, icode=coring.MtrDex.Ed25519_Seed, if creator.salt: pp.salt = (creator.salt if not self.encrypter - else self.encrypter.encrypt(ser=creator.salt).qb64) + else self.encrypter.encrypt(ser=creator.salt, + code=core.MtrDex.X25519_Cipher_Salt).qb64) dt = helping.nowIso8601() ps = PreSit( @@ -1543,7 +1544,8 @@ def ingest(self, secrecies, iridx=0, ncount=1, ncode=coring.MtrDex.Ed25519_Seed, pp = PrePrm(pidx=pidx, algo=algo, salt=(creator.salt if not self.encrypter - else self.encrypter.encrypt(ser=creator.salt).qb64), + else self.encrypter.encrypt(ser=creator.salt, + code=core.MtrDex.X25519_Cipher_Salt).qb64), stem=creator.stem, tier=creator.tier) pre = csigners[0].verfer.qb64b diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index e448bcbc0..3df201ec2 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -12,7 +12,8 @@ from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from cryptography.hazmat.primitives.asymmetric import ec, utils -from ..kering import (EmptyMaterialError, InvalidCodeError, InvalidSizeError) +from ..kering import (EmptyMaterialError, InvalidCodeError, InvalidSizeError, + InvalidValueError) from ..help import helping @@ -766,8 +767,8 @@ def __init__(self, raw=None, code=MtrDex.X25519, verkey=None, **kwa): if not raw and verkey: verfer = Verfer(qb64b=verkey) if verfer.code not in (MtrDex.Ed25519N, MtrDex.Ed25519): - raise ValueError("Unsupported verkey derivation code = {}." - "".format(verfer.code)) + raise InvalidValueError(f"Unsupported verkey derivation code =" + f" {verfer.code}.") # convert signing public key to encryption public key raw = pysodium.crypto_sign_pk_to_box_pk(verfer.raw) @@ -776,7 +777,7 @@ def __init__(self, raw=None, code=MtrDex.X25519, verkey=None, **kwa): if self.code == MtrDex.X25519: self._encrypt = self._x25519 else: - raise ValueError("Unsupported encrypter code = {}.".format(self.code)) + raise InvalidValueError(f"Unsupported encrypter code = {self.code}.") def verifySeed(self, seed): """ @@ -807,23 +808,55 @@ def encrypt(self, ser=None, prim=None, code=None): qb64 or qb2 is to be encrypted based on code code (str): code of plain text type for resultant encrypted cipher """ - if not (ser or prim): - raise EmptyMaterialError(f"Neither bar serialization or primitive " - f"are provided.") + if not ser: - if ser: - prim = Matter(qb64b=ser) + if not prim: + raise EmptyMaterialError(f"Neither bar serialization or primitive " + f"are provided.") - if prim.code == MtrDex.Salt_128: # future other salt codes - code = MtrDex.X25519_Cipher_Salt - elif prim.code == MtrDex.Ed25519_Seed: # future other seed codes - code = MtrDex.X25519_Cipher_Seed - else: - raise ValueError("Unsupported plain text code = {}.".format(prim.code)) + if not code: + if prim.code == MtrDex.Salt_128: # future other salt codes + code = MtrDex.X25519_Cipher_Salt + elif prim.code == MtrDex.Ed25519_Seed: # future other seed codes + code = MtrDex.X25519_Cipher_Seed + else: + raise InvalidValueError(f"Unsupported primitive with code =" + f" {prim.code} when cipher code is " + f"missing.") + + if code in CiXAllQB64Dex: + ser = prim.qb64b + elif code in CiXVarQB2Dex: + ser = prim.qb2 + else: + raise InvalidCodeError(f"Invalide primitive cipher {code=} not " + f"qb64 or qb2.") + + if not code: # assumes default is sniffable stream + code = CiXDex.X25519_Cipher_L0 + + if hasattr(ser, "encode"): + ser = ser.encode() # convert str to bytes + if not isinstance(ser, bytes): + ser = bytes(ser) # convert bytearray and memoryview to bytes + + #if not (ser or prim): + #raise EmptyMaterialError(f"Neither bar serialization or primitive " + #f"are provided.") + + #if ser: + #prim = Matter(qb64b=ser) + + #if prim.code == MtrDex.Salt_128: # future other salt codes + #code = MtrDex.X25519_Cipher_Salt + #elif prim.code == MtrDex.Ed25519_Seed: # future other seed codes + #code = MtrDex.X25519_Cipher_Seed + #else: + #raise ValueError("Unsupported plain text code = {}.".format(prim.code)) # encrypting fully qualified qb64 version of plain text ensures its # derivation code round trips through eventual decryption - return (self._encrypt(ser=prim.qb64b, pubkey=self.raw, code=code)) + return (self._encrypt(ser=ser, pubkey=self.raw, code=code)) @staticmethod def _x25519(ser, pubkey, code): diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index 9b8ec4d16..ef616b5a9 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -779,12 +779,12 @@ def test_encrypter(): assert encrypter.raw == pubkey assert encrypter.verifySeed(seed=cryptsigner.qb64) - cipher = encrypter.encrypt(ser=seedqb64b) + cipher = encrypter.encrypt(ser=seedqb64b, code=MtrDex.X25519_Cipher_Seed) assert cipher.code == MtrDex.X25519_Cipher_Seed uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) assert uncb == seedqb64b - cipher = encrypter.encrypt(ser=saltqb64b) + cipher = encrypter.encrypt(ser=saltqb64b, code=MtrDex.X25519_Cipher_Salt) assert cipher.code == MtrDex.X25519_Cipher_Salt uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) assert uncb == saltqb64b @@ -857,7 +857,7 @@ def test_decrypter(): assert encrypter.raw == pubkey # create cipher of seed - seedcipher = encrypter.encrypt(ser=seedqb64b) + seedcipher = encrypter.encrypt(ser=seedqb64b, code=MtrDex.X25519_Cipher_Seed) assert seedcipher.code == MtrDex.X25519_Cipher_Seed # each encryption uses a nonce so not a stable representation for testing @@ -882,7 +882,7 @@ def test_decrypter(): assert signer.verfer.transferable # create cipher of salt - saltcipher = encrypter.encrypt(ser=saltqb64b) + saltcipher = encrypter.encrypt(ser=saltqb64b, code=MtrDex.X25519_Cipher_Salt) assert saltcipher.code == MtrDex.X25519_Cipher_Salt # each encryption uses a nonce so not a stable representation for testing From 25babd348d47975e0999fa068706d86a6e5ea968 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 3 Aug 2024 15:17:47 -0600 Subject: [PATCH 228/418] started refactoring Decrypter to support variable length codes --- src/keri/core/signing.py | 44 ++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index 3df201ec2..6623a2b5d 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -840,22 +840,9 @@ def encrypt(self, ser=None, prim=None, code=None): if not isinstance(ser, bytes): ser = bytes(ser) # convert bytearray and memoryview to bytes - #if not (ser or prim): - #raise EmptyMaterialError(f"Neither bar serialization or primitive " - #f"are provided.") - - #if ser: - #prim = Matter(qb64b=ser) - - #if prim.code == MtrDex.Salt_128: # future other salt codes - #code = MtrDex.X25519_Cipher_Salt - #elif prim.code == MtrDex.Ed25519_Seed: # future other seed codes - #code = MtrDex.X25519_Cipher_Seed - #else: - #raise ValueError("Unsupported plain text code = {}.".format(prim.code)) - - # encrypting fully qualified qb64 version of plain text ensures its - # derivation code round trips through eventual decryption + # encrypting fully qualified qb64 version of cesr primitive as plain + # text with proper cipher code ensures primitive round trip through eventual + # decryption. Likewise for sniffable stream with sniffible cipher code. return (self._encrypt(ser=ser, pubkey=self.raw, code=code)) @staticmethod @@ -866,7 +853,7 @@ def _x25519(ser, pubkey, code): ser (Union[bytes, str]): qb64b or qb64 serialization of seed or salt to be encrypted. pubkey (bytes): raw binary serialization of encryption public key - code (str): derivation code of serialized plain text seed or salt + code (str): cipher derivation code """ raw = pysodium.crypto_box_seal(ser, pubkey) return Cipher(raw=raw, code=code) @@ -934,7 +921,7 @@ def __init__(self, code=MtrDex.X25519_Private, seed=None, **kwa): else: raise ValueError("Unsupported decrypter code = {}.".format(self.code)) - def decrypt(self, ser=None, cipher=None, transferable=False): + def decrypt(self, *, ser=None, cipher=None, klas=None, transferable=False): """ Returns: Salter or Signer instance derived from plain text decrypted from @@ -944,10 +931,14 @@ def decrypt(self, ser=None, cipher=None, transferable=False): encryption/decryption round trip. Parameters: - ser (Union[bytes,str]): qb64b or qb64 serialization of cipher text + ser (bytes | str): serialization of cipher text cipher (Cipher): optional Cipher instance when ser is None - transferable (bool): True means associated verfer of returned - signer is transferable. False means non-transferable + klas (Matter, Indexer, Streamer): Class used to create instance from + decrypted serialization. + transferable (bool): Modifier of Klas instance creation. When klas + is signer; + True means verfer of returned signer is transferable. + False means non-transferable """ if not (ser or cipher): raise EmptyMaterialError("Neither ser or cipher are provided.") @@ -957,10 +948,11 @@ def decrypt(self, ser=None, cipher=None, transferable=False): return (self._decrypt(cipher=cipher, prikey=self.raw, + klas=klas, transferable=transferable)) @staticmethod - def _x25519(cipher, prikey, transferable=False): + def _x25519(cipher, prikey, klas, transferable=False): """ Returns plain text as Salter or Signer instance depending on the cipher code and the embedded encrypted plain text derivation code. @@ -969,8 +961,12 @@ def _x25519(cipher, prikey, transferable=False): cipher (Cipher): instance of encrypted seed or salt prikey (bytes): raw binary decryption private key derived from signing seed or sigkey - transferable (bool): True means associated verfer of returned - signer is transferable. False means non-transferable + klas (Matter, Indexer, Streamer): Class used to create instance from + decrypted serialization. + transferable (bool): Modifier of Klas instance creation. When klas + is signer; + True means verfer of returned signer is transferable. + False means non-transferable """ pubkey = pysodium.crypto_scalarmult_curve25519_base(prikey) plain = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) # qb64b From 42d51ca74275dd1b104bb487b17d608ba9bee8e3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 3 Aug 2024 15:24:45 -0600 Subject: [PATCH 229/418] more refactoring of Decrypter --- src/keri/core/signing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index 6623a2b5d..52d4f33a3 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -921,7 +921,7 @@ def __init__(self, code=MtrDex.X25519_Private, seed=None, **kwa): else: raise ValueError("Unsupported decrypter code = {}.".format(self.code)) - def decrypt(self, *, ser=None, cipher=None, klas=None, transferable=False): + def decrypt(self, *, cipher=None, ser=None, klas=None, transferable=False): """ Returns: Salter or Signer instance derived from plain text decrypted from @@ -931,8 +931,8 @@ def decrypt(self, *, ser=None, cipher=None, klas=None, transferable=False): encryption/decryption round trip. Parameters: - ser (bytes | str): serialization of cipher text cipher (Cipher): optional Cipher instance when ser is None + ser (bytes | str): serialization of cipher text klas (Matter, Indexer, Streamer): Class used to create instance from decrypted serialization. transferable (bool): Modifier of Klas instance creation. When klas From bec32e05afb7a462f048ae96a152ef2764bb8cf8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 4 Aug 2024 10:16:36 -0600 Subject: [PATCH 230/418] some clarifying refactoring --- src/keri/core/coring.py | 12 ++++----- src/keri/core/signing.py | 58 ++++++++++++++++++++-------------------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 5896ffe09..7e8ee813e 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -350,12 +350,12 @@ class MatterCodex: Bytes_Big_L0: str = '7AAB' # Byte String big lead size 0 Bytes_Big_L1: str = '8AAB' # Byte String big lead size 1 Bytes_Big_L2: str = '9AAB' # Byte String big lead size 2 - X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 - X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 - X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 2 X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index 52d4f33a3..e3fabefc8 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -484,24 +484,24 @@ def signers(self, count=1, start=0, path="", **kwa): # Codes for for ciphers of variable sized sniffable QB2 or QB64 plain text @dataclass(frozen=True) -class CipherX25519VarSnifCodex: +class CipherX25519VarStrmCodex: """ CipherX25519VarCodex is codex all variable sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is Sniffable QB2 or QB64. + for sealed box encryped ciphertext. Plaintext is Sniffable CESR Stream. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 - X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 - X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 2 def __iter__(self): return iter(astuple(self)) -CiXVarSnifDex = CipherX25519VarSnifCodex() # Make instance +CiXVarStrmDex = CipherX25519VarStrmCodex() # Make instance # Codes for for ciphers of variable sized QB64 plain text @@ -593,16 +593,16 @@ def __iter__(self): class CipherX25519AllVarCodex: """ CipherX25519AllVarCodex is codex all variable size codes of cipher bytes - for sealed box encryped ciphertext. Plaintext maybe sniffable or qb64 or qb2. + for sealed box encryped ciphertext. Plaintext maybe sniffable CESR stream or qb64 or qb2. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 - X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 - X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 2 X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 @@ -631,12 +631,12 @@ class CipherX25519AllCodex: Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 - X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 - X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 2 X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 @@ -691,13 +691,13 @@ def __init__(self, raw=None, code=None, **kwa): # code given by raw size. Otherwise provided code fixed or variable size # is handled by Matter superclass. if raw is not None and code is None: - if len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Salt): - code = MtrDex.X25519_Cipher_Salt - elif len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Seed): - code = MtrDex.X25519_Cipher_Seed - else: - raise InvalidSizeError(f"Unsupported fixed raw size" - f" {len(raw)} for {code=}.") + if len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Salt): + code = MtrDex.X25519_Cipher_Salt + elif len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Seed): + code = MtrDex.X25519_Cipher_Seed + else: + raise InvalidSizeError(f"Unsupported fixed raw size" + f" {len(raw)} for {code=}.") if hasattr(raw, "encode"): raw = raw.encode("utf-8") # ensure bytes not str From 359d61aaa7a0d24d85f0f839bc7dc8f035915584 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 10:58:33 -0600 Subject: [PATCH 231/418] Decrypter now supports all cipher codes including variable sizes of qb64, qb2 and sniffable streams --- src/keri/app/keeping.py | 6 +-- src/keri/core/coring.py | 10 ++-- src/keri/core/signing.py | 93 ++++++++++++++++++++++++++------------ src/keri/db/subing.py | 4 +- tests/app/test_keeping.py | 4 +- tests/core/test_signing.py | 10 ++-- 6 files changed, 82 insertions(+), 45 deletions(-) diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index b1d2f1089..839744af4 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -804,7 +804,7 @@ def updateAeid(self, aeid, seed): # re-encrypt root salt secrets by prefix parameters .prms for keys, data in self.ks.prms.getItemIter(): # keys is tuple of pre qb64 if data.salt: - salter = self.decrypter.decrypt(ser=data.salt) + salter = self.decrypter.decrypt(qb64=data.salt) data.salt = (self.encrypter.encrypt(prim=salter).qb64 if self.encrypter else salter.qb64) self.ks.prms.pin(keys, val=data) @@ -889,7 +889,7 @@ def salt(self): """ salt = self.ks.gbls.get('salt') if self.decrypter: # given .decrypt secret salt must be encrypted in db - return self.decrypter.decrypt(ser=salt).qb64 + return self.decrypter.decrypt(qb64=salt).qb64 return salt @@ -1185,7 +1185,7 @@ def rotate(self, pre, ncodes=None, ncount=1, if self.aeid: if not self.decrypter: raise kering.DecryptError("Unauthorized decryption. Aeid but no decrypter.") - salt = self.decrypter.decrypt(ser=salt).qb64 + salt = self.decrypter.decrypt(qb64=salt).qb64 else: salt = core.Salter(qb64=salt).qb64 # ensures salt was unencrypted diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 7e8ee813e..6d933b81c 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -795,9 +795,13 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, rize (int | None): raw size in bytes when variable sized material not including lead bytes if any Otherwise None - qb64b (bytes | None): fully qualified crypto material Base64 - qb64 (str | bytes | None): fully qualified crypto material Base64 - qb2 (bytes | None): fully qualified crypto material Base2 + qb64b (str | bytes | bytearray | memoryview | None): fully qualified + crypto material Base64. When str, encodes as utf-8. Strips when + bytearray and strip is True. + qb64 (str | bytes | bytearray | memoryview | None): fully qualified + crypto material Base64. When str, encodes as utf-8. Ignores strip + qb2 (bytes | bytearray | memoryview | None): fully qualified crypto + material Base2. Strips when bytearray and strip is True. strip (bool): True means strip (delete) matter from input stream bytearray after parsing qb64b or qb2. False means do not strip diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index e3fabefc8..d11e378eb 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -19,7 +19,8 @@ from .coring import (Tiers, ) from .coring import (SmallVrzDex, LargeVrzDex, Matter, MtrDex, Verfer, Cigar) -from .indexing import IdrDex, Siger +from .indexing import IdrDex, Indexer, Siger +from .streaming import Streamer DSS_SIG_MODE = "fips-186-3" @@ -682,6 +683,9 @@ class Cipher(Matter): def __init__(self, raw=None, code=None, **kwa): """ + Inherited Parameters: + (see Matter) + Parmeters: raw (bytes | str): cipher text (not plain text) code (str): cipher suite @@ -725,7 +729,7 @@ def decrypt(self, prikey=None, seed=None): signing key seed used to derive private decryption key """ decrypter = Decrypter(qb64b=prikey, seed=seed) - return decrypter.decrypt(ser=self.qb64b) + return decrypter.decrypt(qb64=self.qb64b) class Encrypter(Matter): @@ -799,11 +803,12 @@ def encrypt(self, ser=None, prim=None, code=None): """ Returns: Cipher instance of cipher text encryption of plain text serialization - provided by either ser or Matter instance when provided. + provided by either ser or prim as CESR primitive instance. Parameters: - ser (Union[bytes,str]): qb64b or qb64 serialization of plain text + ser (str | bytes | bytearray | memoryview): qb64b or qb64 + serialization of plain text prim (Matter | Indexer): CESR primitive instance whose serialization qb64 or qb2 is to be encrypted based on code code (str): code of plain text type for resultant encrypted cipher @@ -893,13 +898,14 @@ def __init__(self, code=MtrDex.X25519_Private, seed=None, **kwa): """ Assign decrypting cipher suite function to ._decrypt - Parameters: See Matter for inheirted parameters - raw (bytes): private decryption key derived from seed (private signing key) - qb64b (bytes): fully qualified private decryption key - qb64 (str): fully qualified private decryption key + Inherited Parameters: + (see Matter) + + Parameters: See Matter for inherited parameters code (str): derivation code for private decryption key - seed (Union[bytes, str]): qb64b or qb64 of signing key seed used to - derive raw which is private decryption key + seed (str | bytes | bytearray | memoryview | None): qb64b or qb64 + of signing key seed used to derive raw which is private + decryption key """ try: super(Decrypter, self).__init__(code=code, **kwa) @@ -921,7 +927,9 @@ def __init__(self, code=MtrDex.X25519_Private, seed=None, **kwa): else: raise ValueError("Unsupported decrypter code = {}.".format(self.code)) - def decrypt(self, *, cipher=None, ser=None, klas=None, transferable=False): + + def decrypt(self, *, cipher=None, qb64=None, qb2=None, klas=None, + transferable=False, **kwa): """ Returns: Salter or Signer instance derived from plain text decrypted from @@ -931,20 +939,29 @@ def decrypt(self, *, cipher=None, ser=None, klas=None, transferable=False): encryption/decryption round trip. Parameters: - cipher (Cipher): optional Cipher instance when ser is None - ser (bytes | str): serialization of cipher text - klas (Matter, Indexer, Streamer): Class used to create instance from + cipher (Cipher): instance. One of cipher, qb64, or qb2 required. + qb64 (str | bytes | bytearray | memoryview | None ): serialization + of cipher text as fully qualified base64. When str, encodes as + utf-8. When bytearray and strip in kwa is True then strips. + qb2 (bytes | bytearray | memoryview | None ): serialization + of cipher text as fully qualified base2. Strips when bytearray + and strip in kwa is True. + klas (Matter | Indexer | Streamer): Class used to create instance from decrypted serialization. - transferable (bool): Modifier of Klas instance creation. When klas - is signer; + transferable (bool): Modifier of Klas instance creation. + When klas init (such as Signer) supports transferabe parm; True means verfer of returned signer is transferable. False means non-transferable """ - if not (ser or cipher): - raise EmptyMaterialError("Neither ser or cipher are provided.") + if not cipher: + if qb64: # create cipher from qb64 + cipher = Cipher(qb64b=qb64, **kwa) - if ser: # create cipher to ensure valid derivation code of material in ser - cipher = Cipher(qb64b=ser) + elif qb2: + cipher = Cipher(qb2=qb2, **kwa) + + else: + raise EmptyMaterialError(f"Need one of cipher, qb64, or qb2.") return (self._decrypt(cipher=cipher, prikey=self.raw, @@ -952,7 +969,7 @@ def decrypt(self, *, cipher=None, ser=None, klas=None, transferable=False): transferable=transferable)) @staticmethod - def _x25519(cipher, prikey, klas, transferable=False): + def _x25519(cipher, prikey, klas=None, transferable=False): """ Returns plain text as Salter or Signer instance depending on the cipher code and the embedded encrypted plain text derivation code. @@ -961,19 +978,35 @@ def _x25519(cipher, prikey, klas, transferable=False): cipher (Cipher): instance of encrypted seed or salt prikey (bytes): raw binary decryption private key derived from signing seed or sigkey - klas (Matter, Indexer, Streamer): Class used to create instance from - decrypted serialization. - transferable (bool): Modifier of Klas instance creation. When klas - is signer; + klas (Matter, Indexer, Streamer | None): Class used to create instance from + decrypted serialization. Default depends on cipher.code. + transferable (bool): Modifier of Klas instance creation. + When klas init (such as Signer) supports transferabe parm; True means verfer of returned signer is transferable. False means non-transferable """ pubkey = pysodium.crypto_scalarmult_curve25519_base(prikey) plain = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) # qb64b # ensure raw plain text is qb64b or qb64 so its derivation code is round tripped - if cipher.code == MtrDex.X25519_Cipher_Salt: - return Salter(qb64b=plain) - elif cipher.code == MtrDex.X25519_Cipher_Seed: - return Signer(qb64b=plain, transferable=transferable) + + if not klas: + if cipher.code == CiXFixQB64Dex.X25519_Cipher_Salt: + #return Salter(qb64b=plain) + klas = Salter + elif cipher.code == CiXFixQB64Dex.X25519_Cipher_Seed: + #return Signer(qb64b=plain, transferable=transferable) + klas = Signer + elif cipher.code in CiXVarStrmDex: + klas = Streamer + else: + raise InvalidCodeError(f"Unsupported cipher code = {cipher.code}" + f" when klas missing.") + + if cipher.code in CiXAllQB64Dex: + return klas(qb64b=plain, transferable=transferable) + elif cipher.code in CiXVarQB2Dex: + return klas(qb2=plain) + elif cipher.code in CiXVarStrmDex: + return klas(stream=plain) else: - raise ValueError("Unsupported cipher text code = {}.".format(cipher.code)) + raise InvalidCodeError(f"Unsupported cipher code = {cipher.code}.") diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 5d092b53c..ec4ae7780 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1008,7 +1008,7 @@ def get(self, keys: Union[str, Iterable], decrypter: core.Decrypter = None): keys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=keys[-1]) # last split if decrypter: - return (decrypter.decrypt(ser=bytes(val), + return (decrypter.decrypt(qb64=bytes(val), transferable=verfer.transferable)) return (self.klas(qb64b=bytes(val), transferable=verfer.transferable)) @@ -1037,7 +1037,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b"", ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split if decrypter: - yield (ikeys, decrypter.decrypt(ser=bytes(val), + yield (ikeys, decrypter.decrypt(qb64=bytes(val), transferable=verfer.transferable)) else: yield (ikeys, self.klas(qb64b=bytes(val), diff --git a/tests/app/test_keeping.py b/tests/app/test_keeping.py index 0d638aef3..87474c493 100644 --- a/tests/app/test_keeping.py +++ b/tests/app/test_keeping.py @@ -1591,7 +1591,7 @@ def test_manager_with_aeid(): pp = manager.ks.prms.get(spre) assert pp.pidx == 0 assert pp.algo == keeping.Algos.salty - assert manager.decrypter.decrypt(ser=pp.salt).qb64 == salt + assert manager.decrypter.decrypt(qb64=pp.salt).qb64 == salt assert pp.stem == '' assert pp.tier == core.Tiers.low @@ -1674,7 +1674,7 @@ def test_manager_with_aeid(): pp = manager.ks.prms.get(spre) assert pp.pidx == 0 assert pp.algo == keeping.Algos.salty - assert manager.decrypter.decrypt(ser=pp.salt).qb64 == salt + assert manager.decrypter.decrypt(qb64=pp.salt).qb64 == salt assert pp.stem == '' assert pp.tier == core.Tiers.low diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index ef616b5a9..16ffc8a17 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -868,7 +868,7 @@ def test_decrypter(): assert decrypter.raw == prikey # decrypt seed cipher using ser - designer = decrypter.decrypt(ser=seedcipher.qb64b, transferable=signer.verfer.transferable) + designer = decrypter.decrypt(qb64=seedcipher.qb64b, transferable=signer.verfer.transferable) assert designer.qb64b == seedqb64b assert designer.code == MtrDex.Ed25519_Seed assert designer.verfer.code == MtrDex.Ed25519 @@ -887,7 +887,7 @@ def test_decrypter(): # each encryption uses a nonce so not a stable representation for testing # decrypt salt cipher using ser - desalter = decrypter.decrypt(ser=saltcipher.qb64b) + desalter = decrypter.decrypt(qb64=saltcipher.qb64b) assert desalter.qb64b == saltqb64b assert desalter.code == MtrDex.Salt_128 @@ -900,7 +900,7 @@ def test_decrypter(): # get from seedcipher above cipherseed = ('PM9jOGWNYfjM_oLXJNaQ8UlFSAV5ACjsUY7J16xfzrlpc9Ve3A5WYrZ4o_' 'NHtP5lhp78Usspl9fyFdnCdItNd5JyqZ6dt8SXOt6TOqOCs-gy0obrwFkPPqBvVkEw') - designer = decrypter.decrypt(ser=cipherseed, transferable=signer.verfer.transferable) + designer = decrypter.decrypt(qb64=cipherseed, transferable=signer.verfer.transferable) assert designer.qb64b == seedqb64b assert designer.code == MtrDex.Ed25519_Seed assert designer.verfer.code == MtrDex.Ed25519 @@ -909,7 +909,7 @@ def test_decrypter(): # get from saltcipher above ciphersalt = ('1AAHjlR2QR9J5Et67Wy-ZaVdTryN6T6ohg44r73GLRPnHw-5S3ABFkhWy' 'IwLOI6TXUB_5CT13S8JvknxLxBaF8ANPK9FSOPD8tYu') - desalter = decrypter.decrypt(ser=ciphersalt) + desalter = decrypter.decrypt(qb64=ciphersalt) assert desalter.qb64b == saltqb64b assert desalter.code == MtrDex.Salt_128 @@ -920,7 +920,7 @@ def test_decrypter(): assert decrypter.raw == prikey # decrypt ciphersalt - desalter = decrypter.decrypt(ser=saltcipher.qb64b) + desalter = decrypter.decrypt(qb64=saltcipher.qb64b) assert desalter.qb64b == saltqb64b assert desalter.code == MtrDex.Salt_128 From 187e4f1e19219498732232ed043b573744258ad9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 11:15:58 -0600 Subject: [PATCH 232/418] cleaned up keeping.Manager.decrypt method --- src/keri/app/habbing.py | 5 ++--- src/keri/app/keeping.py | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index d32a858f5..e67464835 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -1335,9 +1335,8 @@ def decrypt(self, ser, verfers=None, **kwa): if verfers is None: verfers = self.kever.verfers # when group these provide group signing keys - return self.mgr.decrypt(ser=ser, - verfers=verfers, - ) + return self.mgr.decrypt(qb64=ser, verfers=verfers) + def query(self, pre, src, query=None, **kwa): """ Create, sign and return a `qry` message against the attester for the prefix diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index 839744af4..57ac560be 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -1396,12 +1396,13 @@ def sign(self, ser, pubs=None, verfers=None, indexed=True, return cigars - def decrypt(self, ser, pubs=None, verfers=None): + def decrypt(self, qb64, pubs=None, verfers=None): """ - Returns plain text of decrypted serialization + Returns decrypted plaintext of encrypted qb64 ciphertext serialization. Parameters: - ser (bytes): serialization to sign + qb64 (str | bytes | bytearray | memoryview): fully qualified base64 + ciphertext serialization to decrypt pubs (list[str] | None): of qb64 public keys to lookup private keys one of pubs or verfers is required. If both then verfers is ignored. verfers (list[Verfer] | None): Verfer instances of public keys @@ -1410,7 +1411,7 @@ def decrypt(self, ser, pubs=None, verfers=None): private keys Returns: - bytes: decrypted data + plain (bytes): decrypted plaintext """ signers = [] @@ -1435,18 +1436,22 @@ def decrypt(self, ser, pubs=None, verfers=None): raise ValueError("Missing prikey in db for pubkey={}".format(verfer.qb64)) signers.append(signer) - plain = ser + if hasattr(qb64, "encode"): + qb64 = qb64.encode() # convert str to bytes + qb64 = bytes(qb64) # convert bytearray or memoryview to bytes + for signer in signers: sigkey = signer.raw + signer.verfer.raw # sigkey is raw seed + raw verkey prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) # raw private encrypt key pubkey = pysodium.crypto_scalarmult_curve25519_base(prikey) - plain = pysodium.crypto_box_seal_open(plain, pubkey, prikey) # qb64b + plain = pysodium.crypto_box_seal_open(qb64, pubkey, prikey) # qb64b - if plain == ser: - raise ValueError("unable to decrypt data") + if plain == qb64: + raise ValueError(f"Unable to decrypt.") return plain + def ingest(self, secrecies, iridx=0, ncount=1, ncode=coring.MtrDex.Ed25519_Seed, dcode=coring.MtrDex.Blake3_256, algo=Algos.salty, salt=None, stem=None, tier=None, From 370a84210b965cafee6c1aaa5f4d0d3d26e86dfe Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 11:33:46 -0600 Subject: [PATCH 233/418] clean up hab.decrypt. Needs more work --- src/keri/app/cli/commands/decrypt.py | 4 ++-- src/keri/app/cli/commands/witness/authenticate.py | 4 ++-- src/keri/app/habbing.py | 6 +++++- tests/peer/test_exchanging.py | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/keri/app/cli/commands/decrypt.py b/src/keri/app/cli/commands/decrypt.py index de460ded8..063242c4d 100644 --- a/src/keri/app/cli/commands/decrypt.py +++ b/src/keri/app/cli/commands/decrypt.py @@ -55,8 +55,8 @@ def decrypt(tymth, tock=0.0, **opts): else: data = data - m = coring.Matter(qb64=data) - d = coring.Matter(qb64=hab.decrypt(m.raw)) + m = coring.Matter(qb64=data) # should refactor this to use Cipher + d = coring.Matter(qb64=hab.decrypt(ser=m.raw)) print(d.raw) except kering.ConfigurationError: diff --git a/src/keri/app/cli/commands/witness/authenticate.py b/src/keri/app/cli/commands/witness/authenticate.py index 1758093ce..f0a2b4192 100644 --- a/src/keri/app/cli/commands/witness/authenticate.py +++ b/src/keri/app/cli/commands/witness/authenticate.py @@ -127,8 +127,8 @@ def authDo(self, tymth, tock=0.0): data = json.loads(rep.body) totp = data["totp"] - m = coring.Matter(qb64=totp) - d = coring.Matter(qb64=self.hab.decrypt(m.raw)) + m = coring.Matter(qb64=totp) # refactor this to use cipher + d = coring.Matter(qb64=self.hab.decrypt(ser=m.raw)) otpurl = f"otpauth://totp/KERIpy:{self.witness}?secret={d.raw.decode('utf-8')}&issuer=KERIpy" if not self.urlOnly: diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index e67464835..d9431aa4a 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -1320,12 +1320,14 @@ def sign(self, ser, verfers=None, indexed=True, indices=None, ondices=None, **kw indices=indices, ondices=ondices) + def decrypt(self, ser, verfers=None, **kwa): """Decrypt given serialization ser using appropriate keys. Use provided verfers or .kever.verfers to lookup keys to sign. Parameters: - ser (bytes): serialization to decrypt + ser (str | bytes | bytearray | memoryview): serialization to decrypt + verfers (list[Verfer] | None): Verfer instances to get pub verifier keys to lookup and convert to private decryption keys. verfers None means use .kever.verfers. Assumes that when group @@ -1335,6 +1337,8 @@ def decrypt(self, ser, verfers=None, **kwa): if verfers is None: verfers = self.kever.verfers # when group these provide group signing keys + # should not use mgr.decrypt since it assumes qb64. Just lucky its not + # yet a problem return self.mgr.decrypt(qb64=ser, verfers=verfers) diff --git a/tests/peer/test_exchanging.py b/tests/peer/test_exchanging.py index 54115ae60..4a0a0a009 100644 --- a/tests/peer/test_exchanging.py +++ b/tests/peer/test_exchanging.py @@ -84,7 +84,7 @@ def test_essrs(): # Pull the logged ESSR attachment and verify it is the one attached texter = recHby.db.essrs.get(keys=(serder.said,)) - raw = recHab.decrypt(texter[0].raw) + raw = recHab.decrypt(ser=texter[0].raw) assert raw.decode("utf-8") == msg # Test with invalid diger From da477797fe5dda74531d91af53b46bb04e6f8894 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 11:59:43 -0600 Subject: [PATCH 234/418] fix Cipher.decrypt to support new Decrypter.decrypt --- src/keri/core/signing.py | 58 ++++++++++++++++++++++++++++---------- src/keri/core/streaming.py | 14 +++++++-- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index d11e378eb..c1edb91d6 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -712,24 +712,44 @@ def __init__(self, raw=None, code=None, **kwa): raise InvalidCodeError(f"Unsupported cipher code = {self.code}.") - def decrypt(self, prikey=None, seed=None): + def decrypt(self, prikey=None, seed=None, klas=None, transferable=False, + **kwa): """ - Returns plain text as Matter instance (Signer or Salter) of cryptographic - cipher text material given by .raw. Encrypted plain text is fully - qualified (qb64) so derivaton code of plain text preserved through + Returns plain text as klas instance (Matter, Indexer, Streamer). + When klas is None then klas default is based on .code. Maybe Salter, + Signer, or Streamer. Encrypted plain text is fully + qualified (qb64) via self so derivaton code of plain text preserved through encryption/decryption round trip. - Decrypter uses either decryption key given by prikey or derives prikey from - signing key derived from private seed. + The created Decrypter uses either decryption key given by prikey or + when prikey missing derives prikey from signing key derived from private + seed. + + Returns: + decrypted (Matter | Indexer | Streamer): instance of decrypted + cipher text of .raw which is encrypted qb64, qb2, or sniffable + stream depending on .code + + Keyword Parameters: + (see Matter because created Decrypter is Matter subclass) Parameters: prikey (Union[bytes, str]): qb64b or qb64 serialization of private decryption key seed (Union[bytes, str]): qb64b or qb64 serialization of private signing key seed used to derive private decryption key + klas (Matter | Indexer | Streamer): Class used to create instance from + decrypted serialization. + transferable (bool): Modifier of klas instance creation. + When klas init (such as Signer) supports transferabe parm; + True means verfer of returned signer is transferable. + False means non-transferable """ - decrypter = Decrypter(qb64b=prikey, seed=seed) - return decrypter.decrypt(qb64=self.qb64b) + decrypter = Decrypter(qb64b=prikey, seed=seed, **kwa) + return decrypter.decrypt(cipher=self, klas=klas, transferable=transferable) + + #return decrypter.decrypt(qb64=self.qb64b) + class Encrypter(Matter): @@ -930,13 +950,21 @@ def __init__(self, code=MtrDex.X25519_Private, seed=None, **kwa): def decrypt(self, *, cipher=None, qb64=None, qb2=None, klas=None, transferable=False, **kwa): - """ + """Returns plain text as klas instance (Matter, Indexer, Streamer). + When klas is None then klas default is based on cipher.code or inferred + from qb64 or qb2 code. Default maybe Salter, Signer, or Streamer. + Cipher's encrypted plain text is fully qualified (qb64) + so derivaton code of plain text preserved through encryption/decryption + round trip. + + Returns: - Salter or Signer instance derived from plain text decrypted from - encrypted cipher text material given by ser or cipher. Plain text - that is orignally encrypt should always be fully qualified (qb64b) - so that derivaton code of plain text is preserved through - encryption/decryption round trip. + decrypted (Matter | Indexer | Streamer): instance of decrypted + cipher text of .raw which is encrypted qb64, qb2, or sniffable + stream depending on .code + + Keyword Parameters: + (see Matter because created Decrypter is Matter subclass) Parameters: cipher (Cipher): instance. One of cipher, qb64, or qb2 required. @@ -948,7 +976,7 @@ def decrypt(self, *, cipher=None, qb64=None, qb2=None, klas=None, and strip in kwa is True. klas (Matter | Indexer | Streamer): Class used to create instance from decrypted serialization. - transferable (bool): Modifier of Klas instance creation. + transferable (bool): Modifier of klas instance creation. When klas init (such as Signer) supports transferabe parm; True means verfer of returned signer is transferable. False means non-transferable diff --git a/src/keri/core/streaming.py b/src/keri/core/streaming.py index 4ad253ecc..be88485c9 100644 --- a/src/keri/core/streaming.py +++ b/src/keri/core/streaming.py @@ -340,7 +340,7 @@ class Streamer: """ - def __init__(self, stream): + def __init__(self, stream, verify=False): """Initialize instance Holds sniffable CESR stream as byte like string either (bytes, bytearray, or memoryview) @@ -348,7 +348,7 @@ def __init__(self, stream): Parameters: stream (str | bytes | bytearray | memoryview): sniffable CESR stream - + verify (bool): When True raise error if .stream is not sniffable. """ @@ -359,6 +359,15 @@ def __init__(self, stream): self._stream = stream + @property + def _verify(self): + """Returns True if sniffable stream, False otherwise + Returns: + sniffable (bool): True when .stream is sniffable. + False otherwise. + """ + return False + @property def stream(self): @@ -366,6 +375,7 @@ def stream(self): """ return self._stream + @property def text(self): """expanded stream where all primitives and groups in stream are From 40ef9c8dd93caea872089acd751d6e11bff8eea6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 13:18:26 -0600 Subject: [PATCH 235/418] unit tests for Cipher.decrypt with variable sized cipher codes of all three types QB64, QB2, and Sniffable Stream --- src/keri/core/__init__.py | 1 + src/keri/core/signing.py | 9 +- tests/core/test_signing.py | 289 +++++++++++++++++++++++++++++++++---- 3 files changed, 263 insertions(+), 36 deletions(-) diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index ec83d8bbb..b37b9516c 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -16,3 +16,4 @@ from .indexing import Indexer, Siger, IdrDex, IdxSigDex from .signing import Signer, Salter, Cipher, CiXDex, Encrypter, Decrypter from .counting import Counter, Codens, CtrDex_2_0 +from .streaming import Streamer diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index c1edb91d6..f8aeb6ea8 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -734,10 +734,11 @@ def decrypt(self, prikey=None, seed=None, klas=None, transferable=False, (see Matter because created Decrypter is Matter subclass) Parameters: - prikey (Union[bytes, str]): qb64b or qb64 serialization of private - decryption key - seed (Union[bytes, str]): qb64b or qb64 serialization of private - signing key seed used to derive private decryption key + prikey (str | bytes): qb64 or qb64b serialization of private + decryption key. Must be fully qualified with code. + seed (str | bytes): qb64 or qb64b serialization of private + signing key seed used to derive private decryption key. Must be + fully qualified with code. klas (Matter | Indexer | Streamer): Class used to create instance from decrypted serialization. transferable (bool): Modifier of klas instance creation. diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index 16ffc8a17..1d07fd6d4 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -642,18 +642,18 @@ def test_cipher(): texter = core.Texter(text=plain) assert texter.text == plain counter = core.Counter(core.Codens.GenericGroup, count=texter.size) - peb = counter.qb64b + texter.qb64b - raw = pysodium.crypto_box_seal(peb, pubkey) # uses nonce so different everytime + strm = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime assert len(raw) == 108 assert (3 - (len(raw) % 3)) % 3 == 0 cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) assert cipher.code == CiXDex.X25519_Cipher_L0 uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) - assert uncb == peb - peb = bytearray(peb) - counter = core.Counter(qb64b=peb, strip=True) + assert uncb == strm + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) assert counter.code == core.CtrDex_2_0.GenericGroup - texter = core.Texter(qb64b=peb, strip=True) + texter = core.Texter(qb64b=strm, strip=True) assert texter.text == plain # sniffable qb64 lead 1 @@ -661,18 +661,18 @@ def test_cipher(): texter = core.Texter(text=plain) assert texter.text == plain counter = core.Counter(core.Codens.GenericGroup, count=texter.size) - peb = counter.qb64b + texter.qb64b - raw = pysodium.crypto_box_seal(peb, pubkey) # uses nonce so different everytime + strm = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime assert len(raw) == 116 assert (3 - (len(raw) % 3)) % 3 == 1 cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) assert cipher.code == CiXDex.X25519_Cipher_L1 uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) - assert uncb == peb - peb = bytearray(peb) - counter = core.Counter(qb64b=peb, strip=True) + assert uncb == strm + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) assert counter.code == core.CtrDex_2_0.GenericGroup - texter = core.Texter(qb64b=peb, strip=True) + texter = core.Texter(qb64b=strm, strip=True) assert texter.text == plain # sniffable qb64 lead 2 @@ -680,18 +680,18 @@ def test_cipher(): texter = core.Texter(text=plain) assert texter.text == plain counter = core.Counter(core.Codens.GenericGroup, count=texter.size) - peb = counter.qb64b + texter.qb64b - raw = pysodium.crypto_box_seal(peb, pubkey) # uses nonce so different everytime + strm = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime assert len(raw) == 112 assert (3 - (len(raw) % 3)) % 3 == 2 cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) assert cipher.code == CiXDex.X25519_Cipher_L2 uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) - assert uncb == peb - peb = bytearray(peb) - counter = core.Counter(qb64b=peb, strip=True) + assert uncb == strm + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) assert counter.code == core.CtrDex_2_0.GenericGroup - texter = core.Texter(qb64b=peb, strip=True) + texter = core.Texter(qb64b=strm, strip=True) assert texter.text == plain # sniffable qb2 lead 0 @@ -700,19 +700,19 @@ def test_cipher(): assert texter.text == plain counter = core.Counter(core.Codens.GenericGroup, count=texter.size) assert counter.code == core.CtrDex_2_0.GenericGroup - peb = counter.qb64b + texter.qb64b - pebqb2 = decodeB64(peb) - raw = pysodium.crypto_box_seal(pebqb2, pubkey) # uses nonce so different everytime + strm = counter.qb64b + texter.qb64b + strmb2 = decodeB64(strm) + raw = pysodium.crypto_box_seal(strmb2, pubkey) # uses nonce so different everytime assert len(raw) == 93 assert (3 - (len(raw) % 3)) % 3 == 0 cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) assert cipher.code == CiXDex.X25519_Cipher_L0 uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) - assert uncb == pebqb2 - pebqb2 = bytearray(pebqb2) - counter = core.Counter(qb2=pebqb2, strip=True) + assert uncb == strmb2 + strmb2 = bytearray(strmb2) + counter = core.Counter(qb2=strmb2, strip=True) assert counter.code == core.CtrDex_2_0.GenericGroup - texter = core.Texter(qb2=pebqb2, strip=True) + texter = core.Texter(qb2=strmb2, strip=True) assert texter.text == plain @@ -723,20 +723,245 @@ def test_cipher(): assert texter.text == plain counter = core.Counter(core.Codens.GenericGroup, count=texter.size) assert counter.code == core.CtrDex_2_0.BigGenericGroup - peb = counter.qb64b + texter.qb64b - pebqb2 = decodeB64(peb) - raw = pysodium.crypto_box_seal(pebqb2, pubkey) # uses nonce so different everytime + strm = counter.qb64b + texter.qb64b + strmb2 = decodeB64(strm) + raw = pysodium.crypto_box_seal(strmb2, pubkey) # uses nonce so different everytime assert len(raw) == 12696 assert (3 - (len(raw) % 3)) % 3 == 0 assert (len(raw) // 3 ) > (64 ** 2 - 1) # triplets cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) assert cipher.code == CiXDex.X25519_Cipher_Big_L0 uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) - assert uncb == pebqb2 - pebqb2 = bytearray(pebqb2) - counter = core.Counter(qb2=pebqb2, strip=True) + assert uncb == strmb2 + strmb2 = bytearray(strmb2) + counter = core.Counter(qb2=strmb2, strip=True) assert counter.code == core.CtrDex_2_0.BigGenericGroup - texter = core.Texter(qb2=pebqb2, strip=True) + texter = core.Texter(qb2=strmb2, strip=True) + assert texter.text == plain + + # test .decrypt method with variable sized qb64 coded ciphers + # qb64 lead 0 + plain = "The quick brown fox jumps over the lazy " + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 108 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L0 + # test using prikey + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(prikey=prikeyqb64) + texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) + assert texter.text == plain + # test using seed + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(seed=cryptseedqb64) + texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) + assert texter.text == plain + + + # qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dogcats" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 116 + assert (3 - (len(raw) % 3)) % 3 == 1 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L1 + # test using prikey + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(prikey=prikeyqb64) + texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) + assert texter.text == plain + # test using seed + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(seed=cryptseedqb64) + texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) + assert texter.text == plain + + + # qb64 lead 2 + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 112 + assert (3 - (len(raw) % 3)) % 3 == 2 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L2 + # test using prikey + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(prikey=prikeyqb64) + texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) + assert texter.text == plain + # test using seed + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(seed=cryptseedqb64) + texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) + assert texter.text == plain + + # test .decrypt method with variable sized qb2 coded ciphers + # qb2 lead 0 (always lead 0 when qb2 from texter) + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb2, pubkey) # uses nonce so different everytime + assert len(raw) == 96 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB2_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB2_L0 + # test using prikey + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(prikey=prikeyqb64) + texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) + assert texter.text == plain + # test using seed + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(seed=cryptseedqb64) + texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) + assert texter.text == plain + + # sniffable qb64 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + strm = counter.qb64b + texter.qb64b # sniffable stream + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime + assert len(raw) == 108 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + # test using prikey + streamer = cipher.decrypt(prikey=prikeyqb64) # default klas is streamer + assert streamer.stream == strm + streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) + assert streamer.stream == strm + # test using seed + streamer = cipher.decrypt(seed=cryptseedqb64) + assert streamer.stream == strm + streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) + assert streamer.stream == strm + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=strm, strip=True) + assert texter.text == plain + + # sniffable qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + strm = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime + assert len(raw) == 116 + assert (3 - (len(raw) % 3)) % 3 == 1 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L1 + # test using prikey + streamer = cipher.decrypt(prikey=prikeyqb64) # default klas is streamer + assert streamer.stream == strm + streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) + assert streamer.stream == strm + # test using seed + streamer = cipher.decrypt(seed=cryptseedqb64) + assert streamer.stream == strm + streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) + assert streamer.stream == strm + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=strm, strip=True) + assert texter.text == plain + + # sniffable qb64 lead 2 + plain = "The quick brown fox jumps over the lazy " + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + strm = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime + assert len(raw) == 112 + assert (3 - (len(raw) % 3)) % 3 == 2 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L2 + # test using prikey + streamer = cipher.decrypt(prikey=prikeyqb64) # default klas is streamer + assert streamer.stream == strm + streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) + assert streamer.stream == strm + # test using seed + streamer = cipher.decrypt(seed=cryptseedqb64) + assert streamer.stream == strm + streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) + assert streamer.stream == strm + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=strm, strip=True) + assert texter.text == plain + + # sniffable qb2 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + assert counter.code == core.CtrDex_2_0.GenericGroup + strm = counter.qb64b + texter.qb64b + strmb2 = decodeB64(strm) + raw = pysodium.crypto_box_seal(strmb2, pubkey) # uses nonce so different everytime + assert len(raw) == 93 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + # test using prikey + streamer = cipher.decrypt(prikey=prikeyqb64) # default klas is streamer + assert streamer.stream == strmb2 + streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) + assert streamer.stream == strmb2 + # test using seed + streamer = cipher.decrypt(seed=cryptseedqb64) + assert streamer.stream == strmb2 + streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) + assert streamer.stream == strmb2 + strmb2 = bytearray(strmb2) + counter = core.Counter(qb2=strmb2, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb2=strmb2, strip=True) + assert texter.text == plain + + # sniffable qb2 lead 0 Big + + plain = "The quick brown fox jumps over the lazy" * 324 + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + assert counter.code == core.CtrDex_2_0.BigGenericGroup + strm = counter.qb64b + texter.qb64b + strmb2 = decodeB64(strm) + raw = pysodium.crypto_box_seal(strmb2, pubkey) # uses nonce so different everytime + assert len(raw) == 12696 + assert (3 - (len(raw) % 3)) % 3 == 0 + assert (len(raw) // 3 ) > (64 ** 2 - 1) # triplets + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_Big_L0 + # test using prikey + streamer = cipher.decrypt(prikey=prikeyqb64) # default klas is streamer + assert streamer.stream == strmb2 + streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) + assert streamer.stream == strmb2 + # test using seed + streamer = cipher.decrypt(seed=cryptseedqb64) + assert streamer.stream == strmb2 + streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) + assert streamer.stream == strmb2 + strmb2 = bytearray(strmb2) + counter = core.Counter(qb2=strmb2, strip=True) + assert counter.code == core.CtrDex_2_0.BigGenericGroup + texter = core.Texter(qb2=strmb2, strip=True) assert texter.text == plain """ Done Test """ From fc11bc04410508cd599ede9c38d52ce3921db0d9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 13:38:27 -0600 Subject: [PATCH 236/418] set up for round trip encrypt decrypt unit tests using Encrypter, Cipher, and Decrypter --- src/keri/core/signing.py | 5 --- tests/core/test_signing.py | 62 ++++++++++++++++++++++++++++++++------ 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index f8aeb6ea8..e05a9a61a 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -749,9 +749,6 @@ def decrypt(self, prikey=None, seed=None, klas=None, transferable=False, decrypter = Decrypter(qb64b=prikey, seed=seed, **kwa) return decrypter.decrypt(cipher=self, klas=klas, transferable=transferable) - #return decrypter.decrypt(qb64=self.qb64b) - - class Encrypter(Matter): """ @@ -1020,10 +1017,8 @@ def _x25519(cipher, prikey, klas=None, transferable=False): if not klas: if cipher.code == CiXFixQB64Dex.X25519_Cipher_Salt: - #return Salter(qb64b=plain) klas = Salter elif cipher.code == CiXFixQB64Dex.X25519_Cipher_Seed: - #return Signer(qb64b=plain, transferable=transferable) klas = Signer elif cipher.code in CiXVarStrmDex: klas = Streamer diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index 1d07fd6d4..8de9a64f6 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -1004,16 +1004,6 @@ def test_encrypter(): assert encrypter.raw == pubkey assert encrypter.verifySeed(seed=cryptsigner.qb64) - cipher = encrypter.encrypt(ser=seedqb64b, code=MtrDex.X25519_Cipher_Seed) - assert cipher.code == MtrDex.X25519_Cipher_Seed - uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) - assert uncb == seedqb64b - - cipher = encrypter.encrypt(ser=saltqb64b, code=MtrDex.X25519_Cipher_Salt) - assert cipher.code == MtrDex.X25519_Cipher_Salt - uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) - assert uncb == saltqb64b - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) encrypter = Encrypter(verkey=verfer.qb64) @@ -1032,6 +1022,24 @@ def test_encrypter(): assert encrypter.code == MtrDex.X25519 assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' assert encrypter.raw == pubkey + + # Test encrypt method + encrypter = Encrypter(raw=pubkey) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + assert encrypter.verifySeed(seed=cryptsigner.qb64) + + cipher = encrypter.encrypt(ser=seedqb64b, code=MtrDex.X25519_Cipher_Seed) + assert cipher.code == MtrDex.X25519_Cipher_Seed + uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) + assert uncb == seedqb64b + + cipher = encrypter.encrypt(ser=saltqb64b, code=MtrDex.X25519_Cipher_Salt) + assert cipher.code == MtrDex.X25519_Cipher_Salt + uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) + assert uncb == saltqb64b + """ Done Test """ @@ -1149,8 +1157,41 @@ def test_decrypter(): assert desalter.qb64b == saltqb64b assert desalter.code == MtrDex.Salt_128 + + """ Done Test """ +def test_roundtrip(): + """Test round trip encrypt decrypt with variable sized ciphers""" + + # cryptseed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) + cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' + verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) # raw + pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) + prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) + + # create decrypter from prikey + decrypter = Decrypter(raw=prikey) + assert decrypter.code == MtrDex.X25519_Private + assert decrypter.qb64 == 'OLCFxqMz1z1UUS0TEJnvZP_zXHcuYdQsSGBWdOZeY5VQ' + assert decrypter.raw == prikey + + # create encrypter from pubkey + encrypter = Encrypter(raw=pubkey) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + + # create cipher using Encrypter + + + # decrypt cipher using Decrypter + + + + + """End Test""" + if __name__ == "__main__": @@ -1161,4 +1202,5 @@ def test_decrypter(): test_cipher() test_encrypter() test_decrypter() + test_roundtrip() From e41b0f99aa026cd5d4da0edb22a6ff66b827f4c9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 14:22:13 -0600 Subject: [PATCH 237/418] added bare parameter to decrypt methods need unit tests --- src/keri/core/signing.py | 72 +++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index e05a9a61a..254986c9e 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -713,7 +713,7 @@ def __init__(self, raw=None, code=None, **kwa): def decrypt(self, prikey=None, seed=None, klas=None, transferable=False, - **kwa): + bare=False, **kwa): """ Returns plain text as klas instance (Matter, Indexer, Streamer). When klas is None then klas default is based on .code. Maybe Salter, @@ -728,7 +728,8 @@ def decrypt(self, prikey=None, seed=None, klas=None, transferable=False, Returns: decrypted (Matter | Indexer | Streamer): instance of decrypted cipher text of .raw which is encrypted qb64, qb2, or sniffable - stream depending on .code + stream depending on .code when bare is False. Otherwise returns + plaintext itself. Keyword Parameters: (see Matter because created Decrypter is Matter subclass) @@ -745,9 +746,14 @@ def decrypt(self, prikey=None, seed=None, klas=None, transferable=False, When klas init (such as Signer) supports transferabe parm; True means verfer of returned signer is transferable. False means non-transferable + bare (bool): False (default) means returns instance holding plaintext + True means returns plaintext itself """ decrypter = Decrypter(qb64b=prikey, seed=seed, **kwa) - return decrypter.decrypt(cipher=self, klas=klas, transferable=transferable) + return decrypter.decrypt(cipher=self, + klas=klas, + transferable=transferable, + bare=bare) class Encrypter(Matter): @@ -947,7 +953,7 @@ def __init__(self, code=MtrDex.X25519_Private, seed=None, **kwa): def decrypt(self, *, cipher=None, qb64=None, qb2=None, klas=None, - transferable=False, **kwa): + transferable=False, bare=False, **kwa): """Returns plain text as klas instance (Matter, Indexer, Streamer). When klas is None then klas default is based on cipher.code or inferred from qb64 or qb2 code. Default maybe Salter, Signer, or Streamer. @@ -957,9 +963,11 @@ def decrypt(self, *, cipher=None, qb64=None, qb2=None, klas=None, Returns: - decrypted (Matter | Indexer | Streamer): instance of decrypted - cipher text of .raw which is encrypted qb64, qb2, or sniffable - stream depending on .code + decrypted (Matter | Indexer | Streamer | bytes): When bare is False + returns instance of decrypted cipher text of .raw which is + encrypted qb64, qb2, or sniffable stream depending on .code + hhen Bare is True. Otherwise returns decrypted serialization + plaintext whatever that may be. Keyword Parameters: (see Matter because created Decrypter is Matter subclass) @@ -978,6 +986,8 @@ def decrypt(self, *, cipher=None, qb64=None, qb2=None, klas=None, When klas init (such as Signer) supports transferabe parm; True means verfer of returned signer is transferable. False means non-transferable + bare (bool): False (default) means returns instance holding plaintext + True means returns plaintext itself """ if not cipher: if qb64: # create cipher from qb64 @@ -992,10 +1002,11 @@ def decrypt(self, *, cipher=None, qb64=None, qb2=None, klas=None, return (self._decrypt(cipher=cipher, prikey=self.raw, klas=klas, - transferable=transferable)) + transferable=transferable, + bare=bare)) @staticmethod - def _x25519(cipher, prikey, klas=None, transferable=False): + def _x25519(cipher, prikey, klas=None, transferable=False, bare=False): """ Returns plain text as Salter or Signer instance depending on the cipher code and the embedded encrypted plain text derivation code. @@ -1010,27 +1021,34 @@ def _x25519(cipher, prikey, klas=None, transferable=False): When klas init (such as Signer) supports transferabe parm; True means verfer of returned signer is transferable. False means non-transferable + bare (bool): False (default) means CESR instance holding plaintext + True means plaintext """ + # assumes raw plain text is qb64b or qb64 or sniffable stream + # so it's round trippable pubkey = pysodium.crypto_scalarmult_curve25519_base(prikey) plain = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) # qb64b - # ensure raw plain text is qb64b or qb64 so its derivation code is round tripped - if not klas: - if cipher.code == CiXFixQB64Dex.X25519_Cipher_Salt: - klas = Salter - elif cipher.code == CiXFixQB64Dex.X25519_Cipher_Seed: - klas = Signer + if bare: + return plain + + else: + if not klas: + if cipher.code == CiXFixQB64Dex.X25519_Cipher_Salt: + klas = Salter + elif cipher.code == CiXFixQB64Dex.X25519_Cipher_Seed: + klas = Signer + elif cipher.code in CiXVarStrmDex: + klas = Streamer + else: + raise InvalidCodeError(f"Unsupported cipher code = {cipher.code}" + f" when klas missing.") + + if cipher.code in CiXAllQB64Dex: + return klas(qb64b=plain, transferable=transferable) + elif cipher.code in CiXVarQB2Dex: + return klas(qb2=plain) elif cipher.code in CiXVarStrmDex: - klas = Streamer + return klas(stream=plain) else: - raise InvalidCodeError(f"Unsupported cipher code = {cipher.code}" - f" when klas missing.") - - if cipher.code in CiXAllQB64Dex: - return klas(qb64b=plain, transferable=transferable) - elif cipher.code in CiXVarQB2Dex: - return klas(qb2=plain) - elif cipher.code in CiXVarStrmDex: - return klas(stream=plain) - else: - raise InvalidCodeError(f"Unsupported cipher code = {cipher.code}.") + raise InvalidCodeError(f"Unsupported cipher code = {cipher.code}.") From e74476101420872011401b7d3845cf7720c0b8b1 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 14:59:09 -0600 Subject: [PATCH 238/418] added units tests for bare functionality --- tests/core/test_signing.py | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index 8de9a64f6..25de827c8 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -529,9 +529,11 @@ def test_cipher(): # test .decrypt method needs qb64 prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b assert cipher.decrypt(prikey=prikeyqb64).qb64b == seedqb64b + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == seedqb64b cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b assert cipher.decrypt(seed=cryptseedqb64).qb64b == seedqb64b + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == seedqb64b # wrong but shorter code so instance creation succeeds cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_Salt) @@ -557,9 +559,11 @@ def test_cipher(): # test .decrypt method needs qb64 prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b assert cipher.decrypt(prikey=prikeyqb64).qb64b == saltqb64b + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == saltqb64b cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b assert cipher.decrypt(seed=cryptseedqb64).qb64b == saltqb64b + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == saltqb64b with pytest.raises(kering.InvalidCodeError): # bad code cipher = Cipher(raw=raw, code=MtrDex.Ed25519N) @@ -754,11 +758,15 @@ def test_cipher(): result = cipher.decrypt(prikey=prikeyqb64) texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) assert texter.text == plain + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == texter.qb64b # test using seed with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 result = cipher.decrypt(seed=cryptseedqb64) texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) assert texter.text == plain + # test with bare + texter = cipher.decrypt(seed=cryptseedqb64, bare=True) == texter.qb64b # qb64 lead 1 @@ -775,11 +783,15 @@ def test_cipher(): result = cipher.decrypt(prikey=prikeyqb64) texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) assert texter.text == plain + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == texter.qb64b # test using seed with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 result = cipher.decrypt(seed=cryptseedqb64) texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) assert texter.text == plain + # test with bare + texter = cipher.decrypt(seed=cryptseedqb64, bare=True) == texter.qb64b # qb64 lead 2 @@ -796,11 +808,15 @@ def test_cipher(): result = cipher.decrypt(prikey=prikeyqb64) texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) assert texter.text == plain + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == texter.qb64b # test using seed with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 result = cipher.decrypt(seed=cryptseedqb64) texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) assert texter.text == plain + # test with bare + texter = cipher.decrypt(seed=cryptseedqb64, bare=True) == texter.qb64b # test .decrypt method with variable sized qb2 coded ciphers # qb2 lead 0 (always lead 0 when qb2 from texter) @@ -817,11 +833,15 @@ def test_cipher(): result = cipher.decrypt(prikey=prikeyqb64) texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) assert texter.text == plain + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == texter.qb2 # test using seed with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 result = cipher.decrypt(seed=cryptseedqb64) texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) assert texter.text == plain + # test with bare + texter = cipher.decrypt(seed=cryptseedqb64, bare=True) == texter.qb2 # sniffable qb64 lead 0 plain = "The quick brown fox jumps over the lazy" @@ -839,17 +859,22 @@ def test_cipher(): assert streamer.stream == strm streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) assert streamer.stream == strm + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == streamer.stream # test using seed streamer = cipher.decrypt(seed=cryptseedqb64) assert streamer.stream == strm streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) assert streamer.stream == strm + # test with bare + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == streamer.stream strm = bytearray(strm) counter = core.Counter(qb64b=strm, strip=True) assert counter.code == core.CtrDex_2_0.GenericGroup texter = core.Texter(qb64b=strm, strip=True) assert texter.text == plain + # sniffable qb64 lead 1 plain = "The quick brown fox jumps over the lazy dog" texter = core.Texter(text=plain) @@ -866,11 +891,15 @@ def test_cipher(): assert streamer.stream == strm streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) assert streamer.stream == strm + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == streamer.stream # test using seed streamer = cipher.decrypt(seed=cryptseedqb64) assert streamer.stream == strm streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) assert streamer.stream == strm + # test with bare + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == streamer.stream strm = bytearray(strm) counter = core.Counter(qb64b=strm, strip=True) assert counter.code == core.CtrDex_2_0.GenericGroup @@ -893,11 +922,15 @@ def test_cipher(): assert streamer.stream == strm streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) assert streamer.stream == strm + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == streamer.stream # test using seed streamer = cipher.decrypt(seed=cryptseedqb64) assert streamer.stream == strm streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) assert streamer.stream == strm + # test with bare + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == streamer.stream strm = bytearray(strm) counter = core.Counter(qb64b=strm, strip=True) assert counter.code == core.CtrDex_2_0.GenericGroup @@ -922,11 +955,15 @@ def test_cipher(): assert streamer.stream == strmb2 streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) assert streamer.stream == strmb2 + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == streamer.stream # test using seed streamer = cipher.decrypt(seed=cryptseedqb64) assert streamer.stream == strmb2 streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) assert streamer.stream == strmb2 + # test with bare + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == streamer.stream strmb2 = bytearray(strmb2) counter = core.Counter(qb2=strmb2, strip=True) assert counter.code == core.CtrDex_2_0.GenericGroup @@ -953,11 +990,15 @@ def test_cipher(): assert streamer.stream == strmb2 streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) assert streamer.stream == strmb2 + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == streamer.stream # test using seed streamer = cipher.decrypt(seed=cryptseedqb64) assert streamer.stream == strmb2 streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) assert streamer.stream == strmb2 + # test with bare + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == streamer.stream strmb2 = bytearray(strmb2) counter = core.Counter(qb2=strmb2, strip=True) assert counter.code == core.CtrDex_2_0.BigGenericGroup @@ -1107,6 +1148,12 @@ def test_decrypter(): assert designer.verfer.code == MtrDex.Ed25519 assert signer.verfer.transferable + # test bare decryption returns plain not instance + plain = decrypter.decrypt(qb64=seedcipher.qb64b, + transferable=signer.verfer.transferable, + bare=True) + assert plain == seedqb64b + # decrypt seed cipher using cipher designer = decrypter.decrypt(cipher=seedcipher, transferable=signer.verfer.transferable) assert designer.qb64b == seedqb64b @@ -1124,6 +1171,10 @@ def test_decrypter(): assert desalter.qb64b == saltqb64b assert desalter.code == MtrDex.Salt_128 + # test bare decryption returns plain not instance + plain = decrypter.decrypt(qb64=saltcipher.qb64b, bare=True) + assert plain == saltqb64b + # decrypt salt cipher using cipher desalter = decrypter.decrypt(cipher=saltcipher) assert desalter.qb64b == saltqb64b From dbab817d52c5b9c60dd1f4a87f2993432c1f0b58 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 17:08:54 -0600 Subject: [PATCH 239/418] added unit tests of round trip encrypt decrypt --- src/keri/core/signing.py | 9 +++-- tests/core/test_signing.py | 82 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index 254986c9e..d7914523a 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -833,8 +833,9 @@ def encrypt(self, ser=None, prim=None, code=None): ser (str | bytes | bytearray | memoryview): qb64b or qb64 serialization of plain text - prim (Matter | Indexer): CESR primitive instance whose serialization - qb64 or qb2 is to be encrypted based on code + prim (Matter | Indexer | Streamer): CESR primitive instance whose + serialization is qb64 or qb2 or sniffable stream and is to be + encrypted based on code code (str): code of plain text type for resultant encrypted cipher """ if not ser: @@ -857,8 +858,10 @@ def encrypt(self, ser=None, prim=None, code=None): ser = prim.qb64b elif code in CiXVarQB2Dex: ser = prim.qb2 + elif code in CiXVarStrmDex: + ser = prim.stream else: - raise InvalidCodeError(f"Invalide primitive cipher {code=} not " + raise InvalidCodeError(f"Invalid primitive cipher {code=} not " f"qb64 or qb2.") if not code: # assumes default is sniffable stream diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index 25de827c8..d71c93b12 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -1081,6 +1081,9 @@ def test_encrypter(): uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) assert uncb == saltqb64b + # needs tests of encrypter with prim param instead of ser (see roundtrip) + + """ Done Test """ @@ -1233,13 +1236,92 @@ def test_roundtrip(): assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' assert encrypter.raw == pubkey + + # Test cipher qb2 (always L0 when qb2) + + plain = "The quick brown fox jumps over the lazy dog" + tin = core.Texter(text=plain) # texter in + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=tin, code=CiXDex.X25519_Cipher_QB2_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB2_L0 + # decrypt cipher using Decrypter + tout = decrypter.decrypt(cipher=cipher, klas=core.Texter) # texter out + assert tout.text == tin.text + + + # sniffable qb64 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + # sniffable streamer in + sin = core.Streamer(stream=counter.qb64b + texter.qb64b) + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=sin, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + # decrypt cipher using Decrypter + # sniffable stream out + sout = decrypter.decrypt(cipher=cipher, klas=core.Streamer) + assert sin.stream == sout.stream + + + # sniffable qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + # sniffable streamer in + sin = core.Streamer(stream=counter.qb64b + texter.qb64b) # create cipher using Encrypter + cipher = encrypter.encrypt(prim=sin, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L1 + # decrypt cipher using Decrypter + # sniffable stream out + sout = decrypter.decrypt(cipher=cipher, klas=core.Streamer) + assert sin.stream == sout.stream + # sniffable qb64 lead 2 + plain = "The quick brown fox jumps over the lazy " + texter = core.Texter(text=plain) + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + # sniffable streamer in + sin = core.Streamer(stream=counter.qb64b + texter.qb64b) + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=sin, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L2 # decrypt cipher using Decrypter + # sniffable stream out + sout = decrypter.decrypt(cipher=cipher, klas=core.Streamer) + assert sin.stream == sout.stream + # sniffable qb2 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + # sniffable streamer in + sin = core.Streamer(stream=counter.qb2 + texter.qb2) + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=sin, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + # decrypt cipher using Decrypter + # sniffable stream out + sout = decrypter.decrypt(cipher=cipher, klas=core.Streamer) + assert sin.stream == sout.stream + # sniffable qb2 lead 0 Big + plain = "The quick brown fox jumps over the lazy" * 324 + texter = core.Texter(text=plain) + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + assert counter.code == core.CtrDex_2_0.BigGenericGroup + # sniffable streamer in + sin = core.Streamer(stream=counter.qb2 + texter.qb2) + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=sin, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_Big_L0 + # decrypt cipher using Decrypter + # sniffable stream out + sout = decrypter.decrypt(cipher=cipher, klas=core.Streamer) + assert sin.stream == sout.stream """End Test""" From ac5e4df0b43251a5b094539e87a71d46295d4414 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 17:24:48 -0600 Subject: [PATCH 240/418] added support for prim to be Streamer instance --- src/keri/core/signing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py index d7914523a..7a4faa63e 100644 --- a/src/keri/core/signing.py +++ b/src/keri/core/signing.py @@ -823,7 +823,7 @@ def verifySeed(self, seed): pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) return (pubkey == self.raw) - def encrypt(self, ser=None, prim=None, code=None): + def encrypt(self, *, ser=None, prim=None, code=None): """ Returns: Cipher instance of cipher text encryption of plain text serialization @@ -831,8 +831,8 @@ def encrypt(self, ser=None, prim=None, code=None): Parameters: - ser (str | bytes | bytearray | memoryview): qb64b or qb64 - serialization of plain text + ser (str | bytes | bytearray | memoryview): qb64b or qb64 or sniffable + stream serialization of plain text prim (Matter | Indexer | Streamer): CESR primitive instance whose serialization is qb64 or qb2 or sniffable stream and is to be encrypted based on code @@ -872,9 +872,9 @@ def encrypt(self, ser=None, prim=None, code=None): if not isinstance(ser, bytes): ser = bytes(ser) # convert bytearray and memoryview to bytes - # encrypting fully qualified qb64 version of cesr primitive as plain + # encrypting cesr primitive qb64 or qb2 or cesr stream as plain # text with proper cipher code ensures primitive round trip through eventual - # decryption. Likewise for sniffable stream with sniffible cipher code. + # decryption. return (self._encrypt(ser=ser, pubkey=self.raw, code=code)) @staticmethod From cb4bce5c553e81626b32ddfc5f75bec8b4cebd82 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 17:29:16 -0600 Subject: [PATCH 241/418] updated comments on streamer methods --- src/keri/core/streaming.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/keri/core/streaming.py b/src/keri/core/streaming.py index be88485c9..f00e570cc 100644 --- a/src/keri/core/streaming.py +++ b/src/keri/core/streaming.py @@ -362,9 +362,12 @@ def __init__(self, stream, verify=False): @property def _verify(self): """Returns True if sniffable stream, False otherwise - Returns: - sniffable (bool): True when .stream is sniffable. + Returns: + sniffable (bool): True when .stream is sniffable. False otherwise. + Only works for ver 2 CESR because need for all count codes to be + pipelineable in order to simply parse stream + """ return False @@ -384,6 +387,9 @@ def text(self): Returns: stream (bytes): expanded text qb64 version of stream + Only works for ver 2 CESR because need for all count codes to be + pipelineable in order to simply parse and expand stream + """ return self._stream @@ -395,6 +401,9 @@ def binary(self): Returns: stream (bytes): compacted binary qb2 version of stream + Only works for ver 2 CESR because need for all count codes to be + pipelineable in order to simply parse and compact stream + """ return self._stream From 063107ab592a617e4a31a4e990f3d2a52e469980 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 5 Aug 2024 18:23:44 -0600 Subject: [PATCH 242/418] fixed Streamer._verify() method from property --- src/keri/core/streaming.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/keri/core/streaming.py b/src/keri/core/streaming.py index f00e570cc..6314be482 100644 --- a/src/keri/core/streaming.py +++ b/src/keri/core/streaming.py @@ -329,12 +329,14 @@ class Streamer: Has the following public properties: Properties: + stream (bytearray): sniffable CESR stream Methods: Hidden: + _verify() -> bool @@ -359,9 +361,10 @@ def __init__(self, stream, verify=False): self._stream = stream - @property + def _verify(self): - """Returns True if sniffable stream, False otherwise + """Returns True if .stream is sniffable, False otherwise + Returns: sniffable (bool): True when .stream is sniffable. False otherwise. From 2b9ba1584c84b6726f2556c40727e11b4e51a870 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 6 Aug 2024 07:57:42 -0600 Subject: [PATCH 243/418] remove spurious blank lines --- src/keri/core/coring.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 6d933b81c..bb75329e5 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -2451,9 +2451,6 @@ def b64ToVer(b64, *, texted=False): return Versionage(major=b64ToInt(b64[0]), minor=b64ToInt(b64[1:3])) - - - class Texter(Matter): """ Texter is subclass of Matter, cryptographic material, for variable length From 8872e63e44e39831c9a9c924940f28fc75d9dd5d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 6 Aug 2024 14:44:38 -0600 Subject: [PATCH 244/418] more test cases in test_roundtrip of encrypter decrypter --- tests/core/test_signing.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/tests/core/test_signing.py b/tests/core/test_signing.py index d71c93b12..ae9b990c5 100644 --- a/tests/core/test_signing.py +++ b/tests/core/test_signing.py @@ -1211,8 +1211,6 @@ def test_decrypter(): assert desalter.qb64b == saltqb64b assert desalter.code == MtrDex.Salt_128 - - """ Done Test """ def test_roundtrip(): @@ -1237,8 +1235,38 @@ def test_roundtrip(): assert encrypter.raw == pubkey - # Test cipher qb2 (always L0 when qb2) + # Test cipher qb64 lead 0 + plain = "The quick brown fox jumps over the lazy " + tin = core.Texter(text=plain) # texter in + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=tin, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L0 + # decrypt cipher using Decrypter + tout = decrypter.decrypt(cipher=cipher, klas=core.Texter) # texter out + assert tout.text == tin.text + + # Test cipher qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dogcats" + tin = core.Texter(text=plain) # texter in + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=tin, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L1 + # decrypt cipher using Decrypter + tout = decrypter.decrypt(cipher=cipher, klas=core.Texter) # texter out + assert tout.text == tin.text + + # Test cipher qb64 lead 2 + plain = "The quick brown fox jumps over the lazy dog" + tin = core.Texter(text=plain) # texter in + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=tin, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L2 + # decrypt cipher using Decrypter + tout = decrypter.decrypt(cipher=cipher, klas=core.Texter) # texter out + assert tout.text == tin.text + + # Test cipher qb2 (always L0 when qb2) plain = "The quick brown fox jumps over the lazy dog" tin = core.Texter(text=plain) # texter in # create cipher using Encrypter From 5a2c7ed04b401f37c97059d2df12e5443ed44e66 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 7 Aug 2024 14:02:13 -0600 Subject: [PATCH 245/418] refactory Prefixer as most of its functionality has been superseded by SerderKERI --- src/keri/core/coring.py | 41 +--- src/keri/core/eventing.py | 11 +- src/keri/vdr/eventing.py | 20 +- tests/core/test_coring.py | 407 +----------------------------------- tests/core/test_eventing.py | 14 +- tests/vdr/test_eventing.py | 90 -------- 6 files changed, 28 insertions(+), 555 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index bb75329e5..100492d0e 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -484,6 +484,7 @@ def __iter__(self): NonTransDex = NonTransCodex() # Make instance + # When add new to DigCodes update Saider.Digests and Serder.Digests class attr @dataclass(frozen=True) class DigCodex: @@ -3288,40 +3289,14 @@ def __init__(self, raw=None, code=None, ked=None, allows=None, **kwa): the codes allowed to a subset of all supported. """ - try: - super(Prefixer, self).__init__(raw=raw, code=code, **kwa) - except EmptyMaterialError as ex: - if not ked or (not code and "i" not in ked): - raise ex + super(Prefixer, self).__init__(raw=raw, code=code, **kwa) + if self.code not in PreDex: + raise InvalidCodeError(f"Invalid prefixer code = {self.code}.") - if not code: # get code from pre in ked - super(Prefixer, self).__init__(qb64=ked["i"], code=code, **kwa) - code = self.code - - if allows is not None and code not in allows: - raise ValueError("Unallowed code={} for prefixer.".format(code)) - - if code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N]: - self._derive = self._derive_non_transferable - elif code in [MtrDex.Ed25519, MtrDex.ECDSA_256r1, MtrDex.ECDSA_256k1]: - self._derive = self._derive_transferable - elif code == MtrDex.Blake3_256: - self._derive = self._derive_blake3_256 - else: - raise ValueError("Unsupported code = {} for prefixer.".format(code)) - - # use ked and ._derive from code to derive aid prefix and code - raw, code = self.derive(ked=ked) - super(Prefixer, self).__init__(raw=raw, code=code, **kwa) - - if self.code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N]: - self._verify = self._verify_non_transferable - elif self.code in [MtrDex.Ed25519, MtrDex.ECDSA_256r1, MtrDex.ECDSA_256k1]: - self._verify = self._verify_transferable - elif self.code == MtrDex.Blake3_256: - self._verify = self._verify_blake3_256 - else: - raise ValueError("Unsupported code = {} for prefixer.".format(self.code)) + #if self.code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N]: + #self._verify = self._verify_non_transferable + #elif self.code in [MtrDex.Ed25519, MtrDex.ECDSA_256r1, MtrDex.ECDSA_256k1]: + #self._verify = self._verify_transferable def derive(self, ked): """ diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index bef5475d3..b4b9f0e55 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1779,6 +1779,7 @@ def reload(self, state): raise MissingEntryError(f"Corresponding event not found for state=" f"{state}.") self.serder = serdering.SerderKERI(raw=bytes(raw)) + # May want to do additional checks here @@ -1808,10 +1809,10 @@ def incept(self, serder, estOnly=None): self.prefixer = Prefixer(qb64=serder.pre) - if not self.prefixer.verify(ked=ked, prefixed=True): # invalid prefix - raise ValidationError("Invalid prefix = {} for inception evt = {}." - "".format(self.prefixer.qb64, ked)) - + #if not self.prefixer.verify(ked=ked, prefixed=True): # invalid prefix + #raise ValidationError("Invalid prefix = {} for inception evt = {}." + #"".format(self.prefixer.qb64, ked)) + #serder._verify() redundant serder.__init__ defaults to verify self.serder = serder # need whole serder for digest agility comparisons ndigs = serder.ndigs # ked["n"] @@ -1959,6 +1960,7 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # nxt and signatures verify so update state self.sner = sner # sequence number Number instance + #serder._verify() redundant serder.__init__ defaults to verify self.serder = serder # need whole serder for digest agility compare self.ilk = ilk self.tholder = tholder @@ -2019,6 +2021,7 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # validates so update state self.sner = sner # sequence number Number instance + #serder._verify() redundant serder.__init__ defaults to verify self.serder = serder # need for digest agility includes .serder.diger self.ilk = ilk if fn is not None: # first is non-idempotent for fn check mode fn is None diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index bee2c9dcd..5583dc084 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -717,12 +717,6 @@ def __init__(self, cues=None, rsr=None, serder=None, seqner=None, saider=None, raise ValidationError("Expected ilk {} got {} for evt: {}".format(Ilks.vcp, ilk, serder)) self.ilk = ilk - #labels = VCP_LABELS - #for k in labels: - #if k not in serder.ked: - #raise ValidationError("Missing element = {} from {} event for " - #"evt = {}.".format(k, ilk, serder.ked)) - self.incept(serder=serder) self.config(serder=serder, noBackers=noBackers, estOnly=estOnly) @@ -819,14 +813,13 @@ def incept(self, serder): """ ked = serder.ked - self.pre = ked["ii"] - self.prefixer = Prefixer(qb64=serder.pre) - if not self.prefixer.verify(ked=ked, prefixed=True): # invalid prefix - raise ValidationError("Invalid prefix = {} for registry inception evt = {}." - .format(self.prefixer.qb64, ked)) + self.pre = ked["ii"] # which is not the AID of the serder in ked["i"] + self.prefixer = Prefixer(qb64=serder.pre) # this not related to self.pre + #if not self.prefixer.verify(ked=ked, prefixed=True): # invalid prefix + #raise ValidationError("Invalid prefix = {} for registry inception evt = {}." + #.format(self.prefixer.qb64, ked)) + - #sn = ked["s"] - #self.sn = validateSN(sn, inceptive=True) self.sn = Number(numh=ked["s"]).validate(inceptive=True).sn self.cuts = [] # always empty at inception since no prev event @@ -847,6 +840,7 @@ def incept(self, serder): raise ValidationError("Invalid toad = {} for baks = {} for evt = {}." "".format(toad, baks, ked)) self.toad = toad + #serder._verify() redundant serder.__init__ defaults to verify self.serder = serder def config(self, serder, noBackers=None, estOnly=None): diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 2dc1344e7..7ffc5649c 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -4582,8 +4582,8 @@ def test_prefixer(): with pytest.raises(EmptyMaterialError): prefixer = Prefixer(raw=verkey, code='') - with pytest.raises(ValueError): - prefixer = Prefixer(raw=verkey, code=MtrDex.SHA2_256) + #with pytest.raises(InvalidCodeError): + #prefixer = Prefixer(raw=verkey, code=MtrDex.SHA2_256) # test creation given raw and code no derivation prefixer = Prefixer(raw=verkey, code=MtrDex.Ed25519N) # default code is None @@ -4591,421 +4591,20 @@ def test_prefixer(): assert len(prefixer.raw) == Matter._rawSize(prefixer.code) assert len(prefixer.qb64) == Matter.Sizes[prefixer.code].fs - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[prefixer.qb64], # list of qb64 - nt="", - n=[], # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[prefixer.qb64], # list of qb64 - nt="1", - n=["ABCD"], # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - assert prefixer.verify(ked=ked) == False - assert prefixer.verify(ked=ked, prefixed=True) == False prefixer = Prefixer(raw=verkey, code=MtrDex.Ed25519) # defaults provide Ed25519N prefixer assert prefixer.code == MtrDex.Ed25519 assert len(prefixer.raw) == Matter._rawSize(prefixer.code) assert len(prefixer.qb64) == Matter.Sizes[prefixer.code].fs - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[prefixer.qb64], # list of qb64 - nt="1", - n=["ABCD"], # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False + verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) prefixer = Prefixer(raw=verfer.raw, code=MtrDex.Ed25519N) assert prefixer.code == MtrDex.Ed25519N - assert prefixer.verify(ked=ked) == False - assert prefixer.verify(ked=ked, prefixed=True) == False - - # Test basic derivation from ked - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[verfer.qb64], # list of qb64 - nt="", - n=0, # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - prefixer = Prefixer(ked=ked, code=MtrDex.Ed25519) - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - badked = dict(ked) - del badked["i"] - with pytest.raises(EmptyMaterialError): # no pre - prefixer = Prefixer(ked=badked) - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - with pytest.raises(DerivationError): # verfer code not match pre code - prefixer = Prefixer(ked=badked) - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=pre - with pytest.raises(DerivationError): - prefixer = Prefixer(ked=badked, code=MtrDex.Ed25519N) # verfer code not match code - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=pre - prefixer = Prefixer(ked=badked, code=MtrDex.Ed25519N) # verfer code match code but not pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == False - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - prefixer = Prefixer(ked=badked, code=MtrDex.Ed25519N) # verfer code match code and pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == True - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - prefixer = Prefixer(ked=badked) # verfer code match pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == True - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - del badked["i"] - with pytest.raises(EmptyMaterialError): # missing pre - prefixer = Prefixer(ked=badked) - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - with pytest.raises(ShortageError): # empty pre - prefixer = Prefixer(ked=badked) - - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["n"] = "ABCD" - with pytest.raises(DerivationError): # wrong code for transferable - prefixer = Prefixer(ked=badked, code=MtrDex.Ed25519) - - # Test digest derivation from inception ked - vs = versify(version=Version, kind=Kinds.json, size=0) - sn = 0 - ilk = Ilks.icp - sith = "1" - keys = [Prefixer(raw=verkey, code=MtrDex.Ed25519).qb64] - nxt = "" - toad = 0 - wits = [] - cnfg = [] - - ked = dict(v=vs, # version string - t=ilk, - d="", # SAID - i="", # qb64 prefix - s="{:x}".format(sn), # hex string no leading zeros lowercase - kt=sith, # hex string no leading zeros lowercase - k=keys, # list of qb64 - nt=0, - n=[], - bt="{:x}".format(toad), # hex string no leading zeros lowercase - b=wits, # list of qb64 may be empty - c=cnfg, # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EEZn82xRQYFjfkPJ5ECrDNHJ6xSt_hjxybbt_WMpinEF' - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - # test with next digs - ndigs = [Diger(ser=nxtfer.qb64b).qb64] - ked = dict(v=vs, # version string - t=ilk, - d="", # SAID - i="", # qb64 prefix - s="{:x}".format(sn), # hex string no leading zeros lowercase - kt=sith, # hex string no leading zeros lowercase - k=keys, # list of qb64 - nt=1, - n=ndigs, # hash qual Base64 - bt="{:x}".format(toad), # hex string no leading zeros lowercase - b=wits, # list of qb64 may be empty - c=cnfg, # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EHB9-i6jOH6DbK_40vlGF0X78Mg__c3MSzu9AE9ZRrsC' - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - - salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - #secrets = generateSecrets(salt=salt, count=8) - - # test with fractionally weighted sith - secrets = ['AK8F6AAiYDpXlWdj2O5F5-6wNCCNJh2A4XOlqwR_HwwH', - 'AOs8-zNPPh0EhavdrCfCiTk9nGeO8e6VxUCzwdKXJAd0', - 'AHMBU5PsIJN2U9m7j0SGyvs8YD8fkym2noELzxIrzfdG', - 'AJZ7ZLd7unQ4IkMUwE69NXcvDO9rrmmRH_Xk3TPu9BpP', - 'ANfkMQ5LKPfjEdQPK2c_zWsOn4GgLWsnWvIa25EVVbtR', - 'ACrmDHtPQjnM8H9pyKA-QBNdfZ-xixTlRZTS8WXCrrMH', - 'AMRXyU3ErhBNdRSDX1zKlrbZGRp1GfCmkRIa58gF07I8', - 'AC6vsNVCpHa6acGcxk7c-D1mBHlptPrAx8zr-bKvesSW'] - - # create signers from secrets - signers = [core.Signer(qb64=secret) for secret in secrets] # faster - assert [siger.qb64 for siger in signers] == secrets - # each signer has verfer for keys - - # Test with sith with one clause - keys = [signers[0].verfer.qb64, signers[1].verfer.qb64, signers[2].verfer.qb64] - sith = [["1/2", "1/2", "1"]] - ndigs = [Diger(ser=signers[3].verfer.qb64b).qb64] # default limen/sith - - ked = dict(v=vs, # version string - t=ilk, - d="", # SAID - i="", # qb64 prefix - s="{:x}".format(sn), # hex string no leading zeros lowercase - kt=sith, # hex string no leading zeros lowercase - k=keys, # list of qb64 - nt=1, - n=ndigs, # hash qual Base64 - bt="{:x}".format(toad), # hex string no leading zeros lowercase - b=wits, # list of qb64 may be empty - c=cnfg, # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - - prefixer1 = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer1.qb64 == 'EOnpRzJpF1LNdCXl7aQ76BxF7qT94PChM7WGKARhZeKj' - assert prefixer1.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - # now test with different sith but same weights in two clauses - sith = [["1/2", "1/2"], ["1"]] - - ked = dict(v=vs, # version string - t=ilk, - d="", # SAID - i="", # qb64 prefix - s="{:x}".format(sn), # hex string no leading zeros lowercase - kt=sith, # hex string no leading zeros lowercase - k=keys, # list of qb64 - nt=1, - n=ndigs, # hash qual Base64 - bt="{:x}".format(toad), # hex string no leading zeros lowercase - b=wits, # list of qb64 may be empty - c=cnfg, # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - - prefixer2 = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer2.qb64 == 'ECBv9o83MnNYRTdXhwTeR5zgwt8jTr5NIuJ8P00BKySW' - assert prefixer2.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - assert prefixer2.qb64 != prefixer1.qb64 # semantic diff -> syntactic diff - - sith = "1" - seal = dict(i='EBfPkd-A2CQfJmfpmtc1V-yuleSeCcyWBIrTAygUgQ_T', - s='2', - t=Ilks.ixn, - d='EB0_D51cTh_q6uOQ-byFiv5oNXZ-cxdqCqBAa4JmBLtb') - - ked = dict(v=vs, # version string - t=Ilks.dip, - d="", # SAID - i="", # qb64 prefix - s="{:x}".format(sn), # hex string no leading zeros lowercase - kt=sith, # hex string no leading zeros lowercase - k=keys, # list of qb64 - nt=1, - n=ndigs, # hash qual Base64 - bt="{:x}".format(toad), # hex string no leading zeros lowercase - b=wits, # list of qb64 may be empty - c=cnfg, # list of config ordered mappings may be empty - a=[seal], # list of seal dicts - di='EBfPkd-A2CQfJmfpmtc1V-yuleSeCcyWBIrTAygUgQ_T', - ) - - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EEGithHj9A85F9hz1fxlF80U7wvpFoAPj6U4q4YWMehp' - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - # test with allows - with pytest.raises(ValueError): - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256, - allows=[MtrDex.Ed25519N, MtrDex.Ed25519]) - - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256, - allows=[MtrDex.Blake3_256, MtrDex.Ed25519]) - assert prefixer.qb64 == 'EEGithHj9A85F9hz1fxlF80U7wvpFoAPj6U4q4YWMehp' - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - # Secp256r1 - - preN = '1AAIA-KzxCX8SZSl-fpU3vc3z_MBuH06YShJFuiMdAmo37TM' - # 'BrHLayDN-mXKv62DAjFLX1_Y5yEUe0vA9YPe_ihiKYHE' - pre = '1AAJA-KzxCX8SZSl-fpU3vc3z_MBuH06YShJFuiMdAmo37TM' - - # sigkey = ec.generate_private_key(ec.SECP256R1()) - # verkey = sigkey.public_key().public_bytes(encoding=Encoding.X962, format=PublicFormat.CompressedPoint) - verkey = b'\x03\xe2\xb3\xc4%\xfcI\x94\xa5\xf9\xfaT\xde\xf77\xcf\xf3\x01\xb8}:a(I\x16\xe8\x8ct\t\xa8\xdf\xb4\xcc' - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1) - assert verfer.qb64 == '1AAJA-KzxCX8SZSl-fpU3vc3z_MBuH06YShJFuiMdAmo37TM' - nxtkeyqb64 = [coring.Diger(ser=verfer.qb64b).qb64] # dfault sith is 1 - assert nxtkeyqb64 == ['EPrVv1ppjxrtV48cS9Tm49n5xojMlZfhEzExg6Ye_ORN'] - prefixer = Prefixer(raw=verkey, code=MtrDex.ECDSA_256r1) # default code is None - assert prefixer.code == MtrDex.ECDSA_256r1 - assert len(prefixer.raw) == Matter._rawSize(prefixer.code) - assert len(prefixer.qb64) == Matter.Sizes[prefixer.code].fs - - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[prefixer.qb64], # list of qb64 - nt="1", - n=["ABCD"], # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1) - prefixer = Prefixer(raw=verfer.raw, code=MtrDex.ECDSA_256r1N) - assert prefixer.code == MtrDex.ECDSA_256r1N - assert prefixer.verify(ked=ked) == False - assert prefixer.verify(ked=ked, prefixed=True) == False - - # Test basic derivation from ked - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[verfer.qb64], # list of qb64 - nt="", - n=0, # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - prefixer = Prefixer(ked=ked, code=MtrDex.ECDSA_256r1) - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - badked = dict(ked) - del badked["i"] - with pytest.raises(EmptyMaterialError): # no pre - prefixer = Prefixer(ked=badked) - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - with pytest.raises(DerivationError): # verfer code not match pre code - prefixer = Prefixer(ked=badked) - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=pre - with pytest.raises(DerivationError): - prefixer = Prefixer(ked=badked, code=MtrDex.ECDSA_256r1N) # verfer code not match code - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=pre - prefixer = Prefixer(ked=badked, code=MtrDex.ECDSA_256r1N) # verfer code match code but not pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == False - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - prefixer = Prefixer(ked=badked, code=MtrDex.ECDSA_256r1N) # verfer code match code and pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == True - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - prefixer = Prefixer(ked=badked) # verfer code match pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == True """ Done Test """ diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index 7384bbf0f..c50cee10a 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -4460,8 +4460,6 @@ def test_process_nontransferable(): ) _, ked0 = coring.Saider.saidify(sad=ked0) - # verify derivation of aid0 from ked0 - assert aid0.verify(ked=ked0) # Serialize ked0 tser0 = serdering.SerderKERI(sad=ked0) @@ -4500,9 +4498,6 @@ def test_process_nontransferable(): assert verfer.verify(rsig.raw, rser0.raw) del msgb0[:len(rsig.qb64)] - # verify pre - raid0 = Prefixer(qb64=rser0.pre) - assert raid0.verify(ked=rser0.ked) """ Done Test """ @@ -4548,8 +4543,9 @@ def test_process_transferable(): ) - # Use non digestive AID - aid0 = Prefixer(ked=ked0, code=MtrDex.Ed25519) + # Use non digestive AID made from keys[0] + aid0 = Prefixer(raw=Verfer(qb64=keys[0]).raw, code=MtrDex.Ed25519) + #aid0 = Prefixer(ked=ked0, code=MtrDex.Ed25519) assert aid0.code == MtrDex.Ed25519 assert aid0.qb64 == skp0.verfer.qb64 # update ked with pre @@ -4593,10 +4589,6 @@ def test_process_transferable(): assert verfer.verify(rsig.raw, rser0.raw) del msgb0[:len(rsig.qb64)] - # verify pre - raid0 = Prefixer(qb64=rser0.pre) - assert raid0.verify(ked=rser0.ked) - # verify nxt digest from event is still valid digers=rser0.ndigers #assert rnxt1.includes(keys=nxtkeys) diff --git a/tests/vdr/test_eventing.py b/tests/vdr/test_eventing.py index 6cfdf6196..2961c2ee7 100644 --- a/tests/vdr/test_eventing.py +++ b/tests/vdr/test_eventing.py @@ -320,97 +320,7 @@ def test_backer_issue_revoke(mockHelpingNowUTC): """ End Test """ -def test_prefixer(): - pre = "DAtNTPnDFBnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM" - vs = versify(version=Version, kind=Kinds.json, size=0) - - with pytest.raises(EmptyMaterialError): - prefixer = Prefixer() - - # vcp, backers allowed no backers - # ["v", "d", "i", "s", "t", "bt", "b", "c"] - ked = dict(v=vs, - d="", # qb64 SAID - i="", # qb64 pre - ii=pre, - s="{:x}".format(0), - t=Ilks.vcp, - bt=0, - b=[], - c=[], - ) - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EDj0Kq4tFBNGKxpdfX2nCfIvYJ-v1MJ24H1dsPfUqzmB' - assert prefixer.verify(ked=ked) is True - assert prefixer.verify(ked=ked, prefixed=True) is False - - # Invalid event type - #["v", "i", "s", "t", "ri", "dt"] - ked = dict(v=vs, - d="", # qb64 SAID - i="", - s="{:x}".format(0), - t=Ilks.iss, - ri="", - dt="", - ) - #with pytest.raises(DerivationError): - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - - # vcp, no backers allowed - ked = dict(v=vs, - d="", # qb64 SAID - i="", - ii=pre, - s="{:x}".format(0), - t=Ilks.vcp, - bt=0, - b=[], - c=[keventing.TraitDex.NoBackers], - ) - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EDz0QmMxf4Dk0C9uiP-y3okN-Bej2IAXSj8UwQgb3NsL' - assert prefixer.verify(ked=ked) is True - assert prefixer.verify(ked=ked, prefixed=True) is False - - bak1 = "EAvR3p8V95W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc" - bak2 = "DBEpNJeSJjxo6oAxkNE8eCOJg2HRPstqkeHWBAvN9XNU" - bak3 = "DCxo-P4W_Z0xXTfoA3_4DMPn7oi0mLCElOWJDpC0nQXw" - - # vcp, one backer - ked = dict(v=vs, - d="", # qb64 SAID - i="", - ii=pre, - s="{:x}".format(0), - t=Ilks.vcp, - bt=1, - b=[bak1], - c=[], - ) - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EIDRsRwJQNw2ujTeoztpEPxN6XRmBB2bnxWlOzJ9OQHk' - assert prefixer.verify(ked=ked) is True - assert prefixer.verify(ked=ked, prefixed=True) is False - - # vcp, many backers - ked = dict(v=vs, - d="", # qb64 SAID - i="", - ii=pre, - s="{:x}".format(0), - t=Ilks.vcp, - bt=2, - b=[bak1, bak2, bak3], - c=[], - ) - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EMsbtrSOa_lNwDMhpdXphmbItPBcaB0qSboopPo9Ub-s' - assert prefixer.verify(ked=ked) is True - assert prefixer.verify(ked=ked, prefixed=True) is False - - """ End Test """ def test_tever_escrow(mockCoringRandomNonce): From 035d735cbbe02a3515c79bdca90d56e8e5acd004 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 7 Aug 2024 16:18:12 -0600 Subject: [PATCH 246/418] refactored Prefixer to remove verify and derive methods. Functionality moved to Serder. Removed redundant SerderKERI._verify() calls as its init defaults to calling ._verify --- src/keri/core/coring.py | 218 ++++++++--------------------------- src/keri/core/eventing.py | 15 --- src/keri/core/serdering.py | 42 ++++--- src/keri/vdr/eventing.py | 6 - tests/core/test_serdering.py | 64 +++++----- 5 files changed, 111 insertions(+), 234 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 100492d0e..b0f110160 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -466,25 +466,6 @@ def __iter__(self): -@dataclass(frozen=True) -class NonTransCodex: - """ - NonTransCodex is codex all non-transferable derivation codes - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. - ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. - Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. - ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. - - def __iter__(self): - return iter(astuple(self)) - - -NonTransDex = NonTransCodex() # Make instance - - # When add new to DigCodes update Saider.Digests and Serder.Digests class attr @dataclass(frozen=True) class DigCodex: @@ -574,7 +555,7 @@ class PreCodex: Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. - Ed25519: str = 'D' # Ed25519 verification key basic derivation + Ed25519: str = 'D' # Ed25519 verification key, basic derivation. Blake3_256: str = 'E' # Blake3 256 bit digest self-addressing derivation. Blake2b_256: str = 'F' # Blake2b 256 bit digest self-addressing derivation. Blake2s_256: str = 'G' # Blake2s 256 bit digest self-addressing derivation. @@ -586,6 +567,9 @@ class PreCodex: SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation + Ed448N: str = '1AAC' # Ed448 verification key non-transferable, basic derivation. + Ed448: str = '1AAD' # Ed448 verification key, basic derivation. + Ed448_Sig: str = '1AAE' # Ed448 signature. Self-signing derivation. ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. ECDSA_256r1: str = "1AAJ" # ECDSA secp256r1 verification or encryption key, basic derivation @@ -596,6 +580,50 @@ def __iter__(self): PreDex = PreCodex() # Make instance +@dataclass(frozen=True) +class NonTransCodex: + """ + NonTransCodex is codex all non-transferable derivation codes + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. + ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. + Ed448N: str = '1AAC' # Ed448 verification key non-transferable, basic derivation. + ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. + + def __iter__(self): + return iter(astuple(self)) + + +NonTransDex = NonTransCodex() # Make instance + + +@dataclass(frozen=True) +class PreNonDigCodex: + """ + PreNonDigCodex is codex all prefixive but non-digestive derivation codes + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. + Ed25519: str = 'D' # Ed25519 verification key, basic derivation. + ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. + ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation + Ed448N: str = '1AAC' # Ed448 verification key non-transferable, basic derivation. + Ed448: str = '1AAD' # Ed448 verification key, basic derivation. + ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. + ECDSA_256r1: str = "1AAJ" # ECDSA secp256r1 verification or encryption key, basic derivation + + def __iter__(self): + return iter(astuple(self)) + + +PreNonDigDex = PreNonDigCodex() # Make instance + + + + # namedtuple for size entries in Matter and Counter derivation code tables # hs is the hard size int number of chars in hard (stable) part of code # ss is the soft size int number of chars in soft (unstable) part of code @@ -3298,91 +3326,6 @@ def __init__(self, raw=None, code=None, ked=None, allows=None, **kwa): #elif self.code in [MtrDex.Ed25519, MtrDex.ECDSA_256r1, MtrDex.ECDSA_256k1]: #self._verify = self._verify_transferable - def derive(self, ked): - """ - Returns tuple (raw, code) of aid prefix as derived from key event dict ked. - uses a derivation code specific _derive method - - Parameters: - ked is inception key event dict - seed is only used for sig derivation it is the secret key/secret - - """ - ilk = ked["t"] - if ilk not in (Ilks.icp, Ilks.dip, Ilks.vcp, Ilks.iss): - raise ValueError("Nonincepting ilk={} for prefix derivation.".format(ilk)) - - # Serder now does this check - #labels = getattr(Labels, ilk) - #for k in labels: - #if k not in ked: - #raise ValidationError("Missing element = {} from {} event for " - #"evt = {}.".format(k, ilk, ked)) - - return (self._derive(ked=ked)) - - def verify(self, ked, prefixed=False): - """ - Returns True if derivation from ked for .code matches .qb64 and - If prefixed also verifies ked["i"] matches .qb64 - False otherwise - - Parameters: - ked is inception key event dict - """ - ilk = ked["t"] - if ilk not in (Ilks.icp, Ilks.dip, Ilks.vcp, Ilks.iss): - raise ValueError("Nonincepting ilk={} for prefix derivation.".format(ilk)) - - # Serder now does this check - #labels = getattr(Labels, ilk) - #for k in labels: - #if k not in ked: - #raise ValidationError("Missing element = {} from {} event for " - #"evt = {}.".format(k, ilk, ked)) - - return (self._verify(ked=ked, pre=self.qb64, prefixed=prefixed)) - - def _derive_non_transferable(self, ked): - """ - Returns tuple (raw, code) of basic nontransferable Ed25519 prefix (qb64) - as derived from inception key event dict ked keys[0] - """ - ked = dict(ked) # make copy so don't clobber original ked - try: - keys = ked["k"] - if len(keys) != 1: - raise DerivationError("Basic derivation needs at most 1 key " - " got {} keys instead".format(len(keys))) - verfer = Verfer(qb64=keys[0]) - except Exception as ex: - raise DerivationError("Error extracting public key =" - " = {}".format(ex)) - - if verfer.code not in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N]: - raise DerivationError("Mismatch derivation code = {}." - "".format(verfer.code)) - - try: - if verfer.code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N] and ked["n"]: - raise DerivationError("Non-empty nxt = {} for non-transferable" - " code = {}".format(ked["n"], - verfer.code)) - - if verfer.code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N] and "b" in ked and ked["b"]: - raise DerivationError("Non-empty b = {} for non-transferable" - " code = {}".format(ked["b"], - verfer.code)) - - if verfer.code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N] and "a" in ked and ked["a"]: - raise DerivationError("Non-empty a = {} for non-transferable" - " code = {}".format(ked["a"], - verfer.code)) - - except Exception as ex: - raise DerivationError("Error checking nxt = {}".format(ex)) - - return (verfer.raw, verfer.code) def _verify_non_transferable(self, ked, pre, prefixed=False): """ @@ -3412,27 +3355,7 @@ def _verify_non_transferable(self, ked, pre, prefixed=False): return True - def _derive_transferable(self, ked): - """ - Returns tuple (raw, code) of basic Ed25519 prefix (qb64) - as derived from inception key event dict ked keys[0] - """ - ked = dict(ked) # make copy so don't clobber original ked - try: - keys = ked["k"] - if len(keys) != 1: - raise DerivationError("Basic derivation needs at most 1 key " - " got {} keys instead".format(len(keys))) - verfer = Verfer(qb64=keys[0]) - except Exception as ex: - raise DerivationError("Error extracting public key =" - " = {}".format(ex)) - if verfer.code not in [MtrDex.Ed25519, MtrDex.ECDSA_256r1, MtrDex.ECDSA_256k1]: - raise DerivationError("Mismatch derivation code = {}" - "".format(verfer.code)) - - return (verfer.raw, verfer.code) def _verify_transferable(self, ked, pre, prefixed=False): """ @@ -3461,51 +3384,6 @@ def _verify_transferable(self, ked, pre, prefixed=False): return True - def _derive_blake3_256(self, ked): - """ - Returns tuple (raw, code) of pre (qb64) as blake3 digest - as derived from inception key event dict ked - """ - ked = dict(ked) # make copy so don't clobber original ked - ilk = ked["t"] - if ilk not in (Ilks.icp, Ilks.dip, Ilks.vcp, Ilks.iss): - raise DerivationError("Invalid ilk = {} to derive pre.".format(ilk)) - - # put in dummy pre to get size correct - ked["i"] = self.Dummy * Matter.Sizes[MtrDex.Blake3_256].fs - ked["d"] = ked["i"] # must be same dummy - #raw, proto, kind, ked, version = sizeify(ked=ked) - raw, _, _, _, _ = sizeify(ked=ked) - dig = blake3.blake3(raw).digest() # digest with dummy 'i' and 'd' - return (dig, MtrDex.Blake3_256) # dig is derived correct new 'i' and 'd' - - - def _verify_blake3_256(self, ked, pre, prefixed=False): - """ - Returns True if verified False otherwise - Verify derivation of fully qualified Base64 prefix from - inception key event dict (ked) - - Parameters: - ked is inception key event dict - pre is Base64 fully qualified default to .qb64 - """ - try: - raw, code = self._derive_blake3_256(ked=ked) # replace with dummy 'i' - crymat = Matter(raw=raw, code=MtrDex.Blake3_256) - if crymat.qb64 != pre: # derived raw with dummy 'i' must match pre - return False - - if prefixed and ked["i"] != pre: # incoming 'i' must match pre - return False - - if ked["i"] != ked["d"]: # when digestive then SAID must match pre - return False - - except Exception as ex: - return False - - return True diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index b4b9f0e55..23bcbb145 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -761,7 +761,6 @@ def incept(keys, saids = {'i': code} serder = serdering.SerderKERI(sad=ked, makify=True, saids=saids) - serder._verify() # raises error if fails verifications return serder @@ -926,12 +925,8 @@ def rotate(pre, ) serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder - #_, ked = coring.Saider.saidify(sad=ked) - - #return Serder(ked=ked) # return serialized ked def deltate(pre, keys, @@ -1007,13 +1002,8 @@ def interact(pre, ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder - #_, ked = coring.Saider.saidify(sad=ked) - - #return Serder(ked=ked) # return serialized ked - def receipt(pre, sn, @@ -1050,7 +1040,6 @@ def receipt(pre, ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -1105,7 +1094,6 @@ def query(route="", ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder #_, ked = coring.Saider.saidify(sad=ked) @@ -1162,7 +1150,6 @@ def reply(route="", ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -1204,7 +1191,6 @@ def prod(route="", ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder #_, ked = coring.Saider.saidify(sad=ked) @@ -1264,7 +1250,6 @@ def bare(route="", ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder #_, sad = coring.Saider.saidify(sad=sad) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index afe6686d7..2e319136a 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -34,7 +34,8 @@ from . import coring -from .coring import MtrDex, DigDex, PreDex, Saids, Digestage +from .coring import (MtrDex, DigDex, PreDex, NonTransDex, PreNonDigDex, + Saids, Digestage) from .coring import (Matter, Saider, Verfer, Diger, Number, Tholder, Tagger, Ilker, Traitor, Verser, ) @@ -1588,8 +1589,8 @@ def _verify(self, **kwa): if (self.vrsn.major < 2 and self.vrsn.minor < 1 and self.ilk in (Ilks.qry, Ilks.rpy, Ilks.pro, Ilks.bar, Ilks.exn)): - pass - else: # verify pre + pass # non prefixive ilks do not have 'i' field + else: # verify pre 'i' field try: code = Matter(qb64=self.pre).code except Exception as ex: @@ -1604,19 +1605,32 @@ def _verify(self, **kwa): if code not in idex: raise ValidationError(f"Invalid identifier prefix code = {code}.") - # non-transferable pre validations - if code in [PreDex.Ed25519N, PreDex.ECDSA_256r1N, PreDex.ECDSA_256k1N]: - if self.ndigs: - raise ValidationError(f"Non-transferable code = {code} with" - f" non-empty nxt = {self.ndigs}.") - if self.backs: - raise ValidationError("Non-transferable code = {code} with" - f" non-empty backers = {self.backs}.") + if self.ilk in (Ilks.icp, Ilks.dip, Ilks.rot, Ilks.drt): # est event + if self.ilk in (Ilks.icp, Ilks.dip): # inceptive event + if code in PreNonDigDex: + if len(self.keys) != 1: + raise ValidationError(f"Invalid keys for non-digestive " + f"prefix {code=}.") + + if self.pre != self.keys[0]: + raise ValidationError(f"Mismatch prefix = {self.pre} and" + f" zeroth key = {self.keys[0]} for " + f" non-digestive prefix {code=}.") + + # non-transferable pre validations + if code in NonTransDex: + if self.ndigs: # when field missing returns None + raise ValidationError(f"Non-transferable code = {code} with" + f" non-empty nxt = {self.ndigs}.") + + if self.backs: # when field missing returns None + raise ValidationError("Non-transferable code = {code} with" + f" non-empty backers = {self.backs}.") - if self.seals: - raise ValidationError("Non-transferable code = {code} with" - f" non-empty seals = {self.seals}.") + if self.seals: # when field missing returns None + raise ValidationError("Non-transferable code = {code} with" + f" non-empty seals = {self.seals}.") if self.ilk in (Ilks.dip): # validate delpre try: diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 5583dc084..6a9bdf8a8 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -105,7 +105,6 @@ def incept( ) serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -206,7 +205,6 @@ def rotate( ) serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -247,7 +245,6 @@ def issue( ked["dt"] = dt serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -297,7 +294,6 @@ def revoke( _, ked = coring.Saider.saidify(sad=ked) serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -350,7 +346,6 @@ def backerIssue( ked["dt"] = dt serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -405,7 +400,6 @@ def backerRevoke( ked["dt"] = dt serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index bbeebd0e4..097a326a5 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -727,7 +727,7 @@ def test_serderkeri(): b'jON07Rwv","i":"EF6LmlLkfoNVY25RcGTsqKLW5uHq36FbnNEdjON07Rwv","s":"0","kt":"0' b'","k":[],"nt":"0","n":[],"bt":"0","b":[],"c":[],"a":[]}') - assert serder.verify() # because empty prefix 'i' field + assert serder.verify() assert serder.ilk == kering.Ilks.icp assert serder.said == 'EF6LmlLkfoNVY25RcGTsqKLW5uHq36FbnNEdjON07Rwv' assert serder.pre == serder.said # prefix is not saidive @@ -736,12 +736,13 @@ def test_serderkeri(): pre = 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' sad['i'] = pre said = serder.said + sad['k'] = [pre] serder = SerderKERI(sad=sad, makify=True) assert serder.verify() assert serder.ilk == kering.Ilks.icp - assert serder.said == 'EIXK39EgyxshefoCdSpKCkG5FR9s405YI4FAHDvAqO_R' + assert serder.said == 'EOx98RQxHrFkjDxqd8TJv9JFnGyho-r0REi0WYHgbYJ0' assert serder.pre == pre # prefix is not saidive sad = serder.sad @@ -769,8 +770,8 @@ def test_serderkeri(): assert serder.seals == [] assert serder.traits == [] assert serder.tholder.sith == '0' - assert serder.keys == [] - assert [verfer.qb64 for verfer in serder.verfers] == [] + assert serder.keys == ['DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx'] + assert [verfer.qb64 for verfer in serder.verfers] == ['DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx'] assert serder.ntholder.sith == '0' assert [diger.qb64 for diger in serder.ndigers] == [] assert serder.bner.num == 0 @@ -801,8 +802,8 @@ def test_serderkeri(): assert serder.seals == [] assert serder.traits == [] assert serder.tholder.sith == '0' - assert serder.keys == [] - assert [verfer.qb64 for verfer in serder.verfers] == [] + assert serder.keys == ['DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx'] + assert [verfer.qb64 for verfer in serder.verfers] == ['DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx'] assert serder.ntholder.sith == '0' assert [diger.qb64 for diger in serder.ndigers] == [] assert serder.bner.num == 0 @@ -957,25 +958,30 @@ def test_serderkeri_icp(): # test makify with preloaded non-digestive 'i' value in sad pre = 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' sad['i'] = pre + sad['k'] = [pre] serder = SerderKERI(sad=sad, makify=True) - assert serder.sad == {'v': 'KERI10JSON0000cf_', - 't': 'icp', - 'd': 'EIXK39EgyxshefoCdSpKCkG5FR9s405YI4FAHDvAqO_R', - 'i': 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx', - 's': '0', - 'kt': '0', - 'k': [], - 'nt': '0', - 'n': [], - 'bt': '0', - 'b': [], - 'c': [], - 'a': []} + assert serder.sad == \ + { + 'v': 'KERI10JSON0000fd_', + 't': 'icp', + 'd': 'EOx98RQxHrFkjDxqd8TJv9JFnGyho-r0REi0WYHgbYJ0', + 'i': 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx', + 's': '0', + 'kt': '0', + 'k': ['DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx'], + 'nt': '0', + 'n': [], + 'bt': '0', + 'b': [], + 'c': [], + 'a': [] + } - assert serder.raw ==(b'{"v":"KERI10JSON0000cf_","t":"icp","d":"EIXK39EgyxshefoCdSpKCkG5FR9s405YI4FA' - b'HDvAqO_R","i":"DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx","s":"0","kt":"0' - b'","k":[],"nt":"0","n":[],"bt":"0","b":[],"c":[],"a":[]}') + assert serder.raw ==(b'{"v":"KERI10JSON0000fd_","t":"icp","d":"EOx98RQxHrFkjDxqd8TJv9JFnGyho-r0REi0' + b'WYHgbYJ0","i":"DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx","s":"0","kt":"0' + b'","k":["DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx"],"nt":"0","n":[],"bt":' + b'"0","b":[],"c":[],"a":[]}') assert serder.verify() assert serder.ilk == kering.Ilks.icp @@ -1006,8 +1012,8 @@ def test_serderkeri_icp(): assert serder.seals == [] assert serder.traits == [] assert serder.tholder.sith == '0' - assert serder.keys == [] - assert [verfer.qb64 for verfer in serder.verfers] == [] + assert serder.keys == [pre] + assert [verfer.qb64 for verfer in serder.verfers] == [pre] assert serder.ntholder.sith == '0' assert [diger.qb64 for diger in serder.ndigers] == [] assert serder.bner.num == 0 @@ -1042,8 +1048,8 @@ def test_serderkeri_icp(): assert serder.seals == [] assert serder.traits == [] assert serder.tholder.sith == '0' - assert serder.keys == [] - assert [verfer.qb64 for verfer in serder.verfers] == [] + assert serder.keys == [pre] + assert [verfer.qb64 for verfer in serder.verfers] == [pre] assert serder.ntholder.sith == '0' assert [diger.qb64 for diger in serder.ndigers] == [] assert serder.bner.num == 0 @@ -1098,7 +1104,7 @@ def test_serderkeri_rot(): serder = SerderKERI(sad=sad, makify=True) - assert serder.verify() # because pre is empty + assert serder.verify() # because pre is valid assert serder.ilk == kering.Ilks.rot assert serder.pre == pre != serder.said # prefix is not saidive @@ -1206,7 +1212,7 @@ def test_serderkeri_ixn(): serder = SerderKERI(sad=sad, makify=True) - assert serder.verify() # because pre is empty + assert serder.verify() # because pre is now valid assert serder.ilk == kering.Ilks.ixn assert serder.pre == pre != serder.said # prefix is not saidive @@ -1629,7 +1635,7 @@ def test_serderkeri_drt(): serder = SerderKERI(sad=sad, makify=True, verify=False) - assert not serder.verify() # because pre is not digest and delpre is empty + assert not serder.verify() sad = serder.sad pre = 'EF78YGUYCWXptoVVel1TN1F9-KShPHAtEqvf-TEiGvv9' sad['i'] = pre From 3da443857ec399fae5cce6e3c8480f8bd4c62c88 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 7 Aug 2024 17:22:26 -0600 Subject: [PATCH 247/418] fixed up more SerderKERI verify tests clean up stale code in Prefixer All verify functionality of prefixer is now in SerderKERI so Prefixer is legacy. --- src/keri/core/coring.py | 108 ++----------------------- src/keri/core/eventing.py | 6 -- src/keri/core/serdering.py | 33 +++++++- src/keri/vdr/eventing.py | 1 - tests/core/test_serdering.py | 152 +++++++---------------------------- 5 files changed, 66 insertions(+), 234 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index b0f110160..2a55d8790 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -3265,126 +3265,30 @@ def _sha2_256(ser, raw): class Prefixer(Matter): """ - Prefixer is Matter subclass for autonomic identifier prefix using - derivation as determined by code from ked + Prefixer is Matter subclass for autonomic identifier AID prefix Attributes: Inherited Properties: (see Matter) - .pad is int number of pad chars given raw - .code is str derivation code to indicate cypher suite - .raw is bytes crypto material only without code - .index is int count of attached crypto material by context (receipts) - .qb64 is str in Base64 fully qualified with derivation code + crypto mat - .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat - .qb2 is bytes in binary with derivation code + crypto material - .transferable is Boolean, True when transferable derivation code False otherwise Properties: Methods: - verify(): Verifies derivation of aid prefix from a ked Hidden: - ._pad is method to compute .pad property - ._code is str value for .code property - ._raw is bytes value for .raw property - ._index is int value for .index property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 - """ - Dummy = "#" # dummy spaceholder char for pre. Must not be a valid Base64 char - - def __init__(self, raw=None, code=None, ked=None, allows=None, **kwa): - """ - assign ._derive to derive aid prefix from ked - assign ._verify to verify derivation of aid prefix from ked - Default code is None to force EmptyMaterialError when only raw provided but - not code. + """ + def __init__(self, **kwa): + """Checks for .code in PreDex so valid prefixive code Inherited Parameters: - raw is bytes of unqualified crypto material usable for crypto operations - qb64b is bytes of fully qualified crypto material - qb64 is str or bytes of fully qualified crypto material - qb2 is bytes of fully qualified crypto material - code is str of derivation code - index is int of count of attached receipts for CryCntDex codes - - Parameters: - allows (list): allowed codes for prefix. When None then all supported - codes are allowed. This enables a particular use case to restrict - the codes allowed to a subset of all supported. + See Matter """ - super(Prefixer, self).__init__(raw=raw, code=code, **kwa) + super(Prefixer, self).__init__(**kwa) if self.code not in PreDex: raise InvalidCodeError(f"Invalid prefixer code = {self.code}.") - #if self.code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N]: - #self._verify = self._verify_non_transferable - #elif self.code in [MtrDex.Ed25519, MtrDex.ECDSA_256r1, MtrDex.ECDSA_256k1]: - #self._verify = self._verify_transferable - - - def _verify_non_transferable(self, ked, pre, prefixed=False): - """ - Returns True if verified False otherwise - Verify derivation of fully qualified Base64 pre from inception iked dict - - Parameters: - ked is inception key event dict - pre is Base64 fully qualified prefix default to .qb64 - """ - try: - keys = ked["k"] - if len(keys) != 1: - return False - - if keys[0] != pre: - return False - - if prefixed and ked["i"] != pre: - return False - - if ked["n"]: # must be empty - return False - - except Exception as ex: - return False - - return True - - - - def _verify_transferable(self, ked, pre, prefixed=False): - """ - Returns True if verified False otherwise - Verify derivation of fully qualified Base64 prefix from - inception key event dict (ked) - - Parameters: - ked is inception key event dict - pre is Base64 fully qualified prefix default to .qb64 - """ - try: - keys = ked["k"] - if len(keys) != 1: - return False - - if keys[0] != pre: - return False - - if prefixed and ked["i"] != pre: - return False - - except Exception as ex: - return False - - return True - - - # digest algorithm klas, digest size (not default), digest length diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 23bcbb145..6be424a32 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1794,10 +1794,6 @@ def incept(self, serder, estOnly=None): self.prefixer = Prefixer(qb64=serder.pre) - #if not self.prefixer.verify(ked=ked, prefixed=True): # invalid prefix - #raise ValidationError("Invalid prefix = {} for inception evt = {}." - #"".format(self.prefixer.qb64, ked)) - #serder._verify() redundant serder.__init__ defaults to verify self.serder = serder # need whole serder for digest agility comparisons ndigs = serder.ndigs # ked["n"] @@ -1945,7 +1941,6 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # nxt and signatures verify so update state self.sner = sner # sequence number Number instance - #serder._verify() redundant serder.__init__ defaults to verify self.serder = serder # need whole serder for digest agility compare self.ilk = ilk self.tholder = tholder @@ -2006,7 +2001,6 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # validates so update state self.sner = sner # sequence number Number instance - #serder._verify() redundant serder.__init__ defaults to verify self.serder = serder # need for digest agility includes .serder.diger self.ilk = ilk if fn is not None: # first is non-idempotent for fn check mode fn is None diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 2e319136a..b85a9596d 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1569,6 +1569,29 @@ class SerderKERI(Serder): Proto = Protocols.keri # default protocol type + # can't do this override as is because would have to then redo the saidive field + # calculation. To do that would have to first extract the saidive calculations from + # both makify and verify into hidden method called ._saidify and then recall + # ._saidify in order to change defaults via an override + + #def makify(self, **kwa): + #"""Makify given sad dict makes the versions + #Override for ilk and pre specific defaults + #""" + #super(SerderKERI, self).makify(**kwa) # all properties now setup + + #if self.ilk in (Ilks.icp, Ilks.dip) and self.pre: # inceptive with pre + #try: + #code = Matter(qb64=self.pre).code + #except Exception as ex: + #raise ValidationError(f"Invalid identifier prefix = " + #f"{self.pre}.") from ex + + #if code in PreNonDigDex: + #if not self.keys: + #self._sad['k'] = [self.pre] # default for non digestive prefix + #self._sad['kt'] = '1' + def _verify(self, **kwa): """Verifies said(s) in sad against raw @@ -1610,8 +1633,14 @@ def _verify(self, **kwa): if self.ilk in (Ilks.icp, Ilks.dip): # inceptive event if code in PreNonDigDex: if len(self.keys) != 1: - raise ValidationError(f"Invalid keys for non-digestive " - f"prefix {code=}.") + raise ValidationError(f"Invalid keys = {self.keys} " + "for non-digestive prefix " + f"{code=}.") + + if self.tholder.sith != '1': + raise ValidationError(f"Invalid signing threshold =" + f" {self.tholder.sith} for " + f"non-digestive prefix {code=}.") if self.pre != self.keys[0]: raise ValidationError(f"Mismatch prefix = {self.pre} and" diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 6a9bdf8a8..0d46f9f16 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -834,7 +834,6 @@ def incept(self, serder): raise ValidationError("Invalid toad = {} for baks = {} for evt = {}." "".format(toad, baks, ked)) self.toad = toad - #serder._verify() redundant serder.__init__ defaults to verify self.serder = serder def config(self, serder, noBackers=None, estOnly=None): diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 097a326a5..419103865 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -737,12 +737,13 @@ def test_serderkeri(): sad['i'] = pre said = serder.said sad['k'] = [pre] + sad['kt'] = '1' serder = SerderKERI(sad=sad, makify=True) assert serder.verify() assert serder.ilk == kering.Ilks.icp - assert serder.said == 'EOx98RQxHrFkjDxqd8TJv9JFnGyho-r0REi0WYHgbYJ0' + assert serder.said == 'EDjTI-CYL5U9jgeDNMRmbB79zogNOYmMdSUeAxmg5F9k' assert serder.pre == pre # prefix is not saidive sad = serder.sad @@ -769,7 +770,7 @@ def test_serderkeri(): assert serder.sn == 0 assert serder.seals == [] assert serder.traits == [] - assert serder.tholder.sith == '0' + assert serder.tholder.sith == '1' assert serder.keys == ['DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx'] assert [verfer.qb64 for verfer in serder.verfers] == ['DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx'] assert serder.ntholder.sith == '0' @@ -780,8 +781,7 @@ def test_serderkeri(): assert [verfer.qb64 for verfer in serder.berfers] == [] assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + serder = SerderKERI(raw=raw) @@ -801,7 +801,7 @@ def test_serderkeri(): assert serder.sn == 0 assert serder.seals == [] assert serder.traits == [] - assert serder.tholder.sith == '0' + assert serder.tholder.sith == '1' assert serder.keys == ['DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx'] assert [verfer.qb64 for verfer in serder.verfers] == ['DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx'] assert serder.ntholder.sith == '0' @@ -812,8 +812,7 @@ def test_serderkeri(): assert [verfer.qb64 for verfer in serder.berfers] == [] assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + def test_serderkeri_icp(): @@ -885,8 +884,7 @@ def test_serderkeri_icp(): assert serder.adds == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + serder = SerderKERI(raw=raw) assert serder.raw == raw @@ -921,8 +919,7 @@ def test_serderkeri_icp(): assert serder.adds == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + # Test with non-digestive code for 'i' saidive field no sad @@ -959,16 +956,17 @@ def test_serderkeri_icp(): pre = 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' sad['i'] = pre sad['k'] = [pre] + sad['kt'] = '1' serder = SerderKERI(sad=sad, makify=True) assert serder.sad == \ { 'v': 'KERI10JSON0000fd_', 't': 'icp', - 'd': 'EOx98RQxHrFkjDxqd8TJv9JFnGyho-r0REi0WYHgbYJ0', + 'd': 'EDjTI-CYL5U9jgeDNMRmbB79zogNOYmMdSUeAxmg5F9k', 'i': 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx', 's': '0', - 'kt': '0', + 'kt': '1', 'k': ['DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx'], 'nt': '0', 'n': [], @@ -978,10 +976,12 @@ def test_serderkeri_icp(): 'a': [] } - assert serder.raw ==(b'{"v":"KERI10JSON0000fd_","t":"icp","d":"EOx98RQxHrFkjDxqd8TJv9JFnGyho-r0REi0' - b'WYHgbYJ0","i":"DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx","s":"0","kt":"0' - b'","k":["DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx"],"nt":"0","n":[],"bt":' - b'"0","b":[],"c":[],"a":[]}') + + assert serder.raw ==(b'{"v":"KERI10JSON0000fd_","t":"icp","d":"EDjTI-CYL5U9jgeDNMRmbB79zogNOYmMdSUe' + b'Axmg5F9k","i":"DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx","s":"0","kt":"1' + b'","k":["DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx"],"nt":"0","n":[],"bt":' + b'"0","b":[],"c":[],"a":[]}') + assert serder.verify() assert serder.ilk == kering.Ilks.icp @@ -1011,7 +1011,7 @@ def test_serderkeri_icp(): assert serder.sn == 0 assert serder.seals == [] assert serder.traits == [] - assert serder.tholder.sith == '0' + assert serder.tholder.sith == '1' assert serder.keys == [pre] assert [verfer.qb64 for verfer in serder.verfers] == [pre] assert serder.ntholder.sith == '0' @@ -1026,8 +1026,7 @@ def test_serderkeri_icp(): assert serder.adds == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + serder = SerderKERI(raw=raw) assert serder.raw == raw @@ -1047,7 +1046,7 @@ def test_serderkeri_icp(): assert serder.sn == 0 assert serder.seals == [] assert serder.traits == [] - assert serder.tholder.sith == '0' + assert serder.tholder.sith == '1' assert serder.keys == [pre] assert [verfer.qb64 for verfer in serder.verfers] == [pre] assert serder.ntholder.sith == '0' @@ -1062,8 +1061,7 @@ def test_serderkeri_icp(): assert serder.adds == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + """End Test""" @@ -1143,8 +1141,7 @@ def test_serderkeri_rot(): assert serder.berfers == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + serder = SerderKERI(raw=raw) assert serder.raw == raw @@ -1179,8 +1176,7 @@ def test_serderkeri_rot(): assert serder.adds == [] assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + """End Test""" @@ -1254,8 +1250,7 @@ def test_serderkeri_ixn(): assert serder.priorb == b"" assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + serder = SerderKERI(raw=raw) assert serder.raw == raw @@ -1290,8 +1285,7 @@ def test_serderkeri_ixn(): assert serder.adds == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + """End Test""" def test_serderkeri_dip(): @@ -1389,8 +1383,6 @@ def test_serderkeri_dip(): assert serder.adds == None assert serder.delpre == delpre assert serder.delpreb == delpre.encode("utf-8") - #assert serder.fner == None - #assert serder.fn == None serder = SerderKERI(raw=raw) assert serder.raw == raw @@ -1425,8 +1417,6 @@ def test_serderkeri_dip(): assert serder.adds == None assert serder.delpre == delpre assert serder.delpreb == delpre.encode("utf-8") - #assert serder.fner == None - #assert serder.fn == None # Test with non-digestive code for 'i' saidive field no sad @@ -1558,8 +1548,6 @@ def test_serderkeri_dip(): assert serder.adds == None assert serder.delpre == delpre assert serder.delpreb == delpre.encode("utf-8") - #assert serder.fner == None - #assert serder.fn == None serder = SerderKERI(raw=raw) assert serder.raw == raw @@ -1594,8 +1582,6 @@ def test_serderkeri_dip(): assert serder.adds == None assert serder.delpre == delpre assert serder.delpreb == delpre.encode("utf-8") - #assert serder.fner == None - #assert serder.fn == None """End Test""" @@ -1783,8 +1769,6 @@ def test_serderkeri_rct(): assert serder.berfers == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None serder = SerderKERI(raw=raw) assert serder.raw == raw @@ -1814,8 +1798,6 @@ def test_serderkeri_rct(): assert serder.berfers == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None """End Test""" def test_serderkeri_qry(): @@ -1838,18 +1820,6 @@ def test_serderkeri_qry(): assert serder.ilk == kering.Ilks.qry assert serder.pre == None != serder.said # prefix is not saidive - #sad = serder.sad - # test makify with preloaded non-digestive 'i' value in sad - #pre = 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - #sad['i'] = pre - - #serder = SerderKERI(sad=sad, makify=True) - - #assert serder.verify() # because pre is empty - #assert serder.ilk == kering.Ilks.qry - #assert serder.pre == pre != serder.said # prefix is not saidive - - sad = serder.sad raw = serder.raw said = serder.said @@ -1915,8 +1885,7 @@ def test_serderkeri_qry(): assert serder.berfers == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + """End Test""" @@ -1941,18 +1910,6 @@ def test_serderkeri_rpy(): assert serder.ilk == kering.Ilks.rpy assert serder.pre == None != serder.said # prefix is not saidive - #sad = serder.sad - # test makify with preloaded non-digestive 'i' value in sad - #pre = 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - #sad['i'] = pre - - #serder = SerderKERI(sad=sad, makify=True) - - #assert serder.verify() # because pre is empty - #assert serder.ilk == kering.Ilks.qry - #assert serder.pre == pre != serder.said # prefix is not saidive - - sad = serder.sad raw = serder.raw said = serder.said @@ -1986,8 +1943,6 @@ def test_serderkeri_rpy(): assert serder.berfers == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None serder = SerderKERI(raw=raw) assert serder.raw == raw @@ -2017,8 +1972,7 @@ def test_serderkeri_rpy(): assert serder.berfers == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + """End Test""" @@ -2042,18 +1996,6 @@ def test_serderkeri_pro(): assert serder.ilk == kering.Ilks.pro assert serder.pre == None != serder.said # prefix is not saidive - #sad = serder.sad - # test makify with preloaded non-digestive 'i' value in sad - #pre = 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - #sad['i'] = pre - - #serder = SerderKERI(sad=sad, makify=True) - - #assert serder.verify() # because pre is empty - #assert serder.ilk == kering.Ilks.qry - #assert serder.pre == pre != serder.said # prefix is not saidive - - sad = serder.sad raw = serder.raw said = serder.said @@ -2119,8 +2061,7 @@ def test_serderkeri_pro(): assert serder.berfers == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + """End Test""" @@ -2144,18 +2085,6 @@ def test_serderkeri_bar(): assert serder.ilk == kering.Ilks.bar assert serder.pre == None != serder.said # prefix is not saidive - #sad = serder.sad - # test makify with preloaded non-digestive 'i' value in sad - #pre = 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - #sad['i'] = pre - - #serder = SerderKERI(sad=sad, makify=True) - - #assert serder.verify() # because pre is empty - #assert serder.ilk == kering.Ilks.qry - #assert serder.pre == pre != serder.said # prefix is not saidive - - sad = serder.sad raw = serder.raw said = serder.said @@ -2221,8 +2150,7 @@ def test_serderkeri_bar(): assert serder.berfers == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None + """End Test""" @@ -2253,19 +2181,6 @@ def test_serderkeri_exn(): assert serder.pre == '' assert serder.prior == '' - #sad = serder.sad - ## test makify with preloaded non-digestive 'i' value in sad - #pre = 'DKxy2sgzfplyr-tgwIxS19f2OchFHtLwPWD3v4oYimBx' - #sad['i'] = pre - - #serder = SerderKERI(sad=sad, makify=True) - #assert serder.verify() # because pre is empty - #assert serder.ilk == kering.Ilks.exn - ## need to fix this, since exn does not include prefix field which should be - ## required - #assert serder.pre == pre != serder.said # prefix is not saidive - - sad = serder.sad raw = serder.raw said = serder.said @@ -2302,8 +2217,6 @@ def test_serderkeri_exn(): assert serder.berfers == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None serder = SerderKERI(raw=raw) assert serder.raw == raw @@ -2335,9 +2248,6 @@ def test_serderkeri_exn(): assert serder.berfers == None assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None - """End Test""" @@ -2400,8 +2310,6 @@ def test_serderkeri_vcp(): assert serder.berfers == [] assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None assert serder.uuid == None assert serder.nonce == '' @@ -2432,8 +2340,6 @@ def test_serderkeri_vcp(): assert serder.berfers == [] assert serder.delpre == None assert serder.delpreb == None - #assert serder.fner == None - #assert serder.fn == None assert serder.uuid == None assert serder.nonce == '' From 56adab00aaf1c312a8f81e5b7bb44cbc43321b58 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 7 Aug 2024 17:27:57 -0600 Subject: [PATCH 248/418] remove commented out code --- src/keri/core/serdering.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index b85a9596d..08d2280e9 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1569,30 +1569,6 @@ class SerderKERI(Serder): Proto = Protocols.keri # default protocol type - # can't do this override as is because would have to then redo the saidive field - # calculation. To do that would have to first extract the saidive calculations from - # both makify and verify into hidden method called ._saidify and then recall - # ._saidify in order to change defaults via an override - - #def makify(self, **kwa): - #"""Makify given sad dict makes the versions - #Override for ilk and pre specific defaults - #""" - #super(SerderKERI, self).makify(**kwa) # all properties now setup - - #if self.ilk in (Ilks.icp, Ilks.dip) and self.pre: # inceptive with pre - #try: - #code = Matter(qb64=self.pre).code - #except Exception as ex: - #raise ValidationError(f"Invalid identifier prefix = " - #f"{self.pre}.") from ex - - #if code in PreNonDigDex: - #if not self.keys: - #self._sad['k'] = [self.pre] # default for non digestive prefix - #self._sad['kt'] = '1' - - def _verify(self, **kwa): """Verifies said(s) in sad against raw Override for protocol and ilk specific verification behavior. Especially From 795074cb0b709119f30d4b7d48209dd73b8a2252 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 7 Aug 2024 18:10:34 -0600 Subject: [PATCH 249/418] removed obsolete test --- tests/vdr/test_eventing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vdr/test_eventing.py b/tests/vdr/test_eventing.py index 2961c2ee7..d1237dc49 100644 --- a/tests/vdr/test_eventing.py +++ b/tests/vdr/test_eventing.py @@ -713,4 +713,4 @@ def test_tevery_process_escrow(mockCoringRandomNonce): if __name__ == "__main__": #test_tever_escrow() #test_tevery_process_escrow() - test_prefixer() + From 3ae0d486546d645e20422e4adde4614625af2a40 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 7 Aug 2024 18:12:49 -0600 Subject: [PATCH 250/418] fixed empty block --- tests/vdr/test_eventing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/vdr/test_eventing.py b/tests/vdr/test_eventing.py index d1237dc49..7a0f4618f 100644 --- a/tests/vdr/test_eventing.py +++ b/tests/vdr/test_eventing.py @@ -713,4 +713,5 @@ def test_tevery_process_escrow(mockCoringRandomNonce): if __name__ == "__main__": #test_tever_escrow() #test_tevery_process_escrow() + pass From 1f3efa173c1e2c84b007bfda65562c5abe80b53f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 9 Aug 2024 11:28:35 -0600 Subject: [PATCH 251/418] refactor Diger to simplify --- src/keri/core/coring.py | 160 +++++++++++++------------------------- tests/core/test_coring.py | 5 +- 2 files changed, 58 insertions(+), 107 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 2a55d8790..8db1148f1 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -55,6 +55,9 @@ ECDSA_256r1_SEEDBYTES = 32 ECDSA_256k1_SEEDBYTES = 32 +# digest algorithm klas, digest size (not default), digest length +# size and length are needed for some digest types as function parameters +Digestage = namedtuple("Digestage", "klas size length") # SAID field labels Saidage = namedtuple("Saidage", "dollar at id_ i d") @@ -269,6 +272,8 @@ def __delitem__(self, name): + + @dataclass(frozen=True) class MatterCodex: """ @@ -3094,68 +3099,77 @@ class Diger(Matter): """ - def __init__(self, raw=None, ser=None, code=MtrDex.Blake3_256, **kwa): - """ - Assign digest verification function to ._verify + # Maps digest codes to Digestages of algorithms for computing digest. + # Should be based on the same set of codes as in DigestCodex + # so Matter.digestive property works. + # Use unit tests to ensure codex elements sets match - See Matter for inherited parameters + Digests = { + DigDex.Blake3_256: Digestage(klas=blake3.blake3, size=None, length=None), + DigDex.Blake2b_256: Digestage(klas=hashlib.blake2b, size=32, length=None), + DigDex.Blake2s_256: Digestage(klas=hashlib.blake2s, size=None, length=None), + DigDex.SHA3_256: Digestage(klas=hashlib.sha3_256, size=None, length=None), + DigDex.SHA2_256: Digestage(klas=hashlib.sha256, size=None, length=None), + DigDex.Blake3_512: Digestage(klas=blake3.blake3, size=None, length=64), + DigDex.Blake2b_512: Digestage(klas=hashlib.blake2b, size=None, length=None), + DigDex.SHA3_512: Digestage(klas=hashlib.sha3_512, size=None, length=None), + DigDex.SHA2_512: Digestage(klas=hashlib.sha512, size=None, length=None), + } + + def __init__(self, raw=None, ser=None, code=DigDex.Blake3_256, **kwa): + """Initialize attributes Inherited Parameters: - raw is bytes of unqualified crypto material usable for crypto operations - qb64b is bytes of fully qualified crypto material - qb64 is str or bytes of fully qualified crypto material - qb2 is bytes of fully qualified crypto material - code is str of derivation code - index is int of count of attached receipts for CryCntDex codes + See Matter Parameters: - ser is bytes serialization from which raw is computed if not raw + ser (bytes): serialization from which raw is computed if not raw """ - # Should implement all digests in DigCodex instance DigDex + try: super(Diger, self).__init__(raw=raw, code=code, **kwa) except EmptyMaterialError as ex: if not ser: raise ex - if code == MtrDex.Blake3_256: - dig = blake3.blake3(ser).digest() - elif code == MtrDex.Blake2b_256: - dig = hashlib.blake2b(ser, digest_size=32).digest() - elif code == MtrDex.Blake2s_256: - dig = hashlib.blake2s(ser, digest_size=32).digest() - elif code == MtrDex.SHA3_256: - dig = hashlib.sha3_256(ser).digest() - elif code == MtrDex.SHA2_256: - dig = hashlib.sha256(ser).digest() - else: - raise InvalidValueError("Unsupported code={code} for diger.") - - super(Diger, self).__init__(raw=dig, code=code, **kwa) - - if self.code == MtrDex.Blake3_256: - self._verify = self._blake3_256 - elif self.code == MtrDex.Blake2b_256: - self._verify = self._blake2b_256 - elif self.code == MtrDex.Blake2s_256: - self._verify = self._blake2s_256 - elif self.code == MtrDex.SHA3_256: - self._verify = self._sha3_256 - elif self.code == MtrDex.SHA2_256: - self._verify = self._sha2_256 - else: - raise InvalidValueError("Unsupported code={self.code} for diger.") + + raw = self._digest(ser, code=code) + + super(Diger, self).__init__(raw=raw, code=code, **kwa) + + if self.code not in DigDex: + raise InvalidCodeError(f"Unsupported Digest {code=}.") + + + def _digest(self, ser, code=DigDex.Blake3_256): + """Returns raw digest of ser using digest algorithm given by code + + Parameters: + ser (bytes): serialization from which raw digest is computed + code (str): derivation code used to lookup digest algorithm + """ + if code not in self.Digests: + raise InvalidCodeError(f"Unsupported Digest {code=}.") + + klas, size, length = self.Digests[code] # digest algo size & length + ikwa = dict(digest_size=size) if size else dict() # opt digest size + dkwa = dict(length=length) if length else dict() # opt digest length + raw = klas(ser, **ikwa).digest(**dkwa) + return (raw) + def verify(self, ser): """ Returns True if raw digest of ser bytes (serialization) matches .raw - using .raw as reference digest for ._verify digest algorithm determined + using .raw as reference digest for digest algorithm determined by .code Parameters: - ser (bytes): serialization to be digested and compared to .ser + ser (bytes): serialization to be digested and compared to .raw + """ - return (self._verify(ser=ser, raw=self.raw)) + return (self._digest(ser=ser, code=self.code) == self.raw) + def compare(self, ser, dig=None, diger=None): """ @@ -3202,65 +3216,6 @@ def compare(self, ser, dig=None, diger=None): return (False) - @staticmethod - def _blake3_256(ser, raw): - """ - Returns True if verified False otherwise - Verifiy blake3_256 digest of ser matches raw - - Parameters: - ser is bytes serialization - dig is bytes reference digest - """ - return (blake3.blake3(ser).digest() == raw) - - @staticmethod - def _blake2b_256(ser, raw): - """ - Returns True if verified False otherwise - Verifiy blake2b_256 digest of ser matches raw - - Parameters: - ser is bytes serialization - dig is bytes reference digest - """ - return (hashlib.blake2b(ser, digest_size=32).digest() == raw) - - @staticmethod - def _blake2s_256(ser, raw): - """ - Returns True if verified False otherwise - Verifiy blake2s_256 digest of ser matches raw - - Parameters: - ser is bytes serialization - dig is bytes reference digest - """ - return (hashlib.blake2s(ser, digest_size=32).digest() == raw) - - @staticmethod - def _sha3_256(ser, raw): - """ - Returns True if verified False otherwise - Verifiy blake2s_256 digest of ser matches raw - - Parameters: - ser is bytes serialization - dig is bytes reference digest - """ - return (hashlib.sha3_256(ser).digest() == raw) - - @staticmethod - def _sha2_256(ser, raw): - """ - Returns True if verified False otherwise - Verifiy blake2s_256 digest of ser matches raw - - Parameters: - ser is bytes serialization - dig is bytes reference digest - """ - return (hashlib.sha256(ser).digest() == raw) class Prefixer(Matter): @@ -3291,9 +3246,6 @@ def __init__(self, **kwa): -# digest algorithm klas, digest size (not default), digest length -# size and length are needed for some digest types as function parameters -Digestage = namedtuple("Digestage", "klas size length") class Saider(Matter): diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 7ffc5649c..fdd4cffb6 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -4439,11 +4439,9 @@ def test_diger(): ser = b'abcdefghijklmnopqrstuvwxyz0123456789' dig = blake3.blake3(ser).digest() - with pytest.raises(coring.InvalidValueError): + with pytest.raises(kering.InvalidCodeError): diger = Diger(raw=dig, code=MtrDex.Ed25519) - with pytest.raises(coring.InvalidValueError): - diger = Diger(ser=ser, code=MtrDex.Ed25519) diger = Diger(raw=dig) # defaults provide Blake3_256 digester assert diger.code == MtrDex.Blake3_256 @@ -5389,6 +5387,7 @@ def test_tholder(): test_ilker() test_traitor() test_verser() + test_diger() #test_texter() #test_prodex() #test_indexer() From 5969fd511ad78d9330f7fcb16af7569478e3267f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 9 Aug 2024 14:36:53 -0600 Subject: [PATCH 252/418] refactored Saider and Serder to use new Diger and Diger._digest class method so code is much more DRY --- src/keri/core/coring.py | 33 ++++++--------------------------- src/keri/core/serdering.py | 35 ++--------------------------------- tests/core/test_coring.py | 6 ++++-- 3 files changed, 12 insertions(+), 62 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 8db1148f1..643834025 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -3140,18 +3140,18 @@ def __init__(self, raw=None, ser=None, code=DigDex.Blake3_256, **kwa): if self.code not in DigDex: raise InvalidCodeError(f"Unsupported Digest {code=}.") - - def _digest(self, ser, code=DigDex.Blake3_256): + @classmethod + def _digest(cls, ser, code=DigDex.Blake3_256): """Returns raw digest of ser using digest algorithm given by code Parameters: ser (bytes): serialization from which raw digest is computed code (str): derivation code used to lookup digest algorithm """ - if code not in self.Digests: + if code not in cls.Digests: raise InvalidCodeError(f"Unsupported Digest {code=}.") - klas, size, length = self.Digests[code] # digest algo size & length + klas, size, length = cls.Digests[code] # digest algo size & length ikwa = dict(digest_size=size) if size else dict() # opt digest size dkwa = dict(length=length) if length else dict() # opt digest length raw = klas(ser, **ikwa).digest(**dkwa) @@ -3277,19 +3277,6 @@ class Saider(Matter): """ Dummy = "#" # dummy spaceholder char for said. Must not be a valid Base64 char - # should be same set of codes as in coring.DigestCodex coring.DigDex so - # .digestive property works. Unit test ensures code sets match - Digests = { - MtrDex.Blake3_256: Digestage(klas=blake3.blake3, size=None, length=None), - MtrDex.Blake2b_256: Digestage(klas=hashlib.blake2b, size=32, length=None), - MtrDex.Blake2s_256: Digestage(klas=hashlib.blake2s, size=None, length=None), - MtrDex.SHA3_256: Digestage(klas=hashlib.sha3_256, size=None, length=None), - MtrDex.SHA2_256: Digestage(klas=hashlib.sha256, size=None, length=None), - MtrDex.Blake3_512: Digestage(klas=blake3.blake3, size=None, length=64), - MtrDex.Blake2b_512: Digestage(klas=hashlib.blake2b, size=None, length=None), - MtrDex.SHA3_512: Digestage(klas=hashlib.sha3_512, size=None, length=None), - MtrDex.SHA2_512: Digestage(klas=hashlib.sha512, size=None, length=None), - } def __init__(self, raw=None, *, code=None, sad=None, kind=None, label=Saids.d, ignore=None, **kwa): @@ -3434,18 +3421,10 @@ def _derive(clas, sad: dict, *, for f in ignore: del ser[f] - # string now has - # correct size - klas, size, length = clas.Digests[code] + # string now has correct size # sad as 'v' verision string then use its kind otherwise passed in kind cpa = [clas._serialize(ser, kind=kind)] # raw pos arg class - ckwa = dict() # class keyword args - if size: - ckwa.update(digest_size=size) # optional digest_size - dkwa = dict() # digest keyword args - if length: - dkwa.update(length=length) - return klas(*cpa, **ckwa).digest(**dkwa), sad # raw digest and sad + return (Diger._digest(ser=cpa[0], code=code), sad) # raw digest and sad def derive(self, sad, code=None, **kwa): diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 08d2280e9..abb74a9b8 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -348,22 +348,6 @@ class Serder: # Spans dict keyed by version (Versionage instance) of version string span (size) Spans = {Vrsn_1_0: VER1FULLSPAN, Vrsn_2_0: VER2FULLSPAN} - # Maps digest codes to Digestages of algorithms for computing digest. - # Should be based on the same set of codes as in coring.DigestCodex - # coring.DigDex so .digestive property works. - # Use unit tests to ensure codex elements sets match - Digests = { - DigDex.Blake3_256: Digestage(klas=blake3.blake3, size=None, length=None), - DigDex.Blake2b_256: Digestage(klas=hashlib.blake2b, size=32, length=None), - DigDex.Blake2s_256: Digestage(klas=hashlib.blake2s, size=None, length=None), - DigDex.SHA3_256: Digestage(klas=hashlib.sha3_256, size=None, length=None), - DigDex.SHA2_256: Digestage(klas=hashlib.sha256, size=None, length=None), - DigDex.Blake3_512: Digestage(klas=blake3.blake3, size=None, length=64), - DigDex.Blake2b_512: Digestage(klas=hashlib.blake2b, size=None, length=None), - DigDex.SHA3_512: Digestage(klas=hashlib.sha3_512, size=None, length=None), - DigDex.SHA2_512: Digestage(klas=hashlib.sha512, size=None, length=None), - } - # map seal clan names to seal counter code for grouping seals in anchor list ClanCodes = dict() ClanCodes[SClanDom.SealDigest.__name__] = SealDex_2_0.DigestSealSingles @@ -777,14 +761,7 @@ def _verify(self): raw = self.dumps(sad, kind=self.kind) # serialize dummied sad copy for label, code in saids.items(): if code in DigDex: # subclass override if non digestive allowed - klas, size, length = self.Digests[code] # digest algo size & length - ikwa = dict() # digest algo class initi keyword args - if size: - ikwa.update(digest_size=size) # optional digest_size - dkwa = dict() # digest method keyword args - if length: - dkwa.update(length=length) - dig = Matter(raw=klas(raw, **ikwa).digest(**dkwa), code=code).qb64 + dig = Diger(ser=raw, code=code).qb64 if dig != self._sad[label]: # compare to original raise ValidationError(f"Invalid said field '{label}' in sad" f" = {self._sad}, should be {dig}.") @@ -1005,15 +982,7 @@ def makify(self, sad, *, proto=None, vrsn=None, kind=None, raw = self.dumps(sad, kind=kind, proto=proto, vrsn=vrsn) # serialize sized dummied sad for label, code in _saids.items(): if code in DigDex: # subclass override if non digestive allowed - klas, dsize, dlen = self.Digests[code] # digest algo size & length - ikwa = dict() # digest algo class initi keyword args - if dsize: - ikwa.update(digest_size=dsize) # optional digest_size - dkwa = dict() # digest method keyword args - if dlen: - dkwa.update(length=dlen) - dig = Matter(raw=klas(raw, **ikwa).digest(**dkwa), code=code).qb64 - sad[label] = dig + sad[label] = Diger(ser=raw, code=code).qb64 raw = self.dumps(sad, kind=kind, proto=proto, vrsn=vrsn) # compute final raw if kind == Kinds.cesr:# cesr kind version string does not set size diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index fdd4cffb6..9261b0165 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -4432,6 +4432,10 @@ def test_diger(): """ Test the support functionality for Diger subclass of CryMat """ + # Ensure keyspace of Diger.Digests is same as codes in DigDex + assert set(coring.DigDex) == set(Diger.Digests.keys()) + + with pytest.raises(EmptyMaterialError): diger = Diger() @@ -4612,8 +4616,6 @@ def test_saider(): """ Test Saider object """ - # Test class attribute Digest matches DigDex (i.e.DigestCodex) - assert set(Saider.Digests.keys()) == set(code for code in DigDex) code = MtrDex.Blake3_256 kind = Kinds.json From 837ffff17a2c24669797b49fb9f7010f67b74941 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 9 Aug 2024 15:41:39 -0600 Subject: [PATCH 253/418] fixed bug --- src/keri/core/coring.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 643834025..cf7b669ac 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -3407,8 +3407,8 @@ def _derive(clas, sad: dict, *, ignore (list): fields to ignore when generating SAID """ - if code not in DigDex or code not in clas.Digests: - raise ValueError("Unsupported digest code = {}.".format(code)) + if code not in DigDex: + raise ValueError(f"Unsupported digest {code = }.") sad = dict(sad) # make shallow copy so don't clobber original sad # fill id field denoted by label with dummy chars to get size correct @@ -3423,8 +3423,8 @@ def _derive(clas, sad: dict, *, # string now has correct size # sad as 'v' verision string then use its kind otherwise passed in kind - cpa = [clas._serialize(ser, kind=kind)] # raw pos arg class - return (Diger._digest(ser=cpa[0], code=code), sad) # raw digest and sad + cpa = clas._serialize(ser, kind=kind) # raw pos arg class + return (Diger._digest(ser=cpa, code=code), sad) # raw digest and sad def derive(self, sad, code=None, **kwa): From 2dfb282ef5891c5c694c6ebd18e6319870fd4bd9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 9 Aug 2024 15:43:22 -0600 Subject: [PATCH 254/418] fixed some more --- src/keri/core/coring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index cf7b669ac..be89859d1 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -3408,7 +3408,7 @@ def _derive(clas, sad: dict, *, """ if code not in DigDex: - raise ValueError(f"Unsupported digest {code = }.") + raise ValueError(f"Unsupported digest {code=}.") sad = dict(sad) # make shallow copy so don't clobber original sad # fill id field denoted by label with dummy chars to get size correct From 6be8d8f3089a003e832c9a4008bb19d23d3e6b28 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 9 Aug 2024 15:53:16 -0600 Subject: [PATCH 255/418] fix comments --- src/keri/core/coring.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index be89859d1..f4facb5a6 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -3417,13 +3417,11 @@ def _derive(clas, sad: dict, *, raw, proto, kind, sad, version = sizeify(ked=sad, kind=kind) ser = dict(sad) - if ignore: + if ignore: # delete ignore fields in said calculation from ser dict for f in ignore: del ser[f] - # string now has correct size - # sad as 'v' verision string then use its kind otherwise passed in kind - cpa = clas._serialize(ser, kind=kind) # raw pos arg class + cpa = clas._serialize(ser, kind=kind) # serialize ser return (Diger._digest(ser=cpa, code=code), sad) # raw digest and sad From 8c038c9f18d6b29769bedb3eafdea465018e5d08 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 10 Aug 2024 16:23:51 -0600 Subject: [PATCH 256/418] refactor Bexter to add class Method so external Labeler can decode. --- src/keri/core/coring.py | 101 +++++++++++++++++++++++++++++++++++--- tests/core/test_coring.py | 23 +++++++-- 2 files changed, 111 insertions(+), 13 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index f4facb5a6..973f5e50b 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -2013,7 +2013,7 @@ class Tagger(Matter): composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip Properties: - tag (str): B64 primitive without prepad (alias of .soft) + tag (str): B64 .soft portion of code but without prepad Inherited Hidden: (See Matter) @@ -2660,14 +2660,15 @@ def _rawify(self, bext): raw = decodeB64(base)[ls:] # convert and remove leader return raw # raw binary equivalent of text - @property - def bext(self): - """ - Property bext: Base64 text value portion of qualified b64 str - Returns the value portion of .qb64 with text code and leader removed + @classmethod + def _decode(cls, raw, code): + """Returns decoded raw as B64 str aka bext value + + Returns: + bext (str): decoded raw as B64 str aka bext value """ - _, _, _, _, ls = self.Sizes[self.code] - bext = encodeB64(bytes([0] * ls) + self.raw) + _, _, _, _, ls = cls.Sizes[code] + bext = encodeB64(bytes([0] * ls) + raw) ws = 0 if ls == 0 and bext: if bext[0] == ord(b'A'): # strip leading 'A' zero pad @@ -2677,6 +2678,24 @@ def bext(self): return bext.decode('utf-8')[ws:] + @property + def bext(self): + """ + Property bext: Base64 text value portion of qualified b64 str + Returns the value portion of .qb64 with text code and leader removed + """ + return self._decode(raw=self.raw, code=self.code) + #_, _, _, _, ls = self.Sizes[self.code] + #bext = encodeB64(bytes([0] * ls) + self.raw) + #ws = 0 + #if ls == 0 and bext: + #if bext[0] == ord(b'A'): # strip leading 'A' zero pad + #ws = 1 + #else: + #ws = (ls + 1) % 4 + #return bext.decode('utf-8')[ws:] + + class Pather(Bexter): """ Pather is a subclass of Bexter that provides SAD Path language specific functionality @@ -2909,6 +2928,72 @@ def _resolve(self, val, ptr): return self._resolve(cur, ptr) +class Labeler(Matter): + """ + Labeler is subclass of Matter for CESR native field map labels and/or generic + textual field values. Labeler auto sizes the instance code to minimize + the total encoded size of associated field label or textual field value. + + + + Attributes: + + Inherited Properties: + (See Matter) + + + Properties: + label (str): base value without encoding + + Inherited Hidden: + (See Matter) + + Hidden: + _label (str): base value without encoding + + Methods: + + """ + + + def __init__(self, label='', code=None, **kwa): + """ + Inherited Parameters: + (see Matter) + + Parameters: + label (str | bytes): base value before encoding + + """ + self._label = None + if label: + if hasattr(label, "decode"): # make label a str + label = label.decode("utf-8") + + if not Reb64.match(label.encode("utf-8")): + raise InvalidSoftError(f"Non Base64 chars in {tag=}.") + + self._label = label + + + super(Labeler, self).__init__(code=code, **kwa) + + + + if self.code not in LabelDex: + raise InvalidCodeError(f"Invalid code={self.code} for Labeler.") + + @property + def label(self): + """Returns: + label (str): base value without encoding + + getter for ._label. Makes ._label read only + + """ + return self._label + + class Verfer(Matter): """ diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 9261b0165..d4b52f67d 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -40,7 +40,7 @@ from keri.core import coring from keri.core.coring import (Saids, Sadder, Tholder, Seqner, NumDex, Number, Dater, Bexter, Texter, - TagDex, Tagger, Ilker, Traitor, + TagDex, Tagger, Ilker, Traitor, Labeler, Verser, Versage, ) from keri.core.coring import Kindage, Kinds from keri.core.coring import (Sizage, MtrDex, Matter) @@ -4270,6 +4270,19 @@ def test_pather(): """ Done Test """ +def test_labeler(): + """ + Test Labeler subclass of Matter + """ + + + with pytest.raises(EmptyMaterialError): + labeler = Labeler() # defaults + + + """ Done Test """ + + def test_verfer(): """ Test the support functionality for verifier subclass of crymat @@ -5390,14 +5403,14 @@ def test_tholder(): test_traitor() test_verser() test_diger() - #test_texter() + test_texter() + test_bexter() + test_labeler() #test_prodex() - #test_indexer() test_number() #test_seqner() #test_siger() #test_nexter() #test_tholder() - #test_labels() - #test_prefixer() + test_prefixer() From c1798db6653be6265f12a69c57817fcdbf9e69d7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 10 Aug 2024 16:32:30 -0600 Subject: [PATCH 257/418] Finished refactor Bexter so rawify and derawify can be called externally as static and class methods --- src/keri/core/coring.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 973f5e50b..dd36a6916 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -2646,7 +2646,8 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, raise ValidationError("Invalid code = {} for Bexter." "".format(self.code)) - def _rawify(self, bext): + @staticmethod + def _rawify(bext): """Returns raw value equivalent of Base64 text. Suitable for variable sized matter @@ -2661,7 +2662,7 @@ def _rawify(self, bext): return raw # raw binary equivalent of text @classmethod - def _decode(cls, raw, code): + def _derawify(cls, raw, code): """Returns decoded raw as B64 str aka bext value Returns: @@ -2684,16 +2685,7 @@ def bext(self): Property bext: Base64 text value portion of qualified b64 str Returns the value portion of .qb64 with text code and leader removed """ - return self._decode(raw=self.raw, code=self.code) - #_, _, _, _, ls = self.Sizes[self.code] - #bext = encodeB64(bytes([0] * ls) + self.raw) - #ws = 0 - #if ls == 0 and bext: - #if bext[0] == ord(b'A'): # strip leading 'A' zero pad - #ws = 1 - #else: - #ws = (ls + 1) % 4 - #return bext.decode('utf-8')[ws:] + return self._derawify(raw=self.raw, code=self.code) class Pather(Bexter): From 90c30a7841a15ac69aa6fb167199c697419e84b7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 11 Aug 2024 14:23:01 -0600 Subject: [PATCH 258/418] Started creating Labeler class functionality --- src/keri/core/coring.py | 111 ++++++++++++++++++++++++++++++++------- src/keri/help/helping.py | 4 +- 2 files changed, 95 insertions(+), 20 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index dd36a6916..691ae0a9b 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -550,6 +550,46 @@ def __iter__(self): TagDex = TagCodex() # Make instance +@dataclass(frozen=True) +class LabelCodex: + """ + LabelCodex is codex of. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Tag1: str = '0J' # 1 B64 char tag with 1 pre pad + Tag2: str = '0K' # 2 B64 char tag + Tag3: str = 'X' # 3 B64 char tag + Tag4: str = '1AAF' # 4 B64 char tag + Tag5: str = '0L' # 5 B64 char tag with 1 pre pad + Tag6: str = '0M' # 6 B64 char tag + Tag7: str = 'Y' # 7 B64 char tag + Tag8: str = '1AAN' # 8 B64 char tag + Tag9: str = '0N' # 9 B64 char tag with 1 pre pad + Tag10: str = '0O' # 10 B64 char tag + StrB64_L0: str = '4A' # String Base64 Only Leader Size 0 + StrB64_L1: str = '5A' # String Base64 Only Leader Size 1 + StrB64_L2: str = '6A' # String Base64 Only Leader Size 2 + StrB64_Big_L0: str = '7AAA' # String Base64 Only Big Leader Size 0 + StrB64_Big_L1: str = '8AAA' # String Base64 Only Big Leader Size 1 + StrB64_Big_L2: str = '9AAA' # String Base64 Only Big Leader Size 2 + Label1: str = 'V' # Label1 1 bytes for label lead size 1 + Label2: str = 'W' # Label2 2 bytes for label lead size 0 + Bytes_L0: str = '4B' # Byte String lead size 0 + Bytes_L1: str = '5B' # Byte String lead size 1 + Bytes_L2: str = '6B' # Byte String lead size 2 + Bytes_Big_L0: str = '7AAB' # Byte String big lead size 0 + Bytes_Big_L1: str = '8AAB' # Byte String big lead size 1 + Bytes_Big_L2: str = '9AAB' # Byte String big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + + +LabelDex = LabelCodex() # Make instance + + @dataclass(frozen=True) class PreCodex: @@ -2059,16 +2099,13 @@ def __init__(self, tag='', soft='', code=None, **kwa): """ if tag: - if hasattr(tag, "decode"): # make tag str - tag = tag.decode("utf-8") - if not Reb64.match(tag.encode("utf-8")): + if hasattr(tag, "encode"): # make tag bytes for regex + tag = tag.encode("utf-8") + + if not Reb64.match(tag): raise InvalidSoftError(f"Non Base64 chars in {tag=}.") - # TagDex tags appear in order of size 1 to 10, at indices 0 to 9 - codes = astuple(TagDex) - l = len(tag) # soft not empty so l > 0 - if l > len(codes): - raise InvalidSoftError("Oversized tag={soft}.") - code = codes[l-1] # get code for for tag of len where (index = len - 1) + + code = self._codify(tag=tag) soft = tag @@ -2077,6 +2114,27 @@ def __init__(self, tag='', soft='', code=None, **kwa): if (not self._special(self.code)) or self.code not in TagDex: raise InvalidCodeError(f"Invalid code={self.code} for Tagger.") + + @staticmethod + def _codify(tag): + """Returns code for tag when tag is appropriately sized Base64 + + Parameters: + tag (str | bytes): Base64 value + + Returns: + code (str): derivation code for tag + + """ + # TagDex tags appear in order of size 1 to 10, at indices 0 to 9 + codes = astuple(TagDex) + l = len(tag) + if l < 1 or l > len(codes): + raise InvalidSoftError(f"Invalid {tag=} size {l=}, empty or oversized.") + return codes[l-1] # return code at index = len - 1 + + + @property def tag(self): """Returns: @@ -2948,7 +3006,7 @@ class Labeler(Matter): """ - def __init__(self, label='', code=None, **kwa): + def __init__(self, label='', raw=None, code=None, soft=None, **kwa): """ Inherited Parameters: (see Matter) @@ -2958,17 +3016,34 @@ def __init__(self, label='', code=None, **kwa): """ self._label = None - if label: - if hasattr(label, "decode"): # make label a str - label = label.decode("utf-8") - - if not Reb64.match(label.encode("utf-8")): - raise InvalidSoftError(f"Non Base64 chars in {tag=}.") - self._label = label + if label: + if hasattr(label, "encode"): # make label bytes + label = label.encode("utf-8") + + if Reb64.match(label): # candidate for Base64 compact encoding + try: + code = Tagger._codify(tag=label) + soft = label + except InvalidSoftError as ex: # too big + if label[0] != b'A': # use Bexter code + code = MtrDex.StrB64_L0 + raw = Bexter.rawify(label) + else: # use Texter code + code = MtrDex.Bytes_L0 + raw = label + else: + if len(label) == 1: + code = MtrDex.Label1 + elif len(label) == 2: + code = MtrDex.Label2 + else: + code = MtrDex.Bytes_L0 + raw = label + self._label = label.decode() # convert bytes to str - super(Labeler, self).__init__(code=code, **kwa) + super(Labeler, self).__init__(raw=raw, code=code, soft=soft, **kwa) diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index 5333887e5..63bac1a9a 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -264,9 +264,9 @@ def fromIso8601(dts): B64_CHARS = tuple(B64ChrByIdx.values()) # tuple of characters in Base64 -B64REX = rb'^[0-9A-Za-z_-]*\Z' # [A-Za-z0-9\-\_] +B64REX = rb'^[0-9A-Za-z_-]*\Z' # [A-Za-z0-9\-\_] bytes Reb64 = re.compile(B64REX) # compile is faster -# Usage: if Reb64.match(bext): or if not Reb64.match(bext): +# Usage: if Reb64.match(bext): or if not Reb64.match(bext): bext is bytes def intToB64(i, l=1): From 56535e32c9ae6609b8064bd6695a25b05cfe6ae9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 11 Aug 2024 14:29:43 -0600 Subject: [PATCH 259/418] more refactor Labeler --- src/keri/core/coring.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 691ae0a9b..53a7ba2d4 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -3046,10 +3046,11 @@ def __init__(self, label='', raw=None, code=None, soft=None, **kwa): super(Labeler, self).__init__(raw=raw, code=code, soft=soft, **kwa) - if self.code not in LabelDex: raise InvalidCodeError(f"Invalid code={self.code} for Labeler.") + # need way to extract label from raw and code + @property def label(self): """Returns: From 227f9b0cf74577fea6edd4854696004d86d0e459 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 11 Aug 2024 20:12:09 -0600 Subject: [PATCH 260/418] Labeler finished with unit tests --- src/keri/core/coring.py | 43 +++++---- tests/core/test_coring.py | 184 +++++++++++++++++++++++++++++++++++++- 2 files changed, 204 insertions(+), 23 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 53a7ba2d4..99fe7e18a 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -2999,7 +2999,6 @@ class Labeler(Matter): (See Matter) Hidden: - _label (str): base value without encoding Methods: @@ -3015,8 +3014,6 @@ def __init__(self, label='', raw=None, code=None, soft=None, **kwa): label (str | bytes): base value before encoding """ - self._label = None - if label: if hasattr(label, "encode"): # make label bytes label = label.encode("utf-8") @@ -3025,41 +3022,49 @@ def __init__(self, label='', raw=None, code=None, soft=None, **kwa): try: code = Tagger._codify(tag=label) soft = label + except InvalidSoftError as ex: # too big - if label[0] != b'A': # use Bexter code - code = MtrDex.StrB64_L0 - raw = Bexter.rawify(label) - else: # use Texter code - code = MtrDex.Bytes_L0 + if label[0] != ord(b'A'): # use Bexter code + code = LabelDex.StrB64_L0 + raw = Bexter._rawify(label) + + else: # use Texter code since ambiguity if starts with 'A' + code = LabelDex.Bytes_L0 raw = label + else: if len(label) == 1: - code = MtrDex.Label1 + code = LabelDex.Label1 + elif len(label) == 2: - code = MtrDex.Label2 + code = LabelDex.Label2 + else: - code = MtrDex.Bytes_L0 - raw = label + code = LabelDex.Bytes_L0 - self._label = label.decode() # convert bytes to str + raw = label super(Labeler, self).__init__(raw=raw, code=code, soft=soft, **kwa) - if self.code not in LabelDex: raise InvalidCodeError(f"Invalid code={self.code} for Labeler.") - # need way to extract label from raw and code + @property def label(self): - """Returns: + """Extracts and returns label from .code and .soft or .code and .raw + + Returns: label (str): base value without encoding + """ + if self.code in TagDex: # tag + return self.soft # soft part of code - getter for ._label. Makes ._label read only + if self.code in BexDex: # bext + return Bexter._derawify(raw=self.raw, code=self.code) # derawify - """ - return self._label + return self.raw.decode() # everything else is just raw as str diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index d4b52f67d..e41189493 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -33,14 +33,15 @@ from keri.kering import Protocols, Protocolage, Ilkage, Ilks, TraitDex from keri.help import helping -from keri.help.helping import (sceil, intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, +from keri.help.helping import (sceil, intToB64, intToB64b, b64ToInt, + codeB64ToB2, codeB2ToB64, B64_CHARS, Reb64, nabSextets) from keri import core from keri.core import coring from keri.core.coring import (Saids, Sadder, Tholder, Seqner, NumDex, Number, Dater, Bexter, Texter, - TagDex, Tagger, Ilker, Traitor, Labeler, + TagDex, Tagger, Ilker, Traitor, Labeler, LabelDex, Verser, Versage, ) from keri.core.coring import Kindage, Kinds from keri.core.coring import (Sizage, MtrDex, Matter) @@ -4274,11 +4275,186 @@ def test_labeler(): """ Test Labeler subclass of Matter """ - - with pytest.raises(EmptyMaterialError): labeler = Labeler() # defaults + # test taggable label + label = 'z' + raw = b'' + code = LabelDex.Tag1 + qb64 = '0J_z' + qb2 = decodeB64(qb64) + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == label + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code, soft=label) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + + # Test all sizes taggable labels + labels = ('A', 'AB', 'ABC', 'ABCD', 'ABCDE', 'ABCDEF', 'ABCDEFG', 'ABCDEFGH', + 'ABCDEFGHI', 'ABCDEFGHIJ') + + raw = b'' + for i, label in enumerate(labels): + code = astuple(LabelDex)[i] + xs = Matter._xtraSize(code) + qb64 = code + ('_' * xs) + label + qb2 = decodeB64(qb64) + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == label + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code, soft=label) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + + # test bextable labels + label = 'zyxwvutsrqponm' + code = LabelDex.StrB64_L1 + qb64 = '5AAEAAzyxwvutsrqponm' + qb2 = decodeB64(qb64) + raw = qb2[4:] # skip 3 for code and 1 for lead pad + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + rs = (len(label) + len(label) % 4) // 4 + assert labeler.soft == intToB64(rs, 2) == 'AE' + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code, soft=label) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + # test textable labels + # fixed size short + label = '@' + code = LabelDex.Label1 + raw = label.encode() + qb64 = 'VABA' + qb2 = decodeB64(qb64) # b'T\x00@' + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == '' + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + label = '!$' + code = LabelDex.Label2 + raw = label.encode() + qb64 = 'WCEk' + qb2 = decodeB64(qb64) # b'X!$' + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == '' + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + + # variable sized + label = '#yxwvutsrqponm' + code = LabelDex.Bytes_L1 + raw = label.encode() + qb64 = '5BAFACN5eHd2dXRzcnFwb25t' + qb2 = decodeB64(qb64) + + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == 'AF' + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + # test base64 that starts with 'A' get encoded as textable, is not bextable + label = 'Ayxwvutsrqponm' + code = LabelDex.Bytes_L1 + raw = label.encode() + qb64 = '5BAFAEF5eHd2dXRzcnFwb25t' + qb2 = decodeB64(qb64) + + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == 'AF' + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label """ Done Test """ From 8082f7eb50fb7c2997330bcdf959053f04854b27 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 12 Aug 2024 10:46:47 -0400 Subject: [PATCH 261/418] adds docker support for multiple architectures (#832) * adds docker build for multiple architectures Signed-off-by: Kevin Griffin * fix markdown Signed-off-by: Kevin Griffin --------- Signed-off-by: Kevin Griffin --- Makefile | 39 ++++++++++++++++++++++++----------- README.md | 14 +++++++++++++ images/keripy.base.dockerfile | 15 -------------- images/keripy.dockerfile | 2 +- 4 files changed, 42 insertions(+), 28 deletions(-) delete mode 100644 images/keripy.base.dockerfile diff --git a/Makefile b/Makefile index 4ef478940..f34d1bd4e 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,34 @@ .PHONY: build-keri -build-keri: - @docker buildx build --platform=linux/amd64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev12 . - @docker buildx build --platform=linux/arm64 -f images/keripy.dockerfile --tag weboftrust/keri:1.2.0-dev12-arm64 . -.PHONY: build-witness-demo -build-witness-demo: - @@docker buildx build --platform=linux/amd64 -f images/witness.demo.dockerfile --tag weboftrust/keri-witness-demo:1.1.10 . - @@docker buildx build --platform=linux/arm64 -f images/witness.demo.dockerfile --tag weboftrust/keri-witness-demo:1.1.10-arm64 . +VERSION=MULTI_ARCH_TEST -.PHONY: publish-keri -publish-keri: - @docker push weboftrust/keri:1.2.0-dev12 - @docker push weboftrust/keri:1.2.0-dev12-arm64 +define DOCKER_WARNING +In order to use the multi-platform build enable the containerd image store +The containerd image store is not enabled by default. +To enable the feature for Docker Desktop: + Navigate to Settings in Docker Desktop. + In the General tab, check Use containerd for pulling and storing images. + Select Apply and Restart." +endef + +build-keri: .warn + @docker build --platform=linux/amd64,linux/arm64 -f images/keripy.dockerfile -t weboftrust/keri:$(VERSION) . + +.PHONY: build-witness-demo +build-witness-demo: .warn + @docker build --platform=linux/amd64,linux/arm64 -f images/witness.demo.dockerfile -t weboftrust/keri-witness-demo:1.1.10 . .PHONY: publish-keri-witness-demo publish-keri-witness-demo: - @docker push weboftrust/keri-witness-demo --all-tags \ No newline at end of file + @docker push weboftrust/keri-witness-demo --all-tags + +publish-keri: + @docker push weboftrust/keri:$(VERSION) + +.warn: + @echo -e ${RED}"$$DOCKER_WARNING"${NO_COLOUR} + +RED="\033[0;31m" +NO_COLOUR="\033[0m" +export DOCKER_WARNING diff --git a/README.md b/README.md index 935f53a63..a65692358 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,17 @@ pytest tests/demo/ * Build with Sphinx in `/docs`: * `$ make html` +## Publishing containers + +Enable the containerd image store + +The containerd image store isn't enabled by default. To enable the feature for Docker Desktop: + +Navigate to Settings in Docker Desktop. +In the General tab, check Use containerd for pulling and storing images. +Select Apply & Restart. + +```shell +make build-keri +make publish-keri +``` \ No newline at end of file diff --git a/images/keripy.base.dockerfile b/images/keripy.base.dockerfile deleted file mode 100644 index 23de45f8d..000000000 --- a/images/keripy.base.dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# Builder layer -FROM python:3.12-alpine as builder - -# Install compilation dependencies -RUN apk --no-cache add \ - bash \ - alpine-sdk \ - libffi-dev \ - libsodium \ - libsodium-dev - -SHELL ["/bin/bash", "-c"] - -# Setup Rust for blake3 dependency build -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y diff --git a/images/keripy.dockerfile b/images/keripy.dockerfile index 9cc82212c..5591bc801 100644 --- a/images/keripy.dockerfile +++ b/images/keripy.dockerfile @@ -1,6 +1,6 @@ ARG BASE=python:3.12.3-alpine3.20 -FROM ${BASE} as builder +FROM ${BASE} AS builder RUN apk add --no-cache bash From cde9a74cc6f4be887a666d8a6a6e3937b1e2efaf Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Mon, 12 Aug 2024 10:47:40 -0400 Subject: [PATCH 262/418] Adjusted help text for escrow/kevers to better match their intended functionality (#838) --- src/keri/app/cli/commands/escrow.py | 2 +- src/keri/app/cli/commands/kevers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keri/app/cli/commands/escrow.py b/src/keri/app/cli/commands/escrow.py index 62546fa16..4602a6787 100644 --- a/src/keri/app/cli/commands/escrow.py +++ b/src/keri/app/cli/commands/escrow.py @@ -18,7 +18,7 @@ logger = help.ogler.getLogger() -parser = argparse.ArgumentParser(description='Initialize a prefix') +parser = argparse.ArgumentParser(description='Views events in escrow state.') parser.set_defaults(handler=lambda args: handler(args), transferable=True) parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) diff --git a/src/keri/app/cli/commands/kevers.py b/src/keri/app/cli/commands/kevers.py index 4509c476f..a1badf246 100644 --- a/src/keri/app/cli/commands/kevers.py +++ b/src/keri/app/cli/commands/kevers.py @@ -18,7 +18,7 @@ logger = help.ogler.getLogger() -parser = argparse.ArgumentParser(description='Initialize a prefix') +parser = argparse.ArgumentParser(description='Poll events at controller for prefix') parser.set_defaults(handler=lambda args: handler(args), transferable=True) parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) From 388e3924951ef9fa1543ca17c53749e4238766e2 Mon Sep 17 00:00:00 2001 From: Fergal Date: Mon, 12 Aug 2024 15:48:52 +0100 Subject: [PATCH 263/418] feat: raise MissingEntryError for missing credential (#841) --- src/keri/vdr/viring.py | 2 ++ tests/vdr/test_verifying.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 434a2d16e..cf2ed805a 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -485,6 +485,8 @@ def cloneCred(self, said): """ creder = self.creds.get(keys=(said,)) + if creder is None: + raise kering.MissingEntryError(f"no credential found with said {said}") prefixer, seqner, saider = self.cancs.get(keys=(said,)) return creder, prefixer, seqner, saider diff --git a/tests/vdr/test_verifying.py b/tests/vdr/test_verifying.py index ea7bb1bc9..6cd16154e 100644 --- a/tests/vdr/test_verifying.py +++ b/tests/vdr/test_verifying.py @@ -115,6 +115,9 @@ def test_verifier(seeder): for idx, cred in enumerate(creds): assert dcre.sad == cred["sad"] + with pytest.raises(kering.MissingEntryError): + regery.reger.cloneCred(said="nonexistantsaid") + """End Test""" From ee5177ce08dbbb7f932fea0d5e4097701323abc9 Mon Sep 17 00:00:00 2001 From: Rodolfo Date: Mon, 12 Aug 2024 13:55:52 -0300 Subject: [PATCH 264/418] Fix notes datetime ISO 8601 format (#827) * fix notes dt Iso8601 format * add test * use mockHelpingNowUTC in test * iso dt update in signaling --- src/keri/app/notifying.py | 5 ++--- src/keri/app/signaling.py | 6 +++--- tests/app/test_notifying.py | 19 ++++++++++++------- tests/app/test_signaling.py | 2 +- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/keri/app/notifying.py b/src/keri/app/notifying.py index 371815f9b..4ad03d22f 100644 --- a/src/keri/app/notifying.py +++ b/src/keri/app/notifying.py @@ -3,7 +3,6 @@ keri.app.notifying module """ -import datetime from collections.abc import Iterable from typing import Union, Type @@ -71,7 +70,7 @@ def __init__(self, raw=b'', pad=None, note=None): raise ValueError(f"invalid notice, missing attributes in {pad}") if "dt" not in self._pad: - self._pad["dt"] = datetime.datetime.now().isoformat() + self._pad["dt"] = helping.nowIso8601() @property def datetime(self): @@ -386,7 +385,7 @@ def add(self, attrs): """ - note = notice(attrs, dt=datetime.datetime.now()) + note = notice(attrs, dt=helping.nowIso8601()) cig = self.hby.signator.sign(ser=note.raw) if self.noter.add(note, cig): signal = dict( diff --git a/src/keri/app/signaling.py b/src/keri/app/signaling.py index f32e9dbe2..fa44c18fd 100644 --- a/src/keri/app/signaling.py +++ b/src/keri/app/signaling.py @@ -27,7 +27,7 @@ def signal(attrs, topic, ckey=None, dt=None): Notice: Notice instance """ - dt = dt if dt is not None else datetime.datetime.now().isoformat() + dt = dt if dt is not None else helping.nowIso8601() if hasattr(dt, "isoformat"): dt = dt.isoformat() @@ -117,7 +117,7 @@ def push(self, attrs, topic, ckey=None, dt=None): Returns: """ - dt = dt if dt is not None else datetime.datetime.now() + dt = dt if dt is not None else helping.nowIso8601() sig = signal(attrs=attrs, topic=topic, ckey=ckey, dt=dt) if sig.ckey is not None: @@ -146,7 +146,7 @@ def expireDo(self, tymth=None, tock=0.0): _ = (yield self.tock) while True: # loop checking for expired messages - now = datetime.datetime.now() + now = helping.nowUTC() toRemove = [] for sig in self.signals: if now - helping.fromIso8601(sig.dt) > self.SignalTimeout: # Expire messages that are too old diff --git a/tests/app/test_notifying.py b/tests/app/test_notifying.py index b02bd91c2..a62bb620e 100644 --- a/tests/app/test_notifying.py +++ b/tests/app/test_notifying.py @@ -105,7 +105,7 @@ def test_dictersuber(): assert res[2].attrs['a'] == 3 -def test_noter(): +def test_noter(mockHelpingNowUTC): noter = notifying.Noter() assert noter.path.endswith("/not/not") noter.reopen() @@ -137,12 +137,14 @@ def test_noter(): notes = noter.getNotes(start=0) assert len(notes) == 0 - dt = datetime.datetime.now() - note = notifying.notice(attrs=dict(a=1)) + note = notifying.notice(attrs=dict(a=1), + dt=helping.fromIso8601("2022-07-08T15:01:05.453632")) assert noter.add(note, cig) is True - note = notifying.notice(attrs=dict(a=2)) + note = notifying.notice(attrs=dict(a=2), + dt=helping.fromIso8601("2022-07-08T15:01:06.453632")) assert noter.add(note, cig) is True - note = notifying.notice(attrs=dict(a=3)) + note = notifying.notice(attrs=dict(a=3), + dt=helping.fromIso8601("2022-07-08T15:01:07.453632")) assert noter.add(note, cig) is True res = [] @@ -164,12 +166,13 @@ def test_noter(): res.append(note) assert len(res) == 5 + assert res[0][0].datetime == "2021-01-01T00:00:00.000000+00:00" cnt = noter.getNoteCnt() assert cnt == 13 -def test_notifier(): +def test_notifier(mockHelpingNowUTC): with habbing.openHby(name="test") as hby: notifier = notifying.Notifier(hby=hby) assert notifier.signaler is not None @@ -200,7 +203,7 @@ def test_notifier(): assert notifier.rem(note.rid) is True assert notifier.getNotes() == [] - dt = datetime.datetime.now() + dt = helping.nowIso8601() assert notifier.add(attrs=dict(a=1)) is True assert notifier.add(attrs=dict(a=2)) is True assert notifier.add(attrs=dict(a=3)) is True @@ -208,6 +211,8 @@ def test_notifier(): notes = notifier.getNotes() assert len(notes) == 3 + assert notes[2].datetime == "2021-01-01T00:00:00.000000+00:00" + payload = dict(a=1, b=2, c=3) dt = helping.fromIso8601("2022-07-08T15:01:05.453632") cig = coring.Cigar(qb64="AABr1EJXI1sTuI51TXo4F1JjxIJzwPeCxa-Cfbboi7F4Y4GatPEvK629M7G_5c86_Ssvwg8POZWNMV-WreVqBECw") diff --git a/tests/app/test_signaling.py b/tests/app/test_signaling.py index 9010d9e89..fb8f85580 100644 --- a/tests/app/test_signaling.py +++ b/tests/app/test_signaling.py @@ -79,7 +79,7 @@ def test_signaler(): signaler.push(attrs=dict(a=1), topic="/m") signaler.push(attrs=dict(a=2), topic="/m", ckey="abc") signaler.push(attrs=dict(a=3), topic="/m") - now = datetime.datetime.now() - datetime.timedelta(minutes=11) + now = helping.nowUTC() - datetime.timedelta(minutes=11) signaler.push(attrs=dict(a=4), topic="/m", ckey="abc", dt=now) assert len(signaler.signals) == 3 From 970d2fbc1ef7ead32b776d1ccbcf0a42c57fc37c Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Wed, 14 Aug 2024 06:44:18 -0700 Subject: [PATCH 265/418] 2 fixes for delegation: (#844) - Update clone delegation to not raise an exception if the delegator's KEL is not in local kevers - Fix event handling to walk the KEL to find an anchoring event if it exists. These changes close #842 by allowing for a witness to return a KEL for a delegated AID without the AES couple for approved events and without the delegator's KEL. This is the expected behavior for a witness as it should not be assumed that a witness perform watcher functionality. Any validator will have to be introduced to the delegator or OOBI with the delegator before validating the KEL of the delegate. Signed-off-by: pfeairheller --- scripts/demo/basic/delegate.sh | 13 ++++++ src/keri/core/eventing.py | 78 +++++++++++++++++++--------------- src/keri/db/basing.py | 2 +- 3 files changed, 57 insertions(+), 36 deletions(-) diff --git a/scripts/demo/basic/delegate.sh b/scripts/demo/basic/delegate.sh index 4a1e40dfb..9237a5deb 100755 --- a/scripts/demo/basic/delegate.sh +++ b/scripts/demo/basic/delegate.sh @@ -30,3 +30,16 @@ PID_LIST+=" $pid" wait $PID_LIST kli status --name delegate --alias delegate + +kli init --name validator --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis --salt 0ACDEyMzQ1Njc4OWxtbm9vAl + +OOBI=$(kli oobi generate --name delegator --alias delegator --role witness | head -n 1) +kli oobi resolve --name validator --oobi-alias delegator --oobi "${OOBI}" +OOBI=$(kli oobi generate --name delegate --alias delegate --role witness | head -n 1) +kli oobi resolve --name validator --oobi-alias delegate --oobi "${OOBI}" + +AID=$(kli aid --name delegate --alias delegate) +kli kevers --name validator --prefix "${AID}" + + + diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 6be424a32..5443e7d86 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1561,16 +1561,16 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits, - local=local, - delseqner=delseqner, - delsaider=delsaider) + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + local=local, + delseqner=delseqner, + delsaider=delsaider) self.delpre = delpre # may be None self.delegated = True if self.delpre else False @@ -1920,16 +1920,16 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # returned sigers and wigers are verified signatures # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=tholder, - wigers=wigers, - toader=toader, - wits=wits, - local=local, - delseqner=delseqner, - delsaider=delsaider) + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=tholder, + wigers=wigers, + toader=toader, + wits=wits, + local=local, + delseqner=delseqner, + delsaider=delsaider) @@ -1985,18 +1985,18 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=self.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits, - local=local) + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, + sigers=sigers, + verfers=self.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + local=local) # .validateSigsDelWigs above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, first=True if not check else False) # First seen accepted # validates so update state @@ -2314,12 +2314,19 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f"for event={serder.ked}.") if delpre: + if not (delseqner and delsaider): + seal = dict(i=serder.ked["i"], s=serder.snh, d=serder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + self.validateDelegation(serder, sigers=sigers, wigers=wigers, wits=wits, local=local, delpre=delpre, delseqner=delseqner, delsaider=delsaider) - return sigers, wigers, delpre + return sigers, wigers, delpre, delseqner, delsaider def exposeds(self, sigers): @@ -2593,7 +2600,6 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, if self.locallyOwned() or self.locallyWitnessed(wits=wits): return - if self.kevers is None or delpre not in self.kevers: # drop event # ToDo XXXX cue a trigger to get the KEL of the delegator # the processDelegableEvent should also cue a trigger to get KEL @@ -2827,7 +2833,12 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, self.db.putEvt(dgkey, serder.raw) # idempotent (maybe already excrowed) # update event source - if (esr := self.db.esrs.get(keys=dgkeys)): # preexisting esr + + if seqner and saider: # delegation for authorized delegated or issued event + couple = seqner.qb64b + saider.qb64b + self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + + if esr := self.db.esrs.get(keys=dgkeys): # preexisting esr if local and not esr.local: # local overwrites prexisting remote esr.local = local self.db.esrs.pin(keys=dgkeys, val=esr) @@ -2837,9 +2848,6 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, self.db.esrs.put(keys=dgkeys, val=esr) if first: # append event dig to first seen database in order - if seqner and saider: # delegation for authorized delegated or issued event - couple = seqner.qb64b + saider.qb64b - self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal fn = self.db.appendFe(serder.preb, serder.saidb) if firner and fn != firner.sn: # cloned replay but replay fn not match if self.cues is not None: # cue to notice BadCloneFN diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index d50b8b12e..2fc0b8e58 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1617,7 +1617,7 @@ def cloneDelegation(self, kever): kever (Kever): Kever from which to clone the delegator's AID. """ - if kever.delegated: + if kever.delegated and kever.delpre in self.kevers: dkever = self.kevers[kever.delpre] yield from self.cloneDelegation(dkever) From 540945bb27d3aceb463981ebc04ce0a364a7fb8f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 14 Aug 2024 11:57:13 -0600 Subject: [PATCH 266/418] regress without delegation kel walk in event processing since already handled in escrow when source seal not provided --- src/keri/core/eventing.py | 75 ++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 5443e7d86..f098691b8 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1561,16 +1561,16 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits, - local=local, - delseqner=delseqner, - delsaider=delsaider) + sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + local=local, + delseqner=delseqner, + delsaider=delsaider) self.delpre = delpre # may be None self.delegated = True if self.delpre else False @@ -1920,16 +1920,16 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # returned sigers and wigers are verified signatures # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=tholder, - wigers=wigers, - toader=toader, - wits=wits, - local=local, - delseqner=delseqner, - delsaider=delsaider) + sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=tholder, + wigers=wigers, + toader=toader, + wits=wits, + local=local, + delseqner=delseqner, + delsaider=delsaider) @@ -1985,18 +1985,18 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=self.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits, - local=local) + sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, + sigers=sigers, + verfers=self.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + local=local) # .validateSigsDelWigs above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, first=True if not check else False) # First seen accepted # validates so update state @@ -2314,19 +2314,19 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f"for event={serder.ked}.") if delpre: - if not (delseqner and delsaider): - seal = dict(i=serder.ked["i"], s=serder.snh, d=serder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) + #if not (delseqner and delsaider): + #seal = dict(i=serder.ked["i"], s=serder.snh, d=serder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) self.validateDelegation(serder, sigers=sigers, wigers=wigers, wits=wits, local=local, delpre=delpre, - delseqner=delseqner, delsaider=delsaider) + ) # delseqner=delseqner, delsaider=delsaider - return sigers, wigers, delpre, delseqner, delsaider + return (sigers, wigers, delpre) # delseqner, delsaider def exposeds(self, sigers): @@ -2834,6 +2834,7 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, self.db.putEvt(dgkey, serder.raw) # idempotent (maybe already excrowed) # update event source + if seqner and saider: # delegation for authorized delegated or issued event couple = seqner.qb64b + saider.qb64b self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal From 4b6e02d3aa2c9482c80ca1773ba6d1a468d57f18 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 15 Aug 2024 18:13:16 -0600 Subject: [PATCH 267/418] fixed escrow call in Kever.validateDelegation --- src/keri/core/eventing.py | 50 +++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index f098691b8..f6571ab32 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1579,7 +1579,8 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, # .validateSigsDelWigs above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen # returns fn == None if already logged fn log is non idempotent - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, wits=wits, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, + wits=wits, first=True if not check else False, seqner=delseqner, saider=delsaider, firner=firner, dater=dater, local=local) @@ -1935,8 +1936,10 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # .valSigWigsDel above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, wits=wits, - first=True if not check else False, seqner=delseqner, saider=delsaider, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, + wits=wits, + first=True if not check else False, + seqner=delseqner, saider=delsaider, firner=firner, dater=dater, local=local) # nxt and signatures verify so update state @@ -2600,15 +2603,21 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, if self.locallyOwned() or self.locallyWitnessed(wits=wits): return - if self.kevers is None or delpre not in self.kevers: # drop event + if self.kevers is None or delpre not in self.kevers: # missing delegator # ToDo XXXX cue a trigger to get the KEL of the delegator - # the processDelegableEvent should also cue a trigger to get KEL + # the processPSEvent should also cue a trigger to get KEL # of delegator if still missing when processing escrow later. - self.escrowDelegableEvent(serder=serder, sigers=sigers, - wigers=wigers,local=local) - raise MissingDelegableApprovalError(f"Missing Kever for delegator" - f" = {delpre} of event" - f" = {serder.ked}.") + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + local=local) + raise MissingDelegationError(f"Missing KEL of delegator " + f"{delpre} of evt = {serder.ked}.") + + + #self.escrowDelegableEvent(serder=serder, sigers=sigers, + #wigers=wigers,local=local) + #raise MissingDelegableApprovalError(f"Missing Kever for delegator" + #f" = {delpre} of event" + #f" = {serder.ked}.") dkever = self.kevers[delpre] if dkever.doNotDelegate: # drop event if delegation not allowed @@ -2621,7 +2630,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # Once fully receipted, cue in Kevery will then trigger cue to approve # delegation - if delseqner is None or delsaider is None: + if delseqner is None or delsaider is None: # missing delegation seal ref if self.locallyOwned(delpre): # local delegator so escrow. # Won't get to here if not local and locallyOwned(delpre) because # valSigsWigsDel will send nonlocal sourced delegable event to @@ -2834,11 +2843,22 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, self.db.putEvt(dgkey, serder.raw) # idempotent (maybe already excrowed) # update event source - - if seqner and saider: # delegation for authorized delegated or issued event + # delegation for authorized delegated or issued event + # when seqner and saider are provided they are only assured to be valid + # kever for event if kel is delegated and not locallyOwned + # and not locallyWitnessed as the validateDelegation is short circuited + # for non delegated kels, local controllers, and local witnesses. + # These checks prevent ddos via malicious source seal attachments. + # MUST NOT setAes if not delegated or locallyOwned or locallyWitnessed + if (self.delpre and not self.locallyOwned() + and not self.locallyWitnessed(wits=wits) and seqner and saider): couple = seqner.qb64b + saider.qb64b self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + #if seqner and saider: + #couple = seqner.qb64b + saider.qb64b + #self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + if esr := self.db.esrs.get(keys=dgkeys): # preexisting esr if local and not esr.local: # local overwrites prexisting remote esr.local = local @@ -5290,6 +5310,10 @@ def processEscrowPartialSigs(self): else: delpre = eserder.ked["di"] + # consider using here instead todo XXXX + # Kever.fetchDelegatingEvent(delegator=delpre, + # serder=eserder) + # need Kever reference for eserder.pre seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) if srdr is not None: From c983c6f4e5e9572be4a524e16871128032ad023d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 18 Aug 2024 15:28:27 -0600 Subject: [PATCH 268/418] refactoring prep for fixing delegation --- src/keri/core/eventing.py | 235 ++++++++++++++++++++++++++++---------- src/keri/db/basing.py | 66 ++++++----- 2 files changed, 213 insertions(+), 88 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index f6571ab32..857cbdc46 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2397,6 +2397,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, Location Seal is from Delegate's establishment event Assumes state setup + self is last accepted event if any or yet to be accepted event, + serder is received event + Parameters: serder (SerderKERI): instance of delegated event serder sigers (list[Siger]): of Siger instances of indexed controller sigs of @@ -2416,7 +2419,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, If this event is not delegated ignored Returns: - (str | None): qb64 delegator prefix or None if not delegated + None Process Logic: A delegative event is processed differently for each of four different @@ -2525,17 +2528,20 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, events signed by compromised keys. The result of superseded recovery is that the KEL is forked at the sn of the superseding event. All events in the superseded branch of the fork still exist but, by virtue of being superseded, - are disputed. The set of superseding events in the superseding fork forms the authoritative - branch of the KEL. All the already seen superseded events in the superseded fork + are disputed. The set of superseding events in the superseding fork forms + the authoritative branch of the KEL. + + All the already seen superseded events in the superseded fork still remain in the KEL and may be viewed in order of their original acceptance because the database stores all accepted events in order of acceptance and denotes this order using the first seen ordinal number, fn. - The fn is not the same as the sn (sequence number). + + Recall that the fn is not the same as the sn (sequence number). Each event accepted into a KEL has a unique fn but multiple events due to recovery forks may share the same sn. - Superseding Rules for Recovery at given SN (sequence number) + Superseding Rules for Recovery at given 'sn' (sequence number) A0. Any rotation event may supersede an interaction event at the same sn. where that interaction event is not before any other rotation event. @@ -2589,6 +2595,19 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, recovery is no longer possible because the delegatee would no longer control the privete keys needed to verifiably sign a recovery rotation. + + Repair of approval soruce seal couple in 'aes' database on recursive + climb the kel tree. Once an event has been accepted into its kel. + Later adding a source seal couple to 'aes' should then be OK from a + security perspective since its only making discovery less expensive. + + When malicious source seal couple is received but event is validly + delegated and the delegation source seal is repaired then need to replace + malicious source seal couple with repaired seal so repaired seal not + malicous seal gets written to 'aes' db. When the event is valid but + non-delegated then need to nullify malicous source seal couple so it + does not get written to 'aes' datable + """ if not delpre: # not delegable so no delegation validation needed return @@ -2604,20 +2623,18 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, return if self.kevers is None or delpre not in self.kevers: # missing delegator - # ToDo XXXX cue a trigger to get the KEL of the delegator - # the processPSEvent should also cue a trigger to get KEL + # ToDo XXXX cue a trigger to get the KEL of the delegator. This may + # require OOBIing with the delegator. + # The processPSEvent should also cue a trigger to get KEL # of delegator if still missing when processing escrow later. self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) + if delseqner and delsaider: # save in case not attached later + self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError(f"Missing KEL of delegator " f"{delpre} of evt = {serder.ked}.") - #self.escrowDelegableEvent(serder=serder, sigers=sigers, - #wigers=wigers,local=local) - #raise MissingDelegableApprovalError(f"Missing Kever for delegator" - #f" = {delpre} of event" - #f" = {serder.ked}.") dkever = self.kevers[delpre] if dkever.doNotDelegate: # drop event if delegation not allowed @@ -2631,7 +2648,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # delegation if delseqner is None or delsaider is None: # missing delegation seal ref - if self.locallyOwned(delpre): # local delegator so escrow. + if self.locallyOwned(delpre): # local delegator so escrow delegable. # Won't get to here if not local and locallyOwned(delpre) because # valSigsWigsDel will send nonlocal sourced delegable event to # misfit escrow first. Mistfit escrow must first @@ -2642,8 +2659,11 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, f" delegation by {delpre} of" f"event = {serder.ked}.") - else: # not local delegator so escrow + else: # not local delegator so escrow PSEvent self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) + # since delseqner or delsaider is None there is no PACouple to escrow here + #if delseqner and delsaider: # save in case not attached later + #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError(f"No delegation seal for delegator " f"{delpre} of evt = {serder.ked}.") @@ -2653,9 +2673,10 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # get the dig of the delegating event. Using getKeLast ensures delegating # event has not already been superceded key = snKey(pre=delpre, sn=ssn) # database key - raw = self.db.getKeLast(key) # get dig of delegating event + raw = self.db.getKeLast(key) # get dig of delegating event as index last - if raw is None: # no delegating event at key pre, sn + if raw is None: # no index to delegating event at key pre, sn + # Have to wait until delegating event at sn shows up in kel # ToDo XXXX process this cue of query to fetch delegating event from # delegator self.cues.push(dict(kin="query", q=dict(pre=delpre, @@ -2667,7 +2688,8 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, #sn = validateSN(sn=serder.snh, inceptive=inceptive) sn = Number(num=serder.sn).validate(inceptive=inceptive).sn self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + if delseqner and delsaider: # save in case not attached later + self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError("No delegating event from {} at {} for " "evt = {}.".format(delpre, delsaider.qb64, @@ -2676,24 +2698,24 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # get the delegating event from dig ddig = bytes(raw) key = dgKey(pre=delpre, dig=ddig) # database key - raw = self.db.getEvt(key) - if raw is None: # drop event + raw = self.db.getEvt(key) # get actual last event + if raw is None: # drop event should never happen unless database is broken raise ValidationError("Missing delegation from {} at event dig = {} for evt = {}." "".format(delpre, ddig, serder.ked)) - dserder = serdering.SerderKERI(raw=bytes(raw)) # delegating event - + dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event - # compare digests to make sure they match here + # compare digests to make sure they match here. + # should never fail unless database broken if not dserder.compare(said=delsaider.qb64): # drop event raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." "".format(delpre, ddig, serder.ked)) found = False # find event seal of delegated event in delegating data + # purported delegating event from source seal couple # XXXX ToDo need to change logic here to support native CESR seals not just dicts # for JSON, CBOR, MGPK - # may want to try harder here by walking KEL for dseal in dserder.seals: # find delegating seal anchor if tuple(dseal) == SealEvent._fields: seal = SealEvent(**dseal) @@ -2703,29 +2725,32 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, found = True break - if not found: # drop event + if not found: # drop event but may want to try harder here by walking KEL + # source seal couple is bad could be malicious so need to nullify as + # also walk KEL to find if any. raise ValidationError(f"Missing delegation seal in designated event" f"from {delpre} in {dserder.seals} for " f"evt = {serder.ked}.") - # Found anchor so can assume delegation successful unless its one of - # the superseding condidtions. - # Assumes database is reverified each bootup chain-of-custody of disc broken. + # Found anchor so can confirm delegation successful unless its one of + # the superseding conditions. Valid if not superseding drt of drt. + # Assumes database is reverified each bootup or otherwise when + # chain-of-custody of disc has been broken. # Rule for non-supeding delegated rotation of rotation. - # Returning delegator indicates success and eventually results acceptance + # Returning delegator indicates success and eventually results in acceptance # via Kever.logEvent which also writes the delgating event source couple to # db.aess so we can find it later if ((serder.ilk == Ilks.dip) or # delegated inception (serder.sner.num == self.sner.num + 1) or # inorder event (serder.sner.num == self.sner.num and # superseding event - self.ilk == Ilks.ixn and # superseded is ixn + self.ilk == Ilks.ixn and # superseded is ixn and serder.ilk == Ilks.drt)): # recovery rotation superseding ixn - return # delegator # indicates delegation valid with return of delegator + return # indicates delegation valid # get to here means drt rotation superseding another drt rotation # Kever.logEvent saves authorizer (delegator) seal source couple in - # db.aess data base so can use it here to recusively look up delegating - # events + # db.aess (.getAes, .setAes etc) data base so can use it here to + # recusively look up delegating events # set up recursive search for superseding delegations serfn = serder # new potentially superseding delegated event i.e. serf new @@ -2767,34 +2792,110 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # repeat - def fetchDelegatingEvent(self, delegator, serder): - """Returns delegating event by delegator of delegated event given by - serder otherwise raises ValidationError. + def fetchDelegatingEvent(self, delpre, serder, *, + delseqner=None, delsaider=None, eager=False): + """Returns delegating event of delegator given by its aid delpre of + delegated event given by serder otherwise raises ValidationError. + Assumes serder is already delegated event + Returns: + dserder (SerderKERI): key event with delegating seal + Parameters: - delegator (str): qb64 of identifier prefix of delegator + delpre (str): qb64 of identifier prefix of delegator serder (SerderKERI): delegated serder - - """ - dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key - if (couple := self.db.getAes(dgkey)): # delegation source couple + delseqner (Seqner | None): instance of delegating event sequence number. + delsaider (Saider | None): instance of of delegating event digest. + eager (bool): True means do more expensive KEL walk instead of escrow + False means do not do expensive KEL walk now + + Assumes db.aes source seal couple of delegating event cannot be written + unless that event has been accepted (first seen) into delegator's KEL + + ToDo XXXX + Use the db.fons database to lookup delegating events to ensure that + lookup only uses delegating events that were indeed accepted (first seen) + at some point even though they may now be superseded. + + Looking up in the db.evts from dig in db.aes could be malicious escrows + of delegating events?? But a malicious escrow of delegating event would + only write source seal couple to db.pdes not db.aes + + When escrowing .escrowPACouple, the delegating seal source couple goes + in db.pdes indexed by delegating event pre,dig + + Delegating event may have been superceded but delegated event validator + does not know it yet because db.aes keeps original delegating event + source seal from before is was superseded. + + Therefore lookup of delegating event needs to find delegating event via + db.fons and not last key event in .kels which would only return the + superseding event. + + When walking delegation tree a given delegating event may have + been superceded by another delegating event in the same delegator + KEL. This method does not distinguish between superseding and + superseded so can't assume that the delegating event is the last + event at its sn, i.e. the superseding event. + So this method must use db.fons to lookup to ensure that delegating + event was accepted (first seen) even if it has subsequently been + superseded. + This assumes that the delegating source seal in the AES db could + not have been written into the delegate's db.aes unless the delegating + event had been validated and accepted (first seen) into the + delegator's kel even though the delegating event may have later + been superseded at the time of this fetch. + + + Fetch delegating event has two cases: + 1. Delegated event has yet to be accepted (not seen) so can't repair + aes db because aes db must not be written until delegated event has + been accepted (first seen) bu .logEvent. + So must indicate the newly found dserder generates new source couple. + Returned dserder has the info to generate source couple but + does not indicate that a new one is needed. + + 2. Delegated event has been accepted so can repair aes source couple entry + when necessary. + + Need to indicate both if delegated event has been accepted and when + aes has been repaired. + seen=True (delegated event has been accepted first seen) + repair=True found new couple so repair externally if seen == False + or else repair just happened. + + When eager == False then may return None which means to escrow event and + try later with eager = True such as in escrow processor. + """ + dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key of delegate + if (couple := self.db.getAes(dgkey)): # delegation source couple at delegate seqner, saider = deSourceCouple(couple) - dgkey = dgKey(pre=delegator, dig=saider) # event at its said - # get event by dig not by sn at last event because may have been superceded + deldig = saider.qb64 # dig of delegating event + # extra careful double check that .aes is valid by getting + # fner = first seen Number instance index + if not self.db.fons.get(keys=(delpre, deldig)): # None + raise ValidationError(f"Invalid delagation authorizing source " + f"seal couple for {serder.ked}") + + + dgkey = dgKey(pre=delpre, dig=deldig) # event at its said if not (raw := self.db.getEvt(dgkey)): # database broken this should never happen so do not supersede + # ToDo XXXX should repair by deleting the erroneous aes entry and + # returning found one raise ValidationError(f"Missing delegation event for {serder.ked}") - dserder = serdering.SerderKERI(raw=bytes(raw)) # original delegating event i.e. boss original + # original delegating event i.e. boss original + dserder = serdering.SerderKERI(raw=bytes(raw)) - else: #try to find seal the hard way + else: #try to find seal the hard way by walking the delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict - if not (dserder := self.db.findAnchoringSealEvent(pre=serder.delpre, seal=seal)): + if not (dserder := self.db.findAnchoringSealEvent(pre=delpre, seal=seal)): # database broken this should never happen so do not supersede raise ValidationError(f"Missing delegation source seal for {serder.ked}") - return dserder + return dserder # get actually found delseqner, delsaider from dserder def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, @@ -2850,7 +2951,7 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, # for non delegated kels, local controllers, and local witnesses. # These checks prevent ddos via malicious source seal attachments. # MUST NOT setAes if not delegated or locallyOwned or locallyWitnessed - if (self.delpre and not self.locallyOwned() + if (self.delpre and not serder.ilk == Ilks.ixn and not self.locallyOwned() and not self.locallyWitnessed(wits=wits) and seqner and saider): couple = seqner.qb64b + saider.qb64b self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal @@ -4697,15 +4798,17 @@ def fetchEstEvent(self, pre, sn): found = False while not found: - dig = bytes(self.db.getKeLast(key=snKey(pre, sn))) + dig = self.db.getKeLast(key=snKey(pre, sn)) if not dig: return None # retrieve event by dig - raw = bytes(self.db.getEvt(key=dgKey(pre=pre, dig=dig))) + dig = bytes(dig) + raw = self.db.getEvt(key=dgKey(pre=pre, dig=dig)) if not raw: return None + raw = bytes(raw) serder = serdering.SerderKERI(raw=raw) # deserialize event raw if serder.ked["t"] in (Ilks.icp, Ilks.dip, Ilks.rot, Ilks.drt): return serder # establishment event so return @@ -5291,8 +5394,9 @@ def processEscrowPartialSigs(self): raise ValidationError("Missing escrowed evt sigs at " "dig = {}.".format(bytes(edig))) wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - if not wigs: # empty list - # wigs maybe empty while waiting for first witness signature + if not wigs: # empty list wigs witness sigs not wits + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature # which may not arrive until some time after event is fully signed # so just log for debugging but do not unescrow by raising # ValidationError @@ -5309,11 +5413,6 @@ def processEscrowPartialSigs(self): delpre = self.kevers[eserder.pre].delpre else: delpre = eserder.ked["di"] - - # consider using here instead todo XXXX - # Kever.fetchDelegatingEvent(delegator=delpre, - # serder=eserder) - # need Kever reference for eserder.pre seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) if srdr is not None: @@ -5351,6 +5450,7 @@ def processEscrowPartialSigs(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + self.db.delPde(dgkey) # remove escrow if any if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): self.cues.push(dict(kin="psUnescrow", serder=eserder)) @@ -5469,11 +5569,12 @@ def processEscrowPartialWigs(self): raise ValidationError("Missing escrowed evt sigs at " "dig = {}.".format(bytes(edig))) - # get wigs + # get witness signatures (wigs not wits) wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs if not wigs: # empty list - # wigs maybe empty while waiting for first witness signature + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature # which may not arrive until some time after event is fully signed # so just log for debugging but do not unescrow by raising # ValidationError @@ -5492,6 +5593,18 @@ def processEscrowPartialWigs(self): couple = self.db.getPde(dgKey(pre, bytes(edig))) if couple is not None: delseqner, delsaider = deSourceCouple(couple) + elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + if eserder.pre in self.kevers: + delpre = self.kevers[eserder.pre].delpre + else: + delpre = eserder.ked["di"] + seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + couple = delseqner.qb64b + delsaider.qb64b + self.db.putPde(dgkey, couple) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=delseqner, delsaider=delsaider, local=esr.local) @@ -5513,14 +5626,15 @@ def processEscrowPartialWigs(self): # validation will be successful as event would not be in # partially witnessed escrow unless they had already validated - except MissingWitnessSignatureError as ex: - # still waiting on missing witness sigs + except (MissingWitnessSignatureError, MissingDelegationError) as ex: + # still waiting on missing witness sigs or delegation if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + self.db.delPde(dgkey) # remove escrow if any if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) else: @@ -5531,6 +5645,7 @@ def processEscrowPartialWigs(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + self.db.delPde(dgkey) # remove escrow if any logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) logger.debug(f"event=\n{eserder.pretty()}\n") diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 2fc0b8e58..7c7b3f58d 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -611,6 +611,37 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event Only one value per DB key is allowed + .kels is named sub DB of key event log tables that map sequence numbers + to serialized event digests. + Uses sequence number or sn. + snKey + Values are digests used to lookup event in .evts sub DB + DB is keyed by identifier prefix plus sequence number of key event + More than one value per DB key is allowed + + .fels is named sub DB of first seen event log table (FEL) of digests + that indexes events in first 'seen' accepted order for replay and + cloning of event log. + Uses first seen order number or fn. + fnKey + DB is keyed by identifier prefix plus monotonically increasing first + seen order number fn. + Value is digest of serialized event used to lookup event in .evts sub DB + Only one value per DB key is allowed. + Provides append only ordering of accepted first seen events. + + .fons is named subDB CesrSuber + Uses digest + dgKey + Maps prefix and digest to fn value (first seen ordinal number) of + the associated event. So one used pre and event digest, get its fn here + and then use fn to fetch event from .evnts by fn from .fels. + This ensures that any event looked up this way was first seen at + some point in time even if later superseded by a recovery rotation. + Whereas direct lookup in .evts could be escrowed events that may + never have been accepted as first seen. + CesrSuber(db=self, subkey='fons.', klas=core.Number) + .esrs is named sub DB instance of Komer of EventSourceRecord dgKey DB is keyed by identifier prefix plus digest (said) of serialized event @@ -636,8 +667,9 @@ class Baser(dbing.LMDBer): Escrow processing determines if and how to promote event source to local and then reprocess - .delegables is named sub DB instance of CesrIoSetSuber for delegable escrows - subkey "dees." + .delegables is named sub DB instance of CesrIoSetSuber for delegable event + escrows of key event with local delegator that need approval. + subkey "dees." delegable event escrows snKey DB is keyed by event controller prefix plus sn of serialized event where sn is 32 char hex string with leading zeros @@ -648,18 +680,6 @@ class Baser(dbing.LMDBer): source for a delegable event of a local delegator must first pass through the misfit escrow and get promoted to local source. - # delegable events escrows. events with local delegator that need approval - self.delegables = subing.CesrIoSetSuber(db=self, subkey='dees.', klas=coring.Diger) - - .fels is named sub DB of first seen event log table (FEL) of digests - that indexes events in first 'seen' accepted order for replay and - cloning of event log. Only one value per DB key is allowed. - Provides append only ordering of accepted first seen events. - Uses first seen order number or fn. - fnKey - DB is keyed by identifier prefix plus monotonically increasing first - seen order number fn. - Value is digest of serialized event used to lookup event in .evts sub DB .dtss is named sub DB of datetime stamp strings in ISO 8601 format of the datetime when the event was first escrosed and then later first @@ -740,13 +760,6 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event More than one value per DB key is allowed - .kels is named sub DB of key event log tables that map sequence numbers - to serialized event digests. - snKey - Values are digests used to lookup event in .evts sub DB - DB is keyed by identifier prefix plus sequence number of key event - More than one value per DB key is allowed - .pses is named sub DB of partially signed escrowed event tables that map sequence numbers to serialized event digests. snKey @@ -808,10 +821,6 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .fons is named subDB instance of MatterSuber that maps - (prefix, digest) e.g. dgKey to fn value (first seen ordinal number) of - the associated event. So one can lookup event digest, get its fn here - and then use fn to fetch event by fn from .fels. .states (subkey stts.) is named subDB instance of SerderSuber that maps a prefix to the latest keystate for that prefix. Used by ._kevers.db for read @@ -978,6 +987,7 @@ def reopen(self, **kwa): self.evts = self.env.open_db(key=b'evts.') self.fels = self.env.open_db(key=b'fels.') + self.kels = self.env.open_db(key=b'kels.', dupsort=True) self.dtss = self.env.open_db(key=b'dtss.') self.aess = self.env.open_db(key=b'aess.') self.sigs = self.env.open_db(key=b'sigs.', dupsort=True) @@ -986,7 +996,6 @@ def reopen(self, **kwa): self.ures = self.env.open_db(key=b'ures.', dupsort=True) self.vrcs = self.env.open_db(key=b'vrcs.', dupsort=True) self.vres = self.env.open_db(key=b'vres.', dupsort=True) - self.kels = self.env.open_db(key=b'kels.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) self.pdes = self.env.open_db(key=b'pdes.') self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) @@ -996,6 +1005,9 @@ def reopen(self, **kwa): self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) self.qnfs = self.env.open_db(key=b'qnfs.', dupsort=True) + # events as ordered by first seen ordinals + self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=core.Number) + self.migs = subing.CesrSuber(db=self, subkey="migs.", klas=coring.Dater) self.vers = subing.Suber(db=self, subkey="vers.") @@ -1010,8 +1022,6 @@ def reopen(self, **kwa): # delegable events escrows. events with local delegator that need approval self.delegables = subing.IoSetSuber(db=self, subkey='dees.') - # events as ordered by first seen ordinals - self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=core.Number) # Kever state made of KeyStateRecord key states # TODO: clean self.states = koming.Komer(db=self, From 01809fb2e6565cb4d7b2b18d45a155eec4600605 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 20 Aug 2024 07:58:44 -0600 Subject: [PATCH 269/418] refactor delegation seal escrow to catcesrsuber --- src/keri/core/eventing.py | 69 ++++++++++++++++----------- src/keri/db/basing.py | 98 ++++++++++++++++++++++----------------- src/keri/db/subing.py | 1 + tests/core/test_escrow.py | 29 ++++++++---- tests/core/test_kevery.py | 1 + tests/db/test_basing.py | 52 ++++++++++++++++----- tests/db/test_subing.py | 30 ++++++++++++ 7 files changed, 192 insertions(+), 88 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 857cbdc46..23a94b81c 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2820,10 +2820,10 @@ def fetchDelegatingEvent(self, delpre, serder, *, Looking up in the db.evts from dig in db.aes could be malicious escrows of delegating events?? But a malicious escrow of delegating event would - only write source seal couple to db.pdes not db.aes + only write source seal couple to db.udes not db.aes When escrowing .escrowPACouple, the delegating seal source couple goes - in db.pdes indexed by delegating event pre,dig + in db.udes indexed by delegating event pre,dig Delegating event may have been superceded but delegated event validator does not know it yet because db.aes keeps original delegating event @@ -3026,8 +3026,9 @@ def escrowMFEvent(self, serder, sigers, wigers=None, if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent res = self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed @@ -3132,8 +3133,9 @@ def escrowPACouple(self, serder, seqner, saider, local=True): """ local = True if local else False # ignored since not escrowing serder here dgkey = dgKey(serder.preb, serder.saidb) - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent logger.debug("Kever state: Escrowed source couple for partially signed " "or delegated event = %s\n", serder.ked) @@ -3163,8 +3165,9 @@ def escrowPWEvent(self, serder, wigers, sigers=None, if sigers: self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.putEvt(dgkey, serder.raw) # update event source @@ -4851,8 +4854,9 @@ def escrowMFEvent(self, serder, sigers, wigers=None, if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed logger.debug("Kevery process: escrowed misfit event=\n%s", @@ -4891,8 +4895,9 @@ def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None, l if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.addOoe(snKey(serder.preb, serder.sn), serder.saidb) # log escrowed logger.debug("Kevery process: escrowed out of order event=\n%s", @@ -5405,9 +5410,11 @@ def processEscrowPartialSigs(self): # seal source (delegator issuer if any) delseqner = delsaider = None - couple = self.db.getPde(dgkey) - if couple is not None: - delseqner, delsaider = deSourceCouple(couple) + if (couple := self.db.udes.get(keys=dgkey)): + delseqner, delsaider = couple + #couple = self.db.getUde(dgkey) + #if couple is not None: + #delseqner, delsaider = deSourceCouple(couple) elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): if eserder.pre in self.kevers: delpre = self.kevers[eserder.pre].delpre @@ -5418,8 +5425,10 @@ def processEscrowPartialSigs(self): if srdr is not None: delseqner = coring.Seqner(sn=srdr.sn) delsaider = coring.Saider(qb64=srdr.said) - couple = delseqner.qb64b + delsaider.qb64b - self.db.putPde(dgkey, couple) + #couple = delseqner.qb64b + delsaider.qb64b + #self.db.putUde(dgkey, couple) + # idempotent + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) # process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] @@ -5450,7 +5459,8 @@ def processEscrowPartialSigs(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - self.db.delPde(dgkey) # remove escrow if any + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): self.cues.push(dict(kin="psUnescrow", serder=eserder)) @@ -5465,7 +5475,8 @@ def processEscrowPartialSigs(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - self.db.delPde(dgkey) # remove escrow if any + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): self.cues.push(dict(kin="psUnescrow", serder=eserder)) @@ -5590,9 +5601,11 @@ def processEscrowPartialWigs(self): # seal source (delegator issuer if any) delseqner = delsaider = None - couple = self.db.getPde(dgKey(pre, bytes(edig))) - if couple is not None: - delseqner, delsaider = deSourceCouple(couple) + if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + delseqner, delsaider = couple + #couple = self.db.getUde(dgKey(pre, bytes(edig))) + #if couple is not None: + #delseqner, delsaider = deSourceCouple(couple) elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): if eserder.pre in self.kevers: delpre = self.kevers[eserder.pre].delpre @@ -5603,8 +5616,10 @@ def processEscrowPartialWigs(self): if srdr is not None: delseqner = coring.Seqner(sn=srdr.sn) delsaider = coring.Saider(qb64=srdr.said) - couple = delseqner.qb64b + delsaider.qb64b - self.db.putPde(dgkey, couple) + #couple = delseqner.qb64b + delsaider.qb64b + #self.db.putUde(dgkey, couple) + # idempotent + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=delseqner, delsaider=delsaider, local=esr.local) @@ -5634,7 +5649,8 @@ def processEscrowPartialWigs(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - self.db.delPde(dgkey) # remove escrow if any + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) else: @@ -5645,7 +5661,8 @@ def processEscrowPartialWigs(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - self.db.delPde(dgkey) # remove escrow if any + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) logger.debug(f"event=\n{eserder.pretty()}\n") diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 7c7b3f58d..63a456065 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -760,7 +760,15 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event More than one value per DB key is allowed - .pses is named sub DB of partially signed escrowed event tables + .pses is named sub DB of partially signed event escrows + that map sequence numbers to serialized event digests. + snKey + Values are digests used to lookup event in .evts sub DB + DB is keyed by identifier prefix plus sequence number of key event + More than one value per DB key is allowed + + + .pwes is named sub DB of partially witnessed event escrowes that map sequence numbers to serialized event digests. snKey Values are digests used to lookup event in .evts sub DB @@ -777,12 +785,15 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of key event Only one value per DB key is allowed - .pwes is named sub DB of partially witnessed escrowed event tables - that map sequence numbers to serialized event digests. - snKey - Values are digests used to lookup event in .evts sub DB - DB is keyed by identifier prefix plus sequence number of key event - More than one value per DB key is allowed + .udes is named sub DB of unverified delegation seal source couple escrows + that map (pre, digest) of delegated event to delegating seal source + couple (sn, dig) that provides source delegator event seal. + Each couple is concatenation of fully qualified items, snu+dig + of delegating source event in which seal of delegated event appears. + dgKey + Values are couples used to lookup source event in .kels sub DB + DB is keyed by identifier prefix plus digest of key event + Only one value per DB key is allowed .uwes is named sub DB of unverified event indexed escrowed couples from witness signers. Witnesses are from witness list of latest establishment @@ -997,7 +1008,10 @@ def reopen(self, **kwa): self.vrcs = self.env.open_db(key=b'vrcs.', dupsort=True) self.vres = self.env.open_db(key=b'vres.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) - self.pdes = self.env.open_db(key=b'pdes.') + #self.pdes = self.env.open_db(key=b'pdes.') + #self.udes = self.env.open_db(key=b'udes.') + self.udes = subing.CatCesrSuber(db=self, subkey='udes.', + klas=(coring.Seqner, coring.Saider)) self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) self.ooes = self.env.open_db(key=b'ooes.', dupsort=True) @@ -2705,40 +2719,40 @@ def delPse(self, key, val): """ return self.delIoVal(self.pses, key, val) - def putPde(self, key, val): - """ - Use dgKey() - Write serialized event source couple to key (snu+dig) - Does not overwrite existing val if any - Returns True If val successfully written Else False - Returns False if key already exists - """ - return self.putVal(self.pdes, key, val) - - def setPde(self, key, val): - """ - Use dgKey() - Write serialized seal source couple to key (snu+dig) - Overwrites existing val if any - Returns True If val successfully written Else False - """ - return self.setVal(self.pdes, key, val) - - def getPde(self, key): - """ - Use dgKey() - Return seal source couple at key - Returns None if no entry at key - """ - return self.getVal(self.pdes, key) - - def delPde(self, key): - """ - Use dgKey() - Deletes value at key. - Returns True If key exists in database Else False - """ - return self.delVal(self.pdes, key) + #def putUde(self, key, val): + #""" + #Use dgKey() + #Write serialized event source couple to key (snu+dig) + #Does not overwrite existing val if any + #Returns True If val successfully written Else False + #Returns False if key already exists + #""" + #return self.putVal(self.udes, key, val) + + #def setUde(self, key, val): + #""" + #Use dgKey() + #Write serialized seal source couple to key (snu+dig) + #Overwrites existing val if any + #Returns True If val successfully written Else False + #""" + #return self.setVal(self.udes, key, val) + + #def getUde(self, key): + #""" + #Use dgKey() + #Return seal source couple at key + #Returns None if no entry at key + #""" + #return self.getVal(self.udes, key) + + #def delUde(self, key): + #""" + #Use dgKey() + #Deletes value at key. + #Returns True If key exists in database Else False + #""" + #return self.delVal(self.udes, key) def putPwes(self, key, vals): """ diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index ec4ae7780..9232bb873 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -115,6 +115,7 @@ def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], return bytes(keys) # return bytes elif hasattr(keys, "decode"): # bytes return keys + keys = (key.decode() if hasattr(key, "decode") else key for key in keys) return (self.sep.join(keys).encode("utf-8")) diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index d3f419fa6..3b96e7275 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -474,8 +474,12 @@ def test_missing_delegator_escrow(): escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) assert len(escrows) == 1 assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow entry for event + escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb + # verify Kevery process partials escrow is idempotent to previously escrowed events # assuming not stale but nothing else has changed @@ -484,8 +488,11 @@ def test_missing_delegator_escrow(): escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) assert len(escrows) == 1 assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow entry for event + escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb # apply Bob's inception to Dels' Kvy psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) @@ -496,8 +503,11 @@ def test_missing_delegator_escrow(): escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) assert len(escrows) == 1 assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow entry for event + escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb # apply Bob's delegating interaction to Dels' Kvy psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy, local=True) @@ -512,8 +522,11 @@ def test_missing_delegator_escrow(): escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) assert len(escrows) == 0 - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow is None # delegated inception delegation couple + #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + #assert escrow is None # delegated inception delegation couple + #escrow entry for event delegated inception delegation couple + escrow = self.db.udes.get(keys=(delPre, delSrdr.said)) + assert escrow is None # Setup Del rotation event verfers, digers = delMgr.rotate(pre=delPre, temp=True) diff --git a/tests/core/test_kevery.py b/tests/core/test_kevery.py index 97b6dd7d5..f740cc67c 100644 --- a/tests/core/test_kevery.py +++ b/tests/core/test_kevery.py @@ -435,3 +435,4 @@ def test_stale_event_receipts(): if __name__ == "__main__": test_kevery() + test_stale_event_receipts() diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 16a0819eb..17872eb76 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -1009,28 +1009,56 @@ def test_baser(): assert items == [] # empty assert not items - # Test .pdes partial delegated escrow seal source couples + # Test .udes partial delegated escrow seal source couples key = dgKey(preb, digb) assert key == f'{preb.decode("utf-8")}.{digb.decode("utf-8")}'.encode("utf-8") - # test .pdes sub db methods + # test .udes sub db methods + #ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' + #sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' + #ssnu2 = b'0AAAAAAAAAAAAAAAAAAAAAAC' + #sdig2 = b'EBYYJRCCpAGO7WjjsLhtHVR37Pawv67kveIFUPvt38x0' + #val1 = ssnu1 + sdig1 + #val2 = ssnu2 + sdig2 + + #assert db.getUde(key) == None + #assert db.delUde(key) == False + #assert db.putUde(key, val1) == True + #assert db.getUde(key) == val1 + #assert db.putUde(key, val2) == False + #assert db.getUde(key) == val1 + #assert db.setUde(key, val2) == True + #assert db.getUde(key) == val2 + #assert db.delUde(key) == True + #assert db.getUde(key) == None + + # test .udes catcesrsuber sub db methods ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' ssnu2 = b'0AAAAAAAAAAAAAAAAAAAAAAC' sdig2 = b'EBYYJRCCpAGO7WjjsLhtHVR37Pawv67kveIFUPvt38x0' val1 = ssnu1 + sdig1 + tuple1 = (coring.Seqner(qb64b=ssnu1), coring.Saider(qb64b=sdig1)) val2 = ssnu2 + sdig2 + tuple2 = (coring.Seqner(qb64b=ssnu2), coring.Saider(qb64b=sdig2)) + + + assert db.udes.get(keys=key) == None + assert db.udes.rem(keys=key) == False + assert db.udes.put(keys=key, val=tuple1) == True + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val1 + assert db.udes.put(keys=key, val=tuple2) == False + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val1 + assert db.udes.pin(keys=key, val=tuple2) == True + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val2 + assert db.udes.rem(keys=key) == True + assert db.udes.get(keys=key) == None + + - assert db.getPde(key) == None - assert db.delPde(key) == False - assert db.putPde(key, val1) == True - assert db.getPde(key) == val1 - assert db.putPde(key, val2) == False - assert db.getPde(key) == val1 - assert db.setPde(key, val2) == True - assert db.getPde(key) == val2 - assert db.delPde(key) == True - assert db.getPde(key) == None # Partially Witnessed Escrow Events # test .pwes insertion order dup methods. dup vals are insertion order diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 4cee7586e..c046695fc 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -61,6 +61,35 @@ def test_suber(): actual = sdb.get(keys=keys) assert actual == kip + sdb.rem(keys) + actual = sdb.get(keys=keys) + assert actual is None + + sdb.put(keys=keys, val=sue) + actual = sdb.get(keys=keys) + assert actual == sue + + # test with keys as tuple of bytes + keys = (b"test_key", b"0001") + sdb.rem(keys) + actual = sdb.get(keys=keys) + assert actual is None + + sdb.put(keys=keys, val=sue) + actual = sdb.get(keys=keys) + assert actual == sue + + # test with keys as mixed tuple of bytes + keys = (b"test_key", "0001") + sdb.rem(keys) + actual = sdb.get(keys=keys) + assert actual is None + + sdb.put(keys=keys, val=sue) + actual = sdb.get(keys=keys) + assert actual == sue + + # test with keys as string not tuple keys = "keystr" @@ -1637,6 +1666,7 @@ def test_crypt_signer_suber(): if __name__ == "__main__": + test_suber() test_cesr_ioset_suber() test_serder_suber() test_cesr_suber() From a69e7c49da3006027c0449080b923439b21abdff Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Tue, 20 Aug 2024 08:57:12 -0700 Subject: [PATCH 270/418] Update sequence number querying for logs to allow the specification of fn for first seen ordinal as the first event to start streaming back. This is an optimization to allow witnesses and watchers to only stream the events needed and not the entire KEL every time a new event occurs. (#848) Signed-off-by: pfeairheller --- src/keri/app/agenting.py | 5 +++-- src/keri/app/querying.py | 8 ++++++-- src/keri/app/watching.py | 2 +- src/keri/core/eventing.py | 3 ++- tests/app/test_querying.py | 11 +++++++++-- tests/core/test_replay.py | 6 ++++++ 6 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index a6023c3d6..334e3e17d 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -525,7 +525,7 @@ def msgDo(self, tymth=None, tock=1.0, **opts): yield self.tock - def query(self, pre, r="logs", sn='0', src=None, hab=None, anchor=None, wits=None, **kwa): + def query(self, pre, r="logs", sn='0', fn='0', src=None, hab=None, anchor=None, wits=None, **kwa): """ Create, sign and return a `qry` message against the attester for the prefix Parameters: @@ -534,6 +534,7 @@ def query(self, pre, r="logs", sn='0', src=None, hab=None, anchor=None, wits=Non pre (str): qb64 identifier prefix being queried for r (str): query route sn (str): optional specific hex str of sequence number to query for + fn (str): optional specific hex str of sequence number to start with anchor (Seal): anchored Seal to search for wits (list) witnesses to query @@ -541,7 +542,7 @@ def query(self, pre, r="logs", sn='0', src=None, hab=None, anchor=None, wits=Non bytearray: signed query event """ - qry = dict(s=sn) + qry = dict(s=sn, fn=fn) if anchor is not None: qry["a"] = anchor diff --git a/src/keri/app/querying.py b/src/keri/app/querying.py index 11541640f..04b8bf410 100644 --- a/src/keri/app/querying.py +++ b/src/keri/app/querying.py @@ -90,13 +90,17 @@ def recur(self, tyme, deeds=None): class SeqNoQuerier(doing.DoDoer): - def __init__(self, hby, hab, pre, sn, wits=None, **opts): + def __init__(self, hby, hab, pre, sn, fn=None, wits=None, **opts): self.hby = hby self.hab = hab self.pre = pre self.sn = sn + self.fn = fn if fn is not None else 0 self.witq = agenting.WitnessInquisitor(hby=self.hby) - self.witq.query(src=self.hab.pre, pre=self.pre, sn="{:x}".format(self.sn), wits=wits) + self.witq.query(src=self.hab.pre, pre=self.pre, + sn="{:x}".format(self.sn), + fn="{:x}".format(self.fn), + wits=wits) super(SeqNoQuerier, self).__init__(doers=[self.witq], **opts) def recur(self, tyme, deeds=None): diff --git a/src/keri/app/watching.py b/src/keri/app/watching.py index 677ee63f5..5159c0eb5 100644 --- a/src/keri/app/watching.py +++ b/src/keri/app/watching.py @@ -105,7 +105,7 @@ def adjudicate(self, watched, toad=None): for watcher in watchers: saider = self.hab.db.knas.get(keys=(watched, watcher)) if saider is None: - print(f"No key state from watcher {watcher} for {watched}") + logger.info(f"No key state from watcher {watcher} for {watched}") continue ksn = self.hab.db.ksns.get(keys=(saider.qb64,)) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 5443e7d86..cfdfc4128 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -4595,6 +4595,7 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): src = qry["src"] anchor = qry["a"] if "a" in qry else None sn = int(qry["s"], 16) if "s" in qry else None + fn = int(qry["fn"], 16) if "fn" in qry else 0 if pre not in self.kevers: self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) @@ -4612,7 +4613,7 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): raise QueryNotFoundError("Query not found error={}.".format(ked)) msgs = list() # outgoing messages - for msg in self.db.clonePreIter(pre=pre, fn=0): + for msg in self.db.clonePreIter(pre=pre, fn=fn): msgs.append(msg) if kever.delpre: diff --git a/tests/app/test_querying.py b/tests/app/test_querying.py index d723eaa94..fb36461a5 100644 --- a/tests/app/test_querying.py +++ b/tests/app/test_querying.py @@ -37,7 +37,7 @@ def test_querying(): assert msg["src"] == inqHab.pre assert msg["pre"] == subHab.pre assert msg["r"] == "ksn" - assert msg["q"] == {'s': '0'} + assert msg["q"] == {'fn': '0', 's': '0'} assert msg["wits"] is None doist.recur(deeds=deeds) @@ -108,6 +108,12 @@ def test_querying(): doist.recur(deeds=deeds) assert len(sdoer.witq.msgs) == 1 + sdoer = SeqNoQuerier(hby=hby, hab=inqHab, pre=subHab.pre, fn=2, sn=4) + assert len(sdoer.witq.msgs) == 1 + msg = sdoer.witq.msgs.pull() + query = msg['q'] + assert query == {'fn': '2', 's': '4'} + # Test with originally unknown AID sdoer = SeqNoQuerier(hby=hby, hab=inqHab, pre="ExxCHAI9bkl50F5SCKl2AWQbFGKeJtz0uxM2diTMxMQA", sn=1) assert len(sdoer.witq.msgs) == 1 @@ -131,7 +137,8 @@ def test_querying(): assert len(sdoer.witq.msgs) == 1 # Test with originally unknown AID - adoer = AnchorQuerier(hby=hby, hab=inqHab, pre="ExxCHAI9bkl50F5SCKl2AWQbFGKeJtz0uxM2diTMxMQA", anchor={'s': '5'}) + adoer = AnchorQuerier(hby=hby, hab=inqHab, pre="ExxCHAI9bkl50F5SCKl2AWQbFGKeJtz0uxM2diTMxMQA", + anchor={'s': '5'}) assert len(adoer.witq.msgs) == 1 tock = 0.03125 diff --git a/tests/core/test_replay.py b/tests/core/test_replay.py index 70b5ecf02..4902f66f9 100644 --- a/tests/core/test_replay.py +++ b/tests/core/test_replay.py @@ -135,6 +135,12 @@ def test_replay(): b'OOJVAAAjEg-3N4cNT_yot5wWlcKaz-1xPAgteGCsYZhq9dax3sQPD5HFI7M13Bhp' b'kRttBEq92pAaIG') + assert debHab.kever.sn == 6 + msgs = next(debHab.db.clonePreIter(debHab.pre, fn=4)) + serder = serdering.SerderKERI(raw=msgs) + assert serder.ilk == kering.Ilks.ixn + assert serder.sn == 4 + # Play debMsgs to Cam # create non-local kevery for Cam to process msgs from Deb camKevery = eventing.Kevery(db=camHab.db, From 42a3ad6d0938743062d7978d5fa2e23efd3b5630 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 21 Aug 2024 20:34:47 -0600 Subject: [PATCH 271/418] Refactor SerderSuber so can create SerderIoSetSuber with SerderSuberBase so mro works. Create SerderIoSetSuber with unit tests. --- src/keri/db/subing.py | 244 +++++++++++++++++++++++++--------------- tests/db/test_subing.py | 162 +++++++++++++++++++++----- 2 files changed, 285 insertions(+), 121 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 9232bb873..715153130 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -60,8 +60,8 @@ class SuberBase(): db (dbing.LMDBer): base LMDB db sdb (lmdb._Database): instance of lmdb named sub db for this Suber sep (str): separator for combining keys tuple of strs into key bytes - verify (bool): some subclasses want to re-verify when deser from db - default false + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ Sep = '.' # separator for combining key iterables @@ -80,6 +80,8 @@ def __init__(self, db: dbing.LMDBer, *, each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ super(SuberBase, self).__init__() # for multi inheritance self.db = db @@ -115,8 +117,9 @@ def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], return bytes(keys) # return bytes elif hasattr(keys, "decode"): # bytes return keys - keys = (key.decode() if hasattr(key, "decode") else key for key in keys) - return (self.sep.join(keys).encode("utf-8")) + #keys = (key.decode() if hasattr(key, "decode") else key for key in keys) + return (self.sep.join(key.decode() if hasattr(key, "decode") else key + for key in keys).encode("utf-8")) def _tokeys(self, key: Union[str, bytes, memoryview]): @@ -204,6 +207,17 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str = 'docs.', dupsort: bool=False, **kwa): """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key @@ -282,7 +296,7 @@ def rem(self, keys: Union[str, Iterable]): class CesrSuberBase(SuberBase): """ - Sub class of Suber where data is CESR encode/decode ducktyped subclass + Sub class of SuberBase where data is CESR encode/decode ducktyped subclass instance such as Matter, Indexer, Counter with .qb64b property when provided as fully qualified serialization Automatically serializes and deserializes from qb64b to/from CESR instance @@ -291,11 +305,20 @@ class CesrSuberBase(SuberBase): def __init__(self, *pa, klas: Type[coring.Matter] = coring.Matter, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + Parameters: klas (Type[coring.Matter]): Class reference to subclass of Matter or Indexer or Counter or any ducktyped class of Matter + """ super(CesrSuberBase, self).__init__(*pa, **kwa) self.klas = klas @@ -318,7 +341,7 @@ def _des(self, val: Union[str, memoryview, bytes]): """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes - return self.klas(qb64b=val) + return self.klas(qb64b=val) # qb64b parameter accepts str class CesrSuber(CesrSuberBase, Suber): @@ -334,9 +357,16 @@ class CesrSuber(CesrSuberBase, Suber): def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False klas (Type[coring.Matter]): Class reference to subclass of Matter or Indexer or Counter or any ducktyped class of Matter """ @@ -361,7 +391,7 @@ class CatCesrSuberBase(CesrSuberBase): def __init__(self, *pa, klas: Iterable = None, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -369,8 +399,10 @@ def __init__(self, *pa, klas: Iterable = None, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ if klas is None: @@ -378,7 +410,6 @@ def __init__(self, *pa, klas: Iterable = None, **kwa): if not nonStringIterable(klas): # not iterable klas = (klas, ) # make it so super(CatCesrSuberBase, self).__init__(*pa, klas=klas, **kwa) - # self.klas = klas def _ser(self, val: Union[Iterable, coring.Matter]): @@ -423,7 +454,7 @@ class CatCesrSuber(CatCesrSuberBase, Suber): such as Matter, Indexer, Counter Automatically serializes and deserializes from qb64b to/from CESR instances - Attributes: + Attributes: db (dbing.LMDBer): base LMDB db sdb (lmdb._Database): instance of lmdb named sub db for this Suber sep (str): separator for combining keys tuple of strs into key bytes @@ -433,7 +464,7 @@ class CatCesrSuber(CatCesrSuberBase, Suber): def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -441,8 +472,10 @@ def __init__(self, *pa, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(CatCesrSuber, self).__init__(*pa, **kwa) @@ -475,7 +508,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str='docs.', dupsort: bool=False, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -483,6 +516,10 @@ def __init__(self, db: dbing.LMDBer, *, each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(IoSetSuber, self).__init__(db=db, subkey=subkey, dupsort=False, **kwa) @@ -500,6 +537,8 @@ def put(self, keys: Union[str, Iterable], vals: Iterable): result (bool): True If successful, False otherwise. """ + #if not nonStringIterable(vals): # not iterable + #vals = (vals, ) # make iterable return (self.db.putIoSetVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals], @@ -542,6 +581,8 @@ def pin(self, keys: Union[str, Iterable], vals: Iterable): """ key = self._tokey(keys) self.db.delIoSetVals(db=self.sdb, key=key) # delete all values + #if not nonStringIterable(vals): # not iterable + #vals = (vals, ) # make iterable return (self.db.setIoSetVals(db=self.sdb, key=key, vals=[self._ser(val) for val in vals], @@ -779,7 +820,7 @@ class CesrIoSetSuber(CesrSuberBase, IoSetSuber): def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -787,6 +828,8 @@ def __init__(self, *pa, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False klas (Type[coring.Matter]): Class reference to subclass of Matter or Indexer or Counter or any ducktyped class of Matter @@ -829,16 +872,18 @@ class CatCesrIoSetSuber(CatCesrSuberBase, IoSetSuber): """ def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key - False (default) means do not enable duplicates at - each key + False (default) means do not enable duplicates at + each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(CatCesrIoSetSuber, self).__init__(*pa, **kwa) @@ -1044,13 +1089,11 @@ def getItemIter(self, keys: Union[str, Iterable]=b"", yield (ikeys, self.klas(qb64b=bytes(val), transferable=verfer.transferable)) - -class SerderSuber(Suber): +class SerderSuberBase(SuberBase): """ - Sub class of Suber where data is serialized Serder Subclass instance + Sub class of SuberBase where data is serialized Serder Subclass instance given by .klas Automatically serializes and deserializes using .klas Serder methods - """ def __init__(self, *pa, @@ -1060,99 +1103,120 @@ def __init__(self, *pa, Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False - Parameters: + Overridden Parameters: klas (Type[serdering.Serder]): Class reference to subclass of Serder """ - super(SerderSuber, self).__init__(*pa, **kwa) + super(SerderSuberBase, self).__init__(*pa, **kwa) self.klas = klas - def put(self, keys: Union[str, Iterable], val: serdering.SerderKERI): + def _ser(self, val: serdering.Serder): """ - Puts val at key made from keys. Does not overwrite - + Serialize value to bytes to store in db Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Serder): instance - - Returns: - result (bool): True If successful, False otherwise, such as key - already in database. + val (serdering.Serder): instance Serder subclass like SerderKERI """ - return (self.db.putVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) + return val.raw - def pin(self, keys: Union[str, Iterable], val: serdering.SerderKERI): + def _des(self, val: (str | memoryview | bytes)): """ - Pins (sets) val at key made from keys. Overwrites. - + Deserialize val to str Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Serder): instance - - Returns: - result (bool): True If successful. False otherwise. + val (Union[str, memoryview, bytes]): convertable to coring.matter """ - return (self.db.setVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) + if isinstance(val, memoryview): # memoryview is always bytes + val = bytes(val) # convert to bytes + elif hasattr(val, "encode"): # str + val = val.encode() # convert to bytes + return self.klas(raw=val, verify=self.verify) - def get(self, keys: Union[str, Iterable]): +class SerderSuber(SerderSuberBase, Suber): + """ + Sub class of SerderSuberBase, Suber where data is serialized Serder Subclass + instance given by .klas + Automatically serializes and deserializes using .klas Serder methods + """ + + def __init__(self, *pa, **kwa): """ - Gets Serder at keys + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[serdering.Serder]): Class reference to subclass of Serder + """ + super(SerderSuber, self).__init__(*pa, **kwa) - Parameters: - keys (tuple): of key strs to be combined in order to form key - Returns: - Serder: - None: if no entry at keys +class SerderIoSetSuber(SerderSuberBase, IoSetSuber): + """ + Sub class of SerderSuberBase and IoSetSuber that allows multiple Serder + instances to be stored at the same db key in insertion order. + Example use case would be an escrow where the key is a sequence number + based index (such as snKey). - Usage: - Use walrus operator to catch and raise missing entry - if (srder := mydb.get(keys)) is None: - raise ExceptionHere - use srdr here + Sub class of SerderSuberBase where data is serialized Serder Subclass instance + given by .klas + Automatically serializes and deserializes using .klas Serder methods - """ - val = self.db.getVal(db=self.sdb, key=self._tokey(keys)) - return self.klas(raw=bytes(val), verify=self.verify) if val is not None else None + Extends IoSetSuber so that all IoSetSuber methods now work with Serder + subclass for each val. + IoSetSuber stores at each effective key a set of distinct values that + share that same effective key where each member of the set is retrieved in + insertion order (dupsort==False) + The methods allows an Iterable (set valued) of Iterables of separation subclass + instances to be stored at a given effective key in insertion order. - def rem(self, keys: Union[str, Iterable]): - """ - Removes entry at keys + Actual keys include a hidden ordinal key suffix that tracks insertion order. + The suffix is appended and stripped transparently from the keys. The set of + items with duplicate effective keys are retrieved in insertion order when + iterating or as a list of the set elements. The actual iokey for any item + includes the ordinal suffix. - Parameters: - keys (tuple): of key strs to be combined in order to form key + Attributes: + db (dbing.LMDBer): base LMDB db + sdb (lmdb._Database): instance of lmdb named sub db for this Suber + sep (str): separator for combining keys tuple of strs into key bytes + klas (Iterable): of Class references to subclasses of CESR compatible + , each of to Type[coring.Matter etc] - Returns: - result (bool): True if key exists so delete successful. False otherwise - """ - return(self.db.delVal(db=self.sdb, key=self._tokey(keys))) + """ + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[serdering.Serder]): Class reference to subclass of Serder - def getItemIter(self, keys: Union[str, Iterable]=b""): """ - Returns: - iterator (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + super(SerderIoSetSuber, self).__init__(*pa, **kwa) - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield self._tokeys(iokey), self.klas(raw=bytes(val), verify=self.verify) class SchemerSuber(Suber): diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index c046695fc..bd6e10e0d 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -641,79 +641,176 @@ def test_serder_suber(): assert db.name == "test" assert db.opened - sdb = subing.SerderSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.SerderSuber) - assert not sdb.sdb.flags()["dupsort"] + serber = subing.SerderSuber(db=db, subkey='bags.') + assert isinstance(serber, subing.SerderSuber) + assert not serber.sdb.flags()["dupsort"] + assert serber.klas == serdering.SerderKERI pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" srdr0 = eventing.incept(keys=[pre]) keys = (pre, srdr0.said) - sdb.put(keys=keys, val=srdr0) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr0) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - sdb.rem(keys) - actual = sdb.get(keys=keys) + serber.rem(keys) + actual = serber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=srdr0) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr0) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said srdr1 = eventing.rotate(pre=pre, keys=[pre], dig=srdr0.said) - result = sdb.put(keys=keys, val=srdr1) + result = serber.put(keys=keys, val=srdr1) assert not result + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - result = sdb.pin(keys=keys, val=srdr1) + result = serber.pin(keys=keys, val=srdr1) assert result - actual = sdb.get(keys=keys) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said # test with keys as string not tuple keys = "{}.{}".format(pre, srdr1.said) - sdb.put(keys=keys, val=srdr1) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr1) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said - sdb.rem(keys) - actual = sdb.get(keys=keys) + serber.rem(keys) + actual = serber.get(keys=keys) assert actual is None # test missing entry at keys badkey = "badkey" - actual = sdb.get(badkey) + actual = serber.get(badkey) assert actual is None # test iteritems - sdb = subing.SerderSuber(db=db, subkey='pugs.') - assert isinstance(sdb, subing.SerderSuber) - sdb.put(keys=("a","1"), val=srdr0) - sdb.put(keys=("a","2"), val=srdr1) + serber = subing.SerderSuber(db=db, subkey='pugs.') + assert isinstance(serber, subing.SerderSuber) + serber.put(keys=("a","1"), val=srdr0) + serber.put(keys=("a","2"), val=srdr1) - items = [(keys, srdr.said) for keys, srdr in sdb.getItemIter()] + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter()] assert items == [(('a', '1'), srdr0.said), (('a', '2'), srdr1.said)] - assert sdb.put(keys=("b","1"), val=srdr0) - assert sdb.put(keys=("b","2"), val=srdr1) - assert sdb.put(keys=("bc","1"), val=srdr0) + assert serber.put(keys=("b","1"), val=srdr0) + assert serber.put(keys=("b","2"), val=srdr1) + assert serber.put(keys=("bc","1"), val=srdr0) topkeys = ("b", "") # append empty str to force trailing .sep - items = [(keys, srdr.said) for keys, srdr in sdb.getItemIter(keys=topkeys)] + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter(keys=topkeys)] assert items == [(('b', '1'), srdr0.said), (('b', '2'), srdr1.said)] assert not os.path.exists(db.path) assert not db.opened +def test_serder_ioset_suber(): + """ + Test SerderIoSetSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + serber = subing.SerderIoSetSuber(db=db, subkey='bags.') + assert isinstance(serber, subing.SerderIoSetSuber) + assert not serber.sdb.flags()["dupsort"] + assert serber.klas == serdering.SerderKERI + + pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + srdr0 = eventing.incept(keys=[pre]) + + keys = (pre, srdr0.said) + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr0.said + + serber.rem(keys) + actuals = serber.get(keys=keys) + assert not actuals # empty list + + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr0.said + + srdr1 = eventing.rotate(pre=pre, keys=[pre], dig=srdr0.said) + result = serber.put(keys=keys, vals=(srdr1, )) + assert result + actuals = serber.get(keys=keys) + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr0.said + assert isinstance(actuals[1], serdering.SerderKERI) + assert actuals[1].said == srdr1.said + + + result = serber.pin(keys=keys, vals=(srdr1, )) + assert result + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr1.said + + # test with keys as string not tuple + keys = "{}.{}".format(pre, srdr1.said) + + serber.put(keys=keys, vals=(srdr1, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr1.said + + serber.rem(keys) + actuals = serber.get(keys=keys) + assert not actuals + + # test missing entry at keys + badkey = "badkey" + actuals = serber.get(badkey) + assert not actuals + + # test iteritems + serber = subing.SerderIoSetSuber(db=db, subkey='pugs.') + assert isinstance(serber, subing.SerderIoSetSuber) + serber.put(keys=("a","1"), vals=(srdr0, )) + serber.put(keys=("a","1"), vals=(srdr1, )) + serber.put(keys=("a","2"), vals=(srdr1, )) + + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter()] + assert items == [ + (('a', '1'), srdr0.said), + (('a', '1'), srdr1.said), + (('a', '2'), srdr1.said), + ] + + assert serber.put(keys=("b","1"), vals=(srdr0, )) + assert serber.put(keys=("b","2"), vals=(srdr1, )) + assert serber.put(keys=("bc","1"), vals=(srdr0, )) + + topkeys = ("b", "") # append empty str to force trailing .sep + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), srdr0.said), + (('b', '2'), srdr1.said)] + + assert not os.path.exists(db.path) + assert not db.opened def test_cesr_suber(): @@ -996,7 +1093,7 @@ def test_cat_suber(): """Done Test""" -def test_cat__cesr_ioset_suber(): +def test_cat_cesr_ioset_suber(): """ Test CatIoSetSuber LMDBer sub database class """ @@ -1667,11 +1764,14 @@ def test_crypt_signer_suber(): if __name__ == "__main__": test_suber() - test_cesr_ioset_suber() - test_serder_suber() - test_cesr_suber() + test_dup_suber() + test_ioset_suber() test_cat_suber() - test_cat__cesr_ioset_suber() + test_cesr_suber() + test_cesr_ioset_suber() + test_cat_cesr_ioset_suber() test_cesr_dup_suber() + test_serder_suber() + test_serder_ioset_suber() test_signer_suber() test_crypt_signer_suber() From bfd041f2aa951a6e1ffea51df36460a378235b91 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 22 Aug 2024 13:38:06 -0600 Subject: [PATCH 272/418] more unit tests for SerderIoSetSuber --- src/keri/db/subing.py | 5 +++-- tests/db/test_subing.py | 25 +++++++++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 715153130..6a8f005c2 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -664,8 +664,9 @@ def rem(self, keys: Union[str, Iterable], val: Union[str, bytes, memoryview]=b'' Parameters: keys (Iterable): of key strs to be combined in order to form key - val (str): value at key to delete - if val is empty then remove all values at key + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key Returns: result (bool): True if effective key with val exists so delete successful. diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index bd6e10e0d..86e95606b 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -655,7 +655,7 @@ def test_serder_suber(): assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - serber.rem(keys) + assert serber.rem(keys) actual = serber.get(keys=keys) assert actual is None @@ -685,7 +685,7 @@ def test_serder_suber(): assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said - serber.rem(keys) + assert serber.rem(keys) actual = serber.get(keys=keys) assert actual is None @@ -741,7 +741,7 @@ def test_serder_ioset_suber(): assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - serber.rem(keys) + assert serber.rem(keys) actuals = serber.get(keys=keys) assert not actuals # empty list @@ -752,6 +752,7 @@ def test_serder_ioset_suber(): assert actual.said == srdr0.said srdr1 = eventing.rotate(pre=pre, keys=[pre], dig=srdr0.said) + result = serber.put(keys=keys, vals=(srdr1, )) assert result actuals = serber.get(keys=keys) @@ -768,6 +769,22 @@ def test_serder_ioset_suber(): assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said + + # test rem a specific value + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr1.said + assert isinstance(actuals[1], serdering.SerderKERI) + assert actuals[1].said == srdr0.said + + assert serber.rem(keys=keys, val=srdr1) + actuals = serber.get(keys=keys) + assert len(actuals) == 1 + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr0.said + + # test with keys as string not tuple keys = "{}.{}".format(pre, srdr1.said) @@ -777,7 +794,7 @@ def test_serder_ioset_suber(): assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said - serber.rem(keys) + assert serber.rem(keys) actuals = serber.get(keys=keys) assert not actuals From 341e3af4045fdef811ca82e1721673bb19508873 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 22 Aug 2024 14:17:29 -0600 Subject: [PATCH 273/418] added basing.pdes excrow of type IoSetSuber --- src/keri/db/basing.py | 40 ++++++++++++++++++++++------------------ tests/db/test_basing.py | 28 ++++++++-------------------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 63a456065..4bd0c5bf9 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -606,21 +606,24 @@ class Baser(dbing.LMDBer): kevers (dict): Kever instances indexed by identifier prefix qb64 prefixes (OrderedSet): local prefixes corresponding to habitats for this db - .evts is named sub DB whose values are serialized events + .evts is named sub DB whose values are serialized key events dgKey DB is keyed by identifier prefix plus digest of serialized event Only one value per DB key is allowed - .kels is named sub DB of key event log tables that map sequence numbers - to serialized event digests. + .kels is named sub DB of key event logs as indices that map sequence numbers + to serialized key event digests. + Actual serialized key events are stored in .evts by SAID digest Uses sequence number or sn. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .fels is named sub DB of first seen event log table (FEL) of digests - that indexes events in first 'seen' accepted order for replay and + .fels is named sub DB of first seen event logs (FEL) as indices that map + first seen ordinal number to digests. + Actual serialized key events are stored in .evts by SAID digest + This indexes events in first 'seen' accepted order for replay and cloning of event log. Uses first seen order number or fn. fnKey @@ -760,7 +763,7 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event More than one value per DB key is allowed - .pses is named sub DB of partially signed event escrows + .pses is named sub DB of partially signed key event escrows that map sequence numbers to serialized event digests. snKey Values are digests used to lookup event in .evts sub DB @@ -768,22 +771,21 @@ class Baser(dbing.LMDBer): More than one value per DB key is allowed - .pwes is named sub DB of partially witnessed event escrowes + .pwes is named sub DB of partially witnessed key event escrowes that map sequence numbers to serialized event digests. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .pdes is named sub DB of partially delegated escrowed couples - that map digest to seal source couple that provides source - (delegator or issuer) event seal. Each couples is concatenations - of full qualified items, snu+dig of authorizing (delegating or - issuing) source event. - dgKey - Values are couples used to lookup source event in .kels sub DB - DB is keyed by identifier prefix plus digest of key event - Only one value per DB key is allowed + .pdes is named sub DB of partially delegated key event escrows + that map sequence numbers to serialized event digests. This is + used in conjunction with .udes which escrows the associated seal + source couple. + snKey + Values are digests used to lookup delegated event in .evts sub DB + DB is keyed by identifier prefix plus sequence number of key event + More than one value per DB key is allowed .udes is named sub DB of unverified delegation seal source couple escrows that map (pre, digest) of delegated event to delegating seal source @@ -791,7 +793,9 @@ class Baser(dbing.LMDBer): Each couple is concatenation of fully qualified items, snu+dig of delegating source event in which seal of delegated event appears. dgKey - Values are couples used to lookup source event in .kels sub DB + Values are serialized instances of CatCesrSuber as couples + (Seqner.qb64b, Saider.qb64b) used to lookup source event in delegator's + KEL. DB is keyed by identifier prefix plus digest of key event Only one value per DB key is allowed @@ -1009,7 +1013,7 @@ def reopen(self, **kwa): self.vres = self.env.open_db(key=b'vres.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) #self.pdes = self.env.open_db(key=b'pdes.') - #self.udes = self.env.open_db(key=b'udes.') + self.pdes = subing.IoSetSuber(db=self, subkey='pdes.') self.udes = subing.CatCesrSuber(db=self, subkey='udes.', klas=(coring.Seqner, coring.Saider)) self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 17872eb76..a3d8b98b9 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -24,6 +24,7 @@ from keri.db import basing from keri.db import dbing +from keri.db import subing from keri.db.basing import openDB, Baser, KeyStateRecord from keri.db.dbing import (dgKey, onKey, snKey) from keri.db.dbing import openLMDB @@ -1013,26 +1014,13 @@ def test_baser(): key = dgKey(preb, digb) assert key == f'{preb.decode("utf-8")}.{digb.decode("utf-8")}'.encode("utf-8") - # test .udes sub db methods - #ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' - #sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' - #ssnu2 = b'0AAAAAAAAAAAAAAAAAAAAAAC' - #sdig2 = b'EBYYJRCCpAGO7WjjsLhtHVR37Pawv67kveIFUPvt38x0' - #val1 = ssnu1 + sdig1 - #val2 = ssnu2 + sdig2 - - #assert db.getUde(key) == None - #assert db.delUde(key) == False - #assert db.putUde(key, val1) == True - #assert db.getUde(key) == val1 - #assert db.putUde(key, val2) == False - #assert db.getUde(key) == val1 - #assert db.setUde(key, val2) == True - #assert db.getUde(key) == val2 - #assert db.delUde(key) == True - #assert db.getUde(key) == None - - # test .udes catcesrsuber sub db methods + # test .pdes SerderIoSetSuber methods + assert isinstance(db.pdes, subing.IoSetSuber) + + # test .udes CatCesrSuber sub db methods + assert isinstance(db.udes, subing.CatCesrSuber) + assert db.udes.klas == (coring.Seqner, coring.Saider) + ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' ssnu2 = b'0AAAAAAAAAAAAAAAAAAAAAAC' From 8701a2341c9b46e25c68d80bc2441824349391f3 Mon Sep 17 00:00:00 2001 From: arilieb Date: Thu, 22 Aug 2024 15:12:15 -0700 Subject: [PATCH 274/418] Replaced qnfs db with subing.IoSetSuber. Updated usages and removed db specific methods. Added test_query_not_found_escrow to test_querying.py (#849) --- src/keri/app/httping.py | 1 - src/keri/core/eventing.py | 23 +++++----- src/keri/db/basing.py | 93 +------------------------------------- tests/app/test_querying.py | 22 ++++++++- 4 files changed, 34 insertions(+), 105 deletions(-) diff --git a/src/keri/app/httping.py b/src/keri/app/httping.py index 346b870e0..f568ae9fc 100644 --- a/src/keri/app/httping.py +++ b/src/keri/app/httping.py @@ -41,7 +41,6 @@ def process_request(self, req, resp): req: Http request object resp: Http response object - """ sig = req.headers.get("SIGNATURE") ked = req.media diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index cfdfc4128..607048c0a 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -41,7 +41,7 @@ from . import serdering -from ..db import basing, dbing +from ..db import basing, dbing, subing from ..db.basing import KeyStateRecord, StateEERecord, OobiRecord from ..db.dbing import dgKey, snKey, fnKey, splitKeySN, splitKey @@ -3324,6 +3324,7 @@ def __init__(self, *, cues=None, db=None, rvy=None, self.direct = True if direct else False # process as direct mode self.check = True if check else False # process as check mode + @property def kevers(self): """ @@ -4790,7 +4791,7 @@ def escrowQueryNotFoundEvent(self, prefixer, serder, sigers, cigars=None): self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) self.db.putEvt(dgkey, serder.raw) - self.db.addQnf(dgkey, serder.saidb) + self.db.qnfs.add(keys=(prefixer.qb64, serder.said), val=serder.saidb) for cigar in cigars: self.db.addRct(key=dgkey, val=cigar.verfer.qb64b + cigar.qb64b) @@ -5935,7 +5936,7 @@ def processQueryNotFound(self): This allows FIFO processing of events with same prefix and sn but different digest. - Uses .db.addQnf(self, key, val) which is IOVal with dups. + Uses .db.qnfs.add(key=(pre, said), val) which is IOVal with dups Value is dgkey for event stored in .Evt where .Evt has serder.raw of event. @@ -5953,11 +5954,11 @@ def processQueryNotFound(self): pre = b'' sn = 0 while True: # break when done - for ekey, edig in self.db.getQnfItemsNextIter(key=key): + for (pre, said), edig in self.db.qnfs.getItemIter(keys=key): try: - pre, _ = splitKey(ekey) # get pre and sn from escrow item # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(edig))) + dgkey = dgKey(pre.encode("utf-8"), edig.encode("utf-8")) + dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event datetime" @@ -5978,7 +5979,7 @@ def processQueryNotFound(self): "at dig = {}.".format(bytes(edig))) # get the escrowed event using edig - eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + eraw = self.db.getEvt(dgkey) if eraw is None: # no event so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event at." @@ -5990,7 +5991,7 @@ def processQueryNotFound(self): eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event # get sigs and attach - sigs = self.db.getSigs(dgKey(pre, bytes(edig))) + sigs = self.db.getSigs(dgkey) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event sigs at." @@ -6010,7 +6011,7 @@ def processQueryNotFound(self): # get nontrans endorsements cigars = [] - cigs = self.db.getRcts(dgKey(pre, bytes(edig))) # list of wigs + cigs = self.db.getRcts(dgkey) # list of wigs for cig in cigs: (_, cigar) = deReceiptCouple(cig) cigars.append(cigar) @@ -6025,7 +6026,7 @@ def processQueryNotFound(self): except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow - self.db.delQnf(dgKey(pre, edig), edig) # removes one escrow at key val + self.db.qnfs.rem(keys=(pre, said), val=edig) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) else: @@ -6034,7 +6035,7 @@ def processQueryNotFound(self): # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. - self.db.delQnf(dgKey(pre, edig), edig) # removes one escrow at key val + self.db.qnfs.rem(keys=(pre, said), val=edig) # removes one escrow at key val logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) logger.debug(f"event=\n{eserder.pretty()}\n") diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 2fc0b8e58..2fa982e77 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -994,7 +994,7 @@ def reopen(self, **kwa): self.ooes = self.env.open_db(key=b'ooes.', dupsort=True) self.dels = self.env.open_db(key=b'dels.', dupsort=True) self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) - self.qnfs = self.env.open_db(key=b'qnfs.', dupsort=True) + self.qnfs = subing.IoSetSuber(db=self, subkey="qnfs.", dupsort=True) self.migs = subing.CesrSuber(db=self, subkey="migs.", klas=coring.Dater) self.vers = subing.Suber(db=self, subkey="vers.") @@ -3028,97 +3028,6 @@ def delOoe(self, key, val): """ return self.delIoVal(self.ooes, key, val) - def putQnfs(self, key, vals): - """ - Use snKey() - Write each out of order escrow event dig entry from list of bytes vals to key - Adds to existing event indexes at key if any - Returns True If at least one of vals is added as dup, False otherwise - Duplicates are inserted in insertion order. - """ - return self.putIoVals(self.qnfs, key, vals) - - def addQnf(self, key, val): - """ - Use snKey() - Add out of order escrow val bytes as dup to key in db - Adds to existing event indexes at key if any - Returns True if written else False if dup val already exists - Duplicates are inserted in insertion order. - """ - return self.addIoVal(self.qnfs, key, val) - - def getQnfs(self, key): - """ - Use snKey() - Return list of out of order escrow event dig vals at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoVals(self.qnfs, key) - - def getQnfLast(self, key): - """ - Use snKey() - Return last inserted dup val of out of order escrow event dig vals at key - Returns None if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoValLast(self.qnfs, key) - - def getQnfItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of out of order escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.qnfs, key, skip) - - def getQnfItemsNextIter(self, key=b'', skip=True): - """ - Use sgKey() - Return iterator of out of order escrowed event dig items at next key after key. - Items is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNextIter(self.qnfs, key, skip) - - def cntQnfs(self, key): - """ - Use snKey() - Return count of dup event dig at key - Returns zero if no entry at key - """ - return self.cntIoVals(self.qnfs, key) - - def delQnfs(self, key): - """ - Use snKey() - Deletes all values at key. - Returns True If key exists in database Else False - """ - return self.delIoVals(self.qnfs, key) - - def delQnf(self, key, val): - """ - Use snKey() - Deletes dup val at key in db. - Returns True If dup at exists in db Else False - - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace - val is dup val (does not include insertion ordering proem) - """ - return self.delIoVal(self.qnfs, key, val) - def putDes(self, key, vals): """ Use snKey() diff --git a/tests/app/test_querying.py b/tests/app/test_querying.py index fb36461a5..ee6c42f46 100644 --- a/tests/app/test_querying.py +++ b/tests/app/test_querying.py @@ -7,7 +7,8 @@ from keri.app import habbing from keri.app.querying import QueryDoer, KeyStateNoticer, LogQuerier, SeqNoQuerier, AnchorQuerier -from keri.core import parsing, eventing +from keri.core import parsing, eventing, serdering +from keri.db.dbing import dgKey def test_querying(): @@ -148,3 +149,22 @@ def test_querying(): doist.recur(deeds=deeds) assert len(adoer.witq.msgs) == 1 +def test_query_not_found_escrow(): + with habbing.openHby() as hby, \ + habbing.openHby() as hby1: + inqHab = hby.makeHab(name="inquisitor") + subHab = hby1.makeHab(name="subject") + + icp = inqHab.makeOwnInception() + subHab.psr.parseOne(ims=icp) + assert inqHab.pre in subHab.kevers + + qry = inqHab.query(subHab.pre, route="/foo", src=inqHab.pre) + serder = serdering.SerderKERI(raw=qry) + dgkey = dgKey(inqHab.pre, serder.saidb) + + subHab.db.putEvt(dgkey, serder.raw) + subHab.db.qnfs.add(keys=(inqHab.pre, serder.said), val=serder.saidb) + + subHab.kvy.processQueryNotFound() + assert subHab.db.qnfs.get(dgkey) == [] \ No newline at end of file From 13506a29c430baba112d16a58f6bafc2e606b04d Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Thu, 22 Aug 2024 15:50:09 -0700 Subject: [PATCH 275/418] Fixes for anchoring multiple delegates in a single delegator event, exn escrow timeout. (#851) * Updating delegation logic to allow for multiple delegation inception commands to run simultaneously. Ensure proper use of the proxy for delegation inception and rotation kli commands. Add timeout to exn escrow for partial sig escrow of peer-to-peer messages. Signed-off-by: pfeairheller * Add test for exchange escrow timeout. Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller --- src/keri/app/cli/commands/event.py | 84 ++++++++++++++++++++ src/keri/app/cli/commands/incept.py | 10 ++- src/keri/app/cli/commands/interact.py | 17 +++- src/keri/app/cli/commands/multisig/incept.py | 2 +- src/keri/app/cli/commands/multisig/rotate.py | 2 +- src/keri/app/cli/commands/rotate.py | 15 ++-- src/keri/app/delegating.py | 12 ++- src/keri/app/forwarding.py | 5 +- src/keri/db/basing.py | 4 + src/keri/peer/exchanging.py | 69 ++++++++++------ tests/peer/test_exchanging.py | 37 ++++++++- 11 files changed, 213 insertions(+), 44 deletions(-) create mode 100644 src/keri/app/cli/commands/event.py diff --git a/src/keri/app/cli/commands/event.py b/src/keri/app/cli/commands/event.py new file mode 100644 index 000000000..b750d99f6 --- /dev/null +++ b/src/keri/app/cli/commands/event.py @@ -0,0 +1,84 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse +import json + +from hio import help +from hio.base import doing + +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Print an event from an AID, or specific values from an event (defaults to latest event).') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None, + required=True) +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--said', '-S', help='Print the SAID of the event in question', action="store_true") +parser.add_argument('--sn', '-s', help='Print the decimal value of the sequence number of the event in question', + action="store_true") +parser.add_argument('--snh', help='Print the decimal value of the sequence number of the event in question', + action="store_true") +parser.add_argument('--raw', '-r', help='Print the raw, signed value of the event', action="store_true") +parser.add_argument('--json', '-j', help='Pretty print the JSON of the event.', action="store_true") +parser.add_argument('--seal', help='Print an anchorable seal of the event in question.', action="store_true") + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(event, **kwa)] + + +def event(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + alias = args.alias + base = args.base + bran = args.bran + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + if alias is None: + alias = existing.aliasInput(hby) + + hab = hby.habByName(alias) + if hab is None: + print(f"{alias} is not a valid identifier") + + if args.said: + print(hab.kever.serder.said) + + if args.sn: + print(hab.kever.sn) + + if args.snh: + print(hab.kever.serder.snh) + + if args.raw: + print(hab.kever.serder.raw.decode("utf-8")) + + if args.json: + print(hab.kever.serder.pretty()) + + if args.seal: + seal = dict(i=hab.pre, s=hab.kever.serder.snh, d=hab.kever.serder.said) + print(json.dumps(seal)) + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 diff --git a/src/keri/app/cli/commands/incept.py b/src/keri/app/cli/commands/incept.py index 4af084c73..e521d064e 100644 --- a/src/keri/app/cli/commands/incept.py +++ b/src/keri/app/cli/commands/incept.py @@ -138,10 +138,10 @@ def __init__(self, name, base, alias, bran, endpoint, proxy=None, cnfg=None, **k reopen=True, clear=False) self.endpoint = endpoint - self.proxy = proxy self.hby = existing.setupHby(name=name, base=base, bran=bran, cf=cf) + self.proxy = self.hby.habByName(proxy) if proxy is not None else None self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer - self.swain = delegating.Anchorer(hby=self.hby, proxy=self.hby.habByName(self.proxy)) + self.swain = delegating.Anchorer(hby=self.hby, proxy=self.proxy) self.postman = forwarding.Poster(hby=self.hby) self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=['/receipt', "/replay", "/reply"]) doers = [self.hbyDoer, self.postman, self.mbx, self.swain, doing.doify(self.inceptDo)] @@ -185,7 +185,11 @@ def inceptDo(self, tymth, tock=0.0): _ = yield self.tock if hab.kever.delpre: - yield from self.postman.sendEvent(hab=hab, fn=hab.kever.sn) + if self.proxy is not None: + sender = self.proxy + else: + sender = hab + yield from self.postman.sendEventToDelegator(hab=hab, sender=sender, fn=hab.kever.sn) print(f'Prefix {hab.pre}') for idx, verfer in enumerate(hab.kever.verfers): diff --git a/src/keri/app/cli/commands/interact.py b/src/keri/app/cli/commands/interact.py index 53cabce75..07a98ea54 100644 --- a/src/keri/app/cli/commands/interact.py +++ b/src/keri/app/cli/commands/interact.py @@ -26,6 +26,9 @@ dest="endpoint", action='store_true') parser.add_argument("--authenticate", '-z', help="Prompt the controller for authentication codes for each witness", action='store_true') +parser.add_argument('--code', help=': formatted witness auth codes. Can appear multiple times', + default=[], action="append", required=False) +parser.add_argument('--code-time', help='Time the witness codes were captured.', default=None, required=False) def interact(args): @@ -57,7 +60,7 @@ def interact(args): data = None ixnDoer = InteractDoer(name=name, base=base, alias=alias, bran=bran, data=data, authenticate=args.authenticate, - endpoint=args.endpoint) + endpoint=args.endpoint, codes=args.code, codeTime=args.code_time) return [ixnDoer] @@ -68,7 +71,8 @@ class InteractDoer(doing.DoDoer): to all appropriate witnesses """ - def __init__(self, name, base, bran, alias, data: list = None, endpoint=False, authenticate=False): + def __init__(self, name, base, bran, alias, data: list = None, endpoint=False, authenticate=False, + codes=None, codeTime=None): """ Returns DoDoer with all registered Doers needed to perform interaction event. @@ -82,6 +86,8 @@ def __init__(self, name, base, bran, alias, data: list = None, endpoint=False, a self.data = data self.endpoint = endpoint self.authenticate = authenticate + self.codes = codes if codes is not None else [] + self.codeTime = codeTime self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer @@ -105,7 +111,14 @@ def interactDo(self, tymth, tock=0.0, **opts): auths = {} if self.authenticate: + codeTime = helping.fromIso8601(self.codeTime) if self.codeTime is not None else helping.nowIso8601() + for arg in self.codes: + (wit, code) = arg.split(":") + auths[wit] = f"{code}#{codeTime}" + for wit in hab.kever.wits: + if wit in auths: + continue code = input(f"Entire code for {wit}: ") auths[wit] = f"{code}#{helping.nowIso8601()}" diff --git a/src/keri/app/cli/commands/multisig/incept.py b/src/keri/app/cli/commands/multisig/incept.py index 3e24f8f58..57478dfed 100644 --- a/src/keri/app/cli/commands/multisig/incept.py +++ b/src/keri/app/cli/commands/multisig/incept.py @@ -175,7 +175,7 @@ def inceptDo(self, tymth, tock=0.0): yield self.tock if ghab.kever.delpre: - yield from self.postman.sendEvent(hab=ghab, fn=ghab.kever.sn) + yield from self.postman.sendEventToDelegator(hab=ghab, sender=ghab.mhab, fn=ghab.kever.sn) print() displaying.printIdentifier(self.hby, ghab.pre) diff --git a/src/keri/app/cli/commands/multisig/rotate.py b/src/keri/app/cli/commands/multisig/rotate.py index e79e1f6d3..248882903 100644 --- a/src/keri/app/cli/commands/multisig/rotate.py +++ b/src/keri/app/cli/commands/multisig/rotate.py @@ -234,7 +234,7 @@ def rotateDo(self, tymth, tock=0.0, **opts): yield self.tock if ghab.kever.delpre: - yield from self.postman.sendEvent(hab=ghab, fn=ghab.kever.sn) + yield from self.postman.sendEventToDelegator(hab=ghab, sender=ghab.mhab, fn=ghab.kever.sn) print() displaying.printIdentifier(self.hby, ghab.pre) diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index d6f615bcd..6edb0955a 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -33,7 +33,7 @@ default=[], action="append", required=False) parser.add_argument('--code-time', help='Time the witness codes were captured.', default=None, required=False) -parser.add_argument("--proxy", help="alias for delegation communication proxy", default="") +parser.add_argument("--proxy", help="alias for delegation communication proxy", default=None) rotating.addRotationArgs(parser) @@ -148,7 +148,6 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.toad = toad self.data = data self.endpoint = endpoint - self.proxy = proxy self.authenticate = authenticate self.codes = codes if codes is not None else [] self.codeTime = codeTime @@ -159,7 +158,9 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer - self.swain = delegating.Anchorer(hby=self.hby, proxy=self.hby.habByName(self.proxy)) + + self.proxy = self.hby.habByName(proxy) if proxy is not None else None + self.swain = delegating.Anchorer(hby=self.hby, proxy=self.proxy) self.postman = forwarding.Poster(hby=self.hby) self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=['/receipt', "/replay", "/reply"]) doers = [self.hbyDoer, self.mbx, self.swain, self.postman, doing.doify(self.rotateDo)] @@ -213,7 +214,7 @@ def rotateDo(self, tymth, tock=0.0): auths[wit] = f"{code}#{helping.nowIso8601()}" if hab.kever.delpre: - self.swain.delegation(pre=hab.pre, sn=hab.kever.sn, auths=auths) + self.swain.delegation(pre=hab.pre, sn=hab.kever.sn, auths=auths, proxy=self.proxy) print("Waiting for delegation approval...") while not self.swain.complete(hab.kever.prefixer, coring.Seqner(sn=hab.kever.sn)): yield self.tock @@ -237,7 +238,11 @@ def rotateDo(self, tymth, tock=0.0): self.remove([witDoer]) if hab.kever.delpre: - yield from self.postman.sendEvent(hab=hab, fn=hab.kever.sn) + if self.proxy is not None: + sender = self.proxy + else: + sender = hab + yield from self.postman.sendEventToDelegator(hab=hab, sender=sender, fn=hab.kever.sn) print(f'Prefix {hab.pre}') print(f'New Sequence No. {hab.kever.sn}') diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 6b90c53ef..a1cb0a4c8 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -173,10 +173,14 @@ def processPartialWitnessEscrow(self): if not witnessed: continue logger.info(f"Witness receipts complete, waiting for delegation approval.") + if pre not in self.hby.habs: + continue + hab = self.hby.habs[pre] delpre = hab.kever.delpre # get the delegator identifier dkever = hab.kevers[delpre] # and the delegator's kever smids = [] + if isinstance(hab, GroupHab): phab = hab.mhab smids = hab.smids @@ -208,6 +212,9 @@ def processWitnessPublication(self): """ for (pre, said), serder in self.hby.db.dpub.getItemIter(): # group partial witness escrow + if pre not in self.publishers: + continue + publisher = self.publishers[pre] if not publisher.idle: @@ -220,8 +227,11 @@ def processWitnessPublication(self): self.hby.db.cdel.put(keys=(pre, coring.Seqner(sn=serder.sn).qb64), val=coring.Saider(qb64=serder.said)) def publishDelegator(self, pre): - hab = self.hby.habs[pre] + if pre not in self.publishers: + return + publisher = self.publishers[pre] + hab = self.hby.habs[pre] self.extend([publisher]) for msg in hab.db.cloneDelegation(hab.kever): publisher.msgs.append(dict(pre=hab.pre, msg=bytes(msg))) diff --git a/src/keri/app/forwarding.py b/src/keri/app/forwarding.py index b44c67a3a..877a98330 100644 --- a/src/keri/app/forwarding.py +++ b/src/keri/app/forwarding.py @@ -135,15 +135,14 @@ def sent(self, said): return False - def sendEvent(self, hab, fn=0): + def sendEventToDelegator(self, sender, hab, fn=0): """ Returns generator for sending event and waiting until send is complete """ # Send KEL event for processing icp = self.hby.db.cloneEvtMsg(pre=hab.pre, fn=fn, dig=hab.kever.serder.saidb) ser = serdering.SerderKERI(raw=icp) del icp[:ser.size] - sender = hab.mhab.pre if isinstance(hab, GroupHab) else hab.pre - self.send(src=sender, dest=hab.kever.delpre, topic="delegate", serder=ser, attachment=icp) + self.send(src=sender.pre, dest=hab.kever.delpre, topic="delegate", serder=ser, attachment=icp) while True: if self.cues: cue = self.cues.popleft() diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 2fa982e77..3a2b511f4 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1106,6 +1106,10 @@ def reopen(self, **kwa): # exchange message partial signature escrow self.epse = subing.SerderSuber(db=self, subkey="epse.") + # exchange message PS escrow date time of message + self.epsd = subing.CesrSuber(db=self, subkey="epsd.", + klas=coring.Dater) + # exchange messages # TODO: clean self.exns = subing.SerderSuber(db=self, subkey="exns.") diff --git a/src/keri/peer/exchanging.py b/src/keri/peer/exchanging.py index 589fb5a6a..daaf31d45 100644 --- a/src/keri/peer/exchanging.py +++ b/src/keri/peer/exchanging.py @@ -3,6 +3,7 @@ keri.peer.exchanging module """ +import datetime import logging from datetime import timedelta @@ -24,6 +25,8 @@ class Exchanger: Peer to Peer KERI message Exchanger. """ + TimeoutPSE = 10 # seconds to timeout partially signed or delegated escrows + def __init__(self, hby, handlers, cues=None, delta=ExchangeMessageTimeWindow): """ Initialize instance @@ -74,7 +77,7 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): essrs = kwargs["essrs"] if "essrs" in kwargs else [] behavior = self.routes[route] if route in self.routes else None - if tsgs is not None: + if tsgs: for prefixer, seqner, ssaider, sigers in tsgs: # iterate over each tsg if sender != prefixer.qb64: # sig not by aid raise MissingSignatureError(f"Exchange process: skipped signature not from aid=" @@ -96,7 +99,7 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): raise MissingSignatureError(f"Not enough signatures in {indices}" f" for evt = {serder.ked}.") - elif cigars is not None: + elif cigars: for cigar in cigars: if sender != cigar.verfer.qb64: # cig not by aid raise MissingSignatureError(" process: skipped cig not from aid=" @@ -107,6 +110,7 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): " for evt = {}.".format(cigar, serder.ked)) else: + self.escrowPSEvent(serder=serder, tsgs=[], pathed=pathed) raise MissingSignatureError("Failure satisfying exn, no cigs or sigs" " for evt = {}.".format(serder.ked)) @@ -178,35 +182,49 @@ def escrowPSEvent(self, serder, tsgs, pathed): for siger in sigers: self.hby.db.esigs.add(keys=quadkeys, val=siger) + self.hby.db.epsd.put(keys=(dig,), val=coring.Dater()) self.hby.db.epath.pin(keys=(dig,), vals=[bytes(p) for p in pathed]) return self.hby.db.epse.put(keys=(dig,), val=serder) def processEscrowPartialSigned(self): """ Process escrow of partially signed messages """ for (dig,), serder in self.hby.db.epse.getItemIter(): - tsgs = [] - klases = (coring.Prefixer, coring.Seqner, coring.Saider) - args = ("qb64", "snh", "qb64") - sigers = [] - old = None # empty keys - for keys, siger in self.hby.db.esigs.getItemIter(keys=(dig, "")): - quad = keys[1:] - if quad != old: # new tsg - if sigers: # append tsg made for old and sigers - prefixer, seqner, saider = helping.klasify(sers=old, klases=klases, args=args) - - tsgs.append((prefixer, seqner, saider, sigers)) - sigers = [] - old = quad - sigers.append(siger) - if sigers and old: - prefixer, seqner, saider = helping.klasify(sers=old, klases=klases, args=args) - tsgs.append((prefixer, seqner, saider, sigers)) + try: + tsgs = [] + klases = (coring.Prefixer, coring.Seqner, coring.Saider) + args = ("qb64", "snh", "qb64") + sigers = [] - pathed = [bytearray(p.encode("utf-8")) for p in self.hby.db.epath.get(keys=(dig,))] - essrs = [texter for texter in self.hby.db.essrs.get(keys=(dig,))] + dtnow = helping.nowUTC() + dater = self.hby.db.epsd.get(keys=(dig,)) + if dater is None: + raise ValidationError("Missing exn escrowed event datetime " + f"at dig = {dig}.") + + dte = dater.datetime + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): + # escrow stale so raise ValidationError which unescrows below + raise ValidationError("Stale exn event escrow " + f"at dig = {dig}.") + + old = None # empty keys + for keys, siger in self.hby.db.esigs.getItemIter(keys=(dig, "")): + quad = keys[1:] + if quad != old: # new tsg + if sigers: # append tsg made for old and sigers + prefixer, seqner, saider = helping.klasify(sers=old, klases=klases, args=args) + + tsgs.append((prefixer, seqner, saider, sigers)) + sigers = [] + old = quad + sigers.append(siger) + if sigers and old: + prefixer, seqner, saider = helping.klasify(sers=old, klases=klases, args=args) + tsgs.append((prefixer, seqner, saider, sigers)) + + pathed = [bytearray(p.encode("utf-8")) for p in self.hby.db.epath.get(keys=(dig,))] + essrs = [texter for texter in self.hby.db.essrs.get(keys=(dig,))] - try: kwargs = dict() if essrs: kwargs["essrs"] = essrs @@ -219,11 +237,12 @@ def processEscrowPartialSigned(self): logger.info("Exchange partially signed failed: %s", ex.args[0]) except Exception as ex: self.hby.db.epse.rem(dig) + self.hby.db.epsd.rem(dig) self.hby.db.esigs.rem(dig) if logger.isEnabledFor(logging.DEBUG): - logger.info("Exchange partially signed unescrowed: %s", ex.args[0]) + logger.exception("Exchange partially signed unescrowed: %s", ex.args[0]) else: - logger.info("Exchange partially signed unescrowed: %s", ex.args[0]) + logger.error("Exchange partially signed unescrowed: %s", ex.args[0]) else: self.hby.db.epse.rem(dig) self.hby.db.esigs.rem(dig) diff --git a/tests/peer/test_exchanging.py b/tests/peer/test_exchanging.py index 4a0a0a009..50be8ec62 100644 --- a/tests/peer/test_exchanging.py +++ b/tests/peer/test_exchanging.py @@ -4,8 +4,7 @@ """ import pysodium -from base64 import urlsafe_b64encode as encodeB64 -from base64 import urlsafe_b64decode as decodeB64 +import pytest from keri import kering from keri import core @@ -100,8 +99,40 @@ def test_essrs(): assert recHby.db.exns.get(keys=(essr.said,)) is None - def test_exchanger(): + with habbing.openHab(name="sid", base="test", salt=b'0123456789abcdef') as (hby, hab), \ + habbing.openHab(name="rec", base="test", salt=b'0123456789abcdef') as (recHby, recHab): + exc = exchanging.Exchanger(hby=recHby, handlers=[]) + + msg = hab.makeOwnInception() + recHab.psr.parseOne(ims=msg) + + ser, sigs, _ = hab.getOwnEvent(sn=0) + + sadsig = signing.SadPathSigGroup(pather=coring.Pather(path=[]), sigers=sigs) + act = bytearray() + pather = coring.Pather(path=["e"]) + sadsig.transpose(pather) + act.extend(sadsig.proof) + + # create the forward message with payload embedded at `a` field + fwd, _ = exchanging.exchange(route='/fwd', sender=hab.pre, + modifiers=dict(pre="EBCAFG", topic="/delegation"), + payload={}, embeds=dict(evt=ser.raw)) + with pytest.raises(kering.MissingSignatureError): + exc.processEvent(serder=fwd, source=hab.kever.prefixer, tsgs=None) + + assert recHby.db.epse.get(keys=(fwd.said,)) is not None + exc.processEscrowPartialSigned() + assert recHby.db.epse.get(keys=(fwd.said,)) is not None + + # Set the PSE timeout artifically low to trigger removal + exc.TimeoutPSE = 0.00001 + exc.processEscrowPartialSigned() + assert recHby.db.epse.get(keys=(fwd.said,)) is None + + +def test_exchange_ps_escrow_timeout(): with habbing.openHab(name="sid", base="test", salt=b'0123456789abcdef') as (hby, hab), \ habbing.openHab(name="rec", base="test", salt=b'0123456789abcdef') as (recHby, recHab): mbx = storing.Mailboxer(hby=hby) From 9d9a481994e01aaa5a1c7bc637f758a3a2927078 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Thu, 22 Aug 2024 16:04:12 -0700 Subject: [PATCH 276/418] Version Bump --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f34d1bd4e..4bf4ec2fe 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=MULTI_ARCH_TEST +VERSION=1.2.0-dev13 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index 34112fa0d..a704266a4 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev12', # also change in src/keri/__init__.py + version='1.2.0-dev13', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index e4125ad57..09ed18bf2 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev12' # also change in setup.py +__version__ = '1.2.0-dev13' # also change in setup.py From eb53a57eb69b3f52c8323983cd3ef537e865302d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Aug 2024 14:05:41 -0600 Subject: [PATCH 277/418] More refactor clean up of subing --- src/keri/core/eventing.py | 174 +++++++++++++ src/keri/db/basing.py | 34 --- src/keri/db/dbing.py | 132 +++++----- src/keri/db/subing.py | 499 ++++++++++++++++++++------------------ src/keri/vdr/viring.py | 2 +- tests/db/test_dbing.py | 2 +- tests/db/test_subing.py | 145 ++++++++--- 7 files changed, 619 insertions(+), 369 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 23a94b81c..a412c37d4 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5671,6 +5671,180 @@ def processEscrowPartialWigs(self): break key = ekey # setup next while iteration, with key after ekey + def processEscrowPartialDels(self): + """ + Process delgated events escrowed by Kever that were only partially fulfilled + due to missing or unverified delegation seals from delegators. + Events only make into this escrow after fully signed and if witnessed + fully witnessed. + + db.pdes is an instance of subing.IoSetSuber and uses instance methods + for access to the underlying database. + Escrowed items in .pdes are indexed in database table keyed by prefix and + sequence number with multiple entries at same key held in insertion order. + This allows FIFO processing of escrowed events with same prefix and sn. + + Value in each .pdes entry is dgkey (SAID) of event stored in db.evts where + db.evts holds SerderKERI.raw of event. + + Steps: + Each pass (walk index table) + For each prefix,sn + For each escrow item dup at prefix,sn: + Get Event + Get and Attach Signatures + Get and Attach Witness Signatures + Process event as if it came in over the wire + If successful then remove from escrow table + """ + + key = ekey = b'' # both start same. when not same means escrows found + while True: # break when done + for ekey, edig in self.db.getPweItemsNextIter(key=key): + try: + pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) + + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) + + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) + + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) + + # get the escrowed event using edig + eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) + + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + + # get sigs + sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs + if not sigs: # empty list + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) + + # get witness signatures (wigs not wits) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + + if not wigs: # empty list + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) + + # raise ValidationError("Missing escrowed evt wigs at " + # "dig = {}.".format(bytes(edig))) + + # process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + delseqner, delsaider = couple + #couple = self.db.getUde(dgKey(pre, bytes(edig))) + #if couple is not None: + #delseqner, delsaider = deSourceCouple(couple) + elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + if eserder.pre in self.kevers: + delpre = self.kevers[eserder.pre].delpre + else: + delpre = eserder.ked["di"] + seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + #couple = delseqner.qb64b + delsaider.qb64b + #self.db.putUde(dgkey, couple) + # idempotent + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, local=esr.local) + + # If process does NOT validate wigs then process will attempt + # to re-escrow and then raise MissingWitnessSignatureError + # (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pwe escrow is called by + # Kever.self.escrowPWEvent Which calls + # self.db.addPwe(snKey(pre, sn), serder.digb) + # which in turn will NOT enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrowed pwe db. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + # Assumes that controller signature validation and delegation + # validation will be successful as event would not be in + # partially witnessed escrow unless they had already validated + + except (MissingWitnessSignatureError, MissingDelegationError) as ex: + # still waiting on missing witness sigs or delegation + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) + + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs or seal so remove from escrow + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) + + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") + + if ekey == key: # still same so no escrows found on last while iteration + break + key = ekey # setup next while iteration, with key after ekey + + + def processEscrowUnverWitness(self): """ Process escrowed unverified event receipts from witness receiptors diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 4bd0c5bf9..5a1937886 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2723,40 +2723,6 @@ def delPse(self, key, val): """ return self.delIoVal(self.pses, key, val) - #def putUde(self, key, val): - #""" - #Use dgKey() - #Write serialized event source couple to key (snu+dig) - #Does not overwrite existing val if any - #Returns True If val successfully written Else False - #Returns False if key already exists - #""" - #return self.putVal(self.udes, key, val) - - #def setUde(self, key, val): - #""" - #Use dgKey() - #Write serialized seal source couple to key (snu+dig) - #Overwrites existing val if any - #Returns True If val successfully written Else False - #""" - #return self.setVal(self.udes, key, val) - - #def getUde(self, key): - #""" - #Use dgKey() - #Return seal source couple at key - #Returns None if no entry at key - #""" - #return self.getVal(self.udes, key) - - #def delUde(self, key): - #""" - #Use dgKey() - #Deletes value at key. - #Returns True If key exists in database Else False - #""" - #return self.delVal(self.udes, key) def putPwes(self, key, vals): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 1dc1208bc..5d29d242b 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -652,8 +652,39 @@ def delTopVal(self, db, key=b''): # For subdbs with no duplicate values allowed at each key. (dupsort==False) - # and use keys with ordinal as monotonically increasing number part - # such as sn or fn + # and use keys with suffic ordinal that is monotonically increasing number part + # such as fn where no duplicates allowed at a given (pre, on) + + def cntAllOrdValsPre(self, db, pre, on=0): + """ + Returns (int): count of of all ordinal keyed vals with same pre in key + but different on in key in db starting at ordinal number on of pre. + For db with dupsort=False but ordinal number suffix in each key + + Parameters: + db is opened named sub db + pre is bytes of key within sub db's keyspace pre.on + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + key = onKey(pre, on) # start replay at this enty 0 is earliest + count = 0 + if not cursor.set_range(key): # moves to val at key >= key + return count # no values end of db + + for key in cursor.iternext(values=False): # get key only at cursor + try: + cpre, cn = splitKeyON(key) + except ValueError as ex: # not splittable key + break + + if cpre != pre: # prev is now the last event for pre + break # done + count = count+1 + + return count + + def appendOrdValPre(self, db, pre, val): """ Appends val in order after last previous key with same pre in db. @@ -757,6 +788,9 @@ def getAllOrdItemAllPreIter(self, db, key=b''): yield (cpre, cn, val) # (pre, on, dig) of event + + # IoSet insertion order in val so can have effective dups but with + # dupsort = False so val not limited to 511 bytes # For databases that support set of insertion ordered values with apparent # effective duplicate key but with (dupsort==False). Actual key uses hidden # key suffix ordinal to provide insertion ordering of value members of set @@ -810,8 +844,8 @@ def putIoSetVals(self, db, key, vals, *, sep=b'.'): def addIoSetVal(self, db, key, val, *, sep=b'.'): """ - Add val to insertion ordered set of values all with the same apparent - effective key if val not already in set of vals at key. + Add val idempotently to insertion ordered set of values all with the + same apparent effective key if val not already in set of vals at key. A Uses hidden ordinal key suffix for insertion ordering. The suffix is appended and stripped transparently. @@ -845,38 +879,18 @@ def addIoSetVal(self, db, key, val, *, sep=b'.'): return cursor.put(iokey, val, dupdata=False, overwrite=False) - def setIoSetVals(self, db, key, vals, *, sep=b'.'): - """ - Erase all vals at key and then add unique vals as insertion ordered set of - values all with the same apparent effective key. - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. - - Returns: - result (bool): True is added to set. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - vals (abc.Iterable): serialized values to add to set of vals at key - """ - self.delIoSetVals(db=db, key=key, sep=sep) - result = False - vals = oset(vals) # make set - with self.env.begin(db=db, write=True, buffers=True) as txn: - for i, val in enumerate(vals): - iokey = suffix(key, i, sep=sep) # ion is at add on amount - result = txn.put(iokey, val, dupdata=False, overwrite=True) or result - return result - - def appendIoSetVal(self, db, key, val, *, sep=b'.'): """ - Append val to insertion ordered set of values all with the same apparent - effective key. Assumes val is not already in set. + Append val non-idempotently to insertion ordered set of values all with + the same apparent effective key. If val already in set at key then + after append there will be multiple entries in database with val at key + each with different insertion order (iokey). Uses hidden ordinal key suffix for insertion ordering. The suffix is appended and stripped transparently. + Works by walking backward to find last iokey for key instead of reading + all vals for ioky. + Returns: ion (int): hidden insertion ordering ordinal of appended val @@ -923,6 +937,33 @@ def appendIoSetVal(self, db, key, val, *, sep=b'.'): return ion + + def setIoSetVals(self, db, key, vals, *, sep=b'.'): + """ + Erase all vals at key and then add unique vals as insertion ordered set of + values all with the same apparent effective key. + Uses hidden ordinal key suffix for insertion ordering. + The suffix is appended and stripped transparently. + + Returns: + result (bool): True is added to set. + + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + key (bytes): Apparent effective key + vals (abc.Iterable): serialized values to add to set of vals at key + """ + self.delIoSetVals(db=db, key=key, sep=sep) + result = False + vals = oset(vals) # make set + with self.env.begin(db=db, write=True, buffers=True) as txn: + for i, val in enumerate(vals): + iokey = suffix(key, i, sep=sep) # ion is at add on amount + result = txn.put(iokey, val, dupdata=False, overwrite=True) or result + return result + + + def getIoSetVals(self, db, key, *, ion=0, sep=b'.'): """ Returns: @@ -1338,31 +1379,6 @@ def cntVals(self, db, key): return count - def cntValsAllPre(self, db, pre, on=0): - """ - Returns (int): count of of all vals with same pre in key but different - on in key in db starting at ordinal number on of pre - - Does not count dups - - Parameters: - db is opened named sub db - pre is bytes of key within sub db's keyspace pre.on - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest - count = 0 - if not cursor.set_range(key): # moves to val at key >= key - return count # no values end of db - - for val in cursor.iternext(values=False): # get key, val at cursor - cpre, cn = splitKeyON(val) - if cpre != pre: # prev is now the last event for pre - break # done - count = count+1 - - return count def delVals(self, db, key, val=b''): @@ -1385,7 +1401,9 @@ def delVals(self, db, key, val=b''): # For subdbs that support insertion order preserving duplicates at each key. - # dupsort==True and prepends and strips io val proem + # IoDup class IoVals IoItems + # dupsort==True and prepends and strips io val proem to each value. + # because dupsort==True values are limited to 511 bytes including proem def putIoVals(self, db, key, vals): """ Write each entry from list of bytes vals to key in db in insertion order diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 6a8f005c2..54a428f96 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -91,24 +91,30 @@ def __init__(self, db: dbing.LMDBer, *, - def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], + def _tokey(self, keys: str | bytes | memoryview | Iterable[str | bytes], top: bool=False): """ - Converts keys to key str with proper separators and returns key bytes. - If keys is already str then returns. Else If keys is iterable (non-str) - of strs then joins with separator converts to bytes and returns. - top allows partial key from top branch of key space given by partial keys + Converts keys to key bytes with proper separators and returns key bytes. + If keys is already str or bytes then returns key bytes. + Else If keys is iterable (non-str) of strs or bytes then joins with + separator converts to key bytes and returns. When keys is iterable and + top is True then enables partial key from top branch of key space given + by partial keys by appending separator to end of partial key Returns: key (bytes): each element of keys is joined by .sep. If top then last char of key is also .sep Parameters: - keys (Union[str, bytes, Iterable]): str, bytes, or Iterable of str. + keys (str | bytes | memoryview | Iterable[str | bytes]): db key or + Iterable of (str | bytes) to form key. top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep. + key space given by partial keys. Resultant key ends in .sep + character. False means treat as full branch in key space. Resultant key - does not end in .sep + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ if hasattr(keys, "encode"): # str @@ -117,21 +123,22 @@ def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], return bytes(keys) # return bytes elif hasattr(keys, "decode"): # bytes return keys - #keys = (key.decode() if hasattr(key, "decode") else key for key in keys) + if top and keys[-1]: # top and keys is not already partial + keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep return (self.sep.join(key.decode() if hasattr(key, "decode") else key for key in keys).encode("utf-8")) - def _tokeys(self, key: Union[str, bytes, memoryview]): + def _tokeys(self, key: str | bytes | memoryview): """ Converts key bytes to keys tuple of strs by decoding and then splitting - at separator. + at separator .sep. Returns: - keys (iterable): of str + keys (tuple[str]): makes tuple by splitting key at sep Parameters: - key (Union[str, bytes]): str or bytes. + key (str | bytes | memoryview): db key. """ if isinstance(key, memoryview): # memoryview of bytes @@ -141,48 +148,62 @@ def _tokeys(self, key: Union[str, bytes, memoryview]): return tuple(key.split(self.sep)) - def _ser(self, val: Union[str, memoryview, bytes]): + def _ser(self, val: str | bytes | memoryview): """ Serialize value to bytes to store in db Parameters: - val (Union[str, memoryview, bytes]): encodable as bytes + val (str | bytes | memoryview): encodable as bytes """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # return bytes return (val.encode("utf-8") if hasattr(val, "encode") else val) - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: str | bytes | memoryview): """ Deserialize val to str Parameters: - val (Union[str, memoryview, bytes]): decodable as str + val (str | bytes | memoryview): decodable as str """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes return (val.decode("utf-8") if hasattr(val, "decode") else val) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + *, top=False): """ Returns: - items (Iterator): if (key, val) tuples over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + items (Iterator[tuple[key,val]]): (key, val) tuples of each item + over the all the items in subdb whose key startswith key made from + keys. Keys may be keyspace prefix to return branches of key space. + When keys is empty then returns all items in subdb Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. + keys (Iterator[str | bytes | memoryview]): of key parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): yield (self._tokeys(key), self._des(val)) - def trim(self, keys: Union[str, Iterable]=b""): + def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + *, top=False): """ Removes all entries whose keys startswith keys. Enables removal of whole branches of db key space. To ensure that proper separation of a branch @@ -190,17 +211,31 @@ def trim(self, keys: Union[str, Iterable]=b""): 'a.1'and 'a.2' but not 'ab' Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (Iterator[str | bytes | memoryview]): of key parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + Returns: - result (bool): True if key exists so delete successful. False otherwise + result (bool): True if val at key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys))) + return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys, top=top))) class Suber(SuberBase): """ - Sub DB of LMDBer. Subclass of SuberBase + Subclass of SuberBase with no LMDB duplicates (i.e. multiple values at same key). """ def __init__(self, db: dbing.LMDBer, *, @@ -212,7 +247,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key False (default) means do not enable duplicates at - each key + each key. Set to False sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' verify (bool): True means reverify when ._des from db when applicable @@ -294,6 +329,100 @@ def rem(self, keys: Union[str, Iterable]): + + +class OrdSuber(Suber): + """ + Subclass of Suber that adds methods for keys with ordinal numbered suffixes. + Each key consistes of pre joined with .sep to ordinal suffix + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(Suber, self).__init__(*pa, **kwa) + + + def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): + """ + Returns + cnt (int): count of of all ordinal suffix keyed vals with same pre + in key but different on in key in db starting at ordinal number + on of pre where key is formed with onKey(pre,on) + Does not count dups at same on for a given pre, only + unique on at a given pre. + + Parameters: + pre (str | bytes | memoryview): prefix to to be combined with on + to form key + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.cntAllOrdValsPre(db=self.sdb, pre=self._tokey(pre), on=on)) + + # appendOrdPre + + #def appendOrdValPre(self, db, pre, val): + #""" + #Appends val in order after last previous key with same pre in db. + #Returns ordinal number in, on, of appended entry. Appended on is 1 greater + #than previous latest on. + #Uses onKey(pre, on) for entries. + + #Append val to end of db entries with same pre but with on incremented by + #1 relative to last preexisting entry at pre. + + #Parameters: + #db is opened named sub db with dupsort=False + #pre is bytes identifier prefix for event + #val is event digest + #""" + + # getAllOrdItemPreIter + #def getAllOrdItemPreIter(self, db, pre, on=0): + #""" + #Returns iterator of duple item, (on, dig), at each key over all ordinal + #numbered keys with same prefix, pre, in db. Values are sorted by + #onKey(pre, on) where on is ordinal number int. + #Returned items are duples of (on, dig) where on is ordinal number int + #and dig is event digest for lookup in .evts sub db. + + #Raises StopIteration Error when empty. + + #Parameters: + #db is opened named sub db with dupsort=False + #pre is bytes of itdentifier prefix + #on is int ordinal number to resume replay + #""" + + + #def getAllOrdItemAllPreIter(self, db, key=b''): + #""" + #Returns iterator of triple item, (pre, on, dig), at each key over all + #ordinal numbered keys for all prefixes in db. Values are sorted by + #onKey(pre, on) where on is ordinal number int. + #Each returned item is triple (pre, on, dig) where pre is identifier prefix, + #on is ordinal number int and dig is event digest for lookup in .evts sub db. + + #Raises StopIteration Error when empty. + + #Parameters: + #db is opened named sub db with dupsort=False + #key is key location in db to resume replay, + #If empty then start at first key in database + #""" + + class CesrSuberBase(SuberBase): """ Sub class of SuberBase where data is CESR encode/decode ducktyped subclass @@ -524,7 +653,8 @@ def __init__(self, db: dbing.LMDBer, *, super(IoSetSuber, self).__init__(db=db, subkey=subkey, dupsort=False, **kwa) - def put(self, keys: Union[str, Iterable], vals: Iterable): + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Puts all vals at effective key made from keys and hidden ordinal suffix. that are not already in set of vals at key. Does not overwrite. @@ -537,18 +667,21 @@ def put(self, keys: Union[str, Iterable], vals: Iterable): result (bool): True If successful, False otherwise. """ - #if not nonStringIterable(vals): # not iterable - #vals = (vals, ) # make iterable + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putIoSetVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals], sep=self.sep)) - def add(self, keys: Union[str, Iterable], val: Union[bytes, str, memoryview]): + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview): """ - Add val to vals at effective key made from keys and hidden ordinal suffix. - that is not already in set of vals at key. Does not overwrite. + Add val idempotently to vals at effective key made from keys and hidden + ordinal suffix. Idempotent means that added value is not already in set + of vals at key. Does not overwrite or add same value at same key more + than once. Parameters: keys (Iterable): of key strs to be combined in order to form key @@ -564,8 +697,12 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str, memoryview]): val=self._ser(val), sep=self.sep)) + #def append(self, keys: str | bytes | memoryview | Iterable, + # val: str | bytes | memoryview): + - def pin(self, keys: Union[str, Iterable], vals: Iterable): + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Pins (sets) vals at effective key made from keys and hidden ordinal suffix. Overwrites. Removes all pre-existing vals that share same effective keys @@ -581,15 +718,15 @@ def pin(self, keys: Union[str, Iterable], vals: Iterable): """ key = self._tokey(keys) self.db.delIoSetVals(db=self.sdb, key=key) # delete all values - #if not nonStringIterable(vals): # not iterable - #vals = (vals, ) # make iterable + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.setIoSetVals(db=self.sdb, key=key, vals=[self._ser(val) for val in vals], sep=self.sep)) - def get(self, keys: Union[str, Iterable]): + def get(self, keys: str | bytes | memoryview | Iterable): """ Gets vals set list at key made from effective keys @@ -607,7 +744,7 @@ def get(self, keys: Union[str, Iterable]): sep=self.sep)]) - def getLast(self, keys: Union[str, Iterable]): + def getLast(self, keys: str | bytes | memoryview | Iterable): """ Gets last val inserted at effecive key made from keys and hidden ordinal suffix. @@ -624,7 +761,7 @@ def getLast(self, keys: Union[str, Iterable]): - def getIter(self, keys: Union[str, Iterable]): + def getIter(self, keys: str | bytes | memoryview | Iterable): """ Gets vals iterator at effecive key made from keys and hidden ordinal suffix. All vals in set of vals that share same effecive key are retrieved in @@ -644,7 +781,7 @@ def getIter(self, keys: Union[str, Iterable]): - def cnt(self, keys: Union[str, Iterable]): + def cnt(self, keys: str | bytes | memoryview | Iterable): """ Return count of values at effective key made from keys and hidden ordinal suffix. Zero otherwise @@ -657,7 +794,8 @@ def cnt(self, keys: Union[str, Iterable]): sep=self.sep)) - def rem(self, keys: Union[str, Iterable], val: Union[str, bytes, memoryview]=b''): + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): """ Removes entry at effective key made from keys and hidden ordinal suffix that matches val if any. Otherwise deletes all values at effective key. @@ -684,7 +822,8 @@ def rem(self, keys: Union[str, Iterable], val: Union[str, bytes, memoryview]=b'' sep=self.sep) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", + *, top=False): """ Return iterator over all the items in top branch defined by keys where keys may be truncation of full branch. @@ -698,18 +837,28 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): Parameters: keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from + a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. + all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for iokey, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): key, ion = dbing.unsuffix(iokey, sep=self.sep) yield (self._tokeys(key), self._des(val)) - def getIoSetItem(self, keys: Union[str, Iterable]): + def getIoSetItem(self, keys: str | bytes | memoryview | Iterable): """ Gets (iokeys, val) ioitems list at key made from keys where key is apparent effective key and ioitems all have same apparent effective key @@ -730,7 +879,7 @@ def getIoSetItem(self, keys: Union[str, Iterable]): sep=self.sep)]) - def getIoSetItemIter(self, keys: Union[str, Iterable]): + def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key @@ -752,7 +901,7 @@ def getIoSetItemIter(self, keys: Union[str, Iterable]): yield (self._tokeys(iokey), self._des(val)) - def getIoItemIter(self, keys: Union[str, Iterable]=b""): + def getIoItemIter(self, keys: str | bytes | memoryview | Iterable = b""): """ Returns: items (Iterator): tuple (key, val) over the all the items in @@ -773,13 +922,13 @@ def getIoItemIter(self, keys: Union[str, Iterable]=b""): yield (self._tokeys(iokey), self._des(val)) - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): + def remIokey(self, iokeys: str | bytes | memoryview | Iterable): """ - Removes entry at keys + Removes entries at iokeys Parameters: - iokeys (Iterable): of key str or tuple of key strs to be combined - in order to form key + iokeys (str | bytes | memoryview | Iterable): of key str or + tuple of key strs to be combined in order to form key Returns: result (bool): True if key exists so delete successful. False otherwise @@ -1339,7 +1488,8 @@ def __init__(self, db: Type[dbing.LMDBer], *, super(DupSuber, self).__init__(db=db, subkey=subkey, dupsort=True, **kwa) - def put(self, keys: Union[str, Iterable], vals: list): + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Puts all vals at key made from keys. Does not overwrite. Adds to existing dup values at key if any. Duplicate means another entry at the same key @@ -1348,8 +1498,10 @@ def put(self, keys: Union[str, Iterable], vals: list): unless it is a unique value for that key. Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): str or bytes of each value to be written at key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): str or bytes of each + value to be written at key Returns: result (bool): True If successful, False otherwise. @@ -1357,12 +1509,15 @@ def put(self, keys: Union[str, Iterable], vals: list): Apparently always returns True (how .put works with dupsort=True) """ + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals])) - def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview ): """ Add val to vals at key made from keys. Does not overwrite. Adds to existing dup values at key if any. Duplicate means another entry at the same key @@ -1371,8 +1526,8 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): unless it is a unique value for that key. Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Union[str, bytes]): value + keys (str | bytes | memoryview | Iterable): of key strs to be combined in order to form key + val (str | bytes | memoryview): value Returns: result (bool): True means unique value among duplications, @@ -1384,14 +1539,16 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): val=self._ser(val))) - def pin(self, keys: Union[str, Iterable], vals: list): + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Pins (sets) vals at key made from keys. Overwrites. Removes all pre-existing dup vals and replaces them with vals Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): str or bytes values + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): str or bytes values Returns: result (bool): True If successful, False otherwise. @@ -1399,18 +1556,21 @@ def pin(self, keys: Union[str, Iterable], vals: list): """ key = self._tokey(keys) self.db.delVals(db=self.sdb, key=key) # delete all values + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putVals(db=self.sdb, key=key, vals=[self._ser(val) for val in vals])) - def get(self, keys: Union[str, Iterable]): + def get(self, keys: str | bytes | memoryview | Iterable): """ Gets dup vals list at key made from keys Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key Returns: vals (list): each item in list is str @@ -1421,7 +1581,7 @@ def get(self, keys: Union[str, Iterable]): self.db.getValsIter(db=self.sdb, key=self._tokey(keys))] - def getLast(self, keys: Union[str, Iterable]): + def getLast(self, keys: str | bytes | memoryview | Iterable): """ Gets last dup val at key made from keys @@ -1436,14 +1596,15 @@ def getLast(self, keys: Union[str, Iterable]): return self._des(val) if val is not None else val - def getIter(self, keys: Union[str, Iterable]): + def getIter(self, keys: str | bytes | memoryview | Iterable): """ Gets dup vals iterator at key made from keys Duplicates are retrieved in lexocographic order not insertion order. Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key Returns: iterator: vals each of str. Raises StopIteration when done @@ -1453,17 +1614,19 @@ def getIter(self, keys: Union[str, Iterable]): yield self._des(val) - def cnt(self, keys: Union[str, Iterable]): + def cnt(self, keys: str | bytes | memoryview | Iterable): """ Return count of dup values at key made from keys, zero otherwise Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key """ return (self.db.cntVals(db=self.sdb, key=self._tokey(keys))) - def rem(self, keys: Union[str, Iterable], val=b''): + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): """ Removes entry at keys @@ -1476,181 +1639,33 @@ def rem(self, keys: Union[str, Iterable], val=b''): result (bool): True if key exists so delete successful. False otherwise """ - return (self.db.delVals(db=self.sdb, key=self._tokey(keys), val=self._ser(val))) + if val: + return (self.db.delVals(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val))) + else: + return (self.db.delVals(db=self.sdb, + key=self._tokey(keys))) + -class CesrDupSuber(DupSuber): +class CesrDupSuber(CesrSuberBase, DupSuber): """ - Sub class of DupSuber that supports multiple entries at each key (duplicates) - with dupsort==True, where data where data is Matter.qb64b property - which is a fully qualified serialization of matter subclass instance - Automatically serializes and deserializes from qb64b to/from Matter instances + Sub class of DupSuber whose values are CESR ducktypes of Matter subclasses. + serialized to and deserialied from val instance .qb64b property + which is a fully qualified serialization. + Automatically serializes and deserializes from qb64b to/from Matter ducktyped + instances + DupSuber supports multiple entries at each key (duplicates) with dupsort==True Do not use if serialized value is greater than 511 bytes. This is a limitation of dupsort==True sub dbs in LMDB """ - def __init__(self, *pa, klas: Type[coring.Matter] = coring.Matter, **kwa): - """ - Parameters: - db (dbing.LMDBer): base db - subkey (str): LMDB sub database key - klas (Type[coring.Matter]): Class reference to subclass of Matter - """ - super(CesrDupSuber, self).__init__(*pa, **kwa) - self.klas = klas - - - def put(self, keys: Union[str, Iterable], vals: list): - """ - Puts all vals at key made from keys. Does not overwrite. Adds to existing - dup values at key if any. Duplicate means another entry at the same key - but the entry is still a unique value. Duplicates are inserted in - lexocographic order not insertion order. Lmdb does not insert a duplicate - unless it is a unique value for that key. - - Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): instances of coring.Matter (subclass) - - Returns: - result (bool): True If successful, False otherwise. - - Apparently always returns True (how .put works with dupsort=True) - - """ - return (self.db.putVals(db=self.sdb, - key=self._tokey(keys), - vals=[val.qb64b for val in vals])) - - - def add(self, keys: Union[str, Iterable], val: coring.Matter): - """ - Add val to vals at key made from keys. Does not overwrite. Adds to existing - dup values at key if any. Duplicate means another entry at the same key - but the entry is still a unique value. Duplicates are inserted in - lexocographic order not insertion order. Lmdb does not insert a duplicate - unless it is a unique value for that key. - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (coring.Matter): instance (subclass) - - Returns: - result (bool): True means unique value among duplications, - False means duplicte of same value already exists. - - """ - return (self.db.addVal(db=self.sdb, - key=self._tokey(keys), - val=val.qb64b)) - - - def pin(self, keys: Union[str, Iterable], vals: list): - """ - Pins (sets) vals at key made from keys. Overwrites. Removes all - pre-existing dup vals and replaces them with vals - - Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): instances of coring.Matter (subclass) - - Returns: - result (bool): True If successful, False otherwise. - - """ - key = self._tokey(keys) - self.db.delVals(db=self.sdb, key=key) # delete all values - return (self.db.putVals(db=self.sdb, - key=key, - vals=[val.qb64b for val in vals])) - - - def get(self, keys: Union[str, Iterable]): - """ - Gets dup vals list at key made from keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - vals (list): each item in list is instance of self.klas - empty list if no entry at keys - - """ - return [self.klas(qb64b=bytes(val)) for val in - self.db.getValsIter(db=self.sdb, key=self._tokey(keys))] - - - def getLast(self, keys: Union[str, Iterable]): - """ - Gets last dup val at key made from keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - val (str): instance of self.klas else None if no value at key - - """ - val = self.db.getValLast(db=self.sdb, key=self._tokey(keys)) - if val is not None: - val = self.klas(qb64b=bytes(val)) - return val - - - - def getIter(self, keys: Union[str, Iterable]): - """ - Gets dup vals iterator at key made from keys - - Duplicates are retrieved in lexocographic order not insertion order. - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - iterator: vals each of self.klas. Raises StopIteration when done - - """ - for val in self.db.getValsIter(db=self.sdb, key=self._tokey(keys)): - yield self.klas(qb64b=bytes(val)) - - - def rem(self, keys: Union[str, Iterable], val=None): - """ - Removes entry at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (coring.Matter): instance of coring.Matter subclass dup val - at key to delete - if val is None then remove all values at key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - + def __init__(self, *pa, **kwa): """ - if val is not None: - val = val.qb64b - else: - val = b'' - return (self.db.delVals(db=self.sdb, key=self._tokey(keys), val=val)) - + Inherited Parameters: - def getItemIter(self, keys: Union[str, Iterable]=b""): """ - Returns: - iterator (Iteratore: tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + super(CesrDupSuber, self).__init__(*pa, **kwa) - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. - """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield (self._tokeys(key), self.klas(qb64b=bytes(val))) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index cf2ed805a..b8b0e8a74 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntValsAllPre(db=self.tels, pre=pre, on=fn) + return self.cntAllOrdValsPre(db=self.tels, pre=pre, on=fn) def getTibs(self, key): """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index c88cca4f8..f56cb1135 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -377,7 +377,7 @@ def test_lmdber(): assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntValsAllPre(db, preB) == 5 + assert dber.cntAllOrdValsPre(db, preB) == 5 # replay preB events in database items = [item for item in dber.getAllOrdItemPreIter(db, preB)] diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 86e95606b..8dd9e994b 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -146,6 +146,8 @@ def test_suber(): (('b', '2'), 'Green tree'), (('bc', '3'), 'Red apple')] + + # test with top keys for partial tree topkeys = ("b","") # last element empty to force trailing separator items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] assert items == [(('b', '1'), w), @@ -158,6 +160,20 @@ def test_suber(): (('a', '3'), y), (('a', '4'), z)] + # test with top parameter + keys = ("b", ) # last element empty to force trailing separator + items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + assert items == [(('b', '1'), w), + (('b', '2'), x)] + + keys = ("a", ) # last element empty to force trailing separator + items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + assert items == [(('a', '1'), w), + (('a', '2'), x), + (('a', '3'), y), + (('a', '4'), z)] + + # Test trim assert sdb.trim(keys=("b", "")) items = [(keys, val) for keys, val in sdb.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), @@ -171,6 +187,28 @@ def test_suber(): items = [(keys, val) for keys, val in sdb.getItemIter()] assert items == [(('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] + # Test trim with top parameters + sdb.put(keys=("a","1"), val=w) + sdb.put(keys=("a","2"), val=x) + sdb.put(keys=("a","3"), val=y) + sdb.put(keys=("a","4"), val=z) + sdb.put(keys=("b","1"), val=w) + sdb.put(keys=("b","2"), val=x) + + assert sdb.trim(keys=("b",), top=True) + items = [(keys, val) for keys, val in sdb.getItemIter()] + assert items == [(('a', '1'), 'Blue dog'), + (('a', '2'), 'Green tree'), + (('a', '3'), 'Red apple'), + (('a', '4'), 'White snow'), + (('ac', '4'), 'White snow'), + (('bc', '3'), 'Red apple')] + + assert sdb.trim(keys=("a",), top=True) + items = [(keys, val) for keys, val in sdb.getItemIter()] + assert items == [(('ac', '4'), 'White snow'), + (('bc', '3'), 'Red apple')] + assert sdb.trim() items = [(keys, val) for keys, val in sdb.getItemIter()] assert items == [] @@ -191,54 +229,54 @@ def test_dup_suber(): assert db.name == "test" assert db.opened - sdb = subing.DupSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.DupSuber) - assert sdb.sdb.flags()["dupsort"] + dupber = subing.DupSuber(db=db, subkey='bags.') + assert isinstance(dupber, subing.DupSuber) + assert dupber.sdb.flags()["dupsort"] sue = "Hello sailer!" sal = "Not my type." keys0 = ("test_key", "0001") keys1 = ("test_key", "0002") - sdb.put(keys=keys0, vals=[sue, sal]) - actual = sdb.get(keys=keys0) + dupber.put(keys=keys0, vals=[sue, sal]) + actual = dupber.get(keys=keys0) assert actual == [sue, sal] # lexicographic order - assert sdb.cnt(keys0) == 2 + assert dupber.cnt(keys0) == 2 - sdb.rem(keys0) - actual = sdb.get(keys=keys0) + dupber.rem(keys0) + actual = dupber.get(keys=keys0) assert not actual assert actual == [] - assert sdb.cnt(keys0) == 0 + assert dupber.cnt(keys0) == 0 - sdb.put(keys=keys0, vals=[sal, sue]) - actual = sdb.get(keys=keys0) + dupber.put(keys=keys0, vals=[sal, sue]) + actual = dupber.get(keys=keys0) assert actual == [sue, sal] # lexicographic order - actual = sdb.getLast(keys=keys0) + actual = dupber.getLast(keys=keys0) assert actual == sal sam = "A real charmer!" - result = sdb.add(keys=keys0, val=sam) + result = dupber.add(keys=keys0, val=sam) assert result - actual = sdb.get(keys=keys0) + actual = dupber.get(keys=keys0) assert actual == [sam, sue, sal] # lexicographic order zoe = "See ya later." zia = "Hey gorgeous!" - result = sdb.pin(keys=keys0, vals=[zoe, zia]) + result = dupber.pin(keys=keys0, vals=[zoe, zia]) assert result - actual = sdb.get(keys=keys0) + actual = dupber.get(keys=keys0) assert actual == [zia, zoe] # lexi order - sdb.put(keys=keys1, vals=[sal, sue, sam]) - actual = sdb.get(keys=keys1) + dupber.put(keys=keys1, vals=[sal, sue, sam]) + actual = dupber.get(keys=keys1) assert actual == [sam, sue, sal] # lexicographic order - for i, val in enumerate(sdb.getIter(keys=keys1)): + for i, val in enumerate(dupber.getIter(keys=keys1)): assert val == actual[i] - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in dupber.getItemIter()] assert items == [(('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0001'), 'See ya later.'), (('test_key', '0002'), 'A real charmer!'), @@ -246,9 +284,9 @@ def test_dup_suber(): (('test_key', '0002'), 'Not my type.')] - assert sdb.put(keys=("test", "blue"), vals=[sal, sue, sam]) + assert dupber.put(keys=("test", "blue"), vals=[sal, sue, sam]) topkeys = ("test", "") - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in dupber.getItemIter(keys=topkeys)] assert items == [(('test', 'blue'), 'A real charmer!'), (('test', 'blue'), 'Hello sailer!'), (('test', 'blue'), 'Not my type.')] @@ -256,28 +294,54 @@ def test_dup_suber(): # test with keys as string not tuple keys2 = "keystr" bob = "Shove off!" - sdb.put(keys=keys2, vals=[bob]) - actual = sdb.get(keys=keys2) + dupber.put(keys=keys2, vals=[bob]) + actual = dupber.get(keys=keys2) assert actual == [bob] - assert sdb.cnt(keys2) == 1 - sdb.rem(keys2) - actual = sdb.get(keys=keys2) + assert dupber.cnt(keys2) == 1 + dupber.rem(keys2) + actual = dupber.get(keys=keys2) assert actual == [] - assert sdb.cnt(keys2) == 0 + assert dupber.cnt(keys2) == 0 - sdb.put(keys=keys2, vals=[bob]) - actual = sdb.get(keys=keys2) + dupber.put(keys=keys2, vals=[bob]) + actual = dupber.get(keys=keys2) assert actual == [bob] bil = "Go away." - sdb.pin(keys=keys2, vals=[bil]) - actual = sdb.get(keys=keys2) + dupber.pin(keys=keys2, vals=[bil]) + actual = dupber.get(keys=keys2) assert actual == [bil] - sdb.add(keys=keys2, val=bob) - actual = sdb.get(keys=keys2) + dupber.add(keys=keys2, val=bob) + actual = dupber.get(keys=keys2) assert actual == [bil, bob] + + # test cntpre + vals = ["hi", "me", "my"] + for i, val in enumerate(vals): + assert dupber.put(keys=dbing.onKey("bob", i), vals=val) + + vals = ["bye", "guy", "gal"] + for i, val in enumerate(vals): + assert dupber.put(keys=dbing.onKey("bob", i), vals=val) + + items = [(keys, val) for keys, val in dupber.getItemIter(keys="bob")] + assert items == [ + (('bob', '00000000000000000000000000000000'), 'bye'), + (('bob', '00000000000000000000000000000000'), 'hi'), + (('bob', '00000000000000000000000000000001'), 'guy'), + (('bob', '00000000000000000000000000000001'), 'me'), + (('bob', '00000000000000000000000000000002'), 'gal'), + (('bob', '00000000000000000000000000000002'), 'my') + ] + + assert dupber.cnt(keys=dbing.onKey("bob", 1)) == 2 + + vals = [val for val in dupber.getIter(keys=dbing.onKey("bob", 2))] + assert vals == ["gal", "my"] + + assert not os.path.exists(db.path) assert not db.opened @@ -350,6 +414,8 @@ def test_ioset_suber(): (('test_key', '0002'), 'A real charmer!')] + + items = list(sdb.getIoItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), @@ -366,6 +432,9 @@ def test_ioset_suber(): assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + + + # Test with top keys assert sdb.put(keys=("test", "pop"), vals=[sal, sue, sam]) topkeys = ("test", "") items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] @@ -373,6 +442,14 @@ def test_ioset_suber(): (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] + # test with top parameter + keys = ("test", ) + items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # IoItems items = list(sdb.getIoItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), From 696ec3553518ccad6fbb7cb323303cc5b8bbcf33 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Aug 2024 15:36:08 -0600 Subject: [PATCH 278/418] refactor the IoDup methods in dbing to name them IoDup consistently --- src/keri/db/basing.py | 178 ++++++++++++++++++++-------------------- src/keri/db/dbing.py | 143 ++++++++++++++++++++++++++------ src/keri/db/subing.py | 60 ++++++++++++++ src/keri/vdr/viring.py | 14 ++-- tests/db/test_dbing.py | 144 ++++++++++++++++---------------- tests/db/test_subing.py | 163 ++++++++++++++++++++++++++++++++++++ 6 files changed, 507 insertions(+), 195 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 5a1937886..adda4da95 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2231,7 +2231,7 @@ def putUres(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ures, key, vals) + return self.putIoDupVals(self.ures, key, vals) def addUre(self, key, val): """ @@ -2242,7 +2242,7 @@ def addUre(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ures, key, val) + return self.addIoDupVal(self.ures, key, val) def getUres(self, key): """ @@ -2252,7 +2252,7 @@ def getUres(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ures, key) + return self.getIoDupVals(self.ures, key) def getUresIter(self, key): """ @@ -2262,7 +2262,7 @@ def getUresIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.ures, key) + return self.getIoDupValsIter(self.ures, key) def getUreLast(self, key): """ @@ -2272,7 +2272,7 @@ def getUreLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ures, key) + return self.getIoDupValLast(self.ures, key) def getUreItemsNext(self, key=b'', skip=True): """ @@ -2286,7 +2286,7 @@ def getUreItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.ures, key, skip) + return self.getIoDupItemsNext(self.ures, key, skip) def getUreItemsNextIter(self, key=b'', skip=True): """ @@ -2300,7 +2300,7 @@ def getUreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.ures, key, skip) + return self.getIoDupItemsNextIter(self.ures, key, skip) def cntUres(self, key): """ @@ -2308,7 +2308,7 @@ def cntUres(self, key): Return count of receipt triplets at key Returns zero if no entry at key """ - return self.cntIoVals(self.ures, key) + return self.cntIoDupVals(self.ures, key) def delUres(self, key): """ @@ -2316,7 +2316,7 @@ def delUres(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.ures, key) + return self.delIoDupVals(self.ures, key) def delUre(self, key, val): """ @@ -2328,7 +2328,7 @@ def delUre(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ures, key, val) + return self.delIoDupVal(self.ures, key, val) def putVrcs(self, key, vals): """ @@ -2398,7 +2398,7 @@ def putVres(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.vres, key, vals) + return self.putIoDupVals(self.vres, key, vals) def addVre(self, key, val): """ @@ -2409,7 +2409,7 @@ def addVre(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.vres, key, val) + return self.addIoDupVal(self.vres, key, val) def getVres(self, key): """ @@ -2419,7 +2419,7 @@ def getVres(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.vres, key) + return self.getIoDupVals(self.vres, key) def getVresIter(self, key): """ @@ -2429,7 +2429,7 @@ def getVresIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.vres, key) + return self.getIoDupValsIter(self.vres, key) def getVreLast(self, key): """ @@ -2439,7 +2439,7 @@ def getVreLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.vres, key) + return self.getIoDupValLast(self.vres, key) def getVreItemsNext(self, key=b'', skip=True): """ @@ -2453,7 +2453,7 @@ def getVreItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.vres, key, skip) + return self.getIoDupItemsNext(self.vres, key, skip) def getVreItemsNextIter(self, key=b'', skip=True): """ @@ -2467,7 +2467,7 @@ def getVreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.vres, key, skip) + return self.getIoDupItemsNextIter(self.vres, key, skip) def cntVres(self, key): """ @@ -2475,7 +2475,7 @@ def cntVres(self, key): Return count of receipt quinlets at key Returns zero if no entry at key """ - return self.cntIoVals(self.vres, key) + return self.cntIoDupVals(self.vres, key) def delVres(self, key): """ @@ -2483,7 +2483,7 @@ def delVres(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.vres, key) + return self.delIoDupVals(self.vres, key) def delVre(self, key, val): """ @@ -2495,7 +2495,7 @@ def delVre(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.vres, key, val) + return self.delIoDupVal(self.vres, key, val) def putKes(self, key, vals): """ @@ -2505,7 +2505,7 @@ def putKes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.kels, key, vals) + return self.putIoDupVals(self.kels, key, vals) def addKe(self, key, val): """ @@ -2515,7 +2515,7 @@ def addKe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.kels, key, val) + return self.addIoDupVal(self.kels, key, val) def getKes(self, key): """ @@ -2524,7 +2524,7 @@ def getKes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.kels, key) + return self.getIoDupVals(self.kels, key) def getKeLast(self, key): """ @@ -2533,7 +2533,7 @@ def getKeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.kels, key) + return self.getIoDupValLast(self.kels, key) def cntKes(self, key): """ @@ -2541,7 +2541,7 @@ def cntKes(self, key): Return count of dup key event dig val at key Returns zero if no entry at key """ - return self.cntIoVals(self.kels, key) + return self.cntIoDupVals(self.kels, key) def delKes(self, key): """ @@ -2549,7 +2549,7 @@ def delKes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.kels, key) + return self.delIoDupVals(self.kels, key) def getKelIter(self, pre, sn=0): @@ -2632,7 +2632,7 @@ def putPses(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.pses, key, vals) + return self.putIoDupVals(self.pses, key, vals) def addPse(self, key, val): """ @@ -2642,7 +2642,7 @@ def addPse(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.pses, key, val) + return self.addIoDupVal(self.pses, key, val) def getPses(self, key): """ @@ -2651,7 +2651,7 @@ def getPses(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.pses, key) + return self.getIoDupVals(self.pses, key) def getPsesIter(self, key): """ @@ -2660,7 +2660,7 @@ def getPsesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.pses, key) + return self.getIoDupValsIter(self.pses, key) def getPseLast(self, key): """ @@ -2669,7 +2669,7 @@ def getPseLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.pses, key) + return self.getIoDupValLast(self.pses, key) def getPseItemsNext(self, key=b'', skip=True): """ @@ -2681,7 +2681,7 @@ def getPseItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.pses, key, skip) + return self.getIoDupItemsNext(self.pses, key, skip) def getPseItemsNextIter(self, key=b'', skip=True): """ @@ -2693,7 +2693,7 @@ def getPseItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.pses, key, skip) + return self.getIoDupItemsNextIter(self.pses, key, skip) def cntPses(self, key): """ @@ -2701,7 +2701,7 @@ def cntPses(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.pses, key) + return self.cntIoDupVals(self.pses, key) def delPses(self, key): """ @@ -2709,7 +2709,7 @@ def delPses(self, key): Deletes all values at key in db. Returns True If key exists in db Else False """ - return self.delIoVals(self.pses, key) + return self.delIoDupVals(self.pses, key) def delPse(self, key, val): """ @@ -2721,7 +2721,7 @@ def delPse(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.pses, key, val) + return self.delIoDupVal(self.pses, key, val) def putPwes(self, key, vals): @@ -2732,7 +2732,7 @@ def putPwes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.pwes, key, vals) + return self.putIoDupVals(self.pwes, key, vals) def addPwe(self, key, val): """ @@ -2742,7 +2742,7 @@ def addPwe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.pwes, key, val) + return self.addIoDupVal(self.pwes, key, val) def getPwes(self, key): """ @@ -2751,7 +2751,7 @@ def getPwes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.pwes, key) + return self.getIoDupVals(self.pwes, key) def getPwesIter(self, key): """ @@ -2760,7 +2760,7 @@ def getPwesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.pwes, key) + return self.getIoDupValsIter(self.pwes, key) def getPweLast(self, key): """ @@ -2769,7 +2769,7 @@ def getPweLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.pwes, key) + return self.getIoDupValLast(self.pwes, key) def getPweItemsNext(self, key=b'', skip=True): """ @@ -2781,7 +2781,7 @@ def getPweItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.pwes, key, skip) + return self.getIoDupItemsNext(self.pwes, key, skip) def getPweItemsNextIter(self, key=b'', skip=True): """ @@ -2793,7 +2793,7 @@ def getPweItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.pwes, key, skip) + return self.getIoDupItemsNextIter(self.pwes, key, skip) def cntPwes(self, key): """ @@ -2801,7 +2801,7 @@ def cntPwes(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.pwes, key) + return self.cntIoDupVals(self.pwes, key) def delPwes(self, key): """ @@ -2809,7 +2809,7 @@ def delPwes(self, key): Deletes all values at key in db. Returns True If key exists in db Else False """ - return self.delIoVals(self.pwes, key) + return self.delIoDupVals(self.pwes, key) def delPwe(self, key, val): """ @@ -2821,7 +2821,7 @@ def delPwe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.pwes, key, val) + return self.delIoDupVal(self.pwes, key, val) def putUwes(self, key, vals): """ @@ -2832,7 +2832,7 @@ def putUwes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.uwes, key, vals) + return self.putIoDupVals(self.uwes, key, vals) def addUwe(self, key, val): """ @@ -2843,7 +2843,7 @@ def addUwe(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.uwes, key, val) + return self.addIoDupVal(self.uwes, key, val) def getUwes(self, key): """ @@ -2853,7 +2853,7 @@ def getUwes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.uwes, key) + return self.getIoDupVals(self.uwes, key) def getUwesIter(self, key): """ @@ -2863,7 +2863,7 @@ def getUwesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.uwes, key) + return self.getIoDupValsIter(self.uwes, key) def getUweLast(self, key): """ @@ -2873,7 +2873,7 @@ def getUweLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.uwes, key) + return self.getIoDupValLast(self.uwes, key) def getUweItemsNext(self, key=b'', skip=True): """ @@ -2887,7 +2887,7 @@ def getUweItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.uwes, key, skip) + return self.getIoDupItemsNext(self.uwes, key, skip) def getUweItemsNextIter(self, key=b'', skip=True): """ @@ -2901,7 +2901,7 @@ def getUweItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.uwes, key, skip) + return self.getIoDupItemsNextIter(self.uwes, key, skip) def cntUwes(self, key): """ @@ -2909,7 +2909,7 @@ def cntUwes(self, key): Return count of receipt couples at key Returns zero if no entry at key """ - return self.cntIoVals(self.uwes, key) + return self.cntIoDupVals(self.uwes, key) def delUwes(self, key): """ @@ -2917,7 +2917,7 @@ def delUwes(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.uwes, key) + return self.delIoDupVals(self.uwes, key) def delUwe(self, key, val): """ @@ -2929,7 +2929,7 @@ def delUwe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.uwes, key, val) + return self.delIoDupVal(self.uwes, key, val) def putOoes(self, key, vals): """ @@ -2939,7 +2939,7 @@ def putOoes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ooes, key, vals) + return self.putIoDupVals(self.ooes, key, vals) def addOoe(self, key, val): """ @@ -2949,7 +2949,7 @@ def addOoe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ooes, key, val) + return self.addIoDupVal(self.ooes, key, val) def getOoes(self, key): """ @@ -2958,7 +2958,7 @@ def getOoes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ooes, key) + return self.getIoDupVals(self.ooes, key) def getOoeLast(self, key): """ @@ -2967,7 +2967,7 @@ def getOoeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ooes, key) + return self.getIoDupValLast(self.ooes, key) def getOoeItemsNext(self, key=b'', skip=True): """ @@ -2979,7 +2979,7 @@ def getOoeItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.ooes, key, skip) + return self.getIoDupItemsNext(self.ooes, key, skip) def getOoeItemsNextIter(self, key=b'', skip=True): """ @@ -2991,7 +2991,7 @@ def getOoeItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.ooes, key, skip) + return self.getIoDupItemsNextIter(self.ooes, key, skip) def cntOoes(self, key): """ @@ -2999,7 +2999,7 @@ def cntOoes(self, key): Return count of dup event dig at key Returns zero if no entry at key """ - return self.cntIoVals(self.ooes, key) + return self.cntIoDupVals(self.ooes, key) def delOoes(self, key): """ @@ -3007,7 +3007,7 @@ def delOoes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.ooes, key) + return self.delIoDupVals(self.ooes, key) def delOoe(self, key, val): """ @@ -3020,7 +3020,7 @@ def delOoe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ooes, key, val) + return self.delIoDupVal(self.ooes, key, val) def putQnfs(self, key, vals): """ @@ -3030,7 +3030,7 @@ def putQnfs(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.qnfs, key, vals) + return self.putIoDupVals(self.qnfs, key, vals) def addQnf(self, key, val): """ @@ -3040,7 +3040,7 @@ def addQnf(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.qnfs, key, val) + return self.addIoDupVal(self.qnfs, key, val) def getQnfs(self, key): """ @@ -3049,7 +3049,7 @@ def getQnfs(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.qnfs, key) + return self.getIoDupVals(self.qnfs, key) def getQnfLast(self, key): """ @@ -3058,7 +3058,7 @@ def getQnfLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.qnfs, key) + return self.getIoDupValLast(self.qnfs, key) def getQnfItemsNext(self, key=b'', skip=True): """ @@ -3070,7 +3070,7 @@ def getQnfItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.qnfs, key, skip) + return self.getIoDupItemsNext(self.qnfs, key, skip) def getQnfItemsNextIter(self, key=b'', skip=True): """ @@ -3082,7 +3082,7 @@ def getQnfItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.qnfs, key, skip) + return self.getIoDupItemsNextIter(self.qnfs, key, skip) def cntQnfs(self, key): """ @@ -3090,7 +3090,7 @@ def cntQnfs(self, key): Return count of dup event dig at key Returns zero if no entry at key """ - return self.cntIoVals(self.qnfs, key) + return self.cntIoDupVals(self.qnfs, key) def delQnfs(self, key): """ @@ -3098,7 +3098,7 @@ def delQnfs(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.qnfs, key) + return self.delIoDupVals(self.qnfs, key) def delQnf(self, key, val): """ @@ -3111,7 +3111,7 @@ def delQnf(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.qnfs, key, val) + return self.delIoDupVal(self.qnfs, key, val) def putDes(self, key, vals): """ @@ -3121,7 +3121,7 @@ def putDes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.dels, key, vals) + return self.putIoDupVals(self.dels, key, vals) def addDe(self, key, val): """ @@ -3131,7 +3131,7 @@ def addDe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.dels, key, val) + return self.addIoDupVal(self.dels, key, val) def getDes(self, key): """ @@ -3140,7 +3140,7 @@ def getDes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.dels, key) + return self.getIoDupVals(self.dels, key) def getDeLast(self, key): """ @@ -3150,7 +3150,7 @@ def getDeLast(self, key): Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.dels, key) + return self.getIoDupValLast(self.dels, key) def cntDes(self, key): """ @@ -3158,7 +3158,7 @@ def cntDes(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.dels, key) + return self.cntIoDupVals(self.dels, key) def delDes(self, key): """ @@ -3166,7 +3166,7 @@ def delDes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.dels, key) + return self.delIoDupVals(self.dels, key) def getDelIter(self, pre): """ @@ -3195,7 +3195,7 @@ def putLdes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ldes, key, vals) + return self.putIoDupVals(self.ldes, key, vals) def addLde(self, key, val): """ @@ -3205,7 +3205,7 @@ def addLde(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ldes, key, val) + return self.addIoDupVal(self.ldes, key, val) def getLdes(self, key): """ @@ -3214,7 +3214,7 @@ def getLdes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ldes, key) + return self.getIoDupVals(self.ldes, key) def getLdeLast(self, key): """ @@ -3223,7 +3223,7 @@ def getLdeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ldes, key) + return self.getIoDupValLast(self.ldes, key) def getLdeItemsNext(self, key=b'', skip=True): """ @@ -3235,7 +3235,7 @@ def getLdeItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.ldes, key, skip) + return self.getIoDupItemsNext(self.ldes, key, skip) def getLdeItemsNextIter(self, key=b'', skip=True): """ @@ -3247,7 +3247,7 @@ def getLdeItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.ldes, key, skip) + return self.getIoDupItemsNextIter(self.ldes, key, skip) def cntLdes(self, key): """ @@ -3255,7 +3255,7 @@ def cntLdes(self, key): Return count of dup event dig at key Returns zero if no entry at key """ - return self.cntIoVals(self.ldes, key) + return self.cntIoDupVals(self.ldes, key) def delLdes(self, key): """ @@ -3263,7 +3263,7 @@ def delLdes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.ldes, key) + return self.delIoDupVals(self.ldes, key) def delLde(self, key, val): """ @@ -3276,7 +3276,7 @@ def delLde(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ldes, key, val) + return self.delIoDupVal(self.ldes, key, val) class BaserDoer(doing.Doer): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 5d29d242b..f5fbed83b 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1404,7 +1404,7 @@ def delVals(self, db, key, val=b''): # IoDup class IoVals IoItems # dupsort==True and prepends and strips io val proem to each value. # because dupsort==True values are limited to 511 bytes including proem - def putIoVals(self, db, key, vals): + def putIoDupVals(self, db, key, vals): """ Write each entry from list of bytes vals to key in db in insertion order Adds to existing values at key if any @@ -1413,11 +1413,13 @@ def putIoVals(self, db, key, vals): Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order + all values that makes lexocographic order that same as insertion order. + Duplicates are ordered as a pair of key plus value so prepending proem to each value changes duplicate ordering. Proem is 33 characters long. With 32 character hex string followed by '.' for essentiall unlimited number of values which will be limited by memory. + With prepended proem ordinal must explicity check for duplicate values before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). @@ -1429,7 +1431,7 @@ def putIoVals(self, db, key, vals): """ result = False - dups = set(self.getIoVals(db, key)) #get preexisting dups if any + dups = set(self.getIoDupVals(db, key)) #get preexisting dups if any with self.env.begin(db=db, write=True, buffers=True) as txn: idx = 0 cursor = txn.cursor() @@ -1450,7 +1452,7 @@ def putIoVals(self, db, key, vals): return result - def addIoVal(self, db, key, val): + def addIoDupVal(self, db, key, val): """ Add val bytes as dup in insertion order to key in db Adds to existing values at key if any @@ -1458,32 +1460,47 @@ def addIoVal(self, db, key, val): Actual value written include prepended proem ordinal Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). Parameters: db is opened named sub db with dupsort=False key is bytes of key within sub db's keyspace val is bytes of value to be written """ - return self.putIoVals(db, key, [val]) + return self.putIoDupVals(db, key, [val]) - def getIoVals(self, db, key): + def getIoDupVals(self, db, key): """ Return list of duplicate values at key in db in insertion order Returns empty list if no entry at key Removes prepended proem ordinal from each val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True @@ -1503,18 +1520,26 @@ def getIoVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoValsIter(self, db, key): + def getIoDupValsIter(self, db, key): """ Return iterator of all duplicate values at key in db in insertion order Raises StopIteration Error when no remaining dup items = empty. Removes prepended proem ordinal from each val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True @@ -1533,13 +1558,26 @@ def getIoValsIter(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoValLast(self, db, key): + def getIoDupValLast(self, db, key): """ Return last added dup value at key in db in insertion order Returns None no entry at key Removes prepended proem ordinal from val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1558,7 +1596,7 @@ def getIoValLast(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoItemsNext(self, db, key=b"", skip=True): + def getIoDupItemsNext(self, db, key=b"", skip=True): """ Return list of all dup items at next key after key in db in insertion order. Item is (key, val) with proem stripped from val stored in db. @@ -1572,6 +1610,19 @@ def getIoItemsNext(self, db, key=b"", skip=True): Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace or empty string @@ -1592,7 +1643,7 @@ def getIoItemsNext(self, db, key=b"", skip=True): return items - def getIoItemsNextIter(self, db, key=b"", skip=True): + def getIoDupItemsNextIter(self, db, key=b"", skip=True): """ Return iterator of all dup items at next key after key in db in insertion order. Item is (key, val) with proem stripped from val stored in db. @@ -1606,6 +1657,19 @@ def getIoItemsNextIter(self, db, key=b"", skip=True): Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace or empty @@ -1624,11 +1688,24 @@ def getIoItemsNextIter(self, db, key=b"", skip=True): yield (key, val[33:]) # slice off prepended ordering prefix - def cntIoVals(self, db, key): + def cntIoDupVals(self, db, key): """ Return count of dup values at key in db, or zero otherwise Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1646,11 +1723,24 @@ def cntIoVals(self, db, key): return count - def delIoVals(self, db, key): + def delIoDupVals(self, db, key): """ Deletes all values at key in db if key present. Returns True If key exists + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1664,7 +1754,7 @@ def delIoVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def delIoVal(self, db, key, val): + def delIoDupVal(self, db, key, val): """ Deletes dup io val at key in db. Performs strip search to find match. Strips proems and then searches. @@ -1684,8 +1774,7 @@ def delIoVal(self, db, key, val): The problem is that escrow is not fixed buts stuffs gets added and deleted which just adds to the value of the proem. 2**16 is an impossibly large number so the proem will not max out practically. But its not - and elegant solution. So maybe escrows need to use a different approach. - But really didn't want to add another database just for escrows. + an elegant solution. Parameters: db is opened named sub db with dupsort=False diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 54a428f96..bf8755e58 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1374,6 +1374,10 @@ class SchemerSuber(Suber): Sub class of Suber where data is serialized Schemer instance Automatically serializes and deserializes using Schemer methods + ToDo XXXX make this a subclass of SerderSuber since from a ser des interface + Schemer is duck type of Serder, Then can get rid of the redundant put, add, + pin, get etc definitions + """ def __init__(self, *pa, **kwa): @@ -1669,3 +1673,59 @@ def __init__(self, *pa, **kwa): super(CesrDupSuber, self).__init__(*pa, **kwa) +class IoDupSuber(DupSuber): + """ + Sub class of DupSuber that supports Insertion Ordering (IoDup) of duplicates + By automagically prepending and stripping ordinal proem to/from each + duplicate value at a given key. + + IoDupSuber supports insertion ordered multiple entries at each key + (duplicates) with dupsort==True + + Do not use if serialized length key + proem + value, is greater than 511 bytes. + This is a limitation of dupsort==True sub dbs in LMDB + + IoDupSuber may be more performant then IoSetSuber for values that are indices + to other sub dbs that fit the size constraint because LMDB support for + duplicates is more space efficient and code performant. + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + + """ + super(IoDupSuber, self).__init__(*pa, **kwa) + + + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): + """ + Puts all vals at effective key made from keys and hidden ordinal suffix. + that are not already in set of vals at key. Does not overwrite. + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + vals (Iterable): of str serializations + + Returns: + result (bool): True If successful, False otherwise. + + """ + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable + return (self.db.putIoDupVals(db=self.sdb, + key=self._tokey(keys), + vals=[self._ser(val) for val in vals])) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index b8b0e8a74..5a5076e98 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -907,7 +907,7 @@ def putBaks(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.baks, key, vals) + return self.putIoDupVals(self.baks, key, vals) def addBak(self, key, val): @@ -918,7 +918,7 @@ def addBak(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.baks, key, val) + return self.addIoDupVal(self.baks, key, val) def getBaks(self, key): @@ -928,7 +928,7 @@ def getBaks(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.baks, key) + return self.getIoDupVals(self.baks, key) def getBaksIter(self, key): @@ -938,7 +938,7 @@ def getBaksIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.baks, key) + return self.getIoDupValsIter(self.baks, key) def cntBaks(self, key): """ @@ -946,7 +946,7 @@ def cntBaks(self, key): Return count of backer prefixes at key Returns zero if no entry at key """ - return self.cntIoVals(self.baks, key) + return self.cntIoDupVals(self.baks, key) def delBaks(self, key): @@ -955,7 +955,7 @@ def delBaks(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.baks, key) + return self.delIoDupVals(self.baks, key) def delBak(self, key, val): @@ -968,7 +968,7 @@ def delBak(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.baks, key, val) + return self.delIoDupVal(self.baks, key, val) def buildProof(prefixer, seqner, diger, sigers): diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index f56cb1135..fb7357a91 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -445,54 +445,54 @@ def test_lmdber(): vals = [b"z", b"m", b"x", b"a"] db = dber.env.open_db(key=b'peep.', dupsort=True) - assert dber.getIoVals(db, key) == [] - assert dber.getIoValLast(db, key) == None - assert dber.cntIoVals(db, key) == 0 - assert dber.delIoVals(db, key) == False - assert dber.putIoVals(db, key, vals) == True - assert dber.getIoVals(db, key) == vals # preserved insertion order - assert dber.cntIoVals(db, key) == len(vals) == 4 - assert dber.getIoValLast(db, key) == vals[-1] - assert dber.putIoVals(db, key, vals=[b'a']) == False # duplicate - assert dber.getIoVals(db, key) == vals # no change - assert dber.addIoVal(db, key, val=b'b') == True - assert dber.addIoVal(db, key, val=b'a') == False - assert dber.getIoVals(db, key) == [b"z", b"m", b"x", b"a", b"b"] - assert [val for val in dber.getIoValsIter(db, key)] == [b"z", b"m", b"x", b"a", b'b'] - assert dber.delIoVals(db, key) == True - assert dber.getIoVals(db, key) == [] - assert dber.putIoVals(db, key, vals) == True + assert dber.getIoDupVals(db, key) == [] + assert dber.getIoDupValLast(db, key) == None + assert dber.cntIoDupVals(db, key) == 0 + assert dber.delIoDupVals(db, key) == False + assert dber.putIoDupVals(db, key, vals) == True + assert dber.getIoDupVals(db, key) == vals # preserved insertion order + assert dber.cntIoDupVals(db, key) == len(vals) == 4 + assert dber.getIoDupValLast(db, key) == vals[-1] + assert dber.putIoDupVals(db, key, vals=[b'a']) == False # duplicate + assert dber.getIoDupVals(db, key) == vals # no change + assert dber.addIoDupVal(db, key, val=b'b') == True + assert dber.addIoDupVal(db, key, val=b'a') == False + assert dber.getIoDupVals(db, key) == [b"z", b"m", b"x", b"a", b"b"] + assert [val for val in dber.getIoDupValsIter(db, key)] == [b"z", b"m", b"x", b"a", b'b'] + assert dber.delIoDupVals(db, key) == True + assert dber.getIoDupVals(db, key) == [] + assert dber.putIoDupVals(db, key, vals) == True for val in vals: - assert dber.delIoVal(db, key, val) - assert dber.getIoVals(db, key) == [] - assert dber.putIoVals(db, key, vals) == True + assert dber.delIoDupVal(db, key, val) + assert dber.getIoDupVals(db, key) == [] + assert dber.putIoDupVals(db, key, vals) == True for val in sorted(vals): - assert dber.delIoVal(db, key, val) - assert dber.getIoVals(db, key) == [] + assert dber.delIoDupVal(db, key, val) + assert dber.getIoDupVals(db, key) == [] #delete and add in odd order - assert dber.putIoVals(db, key, vals) == True - assert dber.delIoVal(db, key, vals[2]) - assert dber.addIoVal(db, key, b'w') - assert dber.delIoVal(db, key, vals[0]) - assert dber.addIoVal(db, key, b'e') - assert dber.getIoVals(db, key) == [b'm', b'a', b'w', b'e'] + assert dber.putIoDupVals(db, key, vals) == True + assert dber.delIoDupVal(db, key, vals[2]) + assert dber.addIoDupVal(db, key, b'w') + assert dber.delIoDupVal(db, key, vals[0]) + assert dber.addIoDupVal(db, key, b'e') + assert dber.getIoDupVals(db, key) == [b'm', b'a', b'w', b'e'] # Test getIoValsAllPreIter(self, db, pre) vals0 = [b"gamma", b"beta"] sn = 0 key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True + assert dber.addIoDupVal(db, key, vals0[0]) == True + assert dber.addIoDupVal(db, key, vals0[1]) == True vals1 = [b"mary", b"peter", b"john", b"paul"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True + assert dber.putIoDupVals(db, key, vals1) == True vals2 = [b"dog", b"cat", b"bird"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True + assert dber.putIoDupVals(db, key, vals2) == True vals = [bytes(val) for val in dber.getIoValsAllPreIter(db, pre)] allvals = vals0 + vals1 + vals2 @@ -503,18 +503,18 @@ def test_lmdber(): vals0 = [b"gamma", b"beta"] sn = 0 key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True + assert dber.addIoDupVal(db, key, vals0[0]) == True + assert dber.addIoDupVal(db, key, vals0[1]) == True vals1 = [b"mary", b"peter", b"john", b"paul"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True + assert dber.putIoDupVals(db, key, vals1) == True vals2 = [b"dog", b"cat", b"bird"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True + assert dber.putIoDupVals(db, key, vals2) == True vals = [bytes(val) for val in dber.getIoValLastAllPreIter(db, pre)] lastvals = [vals0[-1], vals1[-1], vals2[-1]] @@ -525,18 +525,18 @@ def test_lmdber(): vals0 = [b"gamma", b"beta"] sn = 1 # not start at zero key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True + assert dber.addIoDupVal(db, key, vals0[0]) == True + assert dber.addIoDupVal(db, key, vals0[1]) == True vals1 = [b"mary", b"peter", b"john", b"paul"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True + assert dber.putIoDupVals(db, key, vals1) == True vals2 = [b"dog", b"cat", b"bird"] sn += 2 # gap key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True + assert dber.putIoDupVals(db, key, vals2) == True vals = [bytes(val) for val in dber.getIoValsAnyPreIter(db, pre)] allvals = vals0 + vals1 + vals2 @@ -553,35 +553,35 @@ def test_lmdber(): dKey = snKey(pre=b'A', sn=7) dVals = [b"k", b"b"] - assert dber.putIoVals(edb, key=aKey, vals=aVals) - assert dber.putIoVals(edb, key=bKey, vals=bVals) - assert dber.putIoVals(edb, key=cKey, vals=cVals) - assert dber.putIoVals(edb, key=dKey, vals=dVals) + assert dber.putIoDupVals(edb, key=aKey, vals=aVals) + assert dber.putIoDupVals(edb, key=bKey, vals=bVals) + assert dber.putIoDupVals(edb, key=cKey, vals=cVals) + assert dber.putIoDupVals(edb, key=dKey, vals=dVals) # Test getIoItemsNext(self, db, key=b"") # aVals - items = dber.getIoItemsNext(edb) # get first key in database + items = dber.getIoDupItemsNext(edb) # get first key in database assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = dber.getIoItemsNext(edb, key=aKey, skip=False) # get aKey in database + items = dber.getIoDupItemsNext(edb, key=aKey, skip=False) # get aKey in database assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = dber.getIoItemsNext(edb, key=aKey) # get bKey in database + items = dber.getIoDupItemsNext(edb, key=aKey) # get bKey in database assert items # not empty ikey = items[0][0] assert ikey == bKey vals = [val for key, val in items] assert vals == bVals - items = dber.getIoItemsNext(edb, key=b'', skip=False) # get first key in database + items = dber.getIoDupItemsNext(edb, key=b'', skip=False) # get first key in database assert items # not empty ikey = items[0][0] assert ikey == aKey @@ -589,7 +589,7 @@ def test_lmdber(): assert vals == aVals # bVals - items = dber.getIoItemsNext(edb, key=ikey) + items = dber.getIoDupItemsNext(edb, key=ikey) assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -597,7 +597,7 @@ def test_lmdber(): assert vals == bVals # cVals - items = dber.getIoItemsNext(edb, key=ikey) + items = dber.getIoDupItemsNext(edb, key=ikey) assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -605,7 +605,7 @@ def test_lmdber(): assert vals == cVals # dVals - items = dber.getIoItemsNext(edb, key=ikey) + items = dber.getIoDupItemsNext(edb, key=ikey) assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -613,75 +613,75 @@ def test_lmdber(): assert vals == dVals # none - items = dber.getIoItemsNext(edb, key=ikey) + items = dber.getIoDupItemsNext(edb, key=ikey) assert items == [] # empty assert not items # Test getIoItemsNextIter(self, db, key=b"") # get dups at first key in database # aVals - items = [item for item in dber.getIoItemsNextIter(edb)] + items = [item for item in dber.getIoDupItemsNextIter(edb)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in dber.getIoItemsNextIter(edb, key=aKey, skip=False)] + items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey, skip=False)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in dber.getIoItemsNextIter(edb, key=aKey)] + items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey vals = [val for key, val in items] assert vals == bVals - items = [item for item in dber.getIoItemsNextIter(edb, key=b'', skip=False)] + items = [item for item in dber.getIoDupItemsNextIter(edb, key=b'', skip=False)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True + assert dber.delIoDupVal(edb, ikey, val) == True # bVals - items = [item for item in dber.getIoItemsNextIter(edb, key=ikey)] + items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] assert items # not empty ikey = items[0][0] assert ikey == bKey vals = [val for key, val in items] assert vals == bVals for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True + assert dber.delIoDupVal(edb, ikey, val) == True # cVals - items = [item for item in dber.getIoItemsNextIter(edb, key=ikey)] + items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] assert items # not empty ikey = items[0][0] assert ikey == cKey vals = [val for key, val in items] assert vals == cVals for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True + assert dber.delIoDupVal(edb, ikey, val) == True # dVals - items = [item for item in dber.getIoItemsNext(edb, key=ikey)] + items = [item for item in dber.getIoDupItemsNext(edb, key=ikey)] assert items # not empty ikey = items[0][0] assert ikey == dKey vals = [val for key, val in items] assert vals == dVals for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True + assert dber.delIoDupVal(edb, ikey, val) == True # none - items = [item for item in dber.getIoItemsNext(edb, key=ikey)] + items = [item for item in dber.getIoDupItemsNext(edb, key=ikey)] assert items == [] # empty assert not items @@ -830,21 +830,21 @@ def test_lmdber(): with pytest.raises(KeyError): dber.delVals(db, empty_key) with pytest.raises(KeyError): - dber.putIoVals(db, empty_key, [some_value]) + dber.putIoDupVals(db, empty_key, [some_value]) with pytest.raises(KeyError): - dber.addIoVal(db, empty_key, some_value) + dber.addIoDupVal(db, empty_key, some_value) with pytest.raises(KeyError): - dber.getIoVals(db, empty_key) + dber.getIoDupVals(db, empty_key) with pytest.raises(KeyError): - [_ for _ in dber.getIoValsIter(db, empty_key)] + [_ for _ in dber.getIoDupValsIter(db, empty_key)] with pytest.raises(KeyError): - dber.getIoValLast(db, empty_key) + dber.getIoDupValLast(db, empty_key) with pytest.raises(KeyError): - dber.cntIoVals(db, empty_key) + dber.cntIoDupVals(db, empty_key) with pytest.raises(KeyError): - dber.delIoVals(db, empty_key) + dber.delIoDupVals(db, empty_key) with pytest.raises(KeyError): - dber.delIoVal(db, empty_key, some_value) + dber.delIoDupVal(db, empty_key, some_value) assert not os.path.exists(dber.path) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 8dd9e994b..8cd2a312b 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -346,6 +346,169 @@ def test_dup_suber(): assert not db.opened +def test_iodup_suber(): + """ + Test IoDupSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + sdb = subing.IoSetSuber(db=db, subkey='bags.') + assert isinstance(sdb, subing.IoSetSuber) + assert not sdb.sdb.flags()["dupsort"] + + sue = "Hello sailer!" + sal = "Not my type." + + keys0 = ("test_key", "0001") + keys1 = ("test_key", "0002") + + assert sdb.put(keys=keys0, vals=[sal, sue]) + actuals = sdb.get(keys=keys0) + assert actuals == [sal, sue] # insertion order not lexicographic + assert sdb.cnt(keys0) == 2 + actual = sdb.getLast(keys=keys0) + assert actual == sue + + assert sdb.rem(keys0) + actuals = sdb.get(keys=keys0) + assert not actuals + assert actuals == [] + assert sdb.cnt(keys0) == 0 + + assert sdb.put(keys=keys0, vals=[sue, sal]) + actuals = sdb.get(keys=keys0) + assert actuals == [sue, sal] # insertion order + actual = sdb.getLast(keys=keys0) + assert actual == sal + + sam = "A real charmer!" + result = sdb.add(keys=keys0, val=sam) + assert result + actuals = sdb.get(keys=keys0) + assert actuals == [sue, sal, sam] # insertion order + + zoe = "See ya later." + zia = "Hey gorgeous!" + + result = sdb.pin(keys=keys0, vals=[zoe, zia]) + assert result + actuals = sdb.get(keys=keys0) + assert actuals == [zoe, zia] # insertion order + + assert sdb.put(keys=keys1, vals=[sal, sue, sam]) + actuals = sdb.get(keys=keys1) + assert actuals == [sal, sue, sam] + + for i, val in enumerate(sdb.getIter(keys=keys1)): + assert val == actuals[i] + + items = [(keys, val) for keys, val in sdb.getItemIter()] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] + + + + + items = list(sdb.getIoItemIter()) + assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), + (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), + (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), + (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), + (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + + items = sdb.getIoSetItem(keys=keys1) + assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), + (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), + (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + + items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] + assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), + (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + + + + # Test with top keys + assert sdb.put(keys=("test", "pop"), vals=[sal, sue, sam]) + topkeys = ("test", "") + items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # test with top parameter + keys = ("test", ) + items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # IoItems + items = list(sdb.getIoItemIter(keys=topkeys)) + assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), + (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), + (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] + + # test remove with a specific val + assert sdb.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in sdb.getItemIter()] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!'), + (('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'A real charmer!')] + + assert sdb.trim(keys=("test", "")) + items = [(keys, val) for keys, val in sdb.getItemIter()] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'A real charmer!')] + + for iokeys, val in sdb.getIoItemIter(): + assert sdb.remIokey(iokeys=iokeys) + + assert sdb.cnt(keys=keys0) == 0 + assert sdb.cnt(keys=keys1) == 0 + + + # test with keys as string not tuple + keys2 = "keystr" + bob = "Shove off!" + assert sdb.put(keys=keys2, vals=[bob]) + actuals = sdb.get(keys=keys2) + assert actuals == [bob] + assert sdb.cnt(keys2) == 1 + assert sdb.rem(keys2) + actuals = sdb.get(keys=keys2) + assert actuals == [] + assert sdb.cnt(keys2) == 0 + + assert sdb.put(keys=keys2, vals=[bob]) + actuals = sdb.get(keys=keys2) + assert actuals == [bob] + + bil = "Go away." + assert sdb.pin(keys=keys2, vals=[bil]) + actuals = sdb.get(keys=keys2) + assert actuals == [bil] + + assert sdb.add(keys=keys2, val=bob) + actuals = sdb.get(keys=keys2) + assert actuals == [bil, bob] + + assert not os.path.exists(db.path) + assert not db.opened + + def test_ioset_suber(): """ Test IoSetSuber LMDBer sub database class From c87128d0958f560b0ed44b6b8370fbfa2d4f72c8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Aug 2024 19:15:00 -0600 Subject: [PATCH 279/418] more refactoring clean up technical debt in subing and koming --- src/keri/core/routing.py | 2 +- src/keri/db/dbing.py | 228 ++++++++++++++++---------------- src/keri/db/escrowing.py | 2 +- src/keri/db/koming.py | 48 ++++--- src/keri/db/subing.py | 272 ++++++++++++++++++++++++++++----------- tests/db/test_koming.py | 13 +- tests/db/test_subing.py | 30 +++-- 7 files changed, 372 insertions(+), 223 deletions(-) diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 57d1346a6..4f67c9ce0 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -470,7 +470,7 @@ def processEscrowReply(self): quadruple (prefixer, seqner, diger, siger) """ - for (route, ion), saider in self.db.rpes.getIoItemIter(): + for (route, ion), saider in self.db.rpes.getFullItemIter(): try: tsgs = eventing.fetchTsgs(db=self.db.ssgs, saider=saider) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index f5fbed83b..af146ad71 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1520,11 +1520,11 @@ def getIoDupVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoDupValsIter(self, db, key): + def getIoDupValLast(self, db, key): """ - Return iterator of all duplicate values at key in db in insertion order - Raises StopIteration Error when no remaining dup items = empty. - Removes prepended proem ordinal from each val before returning + Return last added dup value at key in db in insertion order + Returns None no entry at key + Removes prepended proem ordinal from val before returning Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. @@ -1540,7 +1540,6 @@ def getIoDupValsIter(self, db, key): before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). - Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1548,22 +1547,21 @@ def getIoDupValsIter(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - vals = [] + val = None try: - if cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - yield val[33:] # slice off prepended ordering proem + if cursor.set_key(key): # move to first_dup + if cursor.last_dup(): # move to last_dup + val = cursor.value()[33:] # slice off prepended ordering proem + return val except lmdb.BadValsizeError as ex: raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoDupValLast(self, db, key): + def delIoDupVals(self, db, key): """ - Return last added dup value at key in db in insertion order - Returns None no entry at key - Removes prepended proem ordinal from val before returning - Assumes DB opened with dupsort=True + Deletes all values at key in db if key present. + Returns True If key exists Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to @@ -1583,78 +1581,58 @@ def getIoDupValLast(self, db, key): key is bytes of key within sub db's keyspace """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - val = None + with self.env.begin(db=db, write=True, buffers=True) as txn: try: - if cursor.set_key(key): # move to first_dup - if cursor.last_dup(): # move to last_dup - val = cursor.value()[33:] # slice off prepended ordering proem - return val + return (txn.delete(key)) except lmdb.BadValsizeError as ex: raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoDupItemsNext(self, db, key=b"", skip=True): + def delIoDupVal(self, db, key, val): """ - Return list of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key == b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Returns empty list if no entries at next key after key - - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database - + Deletes dup io val at key in db. Performs strip search to find match. + Strips proems and then searches. + Returns True if delete else False if val not present Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - + all values that makes lexocographic order that same as insertion order Duplicates are ordered as a pair of key plus value so prepending proem to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited + With 32 character hex string followed by '.' for essentially unlimited number of values which will be limited by memory. - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Does a linear search so not very efficient when not deleting from the front. + This is hack for supporting escrow which needs to delete individual dup. + The problem is that escrow is not fixed buts stuffs gets added and + deleted which just adds to the value of the proem. 2**16 is an impossibly + large number so the proem will not max out practically. But its not + an elegant solution. Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty string - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass + db is opened named sub db with dupsort=False + key is bytes of key within sub db's keyspace + val is bytes of value to be deleted without intersion ordering proem """ - with self.env.begin(db=db, write=False, buffers=True) as txn: + with self.env.begin(db=db, write=True, buffers=True) as txn: cursor = txn.cursor() - items = [] - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - # slice off prepended ordering prefix on value in item - items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] - return items + try: + if cursor.set_key(key): # move to first_dup + for proval in cursor.iternext_dup(): # value with proem + if val == proval[33:]: # strip of proem + return cursor.delete() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + return False - def getIoDupItemsNextIter(self, db, key=b"", skip=True): + def cntIoDupVals(self, db, key): """ - Return iterator of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key = b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Raises StopIteration Error when no remaining dup items = empty. - - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database - + Return count of dup values at key in db, or zero otherwise Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. @@ -1672,25 +1650,27 @@ def getIoDupItemsNextIter(self, db, key=b"", skip=True): Parameters: db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass + key is bytes of key within sub db's keyspace """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - for key, val in cursor.iternext_dup(keys=True): - yield (key, val[33:]) # slice off prepended ordering prefix + count = 0 + try: + if cursor.set_key(key): # moves to first_dup + count = cursor.count() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + return count - def cntIoDupVals(self, db, key): + + def getIoDupValsIter(self, db, key): """ - Return count of dup values at key in db, or zero otherwise + Return iterator of all duplicate values at key in db in insertion order + Raises StopIteration Error when no remaining dup items = empty. + Removes prepended proem ordinal from each val before returning Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. @@ -1706,6 +1686,7 @@ def cntIoDupVals(self, db, key): before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1713,20 +1694,30 @@ def cntIoDupVals(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - count = 0 + vals = [] try: if cursor.set_key(key): # moves to first_dup - count = cursor.count() + for val in cursor.iternext_dup(): + yield val[33:] # slice off prepended ordering proem except lmdb.BadValsizeError as ex: raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - return count - def delIoDupVals(self, db, key): + + def getIoDupItemsNext(self, db, key=b"", skip=True): """ - Deletes all values at key in db if key present. - Returns True If key exists + Return list of all dup items at next key after key in db in insertion order. + Item is (key, val) with proem stripped from val stored in db. + If key == b'' then returns list of dup items at first key in db. + If skip is False and key is not empty then returns dup items at key + Returns empty list if no entries at next key after key + + If key is empty then gets io items (key, io value) at first key in db + Use the return key from items as next key for next call to function in + order to iterate through the database + + Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to @@ -1743,56 +1734,67 @@ def delIoDupVals(self, db, key): Parameters: db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace + key is bytes of key within sub db's keyspace or empty string + skip is Boolean If True skips to next key if key is not empty string + Othewise don't skip for first pass """ - with self.env.begin(db=db, write=True, buffers=True) as txn: - try: - return (txn.delete(key)) - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + items = [] + if cursor.set_range(key): # moves to first_dup at key + found = True + if skip and key and cursor.key() == key: # skip to next key + found = cursor.next_nodup() # skip to next key not dup if any + if found: + # slice off prepended ordering prefix on value in item + items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] + return items - def delIoDupVal(self, db, key, val): + def getIoDupItemsNextIter(self, db, key=b"", skip=True): """ - Deletes dup io val at key in db. Performs strip search to find match. - Strips proems and then searches. - Returns True if delete else False if val not present + Return iterator of all dup items at next key after key in db in insertion order. + Item is (key, val) with proem stripped from val stored in db. + If key = b'' then returns list of dup items at first key in db. + If skip is False and key is not empty then returns dup items at key + Raises StopIteration Error when no remaining dup items = empty. + + If key is empty then gets io items (key, io value) at first key in db + Use the return key from items as next key for next call to function in + order to iterate through the database + Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order + all values that makes lexocographic order that same as insertion order. + Duplicates are ordered as a pair of key plus value so prepending proem to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentially unlimited + With 32 character hex string followed by '.' for essentiall unlimited number of values which will be limited by memory. - Does a linear search so not very efficient when not deleting from the front. - This is hack for supporting escrow which needs to delete individual dup. - The problem is that escrow is not fixed buts stuffs gets added and - deleted which just adds to the value of the proem. 2**16 is an impossibly - large number so the proem will not max out practically. But its not - an elegant solution. + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). Parameters: - db is opened named sub db with dupsort=False - key is bytes of key within sub db's keyspace - val is bytes of value to be deleted without intersion ordering proem + db is opened named sub db with dupsort=True + key is bytes of key within sub db's keyspace or empty + skip is Boolean If True skips to next key if key is not empty string + Othewise don't skip for first pass """ - with self.env.begin(db=db, write=True, buffers=True) as txn: + with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - try: - if cursor.set_key(key): # move to first_dup - for proval in cursor.iternext_dup(): # value with proem - if val == proval[33:]: # strip of proem - return cursor.delete() - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - return False + if cursor.set_range(key): # moves to first_dup at key + found = True + if skip and key and cursor.key() == key: # skip to next key + found = cursor.next_nodup() # skip to next key not dup if any + if found: + for key, val in cursor.iternext_dup(keys=True): + yield (key, val[33:]) # slice off prepended ordering prefix def getIoValsAllPreIter(self, db, pre, on=0): diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 19b8432f5..1a4ef0282 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -79,7 +79,7 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): extype (Type[Exception]): the expected exception type if the message should remain in escrow """ - for (typ, pre, aid, ion), saider in self.escrowdb.getIoItemIter(keys=(typ,)): + for (typ, pre, aid, ion), saider in self.escrowdb.getFullItemIter(keys=(typ,)): try: tsgs = eventing.fetchTsgs(db=self.tigerdb, saider=saider) diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index f69fd8b1f..b73310da0 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -110,7 +110,34 @@ def _tokeys(self, key: Union[str, bytes, memoryview]): def getItemIter(self, keys: Union[str, Iterable]=b""): + """Iterator over items in .db subclasses that do special hidden transforms + on either the keyspace or valuespace should override this method to hide + hidden parts from the returned items. For example, adding either + a hidden key space suffix or hidden val space proem to ensure insertion + order. Use getFullItemIter instead to return full items with hidden parts + shown for debugging or testing. + + Returns: + items (Iterator): of (key, val) tuples over the all the items in + subdb whose key startswith key made from keys. Keys may be keyspace + prefix to return branches of key space. When keys is empty then + returns all items in subdb + + Parameters: + keys (Iterator): tuple of bytes or strs that may be a truncation of + a full keys tuple in in order to get all the items from + multiple branches of the key space. If keys is empty then gets + all items in database. + """ + for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + yield (self._tokeys(key), self.deserializer(val)) + + + def getFullItemIter(self, keys: Union[str, Iterable]=b""): + """Iterator over items in .db that returns full items with subclass + specific special hidden parts shown for debugging or testing. + Returns: items (Iterator): of (key, val) tuples over the all the items in subdb whose key startswith key made from keys. Keys may be keyspace @@ -616,27 +643,6 @@ def getIoSetItemIter(self, keys: Union[str, Iterable]): yield (self._tokeys(iokey), self.deserializer(val)) - def getIoItemIter(self, keys: Union[str, Iterable]=b""): - """ - Returns: - items (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith effective key made from keys. - Keys may be keyspace prefix to return branches of key space. - When keys is empty then returns all items in subdb. - - - Parameters: - keys (Iterable): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield (self._tokeys(iokey), self.deserializer(val)) - - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): """ Removes entry at iokeys diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index bf8755e58..4c124cda7 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -172,13 +172,57 @@ def _des(self, val: str | bytes | memoryview): def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", *, top=False): - """ + """Iterator over items in .db subclasses that do special hidden transforms + on either the keyspace or valuespace should override this method to hide + hidden parts from the returned items. For example, adding either + a hidden key space suffix or hidden val space proem to ensure insertion + order. Use getFullItemIter instead to return full items with hidden parts + shown for debugging or testing. + Returns: items (Iterator[tuple[key,val]]): (key, val) tuples of each item over the all the items in subdb whose key startswith key made from keys. Keys may be keyspace prefix to return branches of key space. When keys is empty then returns all items in subdb + + + Parameters: + keys (Iterator[str | bytes | memoryview]): of key parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + """ + for key, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): + yield (self._tokeys(key), self._des(val)) + + + def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + *, top=False): + """Iterator over items in .db that returns full items with subclass + specific special hidden parts shown for debugging or testing. + + Returns: + items (Iterator[tuple[key,val]]): (key, val) tuples of each item + over the all the items in subdb whose key startswith key made from + keys. Keys may be keyspace prefix to return branches of key space. + When keys is empty then returns all items in subdb. + This is meant to return full parts of items in both keyspace and + valuespace which may be useful in debugging or testing. + Parameters: keys (Iterator[str | bytes | memoryview]): of key parts that may be a truncation of a full keys tuple in in order to address all the @@ -660,8 +704,9 @@ def put(self, keys: str | bytes | memoryview | Iterable, that are not already in set of vals at key. Does not overwrite. Parameters: - keys (Iterable): of key strs to be combined in order to form key - vals (Iterable): of str serializations + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): of str serializations Returns: result (bool): True If successful, False otherwise. @@ -684,8 +729,9 @@ def add(self, keys: str | bytes | memoryview | Iterable, than once. Parameters: - keys (Iterable): of key strs to be combined in order to form key - val (Union[bytes, str]): serialization + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + val (str | bytes | memoryview): serialization Returns: result (bool): True means unique value added among duplications, @@ -697,8 +743,34 @@ def add(self, keys: str | bytes | memoryview | Iterable, val=self._ser(val), sep=self.sep)) - #def append(self, keys: str | bytes | memoryview | Iterable, - # val: str | bytes | memoryview): + + def append(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview): + """Append val non-idempotently to insertion ordered set of values all with + the same apparent effective key. If val already in set at key then + after append there will be multiple entries in database with val at key + each with different insertion order (iokey). + Uses hidden ordinal key suffix for insertion ordering. + The suffix is appended and stripped transparently. + + Works by walking backward to find last iokey for key instead of reading + all vals for iokey. + + Returns: + ion (int): hidden insertion ordering ordinal of appended val + + Parameters: + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + val (str | bytes | memoryview): serialization + + + """ + return (self.db.appendIoSetVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val), + sep=self.sep)) + def pin(self, keys: str | bytes | memoryview | Iterable, @@ -716,12 +788,10 @@ def pin(self, keys: str | bytes | memoryview | Iterable, result (bool): True If successful, False otherwise. """ - key = self._tokey(keys) - self.db.delIoSetVals(db=self.sdb, key=key) # delete all values if not nonStringIterable(vals): # not iterable vals = (vals, ) # make iterable return (self.db.setIoSetVals(db=self.sdb, - key=key, + key=self._tokey(keys), vals=[self._ser(val) for val in vals], sep=self.sep)) @@ -744,23 +814,6 @@ def get(self, keys: str | bytes | memoryview | Iterable): sep=self.sep)]) - def getLast(self, keys: str | bytes | memoryview | Iterable): - """ - Gets last val inserted at effecive key made from keys and hidden ordinal - suffix. - - Parameters: - keys (Iterable): of key strs to be combined in order to form key - - Returns: - val (str): value str, None if no entry at keys - - """ - val = self.db.getIoSetValLast(db=self.sdb, key=self._tokey(keys)) - return (self._des(val) if val is not None else val) - - - def getIter(self, keys: str | bytes | memoryview | Iterable): """ Gets vals iterator at effecive key made from keys and hidden ordinal suffix. @@ -780,18 +833,21 @@ def getIter(self, keys: str | bytes | memoryview | Iterable): yield self._des(val) - - def cnt(self, keys: str | bytes | memoryview | Iterable): + def getLast(self, keys: str | bytes | memoryview | Iterable): """ - Return count of values at effective key made from keys and hidden ordinal - suffix. Zero otherwise + Gets last val inserted at effecive key made from keys and hidden ordinal + suffix. Parameters: keys (Iterable): of key strs to be combined in order to form key + + Returns: + val (str): value str, None if no entry at keys + """ - return (self.db.cntIoSetVals(db=self.sdb, - key=self._tokey(keys), - sep=self.sep)) + val = self.db.getIoSetValLast(db=self.sdb, key=self._tokey(keys)) + return (self._des(val) if val is not None else val) + def rem(self, keys: str | bytes | memoryview | Iterable, @@ -822,6 +878,35 @@ def rem(self, keys: str | bytes | memoryview | Iterable, sep=self.sep) + def remIokey(self, iokeys: str | bytes | memoryview | Iterable): + """ + Removes entries at iokeys + + Parameters: + iokeys (str | bytes | memoryview | Iterable): of key str or + tuple of key strs to be combined in order to form key + + Returns: + result (bool): True if key exists so delete successful. False otherwise + + """ + return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) + + + def cnt(self, keys: str | bytes | memoryview | Iterable): + """ + Return count of values at effective key made from keys and hidden ordinal + suffix. Zero otherwise + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + """ + return (self.db.cntIoSetVals(db=self.sdb, + key=self._tokey(keys), + sep=self.sep)) + + + def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", *, top=False): """ @@ -901,42 +986,6 @@ def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable): yield (self._tokeys(iokey), self._des(val)) - def getIoItemIter(self, keys: str | bytes | memoryview | Iterable = b""): - """ - Returns: - items (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith effective key made from keys. - Keys may be keyspace prefix to return branches of key space. - When keys is empty then returns all items in subdb. - - - Parameters: - keys (Iterable): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield (self._tokeys(iokey), self._des(val)) - - - def remIokey(self, iokeys: str | bytes | memoryview | Iterable): - """ - Removes entries at iokeys - - Parameters: - iokeys (str | bytes | memoryview | Iterable): of key str or - tuple of key strs to be combined in order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - - """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - - class CesrIoSetSuber(CesrSuberBase, IoSetSuber): """ Subclass of CesrSuber and IoSetSuber. @@ -1713,8 +1762,9 @@ def __init__(self, *pa, **kwa): def put(self, keys: str | bytes | memoryview | Iterable, vals: str | bytes | memoryview | Iterable): """ - Puts all vals at effective key made from keys and hidden ordinal suffix. - that are not already in set of vals at key. Does not overwrite. + Puts all vals idempotently at key made from keys in insertion order using + hidden ordinal proem. Idempotently means do not put any val in vals that is + already in dup vals at key. Does not overwrite. Parameters: keys (Iterable): of key strs to be combined in order to form key @@ -1729,3 +1779,81 @@ def put(self, keys: str | bytes | memoryview | Iterable, return (self.db.putIoDupVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals])) + + + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview): + """ + Add val idempotently at key made from keys in insertion order using hidden + ordinal proem. Idempotently means do not add val that is already in + dup vals at key. Does not overwrite. + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + val (Union[bytes, str]): serialization + + Returns: + result (bool): True means unique value added among duplications, + False means duplicate of same value already exists. + + """ + return (self.db.addIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val))) + + + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): + """ + Pins (sets) vals at key made from keys in insertion order using hidden + ordinal proem. Overwrites. Removes all pre-existing vals that share + same keys and replaces them with vals + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + vals (Iterable): str serializations + + Returns: + result (bool): True If successful, False otherwise. + + """ + key = self._tokey(keys) + self.db.delIoDupVals(db=self.sdb, key=key) # delete all values + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable + return self.db.putIoDupVals(db=self.sdb, + key=keys, + vals=[self._ser(val) for val in vals]) + + + def get(self, keys: str | bytes | memoryview | Iterable): + """ + Gets vals set list at key made from keys in insertion order using + hidden ordinal proem. + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + + Returns: + vals (Iterable): each item in list is str + empty list if no entry at keys + + """ + return ([self._des(val) for val in + self.db.getIoDupVals(db=self.sdb, key=self._tokey(keys))]) + + + def getLast(self, keys: str | bytes | memoryview | Iterable): + """ + Gets last val inserted at key made from keys in insertion order using + hidden ordinal proem. + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + + Returns: + val (str): value str, None if no entry at keys + + """ + val = self.db.getIoDupValLast(db=self.sdb, key=self._tokey(keys)) + return (self._des(val) if val is not None else val) diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index ae1975e36..fb2e9f1d4 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -829,13 +829,13 @@ def __iter__(self): '00000000000000000000000000000000')] i = 0 - for iokeys, end in endDB.getIoItemIter(keys=(cid0, "")): + for iokeys, end in endDB.getFullItemIter(keys=(cid0, "")): assert end == ends[i] assert iokeys == iokeysall[i] i += 1 i = 0 - for iokeys, end in endDB.getIoItemIter(): + for iokeys, end in endDB.getFullItemIter(): assert end == ends[i] assert iokeys == iokeysall[i] i += 1 @@ -850,6 +850,13 @@ def __iter__(self): if __name__ == "__main__": - test_dup_komer() + test_kom_happy_path() test_kom_get_item_iter() + test_put_invalid_dataclass() + test_get_invalid_dataclass() + test_not_found_entity() + test_serialization() + test_custom_serialization() + test_deserialization() + test_dup_komer() test_ioset_komer() diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 8cd2a312b..f68723da2 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -416,7 +416,7 @@ def test_iodup_suber(): - items = list(sdb.getIoItemIter()) + items = list(sdb.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), @@ -450,7 +450,7 @@ def test_iodup_suber(): (('test', 'pop'), 'A real charmer!')] # IoItems - items = list(sdb.getIoItemIter(keys=topkeys)) + items = list(sdb.getFullItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] @@ -473,7 +473,7 @@ def test_iodup_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in sdb.getIoItemIter(): + for iokeys, val in sdb.getFullItemIter(): assert sdb.remIokey(iokeys=iokeys) assert sdb.cnt(keys=keys0) == 0 @@ -579,7 +579,7 @@ def test_ioset_suber(): - items = list(sdb.getIoItemIter()) + items = list(sdb.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), @@ -613,7 +613,7 @@ def test_ioset_suber(): (('test', 'pop'), 'A real charmer!')] # IoItems - items = list(sdb.getIoItemIter(keys=topkeys)) + items = list(sdb.getFullItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] @@ -636,7 +636,7 @@ def test_ioset_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in sdb.getIoItemIter(): + for iokeys, val in sdb.getFullItemIter(): assert sdb.remIokey(iokeys=iokeys) assert sdb.cnt(keys=keys0) == 0 @@ -668,6 +668,12 @@ def test_ioset_suber(): actuals = sdb.get(keys=keys2) assert actuals == [bil, bob] + # ToDo test with append non idempotent + #assert dber.putIoSetVals(db, key1, vals1) == True + #assert dber.getIoSetVals(db, key1) == vals1 + #assert dber.appendIoSetVal(db, key1, val=b"k") == 4 + #assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] + assert not os.path.exists(db.path) assert not db.opened @@ -825,7 +831,7 @@ def test_cesr_ioset_suber(): ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoItemIter()] + items = [(iokeys, val.qb64) for iokeys, val in sdb.getFullItemIter()] assert items == [ ((*keys1, '00000000000000000000000000000000'), said2), ((*keys1, '00000000000000000000000000000002'), said0), @@ -854,13 +860,13 @@ def test_cesr_ioset_suber(): ] topkeys = (seq0, "") - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoItemIter(keys=topkeys)] + items = [(iokeys, val.qb64) for iokeys, val in sdb.getFullItemIter(keys=topkeys)] assert items == [ ((*keys0, '00000000000000000000000000000000'), said1), ((*keys0, '00000000000000000000000000000001'), said2), ] - for iokeys, val in sdb.getIoItemIter(): + for iokeys, val in sdb.getFullItemIter(): assert sdb.remIokey(iokeys=iokeys) assert sdb.cnt(keys=keys0) == 0 @@ -1496,7 +1502,7 @@ def test_cat_cesr_ioset_suber(): ] items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoItemIter()] + for iokeys, vals in sdb.getFullItemIter()] assert items == [ (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), @@ -1526,14 +1532,14 @@ def test_cat_cesr_ioset_suber(): ] items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoItemIter(keys=topkeys)] + for iokeys, vals in sdb.getFullItemIter(keys=topkeys)] assert items == [ (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), ] - for iokeys, val in sdb.getIoItemIter(): + for iokeys, val in sdb.getFullItemIter(): assert sdb.remIokey(iokeys=iokeys) assert sdb.cnt(keys=keys0) == 0 From cf7a71512b0a25bd5690a9c3842bfcd11668dc43 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Aug 2024 20:13:40 -0600 Subject: [PATCH 280/418] even more refactoring cleanup --- src/keri/db/koming.py | 60 +++++++++++++++------------- src/keri/db/subing.py | 86 ++++++++++++++++++++++------------------- tests/db/test_koming.py | 12 ++++++ tests/db/test_subing.py | 6 +++ 4 files changed, 98 insertions(+), 66 deletions(-) diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index b73310da0..ca0459a22 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -576,36 +576,29 @@ def rem(self, keys: Union[str, Iterable], val=None): key=self._tokey(keys), sep=self.sep) - - def getItemIter(self, keys: Union[str, Iterable]=b""): - """Get items iterator - Returns: - items (Iterator): of (key, val) tuples over the all the items in - subdb whose effective key startswith key made from keys. - Keys may be keyspace prefix in order to return branches of key space. - When keys is empty then returns all items in subdb. - Returned key in each item has ordinal suffix removed. + def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): + """ + Removes entry at iokeys Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. + iokeys (tuple): of key str or tuple of key strs to be combined in + order to form key + + Returns: + result (bool): True if key exists so delete successful. False otherwise """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) - yield (self._tokeys(key), self.deserializer(val)) + return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def getIoSetItem(self, keys: Union[str, Iterable]): + def getIoSetItem(self, keys: Union[str, Iterable], *, ion=0): """ Gets (iokeys, val) ioitems list at key made from keys where key is apparent effective key and ioitems all have same apparent effective key Parameters: keys (Iterable): of key strs to be combined in order to form key + ion (int): starting ordinal value, default 0 Returns: ioitems (Iterable): each item in list is tuple (iokeys, val) where each @@ -618,16 +611,18 @@ def getIoSetItem(self, keys: Union[str, Iterable]): return ([(self._tokeys(iokey), self.deserializer(val)) for iokey, val in self.db.getIoSetItems(db=self.sdb, key=self._tokey(keys), + ion=ion, sep=self.sep)]) - def getIoSetItemIter(self, keys: Union[str, Iterable]): + def getIoSetItemIter(self, keys: Union[str, Iterable], *, ion=0): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key Parameters: keys (Iterable): of key strs to be combined in order to form key + ion (int): starting ordinal value, default 0 Returns: ioitems (Iterator): each item iterated is tuple (iokeys, val) where @@ -639,23 +634,34 @@ def getIoSetItemIter(self, keys: Union[str, Iterable]): """ for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, key=self._tokey(keys), + ion=ion, sep=self.sep): yield (self._tokeys(iokey), self.deserializer(val)) - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): - """ - Removes entry at iokeys - Parameters: - iokeys (tuple): of key str or tuple of key strs to be combined in - order to form key + def getItemIter(self, keys: Union[str, Iterable]=b""): + """Get items iterator Returns: - result (bool): True if key exists so delete successful. False otherwise + items (Iterator): of (key, val) tuples over the all the items in + subdb whose effective key startswith key made from keys. + Keys may be keyspace prefix in order to return branches of key space. + When keys is empty then returns all items in subdb. + Returned key in each item has ordinal suffix removed. + + Parameters: + keys (Iterator): tuple of bytes or strs that may be a truncation of + a full keys tuple in in order to get all the items from + multiple branches of the key space. If keys is empty then gets + all items in database. Append "" to end of keys Iterable to + ensure get properly separated top branch key. """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) + for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + key, ion = dbing.unsuffix(iokey, sep=self.sep) + yield (self._tokeys(key), self.deserializer(val)) + diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 4c124cda7..827208fe5 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -906,50 +906,15 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): sep=self.sep)) - - def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", - *, top=False): - """ - Return iterator over all the items in top branch defined by keys where - keys may be truncation of full branch. - - Returns: - items (Iterator): of (key, val) tuples over the all the items in - subdb whose effective key startswith key made from keys. - Keys may be keyspace prefix in order to return branches of key space. - When keys is empty then returns all items in subdb. - Returned key in each item has ordinal suffix removed. - - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to address all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. - Either append "" to end of keys Iterable to ensure get properly - separated top branch key or use top=True. - - top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep - character. - False means treat as full branch in key space. Resultant key - does not end in .sep character. - When last item in keys is empty str then will treat as - partial ending in sep regardless of top value - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) - yield (self._tokeys(key), self._des(val)) - - - def getIoSetItem(self, keys: str | bytes | memoryview | Iterable): + def getIoSetItem(self, keys: str | bytes | memoryview | Iterable, + *, ion=0): """ Gets (iokeys, val) ioitems list at key made from keys where key is apparent effective key and ioitems all have same apparent effective key Parameters: keys (Iterable): of key strs to be combined in order to form key + ion (int): starting ordinal value, default 0 Returns: ioitems (Iterable): each item in list is tuple (iokeys, val) where each @@ -961,16 +926,19 @@ def getIoSetItem(self, keys: str | bytes | memoryview | Iterable): return ([(self._tokeys(iokey), self._des(val)) for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, key=self._tokey(keys), + ion=ion, sep=self.sep)]) - def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable): + def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable, + *, ion=0): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key Parameters: keys (Iterable): of key strs to be combined in order to form key + ion (int): starting ordinal value, default 0 Returns: ioitems (Iterator): each item iterated is tuple (iokeys, val) where @@ -982,10 +950,50 @@ def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable): """ for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, key=self._tokey(keys), + ion=ion, sep=self.sep): yield (self._tokeys(iokey), self._des(val)) + + def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", + *, top=False): + """ + Return iterator over all the items in top branch defined by keys where + keys may be truncation of full branch. + + Returns: + items (Iterator): of (key, val) tuples over the all the items in + subdb whose effective key startswith key made from keys. + Keys may be keyspace prefix in order to return branches of key space. + When keys is empty then returns all items in subdb. + Returned key in each item has ordinal suffix removed. + + Parameters: + keys (Iterator): tuple of bytes or strs that may be a truncation of + a full keys tuple in in order to address all the items from + multiple branches of the key space. If keys is empty then gets + all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + """ + for iokey, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): + key, ion = dbing.unsuffix(iokey, sep=self.sep) + yield (self._tokeys(key), self._des(val)) + + + + class CesrIoSetSuber(CesrSuberBase, IoSetSuber): """ Subclass of CesrSuber and IoSetSuber. diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index fb2e9f1d4..b5877fab6 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -786,6 +786,18 @@ def __iter__(self): assert iokeys == iokeys0[i] i += 1 + i = 1 + for iokeys, end in endDB.getIoSetItem(keys=keys0, ion=i): + assert end == ends[i] + assert iokeys == iokeys0[i] + i += 1 + + i = 1 + for iokeys, end in endDB.getIoSetItemIter(keys=keys0, ion=i): + assert end == ends[i] + assert iokeys == iokeys0[i] + i += 1 + # test getAllItemIter ends = ends + [wit3end] i = 0 diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index f68723da2..db8a5efd0 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -595,6 +595,12 @@ def test_ioset_suber(): assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + items = sdb.getIoSetItem(keys=keys1, ion=1) + assert items ==[(('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), + (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + + items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0, ion=1)] + assert items == [(('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] # Test with top keys From 90216c328af57aea84371bddb243d168aac1696c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 27 Aug 2024 13:04:30 -0600 Subject: [PATCH 281/418] more clean up of subing --- src/keri/core/routing.py | 2 +- src/keri/db/dbing.py | 6 +- src/keri/db/escrowing.py | 2 +- src/keri/db/subing.py | 92 +++++++---------- tests/db/test_subing.py | 209 ++++++++++++++++++++------------------- 5 files changed, 144 insertions(+), 167 deletions(-) diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 4f67c9ce0..1418272de 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -470,7 +470,7 @@ def processEscrowReply(self): quadruple (prefixer, seqner, diger, siger) """ - for (route, ion), saider in self.db.rpes.getFullItemIter(): + for (route,), saider in self.db.rpes.getItemIter(): try: tsgs = eventing.fetchTsgs(db=self.db.ssgs, saider=saider) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index af146ad71..131f6e2fc 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -626,7 +626,7 @@ def delTopVal(self, db, key=b''): db (lmdb._Database): instance of named sub db with dupsort==False key (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is - empty then gets all items in database + empty then deletes all items in database Works for both dupsort==False and dupsort==True Because cursor.iternext() advances cursor after returning item its safe @@ -1189,12 +1189,14 @@ def getIoSetItemsIter(self, db, key, *, ion=0, sep=b'.'): at same apparent effective key where each iteration returns tuple (iokey, val). iokey includes the ordinal key suffix. Uses hidden ordinal key suffix for insertion ordering. + Does not work with partial effective key. Raises StopIteration Error when empty. Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key + key (bytes): Apparent effective key. Does not work with partial key + that is not the full effective key sans the ion part. ion (int): starting ordinal value, default 0 """ with self.env.begin(db=db, write=False, buffers=True) as txn: diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 1a4ef0282..dcf741d07 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -79,7 +79,7 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): extype (Type[Exception]): the expected exception type if the message should remain in escrow """ - for (typ, pre, aid, ion), saider in self.escrowdb.getFullItemIter(keys=(typ,)): + for (typ, pre, aid, ion), saider in self.escrowdb.getFullItemIter(keys=(typ, '')): try: tsgs = eventing.fetchTsgs(db=self.tigerdb, saider=saider) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 827208fe5..fb6edbd62 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -170,32 +170,22 @@ def _des(self, val: str | bytes | memoryview): return (val.decode("utf-8") if hasattr(val, "decode") else val) - def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", - *, top=False): - """Iterator over items in .db subclasses that do special hidden transforms - on either the keyspace or valuespace should override this method to hide - hidden parts from the returned items. For example, adding either - a hidden key space suffix or hidden val space proem to ensure insertion - order. Use getFullItemIter instead to return full items with hidden parts - shown for debugging or testing. - - Returns: - items (Iterator[tuple[key,val]]): (key, val) tuples of each item - over the all the items in subdb whose key startswith key made from - keys. Keys may be keyspace prefix to return branches of key space. - When keys is empty then returns all items in subdb - - + def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + *, top=False): + """ + Removes all entries whose keys startswith keys. Enables removal of whole + branches of db key space. To ensure that proper separation of a branch + include empty string as last key in keys. For example ("a","") deletes + 'a.1'and 'a.2' but not 'ab' Parameters: keys (Iterator[str | bytes | memoryview]): of key parts that may be a truncation of a full keys tuple in in order to address all the items from multiple branches of the key space. - If keys is empty then gets all items in database. + If keys is empty then trims all items in database. Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. - top (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep character. @@ -204,10 +194,11 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", When last item in keys is empty str then will treat as partial ending in sep regardless of top value + + Returns: + result (bool): True if val at key exists so delete successful. False otherwise """ - for key, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): - yield (self._tokeys(key), self._des(val)) + return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys, top=top))) def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", @@ -246,13 +237,22 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", yield (self._tokeys(key), self._des(val)) - def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", - *, top=False): - """ - Removes all entries whose keys startswith keys. Enables removal of whole - branches of db key space. To ensure that proper separation of a branch - include empty string as last key in keys. For example ("a","") deletes - 'a.1'and 'a.2' but not 'ab' + def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + *, top=False): + """Iterator over items in .db subclasses that do special hidden transforms + on either the keyspace or valuespace should override this method to hide + hidden parts from the returned items. For example, adding either + a hidden key space suffix or hidden val space proem to ensure insertion + order. Use getFullItemIter instead to return full items with hidden parts + shown for debugging or testing. + + Returns: + items (Iterator[tuple[key,val]]): (key, val) tuples of each item + over the all the items in subdb whose key startswith key made from + keys. Keys may be keyspace prefix to return branches of key space. + When keys is empty then returns all items in subdb + + Parameters: keys (Iterator[str | bytes | memoryview]): of key parts that may be @@ -262,6 +262,7 @@ def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. + top (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep character. @@ -270,11 +271,10 @@ def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", When last item in keys is empty str then will treat as partial ending in sep regardless of top value - - Returns: - result (bool): True if val at key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys, top=top))) + for key, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): + yield (self._tokeys(key), self._des(val)) class Suber(SuberBase): @@ -374,7 +374,6 @@ def rem(self, keys: Union[str, Iterable]): - class OrdSuber(Suber): """ Subclass of Suber that adds methods for keys with ordinal numbered suffixes. @@ -906,32 +905,7 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): sep=self.sep)) - def getIoSetItem(self, keys: str | bytes | memoryview | Iterable, - *, ion=0): - """ - Gets (iokeys, val) ioitems list at key made from keys where key is - apparent effective key and ioitems all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form key - ion (int): starting ordinal value, default 0 - - Returns: - ioitems (Iterable): each item in list is tuple (iokeys, val) where each - iokeys is actual key tuple including hidden suffix and - each val is str - empty list if no entry at keys - - """ - return ([(self._tokeys(iokey), self._des(val)) for iokey, val in - self.db.getIoSetItemsIter(db=self.sdb, - key=self._tokey(keys), - ion=ion, - sep=self.sep)]) - - - def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable, - *, ion=0): + def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index db8a5efd0..2244c1002 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -356,9 +356,9 @@ def test_iodup_suber(): assert db.name == "test" assert db.opened - sdb = subing.IoSetSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.IoSetSuber) - assert not sdb.sdb.flags()["dupsort"] + ioduber = subing.IoSetSuber(db=db, subkey='bags.') + assert isinstance(ioduber, subing.IoSetSuber) + assert not ioduber.sdb.flags()["dupsort"] sue = "Hello sailer!" sal = "Not my type." @@ -366,47 +366,47 @@ def test_iodup_suber(): keys0 = ("test_key", "0001") keys1 = ("test_key", "0002") - assert sdb.put(keys=keys0, vals=[sal, sue]) - actuals = sdb.get(keys=keys0) + assert ioduber.put(keys=keys0, vals=[sal, sue]) + actuals = ioduber.get(keys=keys0) assert actuals == [sal, sue] # insertion order not lexicographic - assert sdb.cnt(keys0) == 2 - actual = sdb.getLast(keys=keys0) + assert ioduber.cnt(keys0) == 2 + actual = ioduber.getLast(keys=keys0) assert actual == sue - assert sdb.rem(keys0) - actuals = sdb.get(keys=keys0) + assert ioduber.rem(keys0) + actuals = ioduber.get(keys=keys0) assert not actuals assert actuals == [] - assert sdb.cnt(keys0) == 0 + assert ioduber.cnt(keys0) == 0 - assert sdb.put(keys=keys0, vals=[sue, sal]) - actuals = sdb.get(keys=keys0) + assert ioduber.put(keys=keys0, vals=[sue, sal]) + actuals = ioduber.get(keys=keys0) assert actuals == [sue, sal] # insertion order - actual = sdb.getLast(keys=keys0) + actual = ioduber.getLast(keys=keys0) assert actual == sal sam = "A real charmer!" - result = sdb.add(keys=keys0, val=sam) + result = ioduber.add(keys=keys0, val=sam) assert result - actuals = sdb.get(keys=keys0) + actuals = ioduber.get(keys=keys0) assert actuals == [sue, sal, sam] # insertion order zoe = "See ya later." zia = "Hey gorgeous!" - result = sdb.pin(keys=keys0, vals=[zoe, zia]) + result = ioduber.pin(keys=keys0, vals=[zoe, zia]) assert result - actuals = sdb.get(keys=keys0) + actuals = ioduber.get(keys=keys0) assert actuals == [zoe, zia] # insertion order - assert sdb.put(keys=keys1, vals=[sal, sue, sam]) - actuals = sdb.get(keys=keys1) + assert ioduber.put(keys=keys1, vals=[sal, sue, sam]) + actuals = ioduber.get(keys=keys1) assert actuals == [sal, sue, sam] - for i, val in enumerate(sdb.getIter(keys=keys1)): + for i, val in enumerate(ioduber.getIter(keys=keys1)): assert val == actuals[i] - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in ioduber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), @@ -416,48 +416,48 @@ def test_iodup_suber(): - items = list(sdb.getFullItemIter()) + items = list(ioduber.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = sdb.getIoSetItem(keys=keys1) + items = [(iokeys, val) for iokeys, val in ioduber.getIoSetItemIter(keys=keys1)] assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] + items = [(iokeys, val) for iokeys, val in ioduber.getIoSetItemIter(keys=keys0)] assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] # Test with top keys - assert sdb.put(keys=("test", "pop"), vals=[sal, sue, sam]) + assert ioduber.put(keys=("test", "pop"), vals=[sal, sue, sam]) topkeys = ("test", "") - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=topkeys)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] # test with top parameter keys = ("test", ) - items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys, top=True)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] # IoItems - items = list(sdb.getFullItemIter(keys=topkeys)) + items = list(ioduber.getFullItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] # test remove with a specific val - assert sdb.rem(keys=("test_key", "0002"), val=sue) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert ioduber.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in ioduber.getItemIter()] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!'), @@ -466,43 +466,43 @@ def test_iodup_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - assert sdb.trim(keys=("test", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert ioduber.trim(keys=("test", "")) + items = [(keys, val) for keys, val in ioduber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in sdb.getFullItemIter(): - assert sdb.remIokey(iokeys=iokeys) + for iokeys, val in ioduber.getFullItemIter(): + assert ioduber.remIokey(iokeys=iokeys) - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 + assert ioduber.cnt(keys=keys0) == 0 + assert ioduber.cnt(keys=keys1) == 0 # test with keys as string not tuple keys2 = "keystr" bob = "Shove off!" - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert ioduber.put(keys=keys2, vals=[bob]) + actuals = ioduber.get(keys=keys2) assert actuals == [bob] - assert sdb.cnt(keys2) == 1 - assert sdb.rem(keys2) - actuals = sdb.get(keys=keys2) + assert ioduber.cnt(keys2) == 1 + assert ioduber.rem(keys2) + actuals = ioduber.get(keys=keys2) assert actuals == [] - assert sdb.cnt(keys2) == 0 + assert ioduber.cnt(keys2) == 0 - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert ioduber.put(keys=keys2, vals=[bob]) + actuals = ioduber.get(keys=keys2) assert actuals == [bob] bil = "Go away." - assert sdb.pin(keys=keys2, vals=[bil]) - actuals = sdb.get(keys=keys2) + assert ioduber.pin(keys=keys2, vals=[bil]) + actuals = ioduber.get(keys=keys2) assert actuals == [bil] - assert sdb.add(keys=keys2, val=bob) - actuals = sdb.get(keys=keys2) + assert ioduber.add(keys=keys2, val=bob) + actuals = ioduber.get(keys=keys2) assert actuals == [bil, bob] assert not os.path.exists(db.path) @@ -519,9 +519,9 @@ def test_ioset_suber(): assert db.name == "test" assert db.opened - sdb = subing.IoSetSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.IoSetSuber) - assert not sdb.sdb.flags()["dupsort"] + iosuber = subing.IoSetSuber(db=db, subkey='bags.') + assert isinstance(iosuber, subing.IoSetSuber) + assert not iosuber.sdb.flags()["dupsort"] sue = "Hello sailer!" sal = "Not my type." @@ -529,47 +529,47 @@ def test_ioset_suber(): keys0 = ("test_key", "0001") keys1 = ("test_key", "0002") - assert sdb.put(keys=keys0, vals=[sal, sue]) - actuals = sdb.get(keys=keys0) + assert iosuber.put(keys=keys0, vals=[sal, sue]) + actuals = iosuber.get(keys=keys0) assert actuals == [sal, sue] # insertion order not lexicographic - assert sdb.cnt(keys0) == 2 - actual = sdb.getLast(keys=keys0) + assert iosuber.cnt(keys0) == 2 + actual = iosuber.getLast(keys=keys0) assert actual == sue - assert sdb.rem(keys0) - actuals = sdb.get(keys=keys0) + assert iosuber.rem(keys0) + actuals = iosuber.get(keys=keys0) assert not actuals assert actuals == [] - assert sdb.cnt(keys0) == 0 + assert iosuber.cnt(keys0) == 0 - assert sdb.put(keys=keys0, vals=[sue, sal]) - actuals = sdb.get(keys=keys0) + assert iosuber.put(keys=keys0, vals=[sue, sal]) + actuals = iosuber.get(keys=keys0) assert actuals == [sue, sal] # insertion order - actual = sdb.getLast(keys=keys0) + actual = iosuber.getLast(keys=keys0) assert actual == sal sam = "A real charmer!" - result = sdb.add(keys=keys0, val=sam) + result = iosuber.add(keys=keys0, val=sam) assert result - actuals = sdb.get(keys=keys0) + actuals = iosuber.get(keys=keys0) assert actuals == [sue, sal, sam] # insertion order zoe = "See ya later." zia = "Hey gorgeous!" - result = sdb.pin(keys=keys0, vals=[zoe, zia]) + result = iosuber.pin(keys=keys0, vals=[zoe, zia]) assert result - actuals = sdb.get(keys=keys0) + actuals = iosuber.get(keys=keys0) assert actuals == [zoe, zia] # insertion order - assert sdb.put(keys=keys1, vals=[sal, sue, sam]) - actuals = sdb.get(keys=keys1) + assert iosuber.put(keys=keys1, vals=[sal, sue, sam]) + actuals = iosuber.get(keys=keys1) assert actuals == [sal, sue, sam] - for i, val in enumerate(sdb.getIter(keys=keys1)): + for i, val in enumerate(iosuber.getIter(keys=keys1)): assert val == actuals[i] - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), @@ -579,54 +579,54 @@ def test_ioset_suber(): - items = list(sdb.getFullItemIter()) + items = list(iosuber.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = sdb.getIoSetItem(keys=keys1) + items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys1)] assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] + items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys0)] assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] - items = sdb.getIoSetItem(keys=keys1, ion=1) + items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys1, ion=1)] assert items ==[(('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0, ion=1)] + items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys0, ion=1)] assert items == [(('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] # Test with top keys - assert sdb.put(keys=("test", "pop"), vals=[sal, sue, sam]) + assert iosuber.put(keys=("test", "pop"), vals=[sal, sue, sam]) topkeys = ("test", "") - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=topkeys)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] # test with top parameter keys = ("test", ) - items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys, top=True)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] # IoItems - items = list(sdb.getFullItemIter(keys=topkeys)) + items = list(iosuber.getFullItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] # test remove with a specific val - assert sdb.rem(keys=("test_key", "0002"), val=sue) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert iosuber.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!'), @@ -635,50 +635,51 @@ def test_ioset_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - assert sdb.trim(keys=("test", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert iosuber.trim(keys=("test", "")) + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in sdb.getFullItemIter(): - assert sdb.remIokey(iokeys=iokeys) + for iokeys, val in iosuber.getFullItemIter(): + assert iosuber.remIokey(iokeys=iokeys) - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 + assert iosuber.cnt(keys=keys0) == 0 + assert iosuber.cnt(keys=keys1) == 0 # test with keys as string not tuple keys2 = "keystr" bob = "Shove off!" - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert iosuber.put(keys=keys2, vals=[bob]) + actuals = iosuber.get(keys=keys2) assert actuals == [bob] - assert sdb.cnt(keys2) == 1 - assert sdb.rem(keys2) - actuals = sdb.get(keys=keys2) + assert iosuber.cnt(keys2) == 1 + assert iosuber.rem(keys2) + actuals = iosuber.get(keys=keys2) assert actuals == [] - assert sdb.cnt(keys2) == 0 + assert iosuber.cnt(keys2) == 0 - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert iosuber.put(keys=keys2, vals=[bob]) + actuals = iosuber.get(keys=keys2) assert actuals == [bob] bil = "Go away." - assert sdb.pin(keys=keys2, vals=[bil]) - actuals = sdb.get(keys=keys2) + assert iosuber.pin(keys=keys2, vals=[bil]) + actuals = iosuber.get(keys=keys2) assert actuals == [bil] - assert sdb.add(keys=keys2, val=bob) - actuals = sdb.get(keys=keys2) + assert iosuber.add(keys=keys2, val=bob) + actuals = iosuber.get(keys=keys2) assert actuals == [bil, bob] # ToDo test with append non idempotent - #assert dber.putIoSetVals(db, key1, vals1) == True - #assert dber.getIoSetVals(db, key1) == vals1 - #assert dber.appendIoSetVal(db, key1, val=b"k") == 4 - #assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] + assert iosuber.trim() # default trims whole database + assert iosuber.put(keys=keys1, vals=[bob, bil]) + assert iosuber.get(keys=keys1) == [bob, bil] + assert iosuber.append(keys=keys1, val=bob) == 2 + assert iosuber.get(keys=keys1) == [bob, bil, bob] assert not os.path.exists(db.path) assert not db.opened @@ -845,7 +846,7 @@ def test_cesr_ioset_suber(): ((*keys0, '00000000000000000000000000000001'), said2), ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItem(keys=keys1)] + items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItemIter(keys=keys1)] assert items == [ ((*keys1, '00000000000000000000000000000000'), said2), ((*keys1, '00000000000000000000000000000002'), said0), @@ -1518,7 +1519,7 @@ def test_cat_cesr_ioset_suber(): ] items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoSetItem(keys=keys1)] + for iokeys, vals in sdb.getIoSetItemIter(keys=keys1)] assert items == [(keys1 + ('00000000000000000000000000000000', ), [sqr2.qb64, dgr2.qb64])] items = [(iokeys, [val.qb64 for val in vals]) From b512984f8d53f14551c888af2bbd9a7eb26c6eea Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 28 Aug 2024 13:20:25 -0600 Subject: [PATCH 282/418] even more refactoring of subing, dbing, basing --- src/keri/app/storing.py | 2 +- src/keri/core/eventing.py | 148 +------------------------------------- src/keri/db/basing.py | 20 ++++-- src/keri/db/dbing.py | 89 +++++++++++++++-------- src/keri/db/koming.py | 26 +------ src/keri/db/subing.py | 2 +- tests/db/test_dbing.py | 31 +++++--- tests/db/test_koming.py | 4 +- 8 files changed, 101 insertions(+), 221 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index 297dc97e7..b64cf6b67 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -125,7 +125,7 @@ def cloneTopicIter(self, topic, fn=0): if hasattr(topic, 'encode'): topic = topic.encode("utf-8") - for (key, dig) in self.getIoSetItemsIter(self.tpcs, key=topic, ion=fn): + for (key, dig) in self.getIoSetItemIter(self.tpcs, key=topic, ion=fn): topic, ion = dbing.unsuffix(key) if msg := self.msgs.get(keys=dig): yield ion, topic, msg.encode("utf-8") diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index a412c37d4..6b7d16a87 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5527,7 +5527,7 @@ def processEscrowPartialWigs(self): """ key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done + while True: # break when done getPweIoDupItemIter(key=b'') for ekey, edig in self.db.getPweItemsNextIter(key=key): try: pre, sn = splitKeySN(ekey) # get pre and sn from escrow item @@ -5698,151 +5698,7 @@ def processEscrowPartialDels(self): If successful then remove from escrow table """ - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in self.db.getPweItemsNextIter(key=key): - try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) - if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) - - # check date if expired then remove escrow. - dtb = self.db.getDts(dgkey) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) - - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): - # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) - - # get the escrowed event using edig - eraw = self.db.getEvt(dgKey(pre, bytes(edig))) - if eraw is None: - # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) - - eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - - # get sigs - sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs - if not sigs: # empty list - # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) - - # get witness signatures (wigs not wits) - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - - if not wigs: # empty list - # wigs maybe empty if not wits or if wits while waiting - # for first witness signature - # which may not arrive until some time after event is fully signed - # so just log for debugging but do not unescrow by raising - # ValidationError - logger.debug("Kevery unescrow wigs: No event wigs yet at." - "dig = %s", bytes(edig)) - - # raise ValidationError("Missing escrowed evt wigs at " - # "dig = {}.".format(bytes(edig))) - - # process event - sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] - - # seal source (delegator issuer if any) - delseqner = delsaider = None - if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): - delseqner, delsaider = couple - #couple = self.db.getUde(dgKey(pre, bytes(edig))) - #if couple is not None: - #delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - #couple = delseqner.qb64b + delsaider.qb64b - #self.db.putUde(dgkey, couple) - # idempotent - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) - - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) - - # If process does NOT validate wigs then process will attempt - # to re-escrow and then raise MissingWitnessSignatureError - # (subclass of ValidationError) - # so we can distinquish between ValidationErrors that are - # re-escrow vs non re-escrow. We want process to be idempotent - # with respect to processing events that result in escrow items. - # On re-escrow attempt by process, Pwe escrow is called by - # Kever.self.escrowPWEvent Which calls - # self.db.addPwe(snKey(pre, sn), serder.digb) - # which in turn will NOT enter dig as dup if one already exists. - # So re-escrow attempt will not change the escrowed pwe db. - # Non re-escrow ValidationError means some other issue so unescrow. - # No error at all means processed successfully so also unescrow. - # Assumes that controller signature validation and delegation - # validation will be successful as event would not be in - # partially witnessed escrow unless they had already validated - - except (MissingWitnessSignatureError, MissingDelegationError) as ex: - # still waiting on missing witness sigs or delegation - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) - - except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) - - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") - - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey - + pass def processEscrowUnverWitness(self): diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index adda4da95..206506fe0 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2573,7 +2573,7 @@ def getKelIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAllPreIter(self.kels, pre, on=sn) + return self.getIoDupValsAllPreIter(self.kels, pre, on=sn) def getKelBackIter(self, pre, sn=0): @@ -2597,7 +2597,7 @@ def getKelBackIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAllPreBackIter(self.kels, pre, sn) + return self.getIoDupValsAllPreBackIter(self.kels, pre, sn) def getKelLastIter(self, pre, sn=0): @@ -2621,7 +2621,7 @@ def getKelLastIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValLastAllPreIter(self.kels, pre, on=sn) + return self.getIoDupValLastAllPreIter(self.kels, pre, on=sn) def putPses(self, key, vals): @@ -2795,6 +2795,18 @@ def getPweItemsNextIter(self, key=b'', skip=True): """ return self.getIoDupItemsNextIter(self.pwes, key, skip) + def getPweIoDupItemIter(self, key=b''): + """ + Use sgKey() + Return iterator of partial witnessed escrowed event dig items at next key after key. + Items is (key, val) where proem has already been stripped from val + If key is b'' empty then returns dup items at first key. + If skip is False and key is not b'' empty then returns dup items at key + Raises StopIteration Error when empty + Duplicates are retrieved in insertion order. + """ + return self.getIoDupItemsIter(self.pwes, key) + def cntPwes(self, key): """ Use snKey() @@ -3185,7 +3197,7 @@ def getDelIter(self, pre): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAnyPreIter(self.dels, pre) + return self.getIoDupValsAnyPreIter(self.dels, pre) def putLdes(self, key, vals): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 131f6e2fc..9f43f753b 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1156,33 +1156,7 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return False - def getIoSetItems(self, db, key, *, ion=0, sep=b'.'): - """ - Returns: - items (list): list of tuples (iokey, val) of entries in set of with - same apparent effective key. iokey includes the ordinal key suffix - Uses hidden ordinal key suffix for insertion ordering. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - ion (int): starting ordinal value, default 0 - - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - items = [] - iokey = suffix(key, ion, sep=sep) # start ion th value for key zeroth default - cursor = txn.cursor() - if cursor.set_range(iokey): # move to val at key >= iokey if any - for iokey, val in cursor.iternext(): # get iokey, val at cursor - ckey, cion = unsuffix(iokey, sep=sep) - if ckey != key: # prev entry if any was the last entry for key - break # done - items.append((iokey, val)) # another entry at key - return items - - - def getIoSetItemsIter(self, db, key, *, ion=0, sep=b'.'): + def getIoSetItemIter(self, db, key, *, ion=0, sep=b'.'): """ Returns: items (abc.Iterator): iterator over insertion ordered set of values @@ -1754,6 +1728,7 @@ def getIoDupItemsNext(self, db, key=b"", skip=True): return items + def getIoDupItemsNextIter(self, db, key=b"", skip=True): """ Return iterator of all dup items at next key after key in db in insertion order. @@ -1799,7 +1774,59 @@ def getIoDupItemsNextIter(self, db, key=b"", skip=True): yield (key, val[33:]) # slice off prepended ordering prefix - def getIoValsAllPreIter(self, db, pre, on=0): + def getIoDupItemIter(self, db, key=b''): + """ + Iterates over branch of db given by key of IoDup items where each value + has 33 byte insertion ordinal number proem (prefixed) with separator. + Automagically removes (strips) proem before returning items. + + Assumes DB opened with dupsort=True + + Returns: + items (abc.Iterator): iterator of (full key, val) tuples of all + dup items over a branch of the db given by top key where returned + full key is full database key for val not truncated top key. + Item is (key, val) with proem stripped from val stored in db. + If key = b'' then returns list of dup items for all keys in db. + + + Because cursor.iternext() advances cursor after returning item its safe + to delete the item within the iteration loop. curson.iternext() works + for both dupsort==False and dupsort==True + + Raises StopIteration Error when empty. + + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + key (bytes): truncated top key, a key space prefix to get all the items + from multiple branches of the key space. If top key is + empty then gets all items in database + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + if cursor.set_range(key): # move to val at key >= key if any + for ckey, cval in cursor.iternext(): # get key, val first dup at cursor + ckey = bytes(ckey) + if not ckey.startswith(key): # prev entry if any last in branch + break # done + yield (ckey, cval[33:]) # slice off prepended ordering prefix + return # done raises StopIteration + + + def getIoDupValsAllPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries with same prefix across all ordinal numbers in increasing order @@ -1835,7 +1862,7 @@ def getIoValsAllPreIter(self, db, pre, on=0): key = snKey(pre, cnt:=cnt+1) - def getIoValsAllPreBackIter(self, db, pre, on=0): + def getIoDupValsAllPreBackIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries with same prefix across all sequence numbers in decreasing order without gaps @@ -1873,7 +1900,7 @@ def getIoValsAllPreBackIter(self, db, pre, on=0): key = snKey(pre, cnt:=cnt-1) - def getIoValLastAllPreIter(self, db, pre, on=0): + def getIoDupValLastAllPreIter(self, db, pre, on=0): """ Returns iterator of last only of dup vals of each key in insertion order for all entries with same prefix across all sequence numbers in increasing order @@ -1908,7 +1935,7 @@ def getIoValLastAllPreIter(self, db, pre, on=0): key = snKey(pre, cnt:=cnt+1) - def getIoValsAnyPreIter(self, db, pre, on=0): + def getIoDupValsAnyPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for any entries with same prefix across all ordinal numbers in order including gaps diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index ca0459a22..77f0001a6 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -591,30 +591,6 @@ def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def getIoSetItem(self, keys: Union[str, Iterable], *, ion=0): - """ - Gets (iokeys, val) ioitems list at key made from keys where key is - apparent effective key and ioitems all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form key - ion (int): starting ordinal value, default 0 - - Returns: - ioitems (Iterable): each item in list is tuple (iokeys, val) where each - iokeys is actual key tuple including hidden suffix and - each val is str - empty list if no entry at keys - - - """ - return ([(self._tokeys(iokey), self.deserializer(val)) for iokey, val - in self.db.getIoSetItems(db=self.sdb, - key=self._tokey(keys), - ion=ion, - sep=self.sep)]) - - def getIoSetItemIter(self, keys: Union[str, Iterable], *, ion=0): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is @@ -632,7 +608,7 @@ def getIoSetItemIter(self, keys: Union[str, Iterable], *, ion=0): Raises StopIteration when done """ - for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, + for iokey, val in self.db.getIoSetItemIter(db=self.sdb, key=self._tokey(keys), ion=ion, sep=self.sep): diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index fb6edbd62..6615d8d81 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -922,7 +922,7 @@ def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): Raises StopIteration when done """ - for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, + for iokey, val in self.db.getIoSetItemIter(db=self.sdb, key=self._tokey(keys), ion=ion, sep=self.sep): diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index fb7357a91..a12864feb 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -494,7 +494,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoValsAllPreIter(db, pre)] + vals = [bytes(val) for val in dber.getIoDupValsAllPreIter(db, pre)] allvals = vals0 + vals1 + vals2 assert vals == allvals @@ -516,7 +516,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoValLastAllPreIter(db, pre)] + vals = [bytes(val) for val in dber.getIoDupValLastAllPreIter(db, pre)] lastvals = [vals0[-1], vals1[-1], vals2[-1]] assert vals == lastvals @@ -538,7 +538,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoValsAnyPreIter(db, pre)] + vals = [bytes(val) for val in dber.getIoDupValsAnyPreIter(db, pre)] allvals = vals0 + vals1 + vals2 assert vals == allvals @@ -553,11 +553,25 @@ def test_lmdber(): dKey = snKey(pre=b'A', sn=7) dVals = [b"k", b"b"] + + # Test getIoDupItemIter assert dber.putIoDupVals(edb, key=aKey, vals=aVals) assert dber.putIoDupVals(edb, key=bKey, vals=bVals) assert dber.putIoDupVals(edb, key=cKey, vals=cVals) assert dber.putIoDupVals(edb, key=dKey, vals=dVals) + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x'), + (b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z'), + (b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n'), + (b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + # Test getIoItemsNext(self, db, key=b"") # aVals items = dber.getIoDupItemsNext(edb) # get first key in database @@ -769,18 +783,14 @@ def test_lmdber(): assert dber.appendIoSetVal(db, key1, val=b"k") == 4 assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] - assert dber.getIoSetItems(db, key0) == [(b'ABC.ZYX.00000000000000000000000000000000', b'z'), - (b'ABC.ZYX.00000000000000000000000000000001', b'm'), - (b'ABC.ZYX.00000000000000000000000000000002', b'x'), - (b'ABC.ZYX.00000000000000000000000000000003', b'a')] - assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemsIter(db, key0)] == + assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemIter(db, key0)] == [(b'ABC.ZYX.00000000000000000000000000000000', b'z'), (b'ABC.ZYX.00000000000000000000000000000001', b'm'), (b'ABC.ZYX.00000000000000000000000000000002', b'x'), (b'ABC.ZYX.00000000000000000000000000000003', b'a')]) - for iokey, val in dber.getIoSetItemsIter(db, key0): + for iokey, val in dber.getIoSetItemIter(db, key0): assert dber.delIoSetIokey(db, iokey) assert dber.getIoSetVals(db, key0) == [] @@ -811,8 +821,7 @@ def test_lmdber(): dber.cntIoSetVals(db, empty_key) dber.delIoSetVals(db, empty_key) dber.delIoSetVal(db, empty_key, some_value) - dber.getIoSetItems(db, empty_key) - dber.getIoSetItemsIter(db, empty_key) + dber.getIoSetItemIter(db, empty_key) with pytest.raises(KeyError): dber.delIoSetIokey(db, empty_key) with pytest.raises(KeyError): diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index b5877fab6..1359bb280 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -775,7 +775,7 @@ def __iter__(self): i = 0 - for iokeys, end in endDB.getIoSetItem(keys=keys0): + for iokeys, end in endDB.getIoSetItemIter(keys=keys0): assert end == ends[i] assert iokeys == iokeys0[i] i += 1 @@ -787,7 +787,7 @@ def __iter__(self): i += 1 i = 1 - for iokeys, end in endDB.getIoSetItem(keys=keys0, ion=i): + for iokeys, end in endDB.getIoSetItemIter(keys=keys0, ion=i): assert end == ends[i] assert iokeys == iokeys0[i] i += 1 From 695189275d898682f7e44e1c4180ab2988a538c6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 28 Aug 2024 16:35:23 -0600 Subject: [PATCH 283/418] added IoDupSuber with unit tests --- src/keri/core/eventing.py | 254 +++++++++++++++++++------------------- src/keri/db/basing.py | 2 +- src/keri/db/dbing.py | 182 +++++++++++++++++---------- src/keri/db/subing.py | 133 +++++++++++++++++++- tests/db/test_dbing.py | 78 +++++++++++- tests/db/test_subing.py | 70 ++++++----- 6 files changed, 492 insertions(+), 227 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 6b7d16a87..c662095ae 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5526,150 +5526,150 @@ def processEscrowPartialWigs(self): If successful then remove from escrow table """ - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done getPweIoDupItemIter(key=b'') - for ekey, edig in self.db.getPweItemsNextIter(key=key): - try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) - if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) - - # check date if expired then remove escrow. - dtb = self.db.getDts(dgkey) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) - - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): - # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + #key = ekey = b'' # both start same. when not same means escrows found + #while True: # break when done + for ekey, edig in self.db.getPweIoDupItemIter(key=b''): #self.db.getPweItemsNextIter(key=key) + try: + pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) - # get the escrowed event using edig - eraw = self.db.getEvt(dgKey(pre, bytes(edig))) - if eraw is None: - # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) - eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) - # get sigs - sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs - if not sigs: # empty list - # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) + # get the escrowed event using edig + eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) - # get witness signatures (wigs not wits) - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) - if not wigs: # empty list - # wigs maybe empty if not wits or if wits while waiting - # for first witness signature - # which may not arrive until some time after event is fully signed - # so just log for debugging but do not unescrow by raising - # ValidationError - logger.debug("Kevery unescrow wigs: No event wigs yet at." - "dig = %s", bytes(edig)) + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - # raise ValidationError("Missing escrowed evt wigs at " - # "dig = {}.".format(bytes(edig))) + # get sigs + sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs + if not sigs: # empty list + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) - # process event - sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) - # seal source (delegator issuer if any) - delseqner = delsaider = None - if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): - delseqner, delsaider = couple - #couple = self.db.getUde(dgKey(pre, bytes(edig))) - #if couple is not None: - #delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - #couple = delseqner.qb64b + delsaider.qb64b - #self.db.putUde(dgkey, couple) - # idempotent - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + # get witness signatures (wigs not wits) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + if not wigs: # empty list + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) - # If process does NOT validate wigs then process will attempt - # to re-escrow and then raise MissingWitnessSignatureError - # (subclass of ValidationError) - # so we can distinquish between ValidationErrors that are - # re-escrow vs non re-escrow. We want process to be idempotent - # with respect to processing events that result in escrow items. - # On re-escrow attempt by process, Pwe escrow is called by - # Kever.self.escrowPWEvent Which calls - # self.db.addPwe(snKey(pre, sn), serder.digb) - # which in turn will NOT enter dig as dup if one already exists. - # So re-escrow attempt will not change the escrowed pwe db. - # Non re-escrow ValidationError means some other issue so unescrow. - # No error at all means processed successfully so also unescrow. - # Assumes that controller signature validation and delegation - # validation will be successful as event would not be in - # partially witnessed escrow unless they had already validated + # raise ValidationError("Missing escrowed evt wigs at " + # "dig = {}.".format(bytes(edig))) - except (MissingWitnessSignatureError, MissingDelegationError) as ex: - # still waiting on missing witness sigs or delegation - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + # process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] - except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + delseqner, delsaider = couple + #couple = self.db.getUde(dgKey(pre, bytes(edig))) + #if couple is not None: + #delseqner, delsaider = deSourceCouple(couple) + elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + if eserder.pre in self.kevers: + delpre = self.kevers[eserder.pre].delpre else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + delpre = eserder.ked["di"] + seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + #couple = delseqner.qb64b + delsaider.qb64b + #self.db.putUde(dgkey, couple) + # idempotent + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, local=esr.local) + + # If process does NOT validate wigs then process will attempt + # to re-escrow and then raise MissingWitnessSignatureError + # (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pwe escrow is called by + # Kever.self.escrowPWEvent Which calls + # self.db.addPwe(snKey(pre, sn), serder.digb) + # which in turn will NOT enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrowed pwe db. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + # Assumes that controller signature validation and delegation + # validation will be successful as event would not be in + # partially witnessed escrow unless they had already validated + + except (MissingWitnessSignatureError, MissingDelegationError) as ex: + # still waiting on missing witness sigs or delegation + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs or seal so remove from escrow + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") + + #if ekey == key: # still same so no escrows found on last while iteration + #break + #key = ekey # setup next while iteration, with key after ekey def processEscrowPartialDels(self): """ diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 206506fe0..58e320481 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2805,7 +2805,7 @@ def getPweIoDupItemIter(self, key=b''): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsIter(self.pwes, key) + return self.getTopIoDupItemIter(self.pwes, key) def cntPwes(self, key): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 9f43f753b..fbdedee1f 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -601,7 +601,9 @@ def getTopItemIter(self, db, key=b''): db (lmdb._Database): instance of named sub db with dupsort==False key (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is - empty then gets all items in database + empty then gets all items in database. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() @@ -1156,6 +1158,26 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return False + def delIoSetIokey(self, db, iokey): + """ + Deletes val at at actual iokey that includes ordinal key suffix. + + Returns: + result (bool): True if val was deleted at iokey. False otherwise + if no val at iokey + + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + iokey (bytes): actual key with ordinal key suffix + """ + with self.env.begin(db=db, write=True, buffers=True) as txn: + try: + return txn.delete(iokey) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + def getIoSetItemIter(self, db, key, *, ion=0, sep=b'.'): """ Returns: @@ -1185,25 +1207,6 @@ def getIoSetItemIter(self, db, key, *, ion=0, sep=b'.'): return # done raises StopIteration - def delIoSetIokey(self, db, iokey): - """ - Deletes val at at actual iokey that includes ordinal key suffix. - - Returns: - result (bool): True if val was deleted at iokey. False otherwise - if no val at iokey - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - iokey (bytes): actual key with ordinal key suffix - """ - with self.env.begin(db=db, write=True, buffers=True) as txn: - try: - return txn.delete(iokey) - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - # For subdbs that support duplicates at each key (dupsort==True) def putVals(self, db, key, vals): @@ -1496,6 +1499,44 @@ def getIoDupVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + def getIoDupValsIter(self, db, key): + """ + Return iterator of all duplicate values at key in db in insertion order + Raises StopIteration Error when no remaining dup items = empty. + Removes prepended proem ordinal from each val before returning + Assumes DB opened with dupsort=True + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + + + Parameters: + db is opened named sub db with dupsort=True + key is bytes of key within sub db's keyspace + """ + + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + vals = [] + try: + if cursor.set_key(key): # moves to first_dup + for val in cursor.iternext_dup(): + yield val[33:] # slice off prepended ordering proem + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + def getIoDupValLast(self, db, key): """ Return last added dup value at key in db in insertion order @@ -1642,43 +1683,8 @@ def cntIoDupVals(self, db, key): - def getIoDupValsIter(self, db, key): - """ - Return iterator of all duplicate values at key in db in insertion order - Raises StopIteration Error when no remaining dup items = empty. - Removes prepended proem ordinal from each val before returning - Assumes DB opened with dupsort=True - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace - """ - - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - vals = [] - try: - if cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - yield val[33:] # slice off prepended ordering proem - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoDupItemsNext(self, db, key=b"", skip=True): @@ -1774,7 +1780,7 @@ def getIoDupItemsNextIter(self, db, key=b"", skip=True): yield (key, val[33:]) # slice off prepended ordering prefix - def getIoDupItemIter(self, db, key=b''): + def getIoDupItemIter(self, db, key=b'', *, ion=0): """ Iterates over branch of db given by key of IoDup items where each value has 33 byte insertion ordinal number proem (prefixed) with separator. @@ -1783,11 +1789,11 @@ def getIoDupItemIter(self, db, key=b''): Assumes DB opened with dupsort=True Returns: - items (abc.Iterator): iterator of (full key, val) tuples of all - dup items over a branch of the db given by top key where returned - full key is full database key for val not truncated top key. - Item is (key, val) with proem stripped from val stored in db. - If key = b'' then returns list of dup items for all keys in db. + ioitems (Iterator): each item iterated is tuple (keys, ioval) where + each keys is actual keys tuple and each ioval is dup that + includes prefixed insertion ordering proem + empty list if no entry at keys. + Raises StopIteration when done Because cursor.iternext() advances cursor after returning item its safe @@ -1801,6 +1807,9 @@ def getIoDupItemIter(self, db, key=b''): key (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is empty then gets all items in database + ion (int): starting ordinal value, default 0 + + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to @@ -1822,10 +1831,59 @@ def getIoDupItemIter(self, db, key=b''): ckey = bytes(ckey) if not ckey.startswith(key): # prev entry if any last in branch break # done - yield (ckey, cval[33:]) # slice off prepended ordering prefix + cion = int(bytes(cval[:32]), 16) # convert without sep '.' in [33] + if cion < ion: # cion cursor insertion order proem as int + continue # skip dups whose cion is below ion + yield (ckey, cval) # include proem on val return # done raises StopIteration + def getTopIoDupItemIter(self, db, key=b''): + """ + Iterates over top branch of db given by key of IoDup items where each value + has 33 byte insertion ordinal number proem (prefixed) with separator. + Automagically removes (strips) proem before returning items. + + Assumes DB opened with dupsort=True + + Returns: + items (abc.Iterator): iterator of (full key, val) tuples of all + dup items over a branch of the db given by top key where returned + full key is full database key for val not truncated top key. + Item is (key, val) with proem stripped from val stored in db. + If key = b'' then returns list of dup items for all keys in db. + + + Because cursor.iternext() advances cursor after returning item its safe + to delete the item within the iteration loop. curson.iternext() works + for both dupsort==False and dupsort==True + + Raises StopIteration Error when empty. + + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + key (bytes): truncated top key, a key space prefix to get all the items + from multiple branches of the key space. If top key is + empty then gets all items in database + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + for key, val in self.getTopItemIter(db=db, key=key): + val = val[33:] # strip proem + yield (key, val) + + def getIoDupValsAllPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 6615d8d81..2415628df 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -221,6 +221,8 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", If keys is empty then gets all items in database. Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. top (bool): True means treat as partial key tuple from top branch of @@ -261,6 +263,8 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", If keys is empty then gets all items in database. Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. top (bool): True means treat as partial key tuple from top branch of @@ -862,7 +866,7 @@ def rem(self, keys: str | bytes | memoryview | Iterable, if val is empty then remove all values at key Returns: - result (bool): True if effective key with val exists so delete successful. + result (bool): True if effective key with val exists so rem successful. False otherwise """ @@ -950,6 +954,8 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", all items in database. Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. top (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep @@ -1804,14 +1810,14 @@ def pin(self, keys: str | bytes | memoryview | Iterable, if not nonStringIterable(vals): # not iterable vals = (vals, ) # make iterable return self.db.putIoDupVals(db=self.sdb, - key=keys, + key=key, vals=[self._ser(val) for val in vals]) def get(self, keys: str | bytes | memoryview | Iterable): """ - Gets vals set list at key made from keys in insertion order using - hidden ordinal proem. + Gets vals dup list in insertion order using key made from keys and + hidden ordinal proem on dups. Parameters: keys (Iterable): of key strs to be combined in order to form key @@ -1825,6 +1831,24 @@ def get(self, keys: str | bytes | memoryview | Iterable): self.db.getIoDupVals(db=self.sdb, key=self._tokey(keys))]) + def getIter(self, keys: str | bytes | memoryview | Iterable): + """ + Gets vals dup iterator in insertion order using key made from keys and + hidden ordinal proem on dups. + All vals in dups that share same key are retrieved in insertion order. + + Parameters: + keys (str | bytes | memoryview | Iterable): of key parts + + Returns: + vals (Iterator): str values. Raises StopIteration when done + + """ + for val in self.db.getIoDupValsIter(db=self.sdb, + key=self._tokey(keys)): + yield self._des(val) + + def getLast(self, keys: str | bytes | memoryview | Iterable): """ Gets last val inserted at key made from keys in insertion order using @@ -1839,3 +1863,104 @@ def getLast(self, keys: str | bytes | memoryview | Iterable): """ val = self.db.getIoDupValLast(db=self.sdb, key=self._tokey(keys)) return (self._des(val) if val is not None else val) + + + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): + """ + Removes entry at key made from keys and dup val that matches val if any, + notwithstanding hidden ordinal proem. Otherwise deletes all dup values + at key if any. + + Parameters: + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key + + Returns: + result (bool): True if key with dup val exists so rem successful. + False otherwise + + """ + if val: + return self.db.delIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val)) + else: + return self.db.delIoDupVals(db=self.sdb, key=self._tokey(keys)) + + + def cnt(self, keys: str | bytes | memoryview | Iterable): + """ + Return count of dup values at key made from keys with hidden ordinal + proem. Zero otherwise + + Parameters: + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + """ + return (self.db.cntIoDupVals(db=self.sdb, key=self._tokey(keys))) + + + def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): + """ + Gets (iokeys, val) ioitems iterator at key made from keys where key is + apparent effective key and items all have same apparent effective key + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + ion (int): starting ordinal value, default 0 + + Returns: + ioitems (Iterator): each item iterated is tuple (keys, ioval) where + each keys is actual keys tuple and each ioval is dup that + includes prefixed insertion ordering proem. + Empty list if no entry at keys. + Raises StopIteration when done + + """ + for key, ioval in self.db.getIoDupItemIter(db=self.sdb, + key=self._tokey(keys), + ion=ion): + yield (self._tokeys(key), self._des(ioval)) + + + def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", + *, top=False): + """ + Return iterator over all the items including dup items for all keys + in top branch defined by keys where keys may be truncation of full branch. + + Returns: + items (Iterator): of (key, val) tuples over the all the items in + subdb whose key startswith key made from keys and val has its hidden + dup ordinal proem removed. + Keys may be keyspace prefix in order to return branches of key space. + When keys is empty then returns all items in subdb. + + Parameters: + keys (str | bytes | memoryview | Iterable): key or key parts that + may be a truncation of a full keys tuple in in order to address + all the items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + """ + for key, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): + val = val[33:] # strip off proem + yield (self._tokeys(key), self._des(val)) + diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index a12864feb..9680da292 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -554,13 +554,14 @@ def test_lmdber(): dVals = [b"k", b"b"] - # Test getIoDupItemIter + # Test variousItemIter assert dber.putIoDupVals(edb, key=aKey, vals=aVals) assert dber.putIoDupVals(edb, key=bKey, vals=bVals) assert dber.putIoDupVals(edb, key=cKey, vals=cVals) assert dber.putIoDupVals(edb, key=dKey, vals=dVals) - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb)] # default all + # test getTopIoDupItemIter + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb)] # default all assert items == [(b'A.00000000000000000000000000000001', b'z'), (b'A.00000000000000000000000000000001', b'm'), (b'A.00000000000000000000000000000001', b'x'), @@ -572,6 +573,77 @@ def test_lmdber(): (b'A.00000000000000000000000000000007', b'k'), (b'A.00000000000000000000000000000007', b'b')] + # dups at aKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=aKey)] + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x')] + + # dups at bKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=bKey)] + assert items == [(b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z')] + + # dups at cKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=cKey)] + assert items == [(b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n')] + + # dups at dKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=dKey)] + assert items == [(b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + + # dups at missing key + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=b"B.")] + assert not items + + + # test getIoDupItemIter + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000000.z'), + (b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), + (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x'), + (b'A.00000000000000000000000000000002', b'00000000000000000000000000000000.o'), + (b'A.00000000000000000000000000000002', b'00000000000000000000000000000001.r'), + (b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z'), + (b'A.00000000000000000000000000000004', b'00000000000000000000000000000000.h'), + (b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n'), + (b'A.00000000000000000000000000000007', b'00000000000000000000000000000000.k'), + (b'A.00000000000000000000000000000007', b'00000000000000000000000000000001.b')] + + # all keys but ion is non zero + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, ion=1)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), + (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x'), + (b'A.00000000000000000000000000000002', b'00000000000000000000000000000001.r'), + (b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z'), + (b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n'), + (b'A.00000000000000000000000000000007', b'00000000000000000000000000000001.b')] + + # dups at aKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=aKey, ion=1)] + assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), + (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x')] + + # dups at bKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=bKey, ion=2)] + assert items == [(b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z')] + + # dups at cKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=cKey, ion=1)] + assert items ==[(b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n')] + + # dups at dKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=dKey, ion=3)] + assert not items + + # dups at missing key + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=b"B.")] + assert not items + + # Test getIoItemsNext(self, db, key=b"") # aVals items = dber.getIoDupItemsNext(edb) # get first key in database @@ -699,6 +771,8 @@ def test_lmdber(): assert items == [] # empty assert not items + + # test IoSetVals insertion order set of vals methods. key0 = b'ABC.ZYX' key1 = b'DEF.WVU' diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 2244c1002..ffca9aa9e 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -356,9 +356,15 @@ def test_iodup_suber(): assert db.name == "test" assert db.opened - ioduber = subing.IoSetSuber(db=db, subkey='bags.') - assert isinstance(ioduber, subing.IoSetSuber) - assert not ioduber.sdb.flags()["dupsort"] + ioduber = subing.IoDupSuber(db=db, subkey='bags.') + assert isinstance(ioduber, subing.IoDupSuber) + assert ioduber.sdb.flags()["dupsort"] + + sue = "Hello sailer!" + sal = "Not my type." + + keys0 = ("test_key", "0001") + keys1 = ("test_key", "0002") sue = "Hello sailer!" sal = "Not my type." @@ -413,26 +419,28 @@ def test_iodup_suber(): (('test_key', '0002'), 'Hello sailer!'), (('test_key', '0002'), 'A real charmer!')] - - - items = list(ioduber.getFullItemIter()) - assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), - (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), - (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), - (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + assert items == [(('test_key', '0001'), '00000000000000000000000000000000.See ya later.'), + (('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!'), + (('test_key', '0002'), '00000000000000000000000000000000.Not my type.'), + (('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), + (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] - items = [(iokeys, val) for iokeys, val in ioduber.getIoSetItemIter(keys=keys1)] - assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), - (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), '00000000000000000000000000000000.Not my type.'), + (('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), + (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] - items = [(iokeys, val) for iokeys, val in ioduber.getIoSetItemIter(keys=keys0)] - assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), - (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), '00000000000000000000000000000000.See ya later.'), + (('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!')] + items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys1, ion=1)] + assert items ==[(('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), + (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] + items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys0, ion=1)] + assert items == [(('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!')] # Test with top keys assert ioduber.put(keys=("test", "pop"), vals=[sal, sue, sam]) @@ -451,9 +459,9 @@ def test_iodup_suber(): # IoItems items = list(ioduber.getFullItemIter(keys=topkeys)) - assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), - (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] + assert items == [(('test', 'pop'), '00000000000000000000000000000000.Not my type.'), + (('test', 'pop'), '00000000000000000000000000000001.Hello sailer!'), + (('test', 'pop'), '00000000000000000000000000000002.A real charmer!')] # test remove with a specific val assert ioduber.rem(keys=("test_key", "0002"), val=sue) @@ -473,12 +481,8 @@ def test_iodup_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in ioduber.getFullItemIter(): - assert ioduber.remIokey(iokeys=iokeys) - - assert ioduber.cnt(keys=keys0) == 0 - assert ioduber.cnt(keys=keys1) == 0 - + assert ioduber.cnt(keys=keys0) == 2 + assert ioduber.cnt(keys=keys1) == 2 # test with keys as string not tuple keys2 = "keystr" @@ -505,6 +509,12 @@ def test_iodup_suber(): actuals = ioduber.get(keys=keys2) assert actuals == [bil, bob] + # Test trim + assert ioduber.trim() # default trims whole database + assert ioduber.put(keys=keys1, vals=[bob, bil]) + assert ioduber.get(keys=keys1) == [bob, bil] + + assert not os.path.exists(db.path) assert not db.opened @@ -576,9 +586,6 @@ def test_ioset_suber(): (('test_key', '0002'), 'Hello sailer!'), (('test_key', '0002'), 'A real charmer!')] - - - items = list(iosuber.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), @@ -674,7 +681,7 @@ def test_ioset_suber(): actuals = iosuber.get(keys=keys2) assert actuals == [bil, bob] - # ToDo test with append non idempotent + # Test trim and append assert iosuber.trim() # default trims whole database assert iosuber.put(keys=keys1, vals=[bob, bil]) assert iosuber.get(keys=keys1) == [bob, bil] @@ -2035,6 +2042,7 @@ def test_crypt_signer_suber(): if __name__ == "__main__": test_suber() test_dup_suber() + test_iodup_suber() test_ioset_suber() test_cat_suber() test_cesr_suber() From 1ead2a24c54b7287fb4ecd8a1644bc2e527976e6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 28 Aug 2024 16:59:35 -0600 Subject: [PATCH 284/418] added comments todo --- src/keri/db/subing.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 2415628df..df3e8e6e0 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1964,3 +1964,11 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", val = val[33:] # strip off proem yield (self._tokeys(key), self._des(val)) +# ToDo + # these seem like a mixture of OrdSuber and IoDupSuber since ord suber + # has onkey for its key and IoDupSuber has dupsort==True but OrdSuber does + # not. .kels uses these methods + # getIoDupValsAllPreIter(self, db, pre, on=0): + # getIoDupValsAllPreBackIter(self, db, pre, on=0): + # used by .dels but maybe someother method works as well + # getIoDupValsAnyPreIter(self, db, pre, on=0) From f3af009ae6d77187dede3dd08e2cede7bf440ab9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 29 Aug 2024 08:00:20 -0600 Subject: [PATCH 285/418] preliminary refactoring to revise Ordinal Numbered db keys --- src/keri/db/basing.py | 6 +- src/keri/db/dbing.py | 8 +- src/keri/db/subing.py | 204 ++++++++++++++++++++++------------------- src/keri/vdr/viring.py | 4 +- tests/db/test_dbing.py | 30 +++--- 5 files changed, 132 insertions(+), 120 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 58e320481..9278de2f2 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1944,7 +1944,7 @@ def appendFe(self, pre, val): pre is bytes identifier prefix for event val is event digest """ - return self.appendOrdValPre(db=self.fels, pre=pre, val=val) + return self.appendOnValPre(db=self.fels, pre=pre, val=val) def getFelItemPreIter(self, pre, fn=0): """ @@ -1961,7 +1961,7 @@ def getFelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getAllOrdItemPreIter(db=self.fels, pre=pre, on=fn) + return self.getAllOnItemPreIter(db=self.fels, pre=pre, on=fn) def getFelItemAllPreIter(self, key=b''): @@ -1980,7 +1980,7 @@ def getFelItemAllPreIter(self, key=b''): key is key location in db to resume replay, If empty then start at first key in database """ - return self.getAllOrdItemAllPreIter(db=self.fels, key=key) + return self.getAllOnItemAllPreIter(db=self.fels, key=key) def putDts(self, key, val): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index fbdedee1f..71a76e755 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -657,7 +657,7 @@ def delTopVal(self, db, key=b''): # and use keys with suffic ordinal that is monotonically increasing number part # such as fn where no duplicates allowed at a given (pre, on) - def cntAllOrdValsPre(self, db, pre, on=0): + def cntAllOnValsPre(self, db, pre, on=0): """ Returns (int): count of of all ordinal keyed vals with same pre in key but different on in key in db starting at ordinal number on of pre. @@ -687,7 +687,7 @@ def cntAllOrdValsPre(self, db, pre, on=0): return count - def appendOrdValPre(self, db, pre, val): + def appendOnValPre(self, db, pre, val): """ Appends val in order after last previous key with same pre in db. Returns ordinal number in, on, of appended entry. Appended on is 1 greater @@ -737,7 +737,7 @@ def appendOrdValPre(self, db, pre, val): return on - def getAllOrdItemPreIter(self, db, pre, on=0): + def getAllOnItemPreIter(self, db, pre, on=0): """ Returns iterator of duple item, (on, dig), at each key over all ordinal numbered keys with same prefix, pre, in db. Values are sorted by @@ -765,7 +765,7 @@ def getAllOrdItemPreIter(self, db, pre, on=0): yield (cn, val) # (on, dig) of event - def getAllOrdItemAllPreIter(self, db, key=b''): + def getAllOnItemAllPreIter(self, db, key=b''): """ Returns iterator of triple item, (pre, on, dig), at each key over all ordinal numbered keys for all prefixes in db. Values are sorted by diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index df3e8e6e0..3d9c012c0 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -378,98 +378,6 @@ def rem(self, keys: Union[str, Iterable]): -class OrdSuber(Suber): - """ - Subclass of Suber that adds methods for keys with ordinal numbered suffixes. - Each key consistes of pre joined with .sep to ordinal suffix - - """ - - def __init__(self, *pa, **kwa): - """ - Inherited Parameters: - db (dbing.LMDBer): base db - subkey (str): LMDB sub database key - dupsort (bool): True means enable duplicates at each key - False (default) means do not enable duplicates at - each key. Set to False - sep (str): separator to convert keys iterator to key bytes for db key - default is self.Sep == '.' - verify (bool): True means reverify when ._des from db when applicable - False means do not reverify. Default False - """ - super(Suber, self).__init__(*pa, **kwa) - - - def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): - """ - Returns - cnt (int): count of of all ordinal suffix keyed vals with same pre - in key but different on in key in db starting at ordinal number - on of pre where key is formed with onKey(pre,on) - Does not count dups at same on for a given pre, only - unique on at a given pre. - - Parameters: - pre (str | bytes | memoryview): prefix to to be combined with on - to form key - on (int): ordinal number used with onKey(pre,on) to form key. - """ - return (self.db.cntAllOrdValsPre(db=self.sdb, pre=self._tokey(pre), on=on)) - - # appendOrdPre - - #def appendOrdValPre(self, db, pre, val): - #""" - #Appends val in order after last previous key with same pre in db. - #Returns ordinal number in, on, of appended entry. Appended on is 1 greater - #than previous latest on. - #Uses onKey(pre, on) for entries. - - #Append val to end of db entries with same pre but with on incremented by - #1 relative to last preexisting entry at pre. - - #Parameters: - #db is opened named sub db with dupsort=False - #pre is bytes identifier prefix for event - #val is event digest - #""" - - # getAllOrdItemPreIter - #def getAllOrdItemPreIter(self, db, pre, on=0): - #""" - #Returns iterator of duple item, (on, dig), at each key over all ordinal - #numbered keys with same prefix, pre, in db. Values are sorted by - #onKey(pre, on) where on is ordinal number int. - #Returned items are duples of (on, dig) where on is ordinal number int - #and dig is event digest for lookup in .evts sub db. - - #Raises StopIteration Error when empty. - - #Parameters: - #db is opened named sub db with dupsort=False - #pre is bytes of itdentifier prefix - #on is int ordinal number to resume replay - #""" - - - #def getAllOrdItemAllPreIter(self, db, key=b''): - #""" - #Returns iterator of triple item, (pre, on, dig), at each key over all - #ordinal numbered keys for all prefixes in db. Values are sorted by - #onKey(pre, on) where on is ordinal number int. - #Each returned item is triple (pre, on, dig) where pre is identifier prefix, - #on is ordinal number int and dig is event digest for lookup in .evts sub db. - - #Raises StopIteration Error when empty. - - #Parameters: - #db is opened named sub db with dupsort=False - #key is key location in db to resume replay, - #If empty then start at first key in database - #""" - - class CesrSuberBase(SuberBase): """ Sub class of SuberBase where data is CESR encode/decode ducktyped subclass @@ -1965,10 +1873,114 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", yield (self._tokeys(key), self._des(val)) # ToDo - # these seem like a mixture of OrdSuber and IoDupSuber since ord suber - # has onkey for its key and IoDupSuber has dupsort==True but OrdSuber does - # not. .kels uses these methods +# OnIoDupSuber methods instead of pre just use key prefix parts top val so +# can work with any key space that has last part has ordinal +# try refactoring these to use cursor.iternext cursor.iterpre cursor.next cursor.prev +# so they work across prefixes without having the extra burden doing repeated +# iter within a given pre dups. This way they could work with or without duplicates +# and then the IoDupIter could just strip the proem whereas OrdSuber would not care +# OnSuber + #Although these are similar to OrdSuber in that they both have ordinal + # not hidden as last part of keys. OnIoDup suber also has proem that + # must be prefixed and stripped so not able to mixin with each other + # in multiple inheritance + # used by .kels # getIoDupValsAllPreIter(self, db, pre, on=0): # getIoDupValsAllPreBackIter(self, db, pre, on=0): - # used by .dels but maybe someother method works as well + # getIoDupValLastAllPreIter(self.kels, pre, on=sn) + # used by .dels # getIoDupValsAnyPreIter(self, db, pre, on=0) + + + + +class OnSuber(Suber): + """ + Subclass of Suber that adds methods for keys with ordinal numbered suffixes. + Each key consistes of pre joined with .sep to ordinal suffix + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(OnSuber, self).__init__(*pa, **kwa) + + + def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): + """ + Returns + cnt (int): count of of all ordinal suffix keyed vals with same pre + in key but different on in key in db starting at ordinal number + on of pre where key is formed with onKey(pre,on) + Does not count dups at same on for a given pre, only + unique on at a given pre. + + Parameters: + pre (str | bytes | memoryview): prefix to to be combined with on + to form key + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.cntAllOnValsPre(db=self.sdb, pre=self._tokey(pre), on=on)) + + # appendOrdPre + + #def appendOrdValPre(self, db, pre, val): + #""" + #Appends val in order after last previous key with same pre in db. + #Returns ordinal number in, on, of appended entry. Appended on is 1 greater + #than previous latest on. + #Uses onKey(pre, on) for entries. + + #Append val to end of db entries with same pre but with on incremented by + #1 relative to last preexisting entry at pre. + + #Parameters: + #db is opened named sub db with dupsort=False + #pre is bytes identifier prefix for event + #val is event digest + #""" + + # getAllOrdItemPreIter + #def getAllOrdItemPreIter(self, db, pre, on=0): + #""" + #Returns iterator of duple item, (on, dig), at each key over all ordinal + #numbered keys with same prefix, pre, in db. Values are sorted by + #onKey(pre, on) where on is ordinal number int. + #Returned items are duples of (on, dig) where on is ordinal number int + #and dig is event digest for lookup in .evts sub db. + + #Raises StopIteration Error when empty. + + #Parameters: + #db is opened named sub db with dupsort=False + #pre is bytes of itdentifier prefix + #on is int ordinal number to resume replay + #""" + + + #def getAllOrdItemAllPreIter(self, db, key=b''): + #""" + #Returns iterator of triple item, (pre, on, dig), at each key over all + #ordinal numbered keys for all prefixes in db. Values are sorted by + #onKey(pre, on) where on is ordinal number int. + #Each returned item is triple (pre, on, dig) where pre is identifier prefix, + #on is ordinal number int and dig is event digest for lookup in .evts sub db. + + #Raises StopIteration Error when empty. + + #Parameters: + #db is opened named sub db with dupsort=False + #key is key location in db to resume replay, + #If empty then start at first key in database + #""" diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 5a5076e98..ebbf600d8 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -671,7 +671,7 @@ def getTelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getAllOrdItemPreIter(db=self.tels, pre=pre, on=fn) + return self.getAllOnItemPreIter(db=self.tels, pre=pre, on=fn) def cntTels(self, pre, fn=0): """ @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntAllOrdValsPre(db=self.tels, pre=pre, on=fn) + return self.cntAllOnValsPre(db=self.tels, pre=pre, on=fn) def getTibs(self, key): """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 9680da292..7a825ab20 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -322,7 +322,7 @@ def test_lmdber(): # test appendOrdValPre # empty database assert dber.getVal(db, keyB0) == None - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnValPre(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -330,7 +330,7 @@ def test_lmdber(): # earlier pre in database only assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnValPre(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -339,7 +339,7 @@ def test_lmdber(): # earlier and later pre in db but not same pre assert dber.getVal(db, keyA0) == digA assert dber.putVal(db, keyC0, val=digC) == True - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnValPre(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -349,13 +349,13 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None assert dber.getVal(db, keyC0) == digC - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnValPre(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU # earlier pre and later pre and earlier entry for same pre assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOrdValPre(db, preB, digV) + on = dber.appendOnValPre(db, preB, digV) assert on == 1 assert dber.getVal(db, keyB1) == digV @@ -365,49 +365,49 @@ def test_lmdber(): assert dber.delVal(db, keyC0) == True assert dber.getVal(db, keyC0) == None # another value for preB - on = dber.appendOrdValPre(db, preB, digW) + on = dber.appendOnValPre(db, preB, digW) assert on == 2 assert dber.getVal(db, keyB2) == digW # yet another value for preB - on = dber.appendOrdValPre(db, preB, digX) + on = dber.appendOnValPre(db, preB, digX) assert on == 3 assert dber.getVal(db, keyB3) == digX # yet another value for preB - on = dber.appendOrdValPre(db, preB, digY ) + on = dber.appendOnValPre(db, preB, digY ) assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntAllOrdValsPre(db, preB) == 5 + assert dber.cntAllOnValsPre(db, preB) == 5 # replay preB events in database - items = [item for item in dber.getAllOrdItemPreIter(db, preB)] + items = [item for item in dber.getAllOnItemPreIter(db, preB)] assert items == [(0, digU), (1, digV), (2, digW), (3, digX), (4, digY)] # resume replay preB events at on = 3 - items = [item for item in dber.getAllOrdItemPreIter(db, preB, on=3)] + items = [item for item in dber.getAllOnItemPreIter(db, preB, on=3)] assert items == [(3, digX), (4, digY)] # resume replay preB events at on = 5 - items = [item for item in dber.getAllOrdItemPreIter(db, preB, on=5)] + items = [item for item in dber.getAllOnItemPreIter(db, preB, on=5)] assert items == [] # replay all events in database with pre events before and after assert dber.putVal(db, keyA0, val=digA) == True assert dber.putVal(db, keyC0, val=digC) == True - items = [item for item in dber.getAllOrdItemAllPreIter(db)] + items = [item for item in dber.getAllOnItemAllPreIter(db)] assert items == [(preA, 0, digA), (preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY), (preC, 0, digC)] # resume replay all starting at preB on=2 - items = [item for item in dber.getAllOrdItemAllPreIter(db, key=keyB2)] + items = [item for item in dber.getAllOnItemAllPreIter(db, key=keyB2)] assert items == [(preB, 2, digW), (preB, 3, digX), (preB, 4, digY), (preC, 0, digC)] # resume replay all starting at preC on=1 - items = [item for item in dber.getAllOrdItemAllPreIter(db, key=onKey(preC, 1))] + items = [item for item in dber.getAllOnItemAllPreIter(db, key=onKey(preC, 1))] assert items == [] From c86a4c7758baa1e66e00404fc2fc2dde0fb0b4db Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 29 Aug 2024 08:15:33 -0600 Subject: [PATCH 286/418] some comments to remind what to refactor --- src/keri/db/dbing.py | 67 ++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 71a76e755..eacba003e 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -107,7 +107,8 @@ def dtKey(pre, dts): dts = dts.encode("utf-8") # convert str to bytes return (b'%s|%s' % (pre, dts)) - +# ToDo right split so key prefix could be top of key space with more than one +# part def splitKey(key, sep=b'.'): """ Returns duple of pre and either dig or on, sn, fn str or dts datetime str by @@ -656,36 +657,8 @@ def delTopVal(self, db, key=b''): # For subdbs with no duplicate values allowed at each key. (dupsort==False) # and use keys with suffic ordinal that is monotonically increasing number part # such as fn where no duplicates allowed at a given (pre, on) - - def cntAllOnValsPre(self, db, pre, on=0): - """ - Returns (int): count of of all ordinal keyed vals with same pre in key - but different on in key in db starting at ordinal number on of pre. - For db with dupsort=False but ordinal number suffix in each key - - Parameters: - db is opened named sub db - pre is bytes of key within sub db's keyspace pre.on - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest - count = 0 - if not cursor.set_range(key): # moves to val at key >= key - return count # no values end of db - - for key in cursor.iternext(values=False): # get key only at cursor - try: - cpre, cn = splitKeyON(key) - except ValueError as ex: # not splittable key - break - - if cpre != pre: # prev is now the last event for pre - break # done - count = count+1 - - return count - +# ToDo change pre to top so can use in suber for any top branch key space whose last +# part is ordinal number def appendOnValPre(self, db, pre, val): """ @@ -737,6 +710,38 @@ def appendOnValPre(self, db, pre, val): return on + def cntAllOnValsPre(self, db, pre, on=0): + """ + Returns (int): count of of all ordinal keyed vals with same pre in key + but different on in key in db starting at ordinal number on of pre. + For db with dupsort=False but ordinal number suffix in each key + + Parameters: + db is opened named sub db + pre is bytes of key within sub db's keyspace pre.on + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + key = onKey(pre, on) # start replay at this enty 0 is earliest + count = 0 + if not cursor.set_range(key): # moves to val at key >= key + return count # no values end of db + + for key in cursor.iternext(values=False): # get key only at cursor + try: + cpre, cn = splitKeyON(key) + except ValueError as ex: # not splittable key + break + + if cpre != pre: # prev is now the last event for pre + break # done + count = count+1 + + return count + + +#Returned item should be (top, on, val) + def getAllOnItemPreIter(self, db, pre, on=0): """ Returns iterator of duple item, (on, dig), at each key over all ordinal From eeadba63438b552a852aa6e8520e1438ff0cd198 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 29 Aug 2024 11:01:53 -0600 Subject: [PATCH 287/418] refactor paramters to top key to reflect more accurately its usage --- src/keri/app/connecting.py | 2 +- src/keri/db/dbing.py | 26 +++++++++++++------------- src/keri/db/koming.py | 8 ++++---- src/keri/db/subing.py | 16 ++++++++-------- tests/db/test_dbing.py | 7 ++++--- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/keri/app/connecting.py b/src/keri/app/connecting.py index 7ab81dd13..e6a2f545c 100644 --- a/src/keri/app/connecting.py +++ b/src/keri/app/connecting.py @@ -201,7 +201,7 @@ def setImg(self, pre, typ, stream): stream (file): file-like stream of image data """ - self.hby.db.delTopVal(db=self.hby.db.imgs, key=pre.encode("utf-8")) + self.hby.db.delTopVal(db=self.hby.db.imgs, top=pre.encode("utf-8")) key = f"{pre}.content-type".encode("utf-8") self.hby.db.setVal(db=self.hby.db.imgs, key=key, val=typ.encode("utf-8")) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index eacba003e..22100e3bc 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -128,9 +128,9 @@ def splitKey(key, sep=b'.'): else: if hasattr(sep, 'encode'): # make sep match bytes or str sep = sep.encode("utf-8") - splits = key.split(sep) + splits = key.rsplit(sep, 1) if len(splits) != 2: - raise ValueError("Unsplittable key = {}".format(key)) + raise ValueError(f"Unsplittable {key=} at {sep=}.") return tuple(splits) @@ -550,7 +550,7 @@ def cnt(self, db): count += 1 return count - + # Suber subclasses don't need this since they always split keys int tuple to return def getAllItemIter(self, db, key=b'', split=True, sep=b'.'): """ Returns iterator of item duple (key, val), at each key over all @@ -583,9 +583,9 @@ def getAllItemIter(self, db, key=b'', split=True, sep=b'.'): yield tuple(splits) - def getTopItemIter(self, db, key=b''): + def getTopItemIter(self, db, top=b''): """ - Iterates over branch of db given by key + Iterates over branch of db given by top key Returns: items (abc.Iterator): iterator of (full key, val) tuples over a @@ -600,7 +600,7 @@ def getTopItemIter(self, db, key=b''): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items + top (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is empty then gets all items in database. In Python str.startswith('') always returns True so if branch @@ -608,16 +608,16 @@ def getTopItemIter(self, db, key=b''): """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any + if cursor.set_range(top): # move to val at key >= key if any for ckey, cval in cursor.iternext(): # get key, val at cursor ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch + if not ckey.startswith(top): # prev entry if any last in branch break # done yield (ckey, cval) # another entry in branch startswith key return # done raises StopIteration - def delTopVal(self, db, key=b''): + def delTopVal(self, db, top=b''): """ Deletes all values in branch of db given top key. @@ -627,7 +627,7 @@ def delTopVal(self, db, key=b''): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items + top (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is empty then deletes all items in database @@ -643,11 +643,11 @@ def delTopVal(self, db, key=b''): with self.env.begin(db=db, write=True, buffers=True) as txn: result = False cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any + if cursor.set_range(top): # move to val at key >= key if any ckey, cval = cursor.item() while ckey: # end of database key == b'' ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch + if not ckey.startswith(top): # prev entry if any last in branch break # done result = cursor.delete() or result # delete moves cursor to next item ckey, cval = cursor.item() # cursor now at next item after deleted @@ -1884,7 +1884,7 @@ def getTopIoDupItemIter(self, db, key=b''): before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). """ - for key, val in self.getTopItemIter(db=db, key=key): + for key, val in self.getTopItemIter(db=db, top=key): val = val[33:] # strip proem yield (key, val) diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index 77f0001a6..05ac15a25 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -130,7 +130,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): all items in database. """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): yield (self._tokeys(key), self.deserializer(val)) @@ -151,7 +151,7 @@ def getFullItemIter(self, keys: Union[str, Iterable]=b""): all items in database. """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): yield (self._tokeys(key), self.deserializer(val)) @@ -355,7 +355,7 @@ def trim(self, keys: Union[str, Iterable]=b""): Returns: result (bool): True if key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys))) + return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys))) def cntAll(self): @@ -634,7 +634,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): ensure get properly separated top branch key. """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for iokey, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): key, ion = dbing.unsuffix(iokey, sep=self.sep) yield (self._tokeys(key), self.deserializer(val)) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 3d9c012c0..df03c981d 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -198,7 +198,7 @@ def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", Returns: result (bool): True if val at key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys, top=top))) + return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys, top=top))) def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", @@ -235,7 +235,7 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", """ for key, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): + top=self._tokey(keys, top=top)): yield (self._tokeys(key), self._des(val)) @@ -277,7 +277,7 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", """ for key, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): + top=self._tokey(keys, top=top)): yield (self._tokeys(key), self._des(val)) @@ -875,7 +875,7 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", """ for iokey, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): + top=self._tokey(keys, top=top)): key, ion = dbing.unsuffix(iokey, sep=self.sep) yield (self._tokeys(key), self._des(val)) @@ -1054,7 +1054,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): all items in database. """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split yield (ikeys, self.klas(qb64b=bytes(val), @@ -1174,7 +1174,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b"", all items in database. """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split if decrypter: @@ -1412,7 +1412,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): all items in database. """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for iokey, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): yield self._tokeys(iokey), scheming.Schemer(raw=bytes(val)) @@ -1868,7 +1868,7 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", """ for key, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): + top=self._tokey(keys, top=top)): val = val[33:] # strip off proem yield (self._tokeys(key), self._des(val)) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 7a825ab20..cabf2b9a8 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -83,8 +83,7 @@ def test_key_funcs(): with pytest.raises(ValueError): splitKey(pre) - with pytest.raises(ValueError): - splitKey(dgKey(pre, dgKey(pre, dig))) + assert splitKey(dgKey(pre, dgKey(pre, dig))) # rsplit on gets trailing part # memoryview # Bytes @@ -278,7 +277,7 @@ def test_lmdber(): (b'a', b'2', b'wee'), (b'b', b'1', b'woo')] - assert dber.delTopVal(db, key=b"a.") + assert dber.delTopVal(db, top=b"a.") items = [ (key, bytes(val)) for key, val in dber.getTopItemIter(db=db )] assert items == [(b'b.1', b'woo')] @@ -936,4 +935,6 @@ def test_lmdber(): if __name__ == "__main__": test_key_funcs() + test_suffix() test_lmdber() + test_opendatabaser() From a071a72f4c6b5d49d3423367d9480dcdcf66254e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 29 Aug 2024 14:48:29 -0600 Subject: [PATCH 288/418] refactored fnKey snKey splitOnKey etc --- src/keri/app/cli/commands/escrow.py | 8 +-- src/keri/core/eventing.py | 20 +++--- src/keri/db/dbing.py | 102 +++++++++++++++++++--------- src/keri/db/subing.py | 2 +- src/keri/vdr/viring.py | 2 +- tests/core/test_replay.py | 7 ++ tests/db/test_dbing.py | 40 +++++++++-- 7 files changed, 128 insertions(+), 53 deletions(-) diff --git a/src/keri/app/cli/commands/escrow.py b/src/keri/app/cli/commands/escrow.py index 4602a6787..014daf541 100644 --- a/src/keri/app/cli/commands/escrow.py +++ b/src/keri/app/cli/commands/escrow.py @@ -57,7 +57,7 @@ def escrows(tymth, tock=0.0, **opts): key = ekey = b'' # both start same. when not same means escrows found while True: for ekey, edig in hby.db.getOoeItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: oots.append(eventing.loadEvent(hby.db, pre, edig)) @@ -75,7 +75,7 @@ def escrows(tymth, tock=0.0, **opts): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done for ekey, edig in hby.db.getPweItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: pwes.append(eventing.loadEvent(hby.db, pre, edig)) @@ -93,7 +93,7 @@ def escrows(tymth, tock=0.0, **opts): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done for ekey, edig in hby.db.getPseItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: pses.append(eventing.loadEvent(hby.db, pre, edig)) @@ -111,7 +111,7 @@ def escrows(tymth, tock=0.0, **opts): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done for ekey, edig in hby.db.getLdeItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: ldes.append(eventing.loadEvent(hby.db, pre, edig)) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index c662095ae..e7c597683 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -43,7 +43,7 @@ from ..db import basing, dbing from ..db.basing import KeyStateRecord, StateEERecord, OobiRecord -from ..db.dbing import dgKey, snKey, fnKey, splitKeySN, splitKey +from ..db.dbing import dgKey, snKey, fnKey, splitSnKey, splitKey logger = help.ogler.getLogger() @@ -3929,7 +3929,7 @@ def processAttachedReceiptCouples(self, serder, cigars, firner=None, local=None) # Only accept receipt if event is latest event at sn. Means its been # first seen and is the most recent first seen with that sn if firner: - ldig = self.db.getFe(key=fnKey(pre=pre, sn=firner.sn)) + ldig = self.db.getFe(key=fnKey(pre=pre, fn=firner.sn)) else: ldig = self.db.getKeLast(key=snKey(pre=pre, sn=sn)) # retrieve dig of last event at sn. @@ -4106,7 +4106,7 @@ def processAttachedReceiptQuadruples(self, serder, trqs, firner=None, local=None sn = serder.sn if firner: # retrieve last event by fn ordinal - ldig = self.db.getFe(key=fnKey(pre=pre, sn=firner.sn)) + ldig = self.db.getFe(key=fnKey(pre=pre, fn=firner.sn)) else: # Only accept receipt if for last seen version of receipted event at sn ldig = self.db.getKeLast(key=snKey(pre=pre, sn=sn)) # retrieve dig of last event at sn. @@ -5211,7 +5211,7 @@ def processEscrowOutOfOrders(self): while True: # break when done for ekey, edig in self.db.getOoeItemsNextIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below @@ -5350,7 +5350,7 @@ def processEscrowPartialSigs(self): for ekey, edig in self.db.getPseItemsNextIter(key=key): eserder = None try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below @@ -5530,7 +5530,7 @@ def processEscrowPartialWigs(self): #while True: # break when done for ekey, edig in self.db.getPweIoDupItemIter(key=b''): #self.db.getPweItemsNextIter(key=key) try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below @@ -5755,7 +5755,7 @@ def processEscrowUnverWitness(self): while True: # break when done for ekey, ecouple in self.db.getUweItemsNextIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow db key + pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key # get escrowed receipt's rdiger of receipted event and # wiger indexed signature of receipted event @@ -5875,7 +5875,7 @@ def processEscrowUnverNonTrans(self): while True: # break when done for ekey, etriplet in self.db.getUreItemsNextIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item rsaider, sprefixer, cigar = deReceiptTriple(etriplet) cigar.verfer = Verfer(qb64b=sprefixer.qb64b) @@ -6388,7 +6388,7 @@ def processEscrowUnverTrans(self): while True: # break when done for ekey, equinlet in self.db.getVreItemsNextIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) # check date if expired then remove escrow. @@ -6558,7 +6558,7 @@ def processEscrowDuplicitous(self): while True: # break when done for ekey, edig in self.db.getLdeItemsNextIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 22100e3bc..6420af796 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -65,6 +65,51 @@ SuffixSize = 32 # does not include trailing separator MaxSuffix = int("f"*(SuffixSize), 16) +def onKey(top, on, *, sep=b'.'): + """ + Returns: + onkey (bytes): key formed by joining top key and hex str conversion of + int ordinal number on with sep character. + + Parameters: + top (str | bytes): top key prefix to be joined with hex version of on using sep + on (int): ordinal number to be converted to 32 hex bytes + sep (bytes): separator character for join + + """ + if hasattr(top, "encode"): + top = top.encode("utf-8") # convert str to bytes + return (b'%s%s%032x' % (top, sep, on)) + + +def snKey(pre, sn): + """ + Returns: + snkey (bytes): key formed by joining pre and hex str conversion of int + sequence ordinal number sn with sep character b".". + + Parameters: + pre (str | bytes): key prefix to be joined with hex version of on using + b"." sep + sn (int): sequence number to be converted to 32 hex bytes + """ + + return onKey(pre, sn, sep=b'.') + +def fnKey(pre, fn): + """ + Returns: + fnkey (bytes): key formed by joining pre and hex str conversion of int + first seen ordinal number fn with sep character b".". + + Parameters: + pre (str | bytes): key prefix to be joined with hex version of on using + b"." sep + fn (int): first seen ordinal number to be converted to 32 hex bytes + """ + return onKey(pre, fn, sep=b'.') + + def dgKey(pre, dig): """ Returns bytes DB key from concatenation of '.' with qualified Base64 prefix @@ -78,20 +123,6 @@ def dgKey(pre, dig): return (b'%s.%s' % (pre, dig)) -def onKey(pre, sn): - """ - Returns bytes DB key from concatenation with '.' of qualified Base64 prefix - bytes pre and int ordinal number of event, such as sequence number or first - seen order number. - """ - if hasattr(pre, "encode"): - pre = pre.encode("utf-8") # convert str to bytes - return (b'%s.%032x' % (pre, sn)) - -snKey = onKey # alias so intent is clear, sn vs fn -fnKey = onKey # alias so intent is clear, sn vs fn - - def dtKey(pre, dts): """ Returns bytes DB key from concatenation of '|' qualified Base64 prefix @@ -134,7 +165,7 @@ def splitKey(key, sep=b'.'): return tuple(splits) -def splitKeyON(key): +def splitOnKey(key, *, sep=b'.'): """ Returns list of pre and int on from key Accepts either bytes or str key @@ -142,12 +173,17 @@ def splitKeyON(key): """ if isinstance(key, memoryview): key = bytes(key) - pre, on = splitKey(key) + top, on = splitKey(key, sep=sep) on = int(on, 16) - return (pre, on) + return (top, on) -splitKeySN = splitKeyON # alias so intent is clear, sn vs fn -splitKeyFN = splitKeyON # alias so intent is clear, sn vs fn + +splitSnKey = splitOnKey # alias so intent is clear, sn vs fn +splitFnKey = splitOnKey # alias so intent is clear, sn vs fn + +splitKeyON = splitOnKey # backwards compatible alias +splitKeySN = splitSnKey # backwards compatible alias +splitKeyFN = splitFnKey # backwards compatible alias def splitKeyDT(key): @@ -686,12 +722,12 @@ def appendOnValPre(self, db, pre, val): # last is last entry at same pre if cursor.last(): # not empty db. last entry earlier than max ckey = cursor.key() - cpre, cn = splitKeyON(ckey) + cpre, cn = splitOnKey(ckey) if cpre == pre: # last is last entry for same pre on = cn + 1 # increment else: # not past end so not empty either later pre or max entry at pre ckey = cursor.key() - cpre, cn = splitKeyON(ckey) + cpre, cn = splitOnKey(ckey) if cpre == pre: # last entry for pre is already at max raise ValueError("Number part of key {} exceeds maximum" " size.".format(ckey)) @@ -699,7 +735,7 @@ def appendOnValPre(self, db, pre, val): # either no entry before last or earlier pre with entry if cursor.prev(): # prev entry, maybe same or earlier pre ckey = cursor.key() - cpre, cn = splitKeyON(ckey) + cpre, cn = splitOnKey(ckey) if cpre == pre: # last entry at pre on = cn + 1 # increment @@ -710,30 +746,32 @@ def appendOnValPre(self, db, pre, val): return on - def cntAllOnValsPre(self, db, pre, on=0): + def cntAllOnValsPre(self, db, top, on=0, *, sep=b'.'): """ - Returns (int): count of of all ordinal keyed vals with same pre in key - but different on in key in db starting at ordinal number on of pre. - For db with dupsort=False but ordinal number suffix in each key + Returns (int): count of of all ordinal keyed vals with top key + but different on tail in db starting at ordinal number on of top. + Full key is composed of top+sep+ + When dupsort==true then duplicates are included in count Parameters: db is opened named sub db - pre is bytes of key within sub db's keyspace pre.on + top (bytes): top key within sub db's keyspace with trailing part on + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest + key = onKey(top, on) # start replay at this enty 0 is earliest count = 0 if not cursor.set_range(key): # moves to val at key >= key return count # no values end of db for key in cursor.iternext(values=False): # get key only at cursor try: - cpre, cn = splitKeyON(key) + cpre, cn = splitOnKey(key) except ValueError as ex: # not splittable key break - if cpre != pre: # prev is now the last event for pre + if cpre != top: # prev is now the last event for pre break # done count = count+1 @@ -764,7 +802,7 @@ def getAllOnItemPreIter(self, db, pre, on=0): return # no values end of db for key, val in cursor.iternext(): # get key, val at cursor - cpre, cn = splitKeyON(key) + cpre, cn = splitOnKey(key) if cpre != pre: # prev is now the last event for pre break # done yield (cn, val) # (on, dig) of event @@ -791,7 +829,7 @@ def getAllOnItemAllPreIter(self, db, key=b''): return # no values end of db for key, val in cursor.iternext(): # return key, val at cursor - cpre, cn = splitKeyON(key) + cpre, cn = splitOnKey(key) yield (cpre, cn, val) # (pre, on, dig) of event diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index df03c981d..bd2c44632 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1931,7 +1931,7 @@ def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): to form key on (int): ordinal number used with onKey(pre,on) to form key. """ - return (self.db.cntAllOnValsPre(db=self.sdb, pre=self._tokey(pre), on=on)) + return (self.db.cntAllOnValsPre(db=self.sdb, top=self._tokey(pre), on=on)) # appendOrdPre diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index ebbf600d8..a6489cee2 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntAllOnValsPre(db=self.tels, pre=pre, on=fn) + return self.cntAllOnValsPre(db=self.tels, top=pre, on=fn) def getTibs(self, key): """ diff --git a/tests/core/test_replay.py b/tests/core/test_replay.py index 70b5ecf02..f2aed2cba 100644 --- a/tests/core/test_replay.py +++ b/tests/core/test_replay.py @@ -232,6 +232,7 @@ def test_replay(): # get disjoints receipts (vrcs) from Deb of Cam's events by processing Deb's cues debCamVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 assert debCamVrcs == (b'{"v":"KERI10JSON000091_","t":"rct","d":"EBp-SQb9fTgeoQkIkOd2xegv' b'Xy3epjOskiPrf6JDIEuj","i":"EBp-SQb9fTgeoQkIkOd2xegvXy3epjOskiPrf' b'6JDIEuj","s":"0"}-FABELfp9ZhqQCGov3wPRLa6vn5VkIQjug2sb2QD17T-TIp' @@ -259,6 +260,7 @@ def test_replay(): # get disjoints receipts (rcts) from Bev of Deb's events by processing Bevs's cues bevMsgs = bevHab.processCues(bevKevery.cues) + assert len(bevKevery.cues) == 0 assert bevMsgs == (b'{"v":"KERI10JSON0000fd_","t":"icp","d":"EBXqe7Xzsw2aolT09Ouh5Zw9' b'kNn2sgoHmo4zCn7Q7ZSC","i":"BAqph4mAWcf7mkIgk1Xrpvr7dWT7YvHIam_hq' b'UAT2rqw","s":"0","kt":"1","k":["BAqph4mAWcf7mkIgk1Xrpvr7dWT7YvHI' @@ -306,6 +308,7 @@ def test_replay(): # get disjoints receipts (vrcs) from Deb of Bev's events by processing Deb's cues debBevVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 assert debBevVrcs == (b'{"v":"KERI10JSON000091_","t":"rct","d":"EBXqe7Xzsw2aolT09Ouh5Zw9' b'kNn2sgoHmo4zCn7Q7ZSC","i":"BAqph4mAWcf7mkIgk1Xrpvr7dWT7YvHIam_hq' b'UAT2rqw","s":"0"}-FABELfp9ZhqQCGov3wPRLa6vn5VkIQjug2sb2QD17T-TIp' @@ -552,6 +555,7 @@ def test_replay_all(): # get disjoints receipts (vrcs) from Cam of Deb's events by processing Cam's cues camMsgs = camHab.processCues(camKevery.cues) + assert len(camKevery.cues) == 0 # Play camMsgs to Deb # create non-local kevery for Deb to process msgs from Cam @@ -566,6 +570,7 @@ def test_replay_all(): # get disjoints receipts (vrcs) from Deb of Cam's events by processing Deb's cues debCamVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 # Play disjoints debCamVrcs to Cam parsing.Parser().parseOne(ims=bytearray(debCamVrcs), kvy=camKevery) @@ -584,6 +589,7 @@ def test_replay_all(): # get disjoints receipts (rcts) from Bev of Deb's events by processing Bevs's cues bevMsgs = bevHab.processCues(bevKevery.cues) + assert len(bevKevery.cues) == 0 # Play bevMsgs to Deb parsing.Parser().parse(ims=bytearray(bevMsgs), kvy=debKevery) @@ -594,6 +600,7 @@ def test_replay_all(): # get disjoints receipts (vrcs) from Deb of Bev's events by processing Deb's cues debBevVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 # Play disjoints debBevVrcs to Bev parsing.Parser().parseOne(ims=bytearray(debBevVrcs), kvy=bevKevery) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index cabf2b9a8..3d24c7a16 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -17,7 +17,7 @@ from keri.db import dbing from keri.db.dbing import clearDatabaserDir, openLMDB from keri.db.dbing import (dgKey, onKey, fnKey, snKey, dtKey, splitKey, - splitKeyON, splitKeyFN, splitKeySN, splitKeyDT) + splitOnKey, splitKeyFN, splitSnKey, splitKeyDT) from keri.db.dbing import LMDBer @@ -35,11 +35,41 @@ def test_key_funcs(): sn = 3 dts = b'2021-02-13T19:16:50.750302+00:00' + + # test onKey generator of key from top key and trailing ordinal number + assert onKey(pre, 0) == pre + b'.' + b"%032x" % 0 + assert onKey(pre, 1) == pre + b'.' + b"%032x" % 1 + assert onKey(pre, 2) == pre + b'.' + b"%032x" % 2 + assert onKey(pre, 3) == pre + b'.' + b"%032x" % 3 + assert onKey(pre, 4) == pre + b'.' + b"%032x" % 4 + + assert onKey(pre, 0, sep=b'|') == pre + b'|' + b"%032x" % 0 + assert onKey(pre, 4, sep=b'|') == pre + b'|' + b"%032x" % 4 + + assert (onkey := onKey(top=pre, on=0)) == pre + b'.' + b"%032x" % 0 + assert splitKey(key=onkey) == (pre, b"%032x" % 0) + assert splitOnKey(onkey) == (pre, 0) + assert (onkey := onKey(top=pre, on=1)) == pre + b'.' + b"%032x" % 1 + assert splitKey(key=onkey) == (pre, b"%032x" % 1) + assert splitOnKey(onkey) == (pre, 1) + assert (onkey := onKey(top=pre, on=15)) == pre + b'.' + b"%032x" % 15 + assert splitKey(key=onkey) == (pre, b"%032x" % 15) + assert splitOnKey(onkey) == (pre, 15) + + assert (onkey := onKey(top=pre, on=0, sep=b'|')) == pre + b'|' + b"%032x" % 0 + assert splitKey(key=onkey, sep=b'|') == (pre, b"%032x" % 0) + assert splitOnKey(onkey, sep=b'|') == (pre, 0) + assert (onkey := onKey(top=pre, on=15, sep=b'|')) == pre + b'|' + b"%032x" % 15 + assert splitKey(key=onkey, sep=b'|') == (pre, b"%032x" % 15) + assert splitOnKey(onkey, sep=b'|') == (pre, 15) + + + # test snKey assert snKey(pre, sn) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.00000000000000000000000000000003') assert splitKey(snKey(pre, sn)) == (pre, b'%032x' % sn) - assert splitKeySN(snKey(pre, sn)) == (pre, sn) + assert splitSnKey(snKey(pre, sn)) == (pre, sn) assert dgKey(pre, dig) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') @@ -63,7 +93,7 @@ def test_key_funcs(): b'.00000000000000000000000000000003') assert splitKey(snKey(pre, sn).decode("utf-8")) == (pre, '%032x' % sn) - assert splitKeySN(snKey(pre, sn).decode("utf-8")) == (pre, sn) + assert splitSnKey(snKey(pre, sn).decode("utf-8")) == (pre, sn) assert dgKey(pre, dig) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') @@ -94,7 +124,7 @@ def test_key_funcs(): key = memoryview(snKey(pre, sn)) assert splitKey(key) == (pre, b'%032x' % sn) - assert splitKeySN(key) == (pre, sn) + assert splitSnKey(key) == (pre, sn) key = memoryview(dgKey(pre, dig)) assert splitKey(key) == (pre, dig) @@ -376,7 +406,7 @@ def test_lmdber(): assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntAllOnValsPre(db, preB) == 5 + assert dber.cntAllOnValsPre(db, top=preB) == 5 # replay preB events in database items = [item for item in dber.getAllOnItemPreIter(db, preB)] From 30d0ac3f0edee00dac8949f75f03dc4965664563 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 30 Aug 2024 17:03:28 -0600 Subject: [PATCH 289/418] refactoring of ordinal db operations --- src/keri/db/basing.py | 4 +- src/keri/db/dbing.py | 125 +++++++++++++++++++++------------------ src/keri/db/subing.py | 2 +- src/keri/vdr/eventing.py | 2 +- src/keri/vdr/viring.py | 6 +- tests/db/test_basing.py | 4 +- tests/db/test_dbing.py | 13 ++-- tests/vdr/test_viring.py | 2 +- 8 files changed, 83 insertions(+), 75 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 9278de2f2..3218d2d3b 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1535,7 +1535,7 @@ def clonePreIter(self, pre, fn=0): if hasattr(pre, 'encode'): pre = pre.encode("utf-8") - for fn, dig in self.getFelItemPreIter(pre, fn=fn): + for _, fn, dig in self.getFelItemPreIter(pre, fn=fn): try: msg = self.cloneEvtMsg(pre=pre, fn=fn, dig=dig) except Exception: @@ -1961,7 +1961,7 @@ def getFelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getAllOnItemPreIter(db=self.fels, pre=pre, on=fn) + return self.getTopOnItemIter(db=self.fels, top=pre, on=fn) def getFelItemAllPreIter(self, key=b''): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 6420af796..e6c8180fd 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -93,9 +93,9 @@ def snKey(pre, sn): b"." sep sn (int): sequence number to be converted to 32 hex bytes """ - return onKey(pre, sn, sep=b'.') + def fnKey(pre, fn): """ Returns: @@ -696,6 +696,71 @@ def delTopVal(self, db, top=b''): # ToDo change pre to top so can use in suber for any top branch key space whose last # part is ordinal number + + def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): + """ + Returns (int): count of of all ordinal keyed vals with top key + but different on tail in db starting at ordinal number on of top. + Full key is composed of top+sep+ + When dupsort==true then duplicates are included in count since .iternext + includes duplicates. + + Parameters: + db (lmdbsubdb): named sub db of lmdb + top (bytes): top key within sub db's keyspace with trailing part on + on (int): ordinal number at which to initiate count + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest + count = 0 + if not cursor.set_range(key): # moves to val at key >= key + return count # no values end of db + + for key in cursor.iternext(values=False): # get key only at cursor + try: + cpre, cn = splitOnKey(key, sep=sep) + except ValueError as ex: # not splittable key + break + + if cpre != top: # prev is now the last event for pre + break # done + count = count+1 + + return count + + + def getTopOnItemIter(self, db, top, on=0, *, sep=b'.'): + """ + Returns iterator of triples (top, on, val), at each key over all ordinal + numbered keys with same top keyin db. Values are sorted by + onKey(top, on) where on is ordinal number int. + Returned items are triples of (top, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + + Raises StopIteration Error when empty. + + Parameters: + db (subdb): named sub db in lmdb + top (bytes): top key within sub db's keyspace with trailing part on + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest + if not cursor.set_range(key): # moves to val at key >= key + return # no values end of db + + for key, val in cursor.iternext(): # get key, val at cursor + cpre, cn = splitOnKey(key, sep=sep) + if cpre != top: # prev is now the last event for pre + break # done + yield (top, cn, val) + + def appendOnValPre(self, db, pre, val): """ Appends val in order after last previous key with same pre in db. @@ -746,66 +811,8 @@ def appendOnValPre(self, db, pre, val): return on - def cntAllOnValsPre(self, db, top, on=0, *, sep=b'.'): - """ - Returns (int): count of of all ordinal keyed vals with top key - but different on tail in db starting at ordinal number on of top. - Full key is composed of top+sep+ - When dupsort==true then duplicates are included in count - Parameters: - db is opened named sub db - top (bytes): top key within sub db's keyspace with trailing part on - sep (bytes): separator character for split - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = onKey(top, on) # start replay at this enty 0 is earliest - count = 0 - if not cursor.set_range(key): # moves to val at key >= key - return count # no values end of db - for key in cursor.iternext(values=False): # get key only at cursor - try: - cpre, cn = splitOnKey(key) - except ValueError as ex: # not splittable key - break - - if cpre != top: # prev is now the last event for pre - break # done - count = count+1 - - return count - - -#Returned item should be (top, on, val) - - def getAllOnItemPreIter(self, db, pre, on=0): - """ - Returns iterator of duple item, (on, dig), at each key over all ordinal - numbered keys with same prefix, pre, in db. Values are sorted by - onKey(pre, on) where on is ordinal number int. - Returned items are duples of (on, dig) where on is ordinal number int - and dig is event digest for lookup in .evts sub db. - - Raises StopIteration Error when empty. - - Parameters: - db is opened named sub db with dupsort=False - pre is bytes of itdentifier prefix - on is int ordinal number to resume replay - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest - if not cursor.set_range(key): # moves to val at key >= key - return # no values end of db - - for key, val in cursor.iternext(): # get key, val at cursor - cpre, cn = splitOnKey(key) - if cpre != pre: # prev is now the last event for pre - break # done - yield (cn, val) # (on, dig) of event def getAllOnItemAllPreIter(self, db, key=b''): diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index bd2c44632..6866f1b03 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1931,7 +1931,7 @@ def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): to form key on (int): ordinal number used with onKey(pre,on) to form key. """ - return (self.db.cntAllOnValsPre(db=self.sdb, top=self._tokey(pre), on=on)) + return (self.db.cntTopOnVals(db=self.sdb, top=self._tokey(pre), on=on)) # appendOrdPre diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 0d46f9f16..2cc89a235 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -1172,7 +1172,7 @@ def vcState(self, vci): status (Serder): transaction event state notification message """ digs = [] - for _, dig in self.reger.getTelItemPreIter(pre=vci.encode("utf-8")): + for _, _, dig in self.reger.getTelItemPreIter(pre=vci.encode("utf-8")): digs.append(dig) if len(digs) == 0: diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index a6489cee2..3314ecc0d 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -508,7 +508,7 @@ def clonePreIter(self, pre, fn=0): if hasattr(pre, 'encode'): pre = pre.encode("utf-8") - for fn, dig in self.getTelItemPreIter(pre, fn=fn): + for _, fn, dig in self.getTelItemPreIter(pre, fn=fn): msg = self.cloneTvt(pre, dig) yield msg @@ -671,7 +671,7 @@ def getTelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getAllOnItemPreIter(db=self.tels, pre=pre, on=fn) + return self.getTopOnItemIter(db=self.tels, top=pre, on=fn) def cntTels(self, pre, fn=0): """ @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntAllOnValsPre(db=self.tels, top=pre, on=fn) + return self.cntTopOnVals(db=self.tels, top=pre, on=fn) def getTibs(self, key): """ diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index a3d8b98b9..825c039de 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -282,11 +282,11 @@ def test_baser(): # replay preB events in database items = [item for item in db.getFelItemPreIter(preB)] - assert items == [(0, digU), (1, digV), (2, digW), (3, digX), (4, digY)] + assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 3 items = [item for item in db.getFelItemPreIter(preB, fn=3)] - assert items == [(3, digX), (4, digY)] + assert items == [(preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 5 items = [item for item in db.getFelItemPreIter(preB, fn=5)] diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 3d24c7a16..e6ff17bf7 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -406,18 +406,19 @@ def test_lmdber(): assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntAllOnValsPre(db, top=preB) == 5 + assert dber.cntTopOnVals(db, top=preB) == 5 # replay preB events in database - items = [item for item in dber.getAllOnItemPreIter(db, preB)] - assert items == [(0, digU), (1, digV), (2, digW), (3, digX), (4, digY)] + items = [item for item in dber.getTopOnItemIter(db, preB)] + assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), + (preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 3 - items = [item for item in dber.getAllOnItemPreIter(db, preB, on=3)] - assert items == [(3, digX), (4, digY)] + items = [item for item in dber.getTopOnItemIter(db, preB, on=3)] + assert items == [(preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 5 - items = [item for item in dber.getAllOnItemPreIter(db, preB, on=5)] + items = [item for item in dber.getTopOnItemIter(db, preB, on=5)] assert items == [] # replay all events in database with pre events before and after diff --git a/tests/vdr/test_viring.py b/tests/vdr/test_viring.py index 8ec90e476..2e98dda6c 100644 --- a/tests/vdr/test_viring.py +++ b/tests/vdr/test_viring.py @@ -231,7 +231,7 @@ def test_issuer(): assert issuer.putTel(snKey(vcdig, sn + 2), val=idig.qb64b) is True assert issuer.putTel(snKey(vcdig, sn + 3), val=rdig.qb64b) is True - result = [(sn, dig) for sn, dig in issuer.getTelItemPreIter(vcdig)] + result = [(sn, dig) for _, sn, dig in issuer.getTelItemPreIter(vcdig)] assert result == [(0, idig.qb64b), (1, rdig.qb64b), (2, idig.qb64b), (3, rdig.qb64b)] bak1 = b'BA1Q98kT0HRn9R62lY-LufjjKdbCeL1mqu9arTgOmbqI' From f8d42c3d69d888e7ab175b96baa81fe8adb68446 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 30 Aug 2024 19:45:08 -0600 Subject: [PATCH 290/418] some more refactoring --- src/keri/db/dbing.py | 164 +++++++++++++++++++++---------------------- 1 file changed, 81 insertions(+), 83 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index e6c8180fd..c09a22f2e 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -733,10 +733,10 @@ def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): def getTopOnItemIter(self, db, top, on=0, *, sep=b'.'): """ - Returns iterator of triples (top, on, val), at each key over all ordinal + Returns iterator of triples (key, on, val), at each key over all ordinal numbered keys with same top keyin db. Values are sorted by - onKey(top, on) where on is ordinal number int. - Returned items are triples of (top, on, val) + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Returned items are triples of (key, on, val) When dupsort==true then duplicates are included in items since .iternext includes duplicates. @@ -750,15 +750,47 @@ def getTopOnItemIter(self, db, top, on=0, *, sep=b'.'): """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest + if top: # not empty + key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest + else: # empty top + key = top # get all values empty key is start of db if not cursor.set_range(key): # moves to val at key >= key return # no values end of db for key, val in cursor.iternext(): # get key, val at cursor - cpre, cn = splitOnKey(key, sep=sep) - if cpre != top: # prev is now the last event for pre - break # done - yield (top, cn, val) + ckey, cn = splitOnKey(key, sep=sep) + if not ckey.startswith(top): # prev is now the last event for pre + break # done + #if ckey != top: # prev is now the last event for pre + #break # done + yield (ckey, cn, val) + + + def getAllOnItemAllPreIter(self, db, key=b''): + """ + Returns iterator of triple item, (pre, on, dig), at each key over all + ordinal numbered keys for all prefixes in db. Values are sorted by + onKey(pre, on) where on is ordinal number int. + Each returned item is triple (pre, on, dig) where pre is identifier prefix, + on is ordinal number int and dig is event digest for lookup in .evts sub db. + + Raises StopIteration Error when empty. + + Parameters: + db is opened named sub db with dupsort=False + key is key location in db to resume replay, + If empty then start at first key in database + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + if not cursor.set_range(key): # moves to val at key >= key, first if empty + return # no values end of db + + for key, val in cursor.iternext(): # return key, val at cursor + cpre, cn = splitOnKey(key) + yield (cpre, cn, val) # (pre, on, dig) of event + + def appendOnValPre(self, db, pre, val): @@ -812,35 +844,6 @@ def appendOnValPre(self, db, pre, val): - - - - def getAllOnItemAllPreIter(self, db, key=b''): - """ - Returns iterator of triple item, (pre, on, dig), at each key over all - ordinal numbered keys for all prefixes in db. Values are sorted by - onKey(pre, on) where on is ordinal number int. - Each returned item is triple (pre, on, dig) where pre is identifier prefix, - on is ordinal number int and dig is event digest for lookup in .evts sub db. - - Raises StopIteration Error when empty. - - Parameters: - db is opened named sub db with dupsort=False - key is key location in db to resume replay, - If empty then start at first key in database - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if not cursor.set_range(key): # moves to val at key >= key, first if empty - return # no values end of db - - for key, val in cursor.iternext(): # return key, val at cursor - cpre, cn = splitOnKey(key) - yield (cpre, cn, val) # (pre, on, dig) of event - - - # IoSet insertion order in val so can have effective dups but with # dupsort = False so val not limited to 511 bytes # For databases that support set of insertion ordered values with apparent @@ -1732,9 +1735,50 @@ def cntIoDupVals(self, db, key): return count + def getTopIoDupItemIter(self, db, key=b''): + """ + Iterates over top branch of db given by key of IoDup items where each value + has 33 byte insertion ordinal number proem (prefixed) with separator. + Automagically removes (strips) proem before returning items. + + Assumes DB opened with dupsort=True + + Returns: + items (abc.Iterator): iterator of (full key, val) tuples of all + dup items over a branch of the db given by top key where returned + full key is full database key for val not truncated top key. + Item is (key, val) with proem stripped from val stored in db. + If key = b'' then returns list of dup items for all keys in db. + + + Because cursor.iternext() advances cursor after returning item its safe + to delete the item within the iteration loop. curson.iternext() works + for both dupsort==False and dupsort==True + Raises StopIteration Error when empty. + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + key (bytes): truncated top key, a key space prefix to get all the items + from multiple branches of the key space. If top key is + empty then gets all items in database + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + for key, val in self.getTopItemIter(db=db, top=key): + val = val[33:] # strip proem + yield (key, val) def getIoDupItemsNext(self, db, key=b"", skip=True): @@ -1888,52 +1932,6 @@ def getIoDupItemIter(self, db, key=b'', *, ion=0): return # done raises StopIteration - def getTopIoDupItemIter(self, db, key=b''): - """ - Iterates over top branch of db given by key of IoDup items where each value - has 33 byte insertion ordinal number proem (prefixed) with separator. - Automagically removes (strips) proem before returning items. - - Assumes DB opened with dupsort=True - - Returns: - items (abc.Iterator): iterator of (full key, val) tuples of all - dup items over a branch of the db given by top key where returned - full key is full database key for val not truncated top key. - Item is (key, val) with proem stripped from val stored in db. - If key = b'' then returns list of dup items for all keys in db. - - - Because cursor.iternext() advances cursor after returning item its safe - to delete the item within the iteration loop. curson.iternext() works - for both dupsort==False and dupsort==True - - Raises StopIteration Error when empty. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items - from multiple branches of the key space. If top key is - empty then gets all items in database - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - """ - for key, val in self.getTopItemIter(db=db, top=key): - val = val[33:] # strip proem - yield (key, val) - - def getIoDupValsAllPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries From 2b497b8b175e63cd22de067bec1e99453d05a468 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 31 Aug 2024 09:19:39 -0600 Subject: [PATCH 291/418] refactor out the old getAllOnItemAllpre --- src/keri/app/cli/commands/migrate.py | 2 +- src/keri/app/habbing.py | 4 ++-- src/keri/db/basing.py | 32 +++++++++++++++++++--------- src/keri/db/dbing.py | 31 +-------------------------- tests/db/test_dbing.py | 11 +++++----- 5 files changed, 31 insertions(+), 49 deletions(-) diff --git a/src/keri/app/cli/commands/migrate.py b/src/keri/app/cli/commands/migrate.py index 36d3b3c0c..878587790 100644 --- a/src/keri/app/cli/commands/migrate.py +++ b/src/keri/app/cli/commands/migrate.py @@ -148,7 +148,7 @@ def migrateKeys(db): digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", klas=(coring.Prefixer, coring.Seqner)) - for pre, fn, dig in db.getFelItemAllPreIter(key=b''): + for pre, fn, dig in db.getFelItemAllPreIter(): dgkey = dbing.dgKey(pre, dig) # get message if not (raw := db.getEvt(key=dgkey)): raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index d9431aa4a..ba2959222 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -1538,7 +1538,7 @@ def replay(self, pre=None, fn=0): return msgs - def replayAll(self, key=b''): + def replayAll(self): """ Returns replay of FEL first seen event log for all pre starting at key @@ -1547,7 +1547,7 @@ def replayAll(self, key=b''): """ msgs = bytearray() - for msg in self.db.cloneAllPreIter(key=key): + for msg in self.db.cloneAllPreIter(): msgs.extend(msg) return msgs diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 3218d2d3b..029abab39 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1531,6 +1531,13 @@ def clonePreIter(self, pre, fn=0): Returns iterator of first seen event messages with attachments for the identifier prefix pre starting at first seen order number, fn. Essentially a replay in first seen order with attachments + + Parameters: + pre is bytes of itdentifier prefix + fn is int fn to resume replay. Earliset is fn=0 + + Returns: + msgs (Iterator): over all items with pre starting at fn """ if hasattr(pre, 'encode'): pre = pre.encode("utf-8") @@ -1543,18 +1550,19 @@ def clonePreIter(self, pre, fn=0): yield msg - def cloneAllPreIter(self, key=b''): + def cloneAllPreIter(self): """ Returns iterator of first seen event messages with attachments for all - identifier prefixes starting at key. If key == b'' then rstart at first + identifier prefixes starting at key. If key == b'' then start at first key in databse. Use key to resume replay. Essentially a replay in first seen order with attachments of entire set of FELs. - Parameters: - key (bytes): fnKey(pre, fn) + Returns: + msgs (Iterator): over all items in db + """ - for pre, fn, dig in self.getFelItemAllPreIter(key=key): + for pre, fn, dig in self.getFelItemAllPreIter(): try: msg = self.cloneEvtMsg(pre=pre, fn=fn, dig=dig) except Exception: @@ -1948,9 +1956,9 @@ def appendFe(self, pre, val): def getFelItemPreIter(self, pre, fn=0): """ - Returns iterator of all (fn, dig) duples in first seen order for all events - with same prefix, pre, in database. Items are sorted by fnKey(pre, fn) - where fn is first seen order number int. + Returns iterator of all (pre, fn, dig) triples in first seen order for + all events with same prefix, pre, in database. Items are sorted by + fnKey(pre, fn) where fn is first seen order number int. Returns a First Seen Event Log FEL. Returned items are duples of (fn, dig): Where fn is first seen order number int and dig is event digest for lookup in .evts sub db. @@ -1960,11 +1968,14 @@ def getFelItemPreIter(self, pre, fn=0): Parameters: pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 + + Returns: + items (Iterator[(pre, fn, val)]): over all items starting at pre, on """ return self.getTopOnItemIter(db=self.fels, top=pre, on=fn) - def getFelItemAllPreIter(self, key=b''): + def getFelItemAllPreIter(self): """ Returns iterator of all (pre, fn, dig) triples in first seen order for all events for all prefixes in database. Items are sorted by @@ -1980,7 +1991,8 @@ def getFelItemAllPreIter(self, key=b''): key is key location in db to resume replay, If empty then start at first key in database """ - return self.getAllOnItemAllPreIter(db=self.fels, key=key) + #return self.getAllOnItemAllPreIter(db=self.fels, key=key) + return self.getTopOnItemIter(db=self.fels, top=b'') def putDts(self, key, val): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index c09a22f2e..903f44024 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -731,7 +731,7 @@ def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): return count - def getTopOnItemIter(self, db, top, on=0, *, sep=b'.'): + def getTopOnItemIter(self, db, top=b'', on=0, *, sep=b'.'): """ Returns iterator of triples (key, on, val), at each key over all ordinal numbered keys with same top keyin db. Values are sorted by @@ -761,38 +761,9 @@ def getTopOnItemIter(self, db, top, on=0, *, sep=b'.'): ckey, cn = splitOnKey(key, sep=sep) if not ckey.startswith(top): # prev is now the last event for pre break # done - #if ckey != top: # prev is now the last event for pre - #break # done yield (ckey, cn, val) - def getAllOnItemAllPreIter(self, db, key=b''): - """ - Returns iterator of triple item, (pre, on, dig), at each key over all - ordinal numbered keys for all prefixes in db. Values are sorted by - onKey(pre, on) where on is ordinal number int. - Each returned item is triple (pre, on, dig) where pre is identifier prefix, - on is ordinal number int and dig is event digest for lookup in .evts sub db. - - Raises StopIteration Error when empty. - - Parameters: - db is opened named sub db with dupsort=False - key is key location in db to resume replay, - If empty then start at first key in database - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if not cursor.set_range(key): # moves to val at key >= key, first if empty - return # no values end of db - - for key, val in cursor.iternext(): # return key, val at cursor - cpre, cn = splitOnKey(key) - yield (cpre, cn, val) # (pre, on, dig) of event - - - - def appendOnValPre(self, db, pre, val): """ Appends val in order after last previous key with same pre in db. diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index e6ff17bf7..0f51ae5d0 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -425,19 +425,18 @@ def test_lmdber(): assert dber.putVal(db, keyA0, val=digA) == True assert dber.putVal(db, keyC0, val=digC) == True - items = [item for item in dber.getAllOnItemAllPreIter(db)] + items = [item for item in dber.getTopOnItemIter(db, top=b'')] assert items == [(preA, 0, digA), (preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY), (preC, 0, digC)] - # resume replay all starting at preB on=2 - items = [item for item in dber.getAllOnItemAllPreIter(db, key=keyB2)] - assert items == [(preB, 2, digW), (preB, 3, digX), (preB, 4, digY), - (preC, 0, digC)] + top, on = splitOnKey(keyB2) + items = [item for item in dber.getTopOnItemIter(db, top=top, on=on)] + assert items == [(top, 2, digW), (top, 3, digX), (top, 4, digY)] # resume replay all starting at preC on=1 - items = [item for item in dber.getAllOnItemAllPreIter(db, key=onKey(preC, 1))] + items = [item for item in dber.getTopOnItemIter(db, top=preC, on=1)] assert items == [] From f95ac48e07e782d38a418c6361ed322eb5e47930 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 31 Aug 2024 17:45:10 -0600 Subject: [PATCH 292/418] create OnSuber with partial unit tests --- src/keri/db/basing.py | 2 +- src/keri/db/dbing.py | 47 +++--- src/keri/db/subing.py | 341 ++++++++++++++++++++++------------------ tests/db/test_dbing.py | 16 +- tests/db/test_subing.py | 200 +++++++++++++++-------- 5 files changed, 360 insertions(+), 246 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 029abab39..140c49093 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1952,7 +1952,7 @@ def appendFe(self, pre, val): pre is bytes identifier prefix for event val is event digest """ - return self.appendOnValPre(db=self.fels, pre=pre, val=val) + return self.appendTopOnVal(db=self.fels, top=pre, val=val) def getFelItemPreIter(self, pre, fn=0): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 903f44024..73430e869 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -690,12 +690,8 @@ def delTopVal(self, db, top=b''): return result - # For subdbs with no duplicate values allowed at each key. (dupsort==False) - # and use keys with suffic ordinal that is monotonically increasing number part - # such as fn where no duplicates allowed at a given (pre, on) -# ToDo change pre to top so can use in suber for any top branch key space whose last -# part is ordinal number - + # For subdbs the use keys with trailing part the is monotonically + # ordinal number serialized as 32 hex bytes def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): """ @@ -742,6 +738,9 @@ def getTopOnItemIter(self, db, top=b'', on=0, *, sep=b'.'): Raises StopIteration Error when empty. + Returns: + items (Iterator[(top, on, val)]): triples of top key, on int, val + Parameters: db (subdb): named sub db in lmdb top (bytes): top key within sub db's keyspace with trailing part on @@ -763,25 +762,33 @@ def getTopOnItemIter(self, db, top=b'', on=0, *, sep=b'.'): break # done yield (ckey, cn, val) + # only valid for dupsort==False such as fn where only one entry per ordinal + # is allowed - def appendOnValPre(self, db, pre, val): + def appendTopOnVal(self, db, top, val, *, sep=b'.'): """ Appends val in order after last previous key with same pre in db. Returns ordinal number in, on, of appended entry. Appended on is 1 greater - than previous latest on. + than previous latest on at pre. Uses onKey(pre, on) for entries. + Assumes dupsort==False so no duplicates at a given onKey. + Append val to end of db entries with same pre but with on incremented by 1 relative to last preexisting entry at pre. + Returns: + on (int): ordinal number of newly appended val + Parameters: - db is opened named sub db with dupsort=False - pre is bytes identifier prefix for event - val is event digest + db (subdb): named sub db in lmdb + top (bytes): top key within sub db's keyspace with trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split """ # set key with fn at max and then walk backwards to find last entry at pre # if any otherwise zeroth entry at pre - key = onKey(pre, MaxON) + key = onKey(top, MaxON, sep=sep) with self.env.begin(db=db, write=True, buffers=True) as txn: on = 0 # unless other cases match then zeroth entry at pre cursor = txn.cursor() @@ -790,24 +797,24 @@ def appendOnValPre(self, db, pre, val): # last is last entry at same pre if cursor.last(): # not empty db. last entry earlier than max ckey = cursor.key() - cpre, cn = splitOnKey(ckey) - if cpre == pre: # last is last entry for same pre + cpre, cn = splitOnKey(ckey, sep=sep) + if cpre == top: # last is last entry for same pre on = cn + 1 # increment else: # not past end so not empty either later pre or max entry at pre ckey = cursor.key() - cpre, cn = splitOnKey(ckey) - if cpre == pre: # last entry for pre is already at max + cpre, cn = splitOnKey(ckey, sep=sep) + if cpre == top: # last entry for pre is already at max raise ValueError("Number part of key {} exceeds maximum" " size.".format(ckey)) else: # later pre so backup one entry # either no entry before last or earlier pre with entry if cursor.prev(): # prev entry, maybe same or earlier pre ckey = cursor.key() - cpre, cn = splitOnKey(ckey) - if cpre == pre: # last entry at pre + cpre, cn = splitOnKey(ckey, sep=sep) + if cpre == top: # last entry at pre on = cn + 1 # increment - key = onKey(pre, on) + key = onKey(top, on, sep=sep) if not cursor.put(key, val, overwrite=False): raise ValueError("Failed appending {} at {}.".format(val, key)) @@ -839,7 +846,7 @@ def putIoSetVals(self, db, key, vals, *, sep=b'.'): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False key (bytes): Apparent effective key - vals (abc.Iterable): serialized values to add to set of vals at key + vals (Iterable): serialized values to add to set of vals at key """ result = False diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 6866f1b03..3f9bb0ffb 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -91,14 +91,14 @@ def __init__(self, db: dbing.LMDBer, *, - def _tokey(self, keys: str | bytes | memoryview | Iterable[str | bytes], - top: bool=False): + def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], + topive: bool=False): """ Converts keys to key bytes with proper separators and returns key bytes. If keys is already str or bytes then returns key bytes. Else If keys is iterable (non-str) of strs or bytes then joins with separator converts to key bytes and returns. When keys is iterable and - top is True then enables partial key from top branch of key space given + topive is True then enables partial key from top branch of key space given by partial keys by appending separator to end of partial key Returns: @@ -108,7 +108,7 @@ def _tokey(self, keys: str | bytes | memoryview | Iterable[str | bytes], Parameters: keys (str | bytes | memoryview | Iterable[str | bytes]): db key or Iterable of (str | bytes) to form key. - top (bool): True means treat as partial key tuple from top branch of + topive (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep character. False means treat as full branch in key space. Resultant key @@ -123,7 +123,7 @@ def _tokey(self, keys: str | bytes | memoryview | Iterable[str | bytes], return bytes(keys) # return bytes elif hasattr(keys, "decode"): # bytes return keys - if top and keys[-1]: # top and keys is not already partial + if topive and keys[-1]: # topive and keys is not already partial keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep return (self.sep.join(key.decode() if hasattr(key, "decode") else key for key in keys).encode("utf-8")) @@ -170,8 +170,7 @@ def _des(self, val: str | bytes | memoryview): return (val.decode("utf-8") if hasattr(val, "decode") else val) - def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", - *, top=False): + def trim(self, keys: str|bytes|memoryview|Iterable=b"", *, topive=False): """ Removes all entries whose keys startswith keys. Enables removal of whole branches of db key space. To ensure that proper separation of a branch @@ -179,14 +178,14 @@ def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", 'a.1'and 'a.2' but not 'ab' Parameters: - keys (Iterator[str | bytes | memoryview]): of key parts that may be + keys (Iteratabke[str | bytes | memoryview]): of key parts that may be a truncation of a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then trims all items in database. Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. - top (bool): True means treat as partial key tuple from top branch of + topive (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep character. False means treat as full branch in key space. Resultant key @@ -198,11 +197,11 @@ def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", Returns: result (bool): True if val at key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys, top=top))) + return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys, topive=topive))) def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", - *, top=False): + *, topive=False): """Iterator over items in .db that returns full items with subclass specific special hidden parts shown for debugging or testing. @@ -215,7 +214,8 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", valuespace which may be useful in debugging or testing. Parameters: - keys (Iterator[str | bytes | memoryview]): of key parts that may be + keys (str|bytes|memoryview|Iteratable[str | bytes | memoryview]): + of key parts that may be a truncation of a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets all items in database. @@ -225,7 +225,7 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", key is empty string it matches all keys in db with startswith. - top (bool): True means treat as partial key tuple from top branch of + topive (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep character. False means treat as full branch in key space. Resultant key @@ -235,12 +235,12 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", """ for key, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, top=top)): + top=self._tokey(keys, topive=topive)): yield (self._tokeys(key), self._des(val)) - def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", - *, top=False): + def getItemIter(self, keys: str|bytes|memoryview|Iterable=b"", + *, topive=False): """Iterator over items in .db subclasses that do special hidden transforms on either the keyspace or valuespace should override this method to hide hidden parts from the returned items. For example, adding either @@ -257,7 +257,8 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", Parameters: - keys (Iterator[str | bytes | memoryview]): of key parts that may be + keys (str|bytes|memoryview|Iterable[str|bytes|memoryview]): of key + parts that may be a truncation of a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets all items in database. @@ -267,17 +268,17 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", key is empty string it matches all keys in db with startswith. - top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep - character. - False means treat as full branch in key space. Resultant key - does not end in .sep character. - When last item in keys is empty str then will treat as - partial ending in sep regardless of top value + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ for key, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, top=top)): + top=self._tokey(keys, topive=topive)): yield (self._tokeys(key), self._des(val)) @@ -376,6 +377,111 @@ def rem(self, keys: Union[str, Iterable]): return(self.db.delVal(db=self.sdb, key=self._tokey(keys))) +class OnSuberBase(SuberBase): + """ + Subclass of SuberBase that adds methods for keys with exposed key part suffix + that is 32 byte serializaton of monotonically increasing ordinal number on + such as sn or fn. + Each key consistes of top key joined with .sep to ordinal suffix + Works with dupsort==True or False + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Default False + sep (str): separator to convert keys iterator to key bytes for db key + Default '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(OnSuberBase, self).__init__(*pa, **kwa) + + + def cntOn(self, keys: str | bytes | memoryview, on: int=0): + """ + Returns + cnt (int): count of of all ordinal suffix keyed vals with same top + in key but different on in key in db starting at ordinal number + on of pre where key is formed with onKey(pre,on) + Does not count dups at same on for a given pre, only + unique on at a given pre. + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form top key + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.cntTopOnVals(db=self.sdb, + top=self._tokey(keys), + on=on, + sep=self.sep.encode())) + + def getOnItemIter(self, keys: str | bytes | memoryview | Iterable, on: int=0): + """ + Returns + items (Iterator[(top keys, on, val)]): triples of (top keys, on int, + deserialized val) + + Parameters: + keys (str | bytes | memoryview | iterator): top keys as prefix to be + combined with serialized on suffix and sep to form key + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getTopOnItemIter(db=self.sdb, + top=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) + + + +class OnSuber(OnSuberBase, Suber): + """ + Subclass of Suber that adds methods for keys with ordinal numbered suffixes. + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(OnSuber, self).__init__(*pa, **kwa) + + + def appendOn(self, keys: str | bytes | memoryview, + val: str | bytes | memoryview): + """ + Returns: + on (int): ordinal number of newly appended val + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form key + val (str | bytes | memoryview): serialization + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.appendTopOnVal(db=self.sdb, + top=self._tokey(keys), + val=self._ser(val), + sep=self.sep.encode())) + class CesrSuberBase(SuberBase): @@ -843,7 +949,7 @@ def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", - *, top=False): + *, topive=False): """ Return iterator over all the items in top branch defined by keys where keys may be truncation of full branch. @@ -856,7 +962,7 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", Returned key in each item has ordinal suffix removed. Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (Iterable): tuple of bytes or strs that may be a truncation of a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets all items in database. @@ -865,17 +971,17 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", In Python str.startswith('') always returns True so if branch key is empty string it matches all keys in db with startswith. - top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep - character. - False means treat as full branch in key space. Resultant key - does not end in .sep character. - When last item in keys is empty str then will treat as - partial ending in sep regardless of top value + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ for iokey, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, top=top)): + top=self._tokey(keys, topive=topive)): key, ion = dbing.unsuffix(iokey, sep=self.sep) yield (self._tokeys(key), self._des(val)) @@ -1039,7 +1145,8 @@ def get(self, keys: Union[str, Iterable]): if val is not None else None) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: str | bytes | memoryview | Iterable =b"", + *, topive=False): """ Returns: iterator (Iteratore: tuple (key, val) over the all the items in @@ -1048,13 +1155,21 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): returns all items in subdb Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (Iterable): tuple of bytes or strs that may be a truncation of a full keys tuple in in order to get all the items from multiple branches of the key space. If keys is empty then gets all items in database. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split yield (ikeys, self.klas(qb64b=bytes(val), @@ -1155,11 +1270,11 @@ def get(self, keys: Union[str, Iterable], decrypter: core.Decrypter = None): - def getItemIter(self, keys: Union[str, Iterable]=b"", - decrypter: core.Decrypter = None): + def getItemIter(self, keys: str|bytes|memoryview|Iterable=b"", + decrypter: core.Decrypter = None, *, topive=False): """ Returns: - iterator (Iterator): of tuples (key, val) over the all the items in + items (Iterator): of tuples (key, val) over the all the items in subdb whose key startswith key made from keys. Keys may be keyspace prefix to return branches of key space. When keys is empty then returns all items in subdb @@ -1168,13 +1283,21 @@ def getItemIter(self, keys: Union[str, Iterable]=b"", db was encrypted and so decrypts before converting to Signer. Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (Iterable): tuple of bytes or strs that may be a truncation of a full keys tuple in in order to get all the items from multiple branches of the key space. If keys is empty then gets all items in database. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split if decrypter: @@ -1397,7 +1520,8 @@ def rem(self, keys: Union[str, Iterable]): """ return self.db.delVal(db=self.sdb, key=self._tokey(keys)) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: str | bytes | memoryview | Iterable =b"", + *, topive=False): """ Returns: iterator (Iterator): tuple (key, val) over the all the items in @@ -1406,13 +1530,22 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): returns all items in subdb Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (str | bytes | memoryview | Iterable): tuple of bytes or + strs that may be a truncation of a full keys tuple in in order to get all the items from multiple branches of the key space. If keys is empty then gets all items in database. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): + for iokey, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): yield self._tokeys(iokey), scheming.Schemer(raw=bytes(val)) @@ -1686,7 +1819,7 @@ def add(self, keys: str | bytes | memoryview | Iterable, Parameters: keys (Iterable): of key strs to be combined in order to form key - val (Union[bytes, str]): serialization + val (str | bytes | memoryview): serialization Returns: result (bool): True means unique value added among duplications, @@ -1818,7 +1951,8 @@ def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): apparent effective key and items all have same apparent effective key Parameters: - keys (Iterable): of key strs to be combined in order to form key + keys (str|bytes|memoryview|Iterable): of key strs to be combined + in order to form key ion (int): starting ordinal value, default 0 Returns: @@ -1836,7 +1970,7 @@ def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", - *, top=False): + *, topive=False): """ Return iterator over all the items including dup items for all keys in top branch defined by keys where keys may be truncation of full branch. @@ -1858,17 +1992,17 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", In Python str.startswith('') always returns True so if branch key is empty string it matches all keys in db with startswith. - top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep - character. - False means treat as full branch in key space. Resultant key - does not end in .sep character. - When last item in keys is empty str then will treat as - partial ending in sep regardless of top value + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ for key, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, top=top)): + top=self._tokey(keys, topive=topive)): val = val[33:] # strip off proem yield (self._tokeys(key), self._des(val)) @@ -1891,96 +2025,3 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", # used by .dels # getIoDupValsAnyPreIter(self, db, pre, on=0) - - - -class OnSuber(Suber): - """ - Subclass of Suber that adds methods for keys with ordinal numbered suffixes. - Each key consistes of pre joined with .sep to ordinal suffix - - """ - - def __init__(self, *pa, **kwa): - """ - Inherited Parameters: - db (dbing.LMDBer): base db - subkey (str): LMDB sub database key - dupsort (bool): True means enable duplicates at each key - False (default) means do not enable duplicates at - each key. Set to False - sep (str): separator to convert keys iterator to key bytes for db key - default is self.Sep == '.' - verify (bool): True means reverify when ._des from db when applicable - False means do not reverify. Default False - """ - super(OnSuber, self).__init__(*pa, **kwa) - - - def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): - """ - Returns - cnt (int): count of of all ordinal suffix keyed vals with same pre - in key but different on in key in db starting at ordinal number - on of pre where key is formed with onKey(pre,on) - Does not count dups at same on for a given pre, only - unique on at a given pre. - - Parameters: - pre (str | bytes | memoryview): prefix to to be combined with on - to form key - on (int): ordinal number used with onKey(pre,on) to form key. - """ - return (self.db.cntTopOnVals(db=self.sdb, top=self._tokey(pre), on=on)) - - # appendOrdPre - - #def appendOrdValPre(self, db, pre, val): - #""" - #Appends val in order after last previous key with same pre in db. - #Returns ordinal number in, on, of appended entry. Appended on is 1 greater - #than previous latest on. - #Uses onKey(pre, on) for entries. - - #Append val to end of db entries with same pre but with on incremented by - #1 relative to last preexisting entry at pre. - - #Parameters: - #db is opened named sub db with dupsort=False - #pre is bytes identifier prefix for event - #val is event digest - #""" - - # getAllOrdItemPreIter - #def getAllOrdItemPreIter(self, db, pre, on=0): - #""" - #Returns iterator of duple item, (on, dig), at each key over all ordinal - #numbered keys with same prefix, pre, in db. Values are sorted by - #onKey(pre, on) where on is ordinal number int. - #Returned items are duples of (on, dig) where on is ordinal number int - #and dig is event digest for lookup in .evts sub db. - - #Raises StopIteration Error when empty. - - #Parameters: - #db is opened named sub db with dupsort=False - #pre is bytes of itdentifier prefix - #on is int ordinal number to resume replay - #""" - - - #def getAllOrdItemAllPreIter(self, db, key=b''): - #""" - #Returns iterator of triple item, (pre, on, dig), at each key over all - #ordinal numbered keys for all prefixes in db. Values are sorted by - #onKey(pre, on) where on is ordinal number int. - #Each returned item is triple (pre, on, dig) where pre is identifier prefix, - #on is ordinal number int and dig is event digest for lookup in .evts sub db. - - #Raises StopIteration Error when empty. - - #Parameters: - #db is opened named sub db with dupsort=False - #key is key location in db to resume replay, - #If empty then start at first key in database - #""" diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 0f51ae5d0..dbafab599 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -351,7 +351,7 @@ def test_lmdber(): # test appendOrdValPre # empty database assert dber.getVal(db, keyB0) == None - on = dber.appendOnValPre(db, preB, digU) + on = dber.appendTopOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -359,7 +359,7 @@ def test_lmdber(): # earlier pre in database only assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOnValPre(db, preB, digU) + on = dber.appendTopOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -368,7 +368,7 @@ def test_lmdber(): # earlier and later pre in db but not same pre assert dber.getVal(db, keyA0) == digA assert dber.putVal(db, keyC0, val=digC) == True - on = dber.appendOnValPre(db, preB, digU) + on = dber.appendTopOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -378,13 +378,13 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None assert dber.getVal(db, keyC0) == digC - on = dber.appendOnValPre(db, preB, digU) + on = dber.appendTopOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU # earlier pre and later pre and earlier entry for same pre assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOnValPre(db, preB, digV) + on = dber.appendTopOnVal(db, preB, digV) assert on == 1 assert dber.getVal(db, keyB1) == digV @@ -394,15 +394,15 @@ def test_lmdber(): assert dber.delVal(db, keyC0) == True assert dber.getVal(db, keyC0) == None # another value for preB - on = dber.appendOnValPre(db, preB, digW) + on = dber.appendTopOnVal(db, preB, digW) assert on == 2 assert dber.getVal(db, keyB2) == digW # yet another value for preB - on = dber.appendOnValPre(db, preB, digX) + on = dber.appendTopOnVal(db, preB, digX) assert on == 3 assert dber.getVal(db, keyB3) == digX # yet another value for preB - on = dber.appendOnValPre(db, preB, digY ) + on = dber.appendTopOnVal(db, preB, digY ) assert on == 4 assert dber.getVal(db, keyB4) == digY diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index ffca9aa9e..f49fa6e7f 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -31,62 +31,62 @@ def test_suber(): assert db.name == "test" assert db.opened - sdb = subing.Suber(db=db, subkey='bags.') - assert isinstance(sdb, subing.Suber) - assert not sdb.sdb.flags()["dupsort"] + suber = subing.Suber(db=db, subkey='bags.') + assert isinstance(suber, subing.Suber) + assert not suber.sdb.flags()["dupsort"] sue = "Hello sailer!" keys = ("test_key", "0001") - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue - sdb.rem(keys) - actual = sdb.get(keys=keys) + suber.rem(keys) + actual = suber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue kip = "Hey gorgeous!" - result = sdb.put(keys=keys, val=kip) + result = suber.put(keys=keys, val=kip) assert not result - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual == sue - result = sdb.pin(keys=keys, val=kip) + result = suber.pin(keys=keys, val=kip) assert result - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual == kip - sdb.rem(keys) - actual = sdb.get(keys=keys) + suber.rem(keys) + actual = suber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue # test with keys as tuple of bytes keys = (b"test_key", b"0001") - sdb.rem(keys) - actual = sdb.get(keys=keys) + suber.rem(keys) + actual = suber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue # test with keys as mixed tuple of bytes keys = (b"test_key", "0001") - sdb.rem(keys) - actual = sdb.get(keys=keys) + suber.rem(keys) + actual = suber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue @@ -95,21 +95,21 @@ def test_suber(): bob = "Shove off!" - sdb.put(keys=keys, val=bob) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=bob) + actual = suber.get(keys=keys) assert actual == bob - sdb.rem(keys) + suber.rem(keys) - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual is None liz = "May life is insane." keys = ("test_key", "0002") - sdb.put(keys=keys, val=liz) - actual = sdb.get(("not_found", "0002")) + suber.put(keys=keys, val=liz) + actual = suber.get(("not_found", "0002")) assert actual is None w = "Blue dog" @@ -117,26 +117,26 @@ def test_suber(): y = "Red apple" z = "White snow" - sdb = subing.Suber(db=db, subkey='pugs.') - assert isinstance(sdb, subing.Suber) + suber = subing.Suber(db=db, subkey='pugs.') + assert isinstance(suber, subing.Suber) - sdb.put(keys=("a","1"), val=w) - sdb.put(keys=("a","2"), val=x) - sdb.put(keys=("a","3"), val=y) - sdb.put(keys=("a","4"), val=z) + suber.put(keys=("a","1"), val=w) + suber.put(keys=("a","2"), val=x) + suber.put(keys=("a","3"), val=y) + suber.put(keys=("a","4"), val=z) - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), w), (('a', '2'), x), (('a', '3'), y), (('a', '4'), z)] - sdb.put(keys=("b","1"), val=w) - sdb.put(keys=("b","2"), val=x) - sdb.put(keys=("bc","3"), val=y) - sdb.put(keys=("ac","4"), val=z) + suber.put(keys=("b","1"), val=w) + suber.put(keys=("b","2"), val=x) + suber.put(keys=("bc","3"), val=y) + suber.put(keys=("ac","4"), val=z) - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), (('a', '2'), 'Green tree'), (('a', '3'), 'Red apple'), @@ -149,12 +149,12 @@ def test_suber(): # test with top keys for partial tree topkeys = ("b","") # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=topkeys)] assert items == [(('b', '1'), w), (('b', '2'), x)] topkeys = ("a","") # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=topkeys)] assert items == [(('a', '1'), w), (('a', '2'), x), (('a', '3'), y), @@ -162,20 +162,20 @@ def test_suber(): # test with top parameter keys = ("b", ) # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=keys, topive=True)] assert items == [(('b', '1'), w), (('b', '2'), x)] keys = ("a", ) # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=keys, topive=True)] assert items == [(('a', '1'), w), (('a', '2'), x), (('a', '3'), y), (('a', '4'), z)] # Test trim - assert sdb.trim(keys=("b", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert suber.trim(keys=("b", "")) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), (('a', '2'), 'Green tree'), (('a', '3'), 'Red apple'), @@ -183,20 +183,20 @@ def test_suber(): (('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] - assert sdb.trim(keys=("a", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert suber.trim(keys=("a", "")) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] # Test trim with top parameters - sdb.put(keys=("a","1"), val=w) - sdb.put(keys=("a","2"), val=x) - sdb.put(keys=("a","3"), val=y) - sdb.put(keys=("a","4"), val=z) - sdb.put(keys=("b","1"), val=w) - sdb.put(keys=("b","2"), val=x) - - assert sdb.trim(keys=("b",), top=True) - items = [(keys, val) for keys, val in sdb.getItemIter()] + suber.put(keys=("a","1"), val=w) + suber.put(keys=("a","2"), val=x) + suber.put(keys=("a","3"), val=y) + suber.put(keys=("a","4"), val=z) + suber.put(keys=("b","1"), val=w) + suber.put(keys=("b","2"), val=x) + + assert suber.trim(keys=("b",), topive=True) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), (('a', '2'), 'Green tree'), (('a', '3'), 'Red apple'), @@ -204,21 +204,86 @@ def test_suber(): (('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] - assert sdb.trim(keys=("a",), top=True) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert suber.trim(keys=("a",), topive=True) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] - assert sdb.trim() - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert suber.trim() + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [] - assert not sdb.trim() + assert not suber.trim() assert not os.path.exists(db.path) assert not db.opened +def test_on_suber(): + """ + Test OnSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.OnSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.OnSuber) + assert not onsuber.sdb.flags()["dupsort"] + + w = "Blue dog" + x = "Green tree" + y = "Red apple" + z = "White snow" + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow')] + + + # test getOnItemIter + + assert 0 == onsuber.appendOn(keys=("b",), val=w) + assert 1 == onsuber.appendOn(keys=("b",), val=x) + assert 0 == onsuber.appendOn(keys=("bc",), val=y) + assert 0 == onsuber.appendOn(keys=("ac",), val=z) + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('ac', '00000000000000000000000000000000'), 'White snow'), + (('b', '00000000000000000000000000000000'), 'Blue dog'), + (('b', '00000000000000000000000000000001'), 'Green tree'), + (('bc', '00000000000000000000000000000000'), 'Red apple')] + + # test getOnItemIter + + + + assert not os.path.exists(db.path) + assert not db.opened + + + def test_dup_suber(): """ Test DubSuber LMDBer sub database class @@ -452,7 +517,7 @@ def test_iodup_suber(): # test with top parameter keys = ("test", ) - items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys, topive=True)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] @@ -620,7 +685,7 @@ def test_ioset_suber(): # test with top parameter keys = ("test", ) - items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys, topive=True)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] @@ -2041,6 +2106,7 @@ def test_crypt_signer_suber(): if __name__ == "__main__": test_suber() + test_on_suber() test_dup_suber() test_iodup_suber() test_ioset_suber() From f884ba26a1f4aafada08fede20dc99abf62f33fd Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 31 Aug 2024 22:15:38 -0600 Subject: [PATCH 293/418] clean up fix texts call signatures --- src/keri/db/basing.py | 4 ++-- src/keri/db/dbing.py | 49 ++++++++++++++++++++++------------------- src/keri/db/subing.py | 26 +++++++++++----------- src/keri/vdr/viring.py | 4 ++-- tests/db/test_dbing.py | 38 ++++++++++++++++++++++++-------- tests/db/test_subing.py | 31 +++++++++++++++++++++++++- 6 files changed, 102 insertions(+), 50 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 140c49093..d39fd641c 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1972,7 +1972,7 @@ def getFelItemPreIter(self, pre, fn=0): Returns: items (Iterator[(pre, fn, val)]): over all items starting at pre, on """ - return self.getTopOnItemIter(db=self.fels, top=pre, on=fn) + return self.getOnItemIter(db=self.fels, key=pre, on=fn) def getFelItemAllPreIter(self): @@ -1992,7 +1992,7 @@ def getFelItemAllPreIter(self): first key in database """ #return self.getAllOnItemAllPreIter(db=self.fels, key=key) - return self.getTopOnItemIter(db=self.fels, top=b'') + return self.getOnItemIter(db=self.fels, key=b'') def putDts(self, key, val): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 73430e869..8637e4741 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -693,7 +693,7 @@ def delTopVal(self, db, top=b''): # For subdbs the use keys with trailing part the is monotonically # ordinal number serialized as 32 hex bytes - def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): + def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): """ Returns (int): count of of all ordinal keyed vals with top key but different on tail in db starting at ordinal number on of top. @@ -703,34 +703,37 @@ def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): Parameters: db (lmdbsubdb): named sub db of lmdb - top (bytes): top key within sub db's keyspace with trailing part on + key (bytes): key within sub db's keyspace plus trailing part on on (int): ordinal number at which to initiate count sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key count = 0 - if not cursor.set_range(key): # moves to val at key >= key + if not cursor.set_range(onkey): # moves to val at key >= key return count # no values end of db - for key in cursor.iternext(values=False): # get key only at cursor + for ckey in cursor.iternext(values=False): # get key only at cursor try: - cpre, cn = splitOnKey(key, sep=sep) + ckey, cn = splitOnKey(ckey, sep=sep) except ValueError as ex: # not splittable key break - if cpre != top: # prev is now the last event for pre + if key and ckey != key: # prev is now the last event for pre break # done count = count+1 return count - def getTopOnItemIter(self, db, top=b'', on=0, *, sep=b'.'): + def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): """ Returns iterator of triples (key, on, val), at each key over all ordinal - numbered keys with same top keyin db. Values are sorted by + numbered keys with same key + sep + on in db. Values are sorted by onKey(key, on) where on is ordinal number int and key is prefix sans on. Returned items are triples of (key, on, val) When dupsort==true then duplicates are included in items since .iternext @@ -739,28 +742,28 @@ def getTopOnItemIter(self, db, top=b'', on=0, *, sep=b'.'): Raises StopIteration Error when empty. Returns: - items (Iterator[(top, on, val)]): triples of top key, on int, val + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: db (subdb): named sub db in lmdb - top (bytes): top key within sub db's keyspace with trailing part on + key (bytes): key within sub db's keyspace plus trailing part on on (int): ordinal number at which to initiate retrieval sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - if top: # not empty - key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest - else: # empty top - key = top # get all values empty key is start of db - if not cursor.set_range(key): # moves to val at key >= key - return # no values end of db - - for key, val in cursor.iternext(): # get key, val at cursor - ckey, cn = splitOnKey(key, sep=sep) - if not ckey.startswith(top): # prev is now the last event for pre - break # done - yield (ckey, cn, val) + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: # empty + onkey = key + if not cursor.set_range(onkey): # moves to val at key >= onkey + return # no values end of db raises StopIteration + + for ckey, cval in cursor.iternext(): # get key, val at cursor + ckey, cn = splitOnKey(ckey, sep=sep) + if key and not ckey == key: + break + yield (ckey, cn, cval) # only valid for dupsort==False such as fn where only one entry per ordinal # is allowed diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 3f9bb0ffb..ea3a8a556 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -200,7 +200,7 @@ def trim(self, keys: str|bytes|memoryview|Iterable=b"", *, topive=False): return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys, topive=topive))) - def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]="", *, topive=False): """Iterator over items in .db that returns full items with subclass specific special hidden parts shown for debugging or testing. @@ -239,7 +239,7 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", yield (self._tokeys(key), self._des(val)) - def getItemIter(self, keys: str|bytes|memoryview|Iterable=b"", + def getItemIter(self, keys: str|bytes|memoryview|Iterable="", *, topive=False): """Iterator over items in .db subclasses that do special hidden transforms on either the keyspace or valuespace should override this method to hide @@ -403,7 +403,7 @@ def __init__(self, *pa, **kwa): super(OnSuberBase, self).__init__(*pa, **kwa) - def cntOn(self, keys: str | bytes | memoryview, on: int=0): + def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): """ Returns cnt (int): count of of all ordinal suffix keyed vals with same top @@ -417,12 +417,12 @@ def cntOn(self, keys: str | bytes | memoryview, on: int=0): combined with serialized on suffix and sep to form top key on (int): ordinal number used with onKey(pre,on) to form key. """ - return (self.db.cntTopOnVals(db=self.sdb, - top=self._tokey(keys), + return (self.db.cntOnVals(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())) - def getOnItemIter(self, keys: str | bytes | memoryview | Iterable, on: int=0): + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns items (Iterator[(top keys, on, val)]): triples of (top keys, on int, @@ -434,8 +434,8 @@ def getOnItemIter(self, keys: str | bytes | memoryview | Iterable, on: int=0): on (int): ordinal number used with onKey(pre,on) to form key. sep (bytes): separator character for split """ - for keys, on, val in (self.db.getTopOnItemIter(db=self.sdb, - top=self._tokey(keys), on=on, sep=self.sep.encode())): + for keys, on, val in (self.db.getOnItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): yield (self._tokeys(keys), on, self._des(val)) @@ -948,7 +948,7 @@ def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): - def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ Return iterator over all the items in top branch defined by keys where @@ -1145,7 +1145,7 @@ def get(self, keys: Union[str, Iterable]): if val is not None else None) - def getItemIter(self, keys: str | bytes | memoryview | Iterable =b"", + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ Returns: @@ -1270,7 +1270,7 @@ def get(self, keys: Union[str, Iterable], decrypter: core.Decrypter = None): - def getItemIter(self, keys: str|bytes|memoryview|Iterable=b"", + def getItemIter(self, keys: str|bytes|memoryview|Iterable= "", decrypter: core.Decrypter = None, *, topive=False): """ Returns: @@ -1520,7 +1520,7 @@ def rem(self, keys: Union[str, Iterable]): """ return self.db.delVal(db=self.sdb, key=self._tokey(keys)) - def getItemIter(self, keys: str | bytes | memoryview | Iterable =b"", + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ Returns: @@ -1969,7 +1969,7 @@ def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): yield (self._tokeys(key), self._des(ioval)) - def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ Return iterator over all the items including dup items for all keys diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 3314ecc0d..11123c57b 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -671,7 +671,7 @@ def getTelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getTopOnItemIter(db=self.tels, top=pre, on=fn) + return self.getOnItemIter(db=self.tels, key=pre, on=fn) def cntTels(self, pre, fn=0): """ @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntTopOnVals(db=self.tels, top=pre, on=fn) + return self.cntOnVals(db=self.tels, key=pre, on=fn) def getTibs(self, key): """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index dbafab599..68cb272c9 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -317,6 +317,7 @@ def test_lmdber(): preA = b'BBKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc' preB = b'EH7Oq9oxCgYa-nnNLvwhp9sFZpALILlRYyB-6n4WDi7w' preC = b'EIDA1n-WiBA0A8YOqnKrB-wWQYYC49i5zY_qrIZIicQg' + preD = b'EAYC49i5zY_qrIZIicQgIDA1n-WiBA0A8YOqnKrB-wWQ' keyA0 = onKey(preA, 0) @@ -406,37 +407,56 @@ def test_lmdber(): assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntTopOnVals(db, top=preB) == 5 + assert dber.appendTopOnVal(db, preD, digY ) == 0 + + assert dber.cntOnVals(db, key=preB) == 5 + assert dber.cntOnVals(db, key=b'') == 6 # all keys + assert dber.cntOnVals(db) == 6 # all keys # replay preB events in database - items = [item for item in dber.getTopOnItemIter(db, preB)] + items = [item for item in dber.getOnItemIter(db, preB)] assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 3 - items = [item for item in dber.getTopOnItemIter(db, preB, on=3)] + items = [item for item in dber.getOnItemIter(db, preB, on=3)] assert items == [(preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 5 - items = [item for item in dber.getTopOnItemIter(db, preB, on=5)] + items = [item for item in dber.getOnItemIter(db, preB, on=5)] assert items == [] # replay all events in database with pre events before and after assert dber.putVal(db, keyA0, val=digA) == True assert dber.putVal(db, keyC0, val=digC) == True - items = [item for item in dber.getTopOnItemIter(db, top=b'')] - assert items == [(preA, 0, digA), (preB, 0, digU), (preB, 1, digV), - (preB, 2, digW), (preB, 3, digX), (preB, 4, digY), + items = [item for item in dber.getOnItemIter(db, key=b'')] + assert items == [(preA, 0, digA), + (preD, 0, digY), + (preB, 0, digU), + (preB, 1, digV), + (preB, 2, digW), + (preB, 3, digX), + (preB, 4, digY), + (preC, 0, digC)] + + items = [item for item in dber.getOnItemIter(db)] + assert items == [(preA, 0, digA), + (preD, 0, digY), + (preB, 0, digU), + (preB, 1, digV), + (preB, 2, digW), + (preB, 3, digX), + (preB, 4, digY), (preC, 0, digC)] # resume replay all starting at preB on=2 top, on = splitOnKey(keyB2) - items = [item for item in dber.getTopOnItemIter(db, top=top, on=on)] + items = [item for item in dber.getOnItemIter(db, key=top, on=on)] assert items == [(top, 2, digW), (top, 3, digX), (top, 4, digY)] # resume replay all starting at preC on=1 - items = [item for item in dber.getTopOnItemIter(db, top=preC, on=1)] + items = [item for item in dber.getOnItemIter(db, key=preC, on=1)] assert items == [] diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index f49fa6e7f..2c285a8f8 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -254,8 +254,17 @@ def test_on_suber(): (('a', '00000000000000000000000000000002'), 'Red apple'), (('a', '00000000000000000000000000000003'), 'White snow')] - # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + assert 0 == onsuber.appendOn(keys=("b",), val=w) assert 1 == onsuber.appendOn(keys=("b",), val=x) @@ -264,6 +273,7 @@ def test_on_suber(): assert onsuber.cntOn(keys=("b",)) == 2 assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 items = [(keys, val) for keys, val in onsuber.getItemIter()] assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), @@ -276,7 +286,26 @@ def test_on_suber(): (('bc', '00000000000000000000000000000000'), 'Red apple')] # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + items = [item for item in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] assert not os.path.exists(db.path) From af1da8fdd62648fa8a3b45b61c78c078f350f961 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 31 Aug 2024 22:27:51 -0600 Subject: [PATCH 294/418] more clean up refactoring --- src/keri/db/basing.py | 2 +- src/keri/db/dbing.py | 30 +++++++++++++++--------------- src/keri/db/subing.py | 4 ++-- tests/db/test_dbing.py | 18 +++++++++--------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index d39fd641c..0fc69cfcf 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1952,7 +1952,7 @@ def appendFe(self, pre, val): pre is bytes identifier prefix for event val is event digest """ - return self.appendTopOnVal(db=self.fels, top=pre, val=val) + return self.appendOnVal(db=self.fels, key=pre, val=val) def getFelItemPreIter(self, pre, fn=0): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 8637e4741..27bcc4ede 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -768,7 +768,7 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): # only valid for dupsort==False such as fn where only one entry per ordinal # is allowed - def appendTopOnVal(self, db, top, val, *, sep=b'.'): + def appendOnVal(self, db, key, val, *, sep=b'.'): """ Appends val in order after last previous key with same pre in db. Returns ordinal number in, on, of appended entry. Appended on is 1 greater @@ -785,42 +785,42 @@ def appendTopOnVal(self, db, top, val, *, sep=b'.'): Parameters: db (subdb): named sub db in lmdb - top (bytes): top key within sub db's keyspace with trailing part on + key (bytes): key within sub db's keyspace plus trailing part on val (bytes): serialized value to append sep (bytes): separator character for split """ # set key with fn at max and then walk backwards to find last entry at pre # if any otherwise zeroth entry at pre - key = onKey(top, MaxON, sep=sep) + onkey = onKey(key, MaxON, sep=sep) with self.env.begin(db=db, write=True, buffers=True) as txn: on = 0 # unless other cases match then zeroth entry at pre cursor = txn.cursor() - if not cursor.set_range(key): # max is past end of database + if not cursor.set_range(onkey): # max is past end of database # so either empty database or last is earlier pre or # last is last entry at same pre if cursor.last(): # not empty db. last entry earlier than max ckey = cursor.key() - cpre, cn = splitOnKey(ckey, sep=sep) - if cpre == top: # last is last entry for same pre + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last is last entry for same pre on = cn + 1 # increment else: # not past end so not empty either later pre or max entry at pre ckey = cursor.key() - cpre, cn = splitOnKey(ckey, sep=sep) - if cpre == top: # last entry for pre is already at max - raise ValueError("Number part of key {} exceeds maximum" - " size.".format(ckey)) + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last entry for pre is already at max + raise ValueError(f"Number part {cn=} for key part {ckey=}" + f"exceeds maximum size.") else: # later pre so backup one entry # either no entry before last or earlier pre with entry if cursor.prev(): # prev entry, maybe same or earlier pre ckey = cursor.key() - cpre, cn = splitOnKey(ckey, sep=sep) - if cpre == top: # last entry at pre + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last entry at pre on = cn + 1 # increment - key = onKey(top, on, sep=sep) + onkey = onKey(key, on, sep=sep) - if not cursor.put(key, val, overwrite=False): - raise ValueError("Failed appending {} at {}.".format(val, key)) + if not cursor.put(onkey, val, overwrite=False): + raise ValueError(f"Failed appending {val=} at {key=}.") return on diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index ea3a8a556..db56269f8 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -477,8 +477,8 @@ def appendOn(self, keys: str | bytes | memoryview, val (str | bytes | memoryview): serialization on (int): ordinal number used with onKey(pre,on) to form key. """ - return (self.db.appendTopOnVal(db=self.sdb, - top=self._tokey(keys), + return (self.db.appendOnVal(db=self.sdb, + key=self._tokey(keys), val=self._ser(val), sep=self.sep.encode())) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 68cb272c9..73701ac2f 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -352,7 +352,7 @@ def test_lmdber(): # test appendOrdValPre # empty database assert dber.getVal(db, keyB0) == None - on = dber.appendTopOnVal(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -360,7 +360,7 @@ def test_lmdber(): # earlier pre in database only assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendTopOnVal(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -369,7 +369,7 @@ def test_lmdber(): # earlier and later pre in db but not same pre assert dber.getVal(db, keyA0) == digA assert dber.putVal(db, keyC0, val=digC) == True - on = dber.appendTopOnVal(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -379,13 +379,13 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None assert dber.getVal(db, keyC0) == digC - on = dber.appendTopOnVal(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU # earlier pre and later pre and earlier entry for same pre assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendTopOnVal(db, preB, digV) + on = dber.appendOnVal(db, preB, digV) assert on == 1 assert dber.getVal(db, keyB1) == digV @@ -395,19 +395,19 @@ def test_lmdber(): assert dber.delVal(db, keyC0) == True assert dber.getVal(db, keyC0) == None # another value for preB - on = dber.appendTopOnVal(db, preB, digW) + on = dber.appendOnVal(db, preB, digW) assert on == 2 assert dber.getVal(db, keyB2) == digW # yet another value for preB - on = dber.appendTopOnVal(db, preB, digX) + on = dber.appendOnVal(db, preB, digX) assert on == 3 assert dber.getVal(db, keyB3) == digX # yet another value for preB - on = dber.appendTopOnVal(db, preB, digY ) + on = dber.appendOnVal(db, preB, digY ) assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.appendTopOnVal(db, preD, digY ) == 0 + assert dber.appendOnVal(db, preD, digY ) == 0 assert dber.cntOnVals(db, key=preB) == 5 assert dber.cntOnVals(db, key=b'') == 6 # all keys From 058ceef990c628ff7458a44fc9653ff68963ec25 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 31 Aug 2024 22:29:30 -0600 Subject: [PATCH 295/418] added test vector --- tests/db/test_subing.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 2c285a8f8..549a6de9d 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -307,6 +307,16 @@ def test_on_suber(): (('b',), 1, 'Green tree'), (('bc',), 0, 'Red apple')] + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + assert not os.path.exists(db.path) assert not db.opened From 1d5bfca96b3106f7895a6f2fd2b4907d04c345a2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 1 Sep 2024 17:28:45 -0600 Subject: [PATCH 296/418] tests for OnIoDupSuber --- src/keri/db/dbing.py | 13 ++-- src/keri/db/subing.py | 83 ++++++++++++++++++----- tests/db/test_subing.py | 145 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 20 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 27bcc4ede..524a37625 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -700,10 +700,12 @@ def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): Full key is composed of top+sep+ When dupsort==true then duplicates are included in count since .iternext includes duplicates. + when key is empty then retrieves whole db Parameters: db (lmdbsubdb): named sub db of lmdb key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db on (int): ordinal number at which to initiate count sep (bytes): separator character for split """ @@ -738,6 +740,7 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): Returned items are triples of (key, on, val) When dupsort==true then duplicates are included in items since .iternext includes duplicates. + when key is empty then retrieves whole db Raises StopIteration Error when empty. @@ -747,6 +750,7 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): Parameters: db (subdb): named sub db in lmdb key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db on (int): ordinal number at which to initiate retrieval sep (bytes): separator character for split """ @@ -765,17 +769,18 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): break yield (ckey, cn, cval) - # only valid for dupsort==False such as fn where only one entry per ordinal - # is allowed + def appendOnVal(self, db, key, val, *, sep=b'.'): """ - Appends val in order after last previous key with same pre in db. + Appends val in order after last previous key with same pre in db where + full key has key prefix and serialized on suffix attached with sep. Returns ordinal number in, on, of appended entry. Appended on is 1 greater than previous latest on at pre. Uses onKey(pre, on) for entries. - Assumes dupsort==False so no duplicates at a given onKey. + Works with either dupsort==True or False since always creates new full + key. Append val to end of db entries with same pre but with on incremented by 1 relative to last preexisting entry at pre. diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index db56269f8..1f1c679c8 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -403,6 +403,25 @@ def __init__(self, *pa, **kwa): super(OnSuberBase, self).__init__(*pa, **kwa) + def appendOn(self, keys: str | bytes | memoryview, + val: str | bytes | memoryview): + """ + Returns: + on (int): ordinal number of newly appended val + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form key + val (str | bytes | memoryview): serialization + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.appendOnVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val), + sep=self.sep.encode())) + + + def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): """ Returns @@ -415,6 +434,7 @@ def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): Parameters: keys (str | bytes | memoryview | Iterable): top keys as prefix to be combined with serialized on suffix and sep to form top key + When keys is empty then counts whole database including duplicates on (int): ordinal number used with onKey(pre,on) to form key. """ return (self.db.cntOnVals(db=self.sdb, @@ -431,6 +451,7 @@ def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): Parameters: keys (str | bytes | memoryview | iterator): top keys as prefix to be combined with serialized on suffix and sep to form key + When keys is empty then retrieves whole database including duplicates on (int): ordinal number used with onKey(pre,on) to form key. sep (bytes): separator character for split """ @@ -465,22 +486,6 @@ def __init__(self, *pa, **kwa): super(OnSuber, self).__init__(*pa, **kwa) - def appendOn(self, keys: str | bytes | memoryview, - val: str | bytes | memoryview): - """ - Returns: - on (int): ordinal number of newly appended val - - Parameters: - keys (str | bytes | memoryview | Iterable): top keys as prefix to be - combined with serialized on suffix and sep to form key - val (str | bytes | memoryview): serialization - on (int): ordinal number used with onKey(pre,on) to form key. - """ - return (self.db.appendOnVal(db=self.sdb, - key=self._tokey(keys), - val=self._ser(val), - sep=self.sep.encode())) @@ -2025,3 +2030,49 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", # used by .dels # getIoDupValsAnyPreIter(self, db, pre, on=0) + +class OnIoDupSuber(OnSuberBase, DupSuber): + """ + Sub class of IoDupSuber and OnSuberBase that supports Insertion Ordering + (IoDup) of duplicates but where the trailing part of the key space is + a serialized monotonically increasing ordinal number. This is useful for + escrows of key events etc where duplicates of likely events are maintained + in insertion order. + Insertion order is maintained by automagically prepending and stripping an + ordinal ordering proem to/from each duplicate value at a given key. + + OnIoDupSuber adds the convenience methods from OnSuberBase to IoDupSuber for + those cases where the keyspace has a trailing ordinal part. + + There are two ordinals, one in the key space and a hidden one in the duplicate + data value space. + + OnIoDupSuber supports insertion ordered multiple entries at each key + (duplicates) with dupsort==True + + Do not use if serialized length key + proem + value, is greater than 511 bytes. + This is a limitation of dupsort==True sub dbs in LMDB + + OnIoDupSuber may be more performant then IoSetSuber for values that are indices + to other sub dbs that fit the size constraint because LMDB support for + duplicates is more space efficient and code performant. + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + + """ + super(OnIoDupSuber, self).__init__(*pa, **kwa) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 549a6de9d..a9f677034 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -623,6 +623,150 @@ def test_iodup_suber(): assert not db.opened +def test_oniodup_suber(): + """ + Test OnIoDupSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.OnIoDupSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.OnIoDupSuber) + assert onsuber.sdb.flags()["dupsort"] + + w = "Blue dog" + x = "Green tree" + y = "Red apple" + z = "White snow" + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow')] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + # test add duplicates + assert onsuber.add(keys=dbing.onKey("b", 0), val=w) + assert onsuber.add(keys=dbing.onKey("b", 1), val=x) + assert onsuber.add(keys=dbing.onKey("bc", 0), val=y) + assert onsuber.add(keys=dbing.onKey("ac", 0), val=z) + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('ac', '00000000000000000000000000000000'), 'White snow'), + (('b', '00000000000000000000000000000000'), 'Blue dog'), + (('b', '00000000000000000000000000000001'), 'Green tree'), + (('bc', '00000000000000000000000000000000'), 'Red apple')] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + + items = [item for item in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + # test with duplicates + assert onsuber.add(keys=dbing.onKey("a", 0), val=z) + assert onsuber.add(keys=dbing.onKey("a", 1), val=y) + assert onsuber.add(keys=dbing.onKey("a", 2), val=x) + assert onsuber.add(keys=dbing.onKey("a", 3), val=w) + + assert onsuber.cntOn(keys=("a",)) == 8 + assert onsuber.cntOn(keys=("a",), on=2) == 4 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter(keys=("a", ""))] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000000'), 'White snow'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000001'), 'Red apple'), + (('a', '00000000000000000000000000000002'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'Blue dog'), + (('a', '00000000000000000000000000000003'), 'White snow')] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 0, 'White snow'), + (('a',), 1, 'Green tree'), + (('a',), 1, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'Blue dog'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'Blue dog'), + (('a',), 3, 'White snow')] + + # test append with duplicates + assert 4 == onsuber.appendOn(keys=("a",), val=x) + assert onsuber.cntOn(keys=("a",)) == 9 + + assert not os.path.exists(db.path) + assert not db.opened + + + def test_ioset_suber(): """ Test IoSetSuber LMDBer sub database class @@ -2148,6 +2292,7 @@ def test_crypt_signer_suber(): test_on_suber() test_dup_suber() test_iodup_suber() + test_oniodup_suber() test_ioset_suber() test_cat_suber() test_cesr_suber() From db25724d8b8dd84693e39225f8a60b5b7342e437 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 1 Sep 2024 19:00:16 -0600 Subject: [PATCH 297/418] fixed class hierarchy for OnIoDupSuber and unit tests refactored dbing support methods more consistent --- src/keri/db/dbing.py | 198 +++++++++++++++++++++++++++------------- src/keri/db/subing.py | 70 ++++++++++---- tests/db/test_dbing.py | 29 ++++-- tests/db/test_subing.py | 21 +++-- 4 files changed, 219 insertions(+), 99 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 524a37625..3c876f87b 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1721,7 +1721,67 @@ def cntIoDupVals(self, db, key): return count - def getTopIoDupItemIter(self, db, key=b''): +# used in IoDupSuber + def getIoDupItemIter(self, db, key=b'', *, ion=0): + """ + Iterates over branch of db given by key of IoDup items where each value + has 33 byte insertion ordinal number proem (prefixed) with separator. + Automagically removes (strips) proem before returning items. + + Assumes DB opened with dupsort=True + + Returns: + ioitems (Iterator): each item iterated is tuple (keys, ioval) where + each keys is actual keys tuple and each ioval is dup that + includes prefixed insertion ordering proem + empty list if no entry at keys. + Raises StopIteration when done + + + Because cursor.iternext() advances cursor after returning item its safe + to delete the item within the iteration loop. curson.iternext() works + for both dupsort==False and dupsort==True + + Raises StopIteration Error when empty. + + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + key (bytes): truncated top key, a key space prefix to get all the items + from multiple branches of the key space. If top key is + empty then gets all items in database + ion (int): starting ordinal value, default 0 + + + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + if cursor.set_range(key): # move to val at key >= key if any + for ckey, cval in cursor.iternext(): # get key, val first dup at cursor + ckey = bytes(ckey) + if not ckey.startswith(key): # prev entry if any last in branch + break # done + cion = int(bytes(cval[:32]), 16) # convert without sep '.' in [33] + if cion < ion: # cion cursor insertion order proem as int + continue # skip dups whose cion is below ion + yield (ckey, cval) # include proem on val + return # done raises StopIteration + + +# used in IoDupSuber.getItemIter + def getTopIoDupItemIter(self, db, top=b''): """ Iterates over top branch of db given by key of IoDup items where each value has 33 byte insertion ordinal number proem (prefixed) with separator. @@ -1745,7 +1805,7 @@ def getTopIoDupItemIter(self, db, key=b''): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items + top (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is empty then gets all items in database @@ -1762,11 +1822,12 @@ def getTopIoDupItemIter(self, db, key=b''): before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). """ - for key, val in self.getTopItemIter(db=db, top=key): + for top, val in self.getTopItemIter(db=db, top=top): val = val[33:] # strip proem - yield (key, val) - + yield (top, val) +# not needed in IoDupSuber since not used anywhere always uses the Iter version +# below. confirm this in general and refactor any that do use to use the iter verio def getIoDupItemsNext(self, db, key=b"", skip=True): """ Return list of all dup items at next key after key in db in insertion order. @@ -1813,7 +1874,8 @@ def getIoDupItemsNext(self, db, key=b"", skip=True): items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] return items - +# don't need this in IoDupSuber because can replace with self.getTopIoDupItemIter +# which in turen can be replaced with IoDupSuber.getItemIter def getIoDupItemsNextIter(self, db, key=b"", skip=True): """ @@ -1859,64 +1921,7 @@ def getIoDupItemsNextIter(self, db, key=b"", skip=True): for key, val in cursor.iternext_dup(keys=True): yield (key, val[33:]) # slice off prepended ordering prefix - - def getIoDupItemIter(self, db, key=b'', *, ion=0): - """ - Iterates over branch of db given by key of IoDup items where each value - has 33 byte insertion ordinal number proem (prefixed) with separator. - Automagically removes (strips) proem before returning items. - - Assumes DB opened with dupsort=True - - Returns: - ioitems (Iterator): each item iterated is tuple (keys, ioval) where - each keys is actual keys tuple and each ioval is dup that - includes prefixed insertion ordering proem - empty list if no entry at keys. - Raises StopIteration when done - - - Because cursor.iternext() advances cursor after returning item its safe - to delete the item within the iteration loop. curson.iternext() works - for both dupsort==False and dupsort==True - - Raises StopIteration Error when empty. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items - from multiple branches of the key space. If top key is - empty then gets all items in database - ion (int): starting ordinal value, default 0 - - - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any - for ckey, cval in cursor.iternext(): # get key, val first dup at cursor - ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch - break # done - cion = int(bytes(cval[:32]), 16) # convert without sep '.' in [33] - if cion < ion: # cion cursor insertion order proem as int - continue # skip dups whose cion is below ion - yield (ckey, cval) # include proem on val - return # done raises StopIteration - +# Should not need this in OnIoDupSuber because can replace with get getIoDupItemIter def getIoDupValsAllPreIter(self, db, pre, on=0): """ @@ -2026,7 +2031,7 @@ def getIoDupValLastAllPreIter(self, db, pre, on=0): yield cursor.value()[33:] # slice off prepended ordering prefix key = snKey(pre, cnt:=cnt+1) - +# used by .dels getIoDupItemIter def getIoDupValsAnyPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for any entries @@ -2068,3 +2073,66 @@ def getIoDupValsAnyPreIter(self, db, pre, on=0): yield val[33:] # slice off prepended ordering prefix cnt = int(back, 16) key = snKey(pre, cnt:=cnt+1) + + + # methods for OnIoDup that combines IoDup value proem with On ordinal numbered + # trailing prefix + # this is so we do the proem add and strip here not in some higher level class + # like suber + + # used in OnIoDupSuber + def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): + """ + Appends val in order after last previous key with same pre in db where + full key has key prefix and serialized on suffix attached with sep and + value has ordinal proem prefixed. + Returns ordinal number in, on, of appended entry. Appended on is 1 greater + than previous latest on at pre. + Uses onKey(pre, on) for entries. + + Works with either dupsort==True or False since always creates new full + key. + + Append val to end of db entries with same pre but with on incremented by + 1 relative to last preexisting entry at pre. + + Returns: + on (int): ordinal number of newly appended val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split + """ + val = (b'%032x.' % (0)) + val # prepend ordering proem + return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) + + # used in OnIoDupSuber + def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnItemIter(db=db, key=key, on=on, sep=sep): + val = val[33:] # strip proem + yield (key, on, val) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 1f1c679c8..7031796c2 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1950,14 +1950,16 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): return (self.db.cntIoDupVals(db=self.sdb, key=self._tokey(keys))) - def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): + def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable = '', + *, ion=0): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key Parameters: keys (str|bytes|memoryview|Iterable): of key strs to be combined - in order to form key + in order to form key. When keys is empty then retrieves + all items in database. ion (int): starting ordinal value, default 0 Returns: @@ -2006,9 +2008,9 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, + + for key, val in self.db.getTopIoDupItemIter(db=self.sdb, top=self._tokey(keys, topive=topive)): - val = val[33:] # strip off proem yield (self._tokeys(key), self._des(val)) # ToDo @@ -2018,20 +2020,16 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", # so they work across prefixes without having the extra burden doing repeated # iter within a given pre dups. This way they could work with or without duplicates # and then the IoDupIter could just strip the proem whereas OrdSuber would not care -# OnSuber - #Although these are similar to OrdSuber in that they both have ordinal - # not hidden as last part of keys. OnIoDup suber also has proem that - # must be prefixed and stripped so not able to mixin with each other - # in multiple inheritance - # used by .kels - # getIoDupValsAllPreIter(self, db, pre, on=0): - # getIoDupValsAllPreBackIter(self, db, pre, on=0): - # getIoDupValLastAllPreIter(self.kels, pre, on=sn) - # used by .dels - # getIoDupValsAnyPreIter(self, db, pre, on=0) - - -class OnIoDupSuber(OnSuberBase, DupSuber): + +# used by .kels + # getIoDupValsAllPreIter(self, db, pre, on=0): + # getIoDupValsAllPreBackIter(self, db, pre, on=0): + # getIoDupValLastAllPreIter(self.kels, pre, on=sn) +# used by .dels + # getIoDupValsAnyPreIter(self, db, pre, on=0) + + +class OnIoDupSuber(OnSuberBase, IoDupSuber): """ Sub class of IoDupSuber and OnSuberBase that supports Insertion Ordering (IoDup) of duplicates but where the trailing part of the key space is @@ -2076,3 +2074,39 @@ def __init__(self, *pa, **kwa): """ super(OnIoDupSuber, self).__init__(*pa, **kwa) + + + def appendOn(self, keys: str | bytes | memoryview, + val: str | bytes | memoryview): + """ + Returns: + on (int): ordinal number of newly appended val + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form key + val (str | bytes | memoryview): serialization + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.appendOnIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val), + sep=self.sep.encode())) + + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + items (Iterator[(top keys, on, val)]): triples of (top keys, on int, + deserialized val) + + Parameters: + keys (str | bytes | memoryview | iterator): top keys as prefix to be + combined with serialized on suffix and sep to form key + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnIoDupItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 73701ac2f..c9d226231 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -349,7 +349,7 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None - # test appendOrdValPre + # test appendOnValPre # empty database assert dber.getVal(db, keyB0) == None on = dber.appendOnVal(db, preB, digU) @@ -526,6 +526,8 @@ def test_lmdber(): assert dber.addIoDupVal(db, key, b'e') assert dber.getIoDupVals(db, key) == [b'm', b'a', b'w', b'e'] + + # Test getIoValsAllPreIter(self, db, pre) vals0 = [b"gamma", b"beta"] sn = 0 @@ -623,29 +625,29 @@ def test_lmdber(): (b'A.00000000000000000000000000000007', b'b')] # dups at aKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=aKey)] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=aKey)] assert items == [(b'A.00000000000000000000000000000001', b'z'), (b'A.00000000000000000000000000000001', b'm'), (b'A.00000000000000000000000000000001', b'x')] # dups at bKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=bKey)] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=bKey)] assert items == [(b'A.00000000000000000000000000000002', b'o'), (b'A.00000000000000000000000000000002', b'r'), (b'A.00000000000000000000000000000002', b'z')] # dups at cKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=cKey)] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=cKey)] assert items == [(b'A.00000000000000000000000000000004', b'h'), (b'A.00000000000000000000000000000004', b'n')] # dups at dKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=dKey)] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=dKey)] assert items == [(b'A.00000000000000000000000000000007', b'k'), (b'A.00000000000000000000000000000007', b'b')] # dups at missing key - items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=b"B.")] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=b"B.")] assert not items @@ -820,6 +822,21 @@ def test_lmdber(): assert items == [] # empty assert not items + # test OnIoDup methods + key = b'Z' + assert 0 == dber.appendOnIoDupVal(edb, key, val=b'k') + assert 1 == dber.appendOnIoDupVal(edb, key, val=b'l') + assert 2 == dber.appendOnIoDupVal(edb, key, val=b'm') + assert 3 == dber.appendOnIoDupVal(edb, key, val=b'n') + + assert dber.cntOnVals(edb, key) == 4 + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(edb, key=key)] + assert items == [(b'Z', 0, b'k'), + (b'Z', 1, b'l'), + (b'Z', 2, b'm'), + (b'Z', 3, b'n')] + # test IoSetVals insertion order set of vals methods. diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index a9f677034..a3db90b1d 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -736,10 +736,10 @@ def test_oniodup_suber(): (('a', '00000000000000000000000000000000'), 'White snow'), (('a', '00000000000000000000000000000001'), 'Green tree'), (('a', '00000000000000000000000000000001'), 'Red apple'), - (('a', '00000000000000000000000000000002'), 'Green tree'), (('a', '00000000000000000000000000000002'), 'Red apple'), - (('a', '00000000000000000000000000000003'), 'Blue dog'), - (('a', '00000000000000000000000000000003'), 'White snow')] + (('a', '00000000000000000000000000000002'), 'Green tree'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('a', '00000000000000000000000000000003'), 'Blue dog')] # test getOnItemIter items = [item for item in onsuber.getOnItemIter(keys='a')] @@ -747,16 +747,17 @@ def test_oniodup_suber(): (('a',), 0, 'White snow'), (('a',), 1, 'Green tree'), (('a',), 1, 'Red apple'), - (('a',), 2, 'Green tree'), (('a',), 2, 'Red apple'), - (('a',), 3, 'Blue dog'), - (('a',), 3, 'White snow')] + (('a',), 2, 'Green tree'), + (('a',), 3, 'White snow'), + (('a',), 3, 'Blue dog')] items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] - assert items == [(('a',), 2, 'Green tree'), - (('a',), 2, 'Red apple'), - (('a',), 3, 'Blue dog'), - (('a',), 3, 'White snow')] + assert items ==[(('a',), 2, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 3, 'White snow'), + (('a',), 3, 'Blue dog')] + # test append with duplicates assert 4 == onsuber.appendOn(keys=("a",), val=x) From d55ecaf394e33cf7c6fd7b2a003465778954f30e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 1 Sep 2024 19:03:21 -0600 Subject: [PATCH 298/418] added test vector --- tests/db/test_dbing.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index c9d226231..7a8c3afe3 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -837,6 +837,11 @@ def test_lmdber(): (b'Z', 2, b'm'), (b'Z', 3, b'n')] + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(edb, key=key, on=2)] + assert items == [ + (b'Z', 2, b'm'), + (b'Z', 3, b'n')] + # test IoSetVals insertion order set of vals methods. From 2fc53a6326d2a13a86c43ca8dac6bb36c0ee0278 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 1 Sep 2024 19:16:30 -0600 Subject: [PATCH 299/418] some clean up --- src/keri/db/dbing.py | 138 ++++++++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 3c876f87b..a002bf462 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1921,6 +1921,71 @@ def getIoDupItemsNextIter(self, db, key=b"", skip=True): for key, val in cursor.iternext_dup(keys=True): yield (key, val[33:]) # slice off prepended ordering prefix + + # methods for OnIoDup that combines IoDup value proem with On ordinal numbered + # trailing prefix + # this is so we do the proem add and strip here not in some higher level class + # like suber + + # used in OnIoDupSuber + def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): + """ + Appends val in order after last previous key with same pre in db where + full key has key prefix and serialized on suffix attached with sep and + value has ordinal proem prefixed. + Returns ordinal number in, on, of appended entry. Appended on is 1 greater + than previous latest on at pre. + Uses onKey(pre, on) for entries. + + Works with either dupsort==True or False since always creates new full + key. + + Append val to end of db entries with same pre but with on incremented by + 1 relative to last preexisting entry at pre. + + Returns: + on (int): ordinal number of newly appended val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split + """ + val = (b'%032x.' % (0)) + val # prepend ordering proem + return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) + + # used in OnIoDupSuber + def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnItemIter(db=db, key=key, on=on, sep=sep): + val = val[33:] # strip proem + yield (key, on, val) + + + # Should not need this in OnIoDupSuber because can replace with get getIoDupItemIter def getIoDupValsAllPreIter(self, db, pre, on=0): @@ -1996,12 +2061,12 @@ def getIoDupValsAllPreBackIter(self, db, pre, on=0): yield val[33:] key = snKey(pre, cnt:=cnt-1) - +# need to fix this so it is not stopped by gaps in on def getIoDupValLastAllPreIter(self, db, pre, on=0): """ Returns iterator of last only of dup vals of each key in insertion order - for all entries with same prefix across all sequence numbers in increasing order - without gaps starting with on (default = 0). Stops if gap or different pre. + for all entries with same key across all sequence numbers in increasing order + without gaps starting with on (default = 0). Stops if gap or different key. Assumes that key is combination of prefix and sequence number given by .snKey(). Removes prepended proem ordinal from each val before returning @@ -2028,10 +2093,11 @@ def getIoDupValLastAllPreIter(self, db, pre, on=0): key = snKey(pre, cnt:=on) while cursor.set_key(key): # moves to first_dup if cursor.last_dup(): # move to last_dup - yield cursor.value()[33:] # slice off prepended ordering prefix + yield cursor.value()[33:] # slice off prepended ordering proem key = snKey(pre, cnt:=cnt+1) -# used by .dels getIoDupItemIter +# used by .dels should be able to replace with self.getTopIoDupItemIter which +# gets used by IoDupSuber.getIoDupItemIter def getIoDupValsAnyPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for any entries @@ -2074,65 +2140,3 @@ def getIoDupValsAnyPreIter(self, db, pre, on=0): cnt = int(back, 16) key = snKey(pre, cnt:=cnt+1) - - # methods for OnIoDup that combines IoDup value proem with On ordinal numbered - # trailing prefix - # this is so we do the proem add and strip here not in some higher level class - # like suber - - # used in OnIoDupSuber - def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): - """ - Appends val in order after last previous key with same pre in db where - full key has key prefix and serialized on suffix attached with sep and - value has ordinal proem prefixed. - Returns ordinal number in, on, of appended entry. Appended on is 1 greater - than previous latest on at pre. - Uses onKey(pre, on) for entries. - - Works with either dupsort==True or False since always creates new full - key. - - Append val to end of db entries with same pre but with on incremented by - 1 relative to last preexisting entry at pre. - - Returns: - on (int): ordinal number of newly appended val - - Parameters: - db (subdb): named sub db in lmdb - key (bytes): key within sub db's keyspace plus trailing part on - val (bytes): serialized value to append - sep (bytes): separator character for split - """ - val = (b'%032x.' % (0)) + val # prepend ordering proem - return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) - - # used in OnIoDupSuber - def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): - """ - Returns iterator of triples (key, on, val), at each key over all ordinal - numbered keys with same key + sep + on in db. Values are sorted by - onKey(key, on) where on is ordinal number int and key is prefix sans on. - Values duplicates are sorted internally by hidden prefixed insertion order - proem ordinal - Returned items are triples of (key, on, val) - When dupsort==true then duplicates are included in items since .iternext - includes duplicates. - when key is empty then retrieves whole db - - Raises StopIteration Error when empty. - - Returns: - items (Iterator[(key, on, val)]): triples of key, on, val - - Parameters: - db (subdb): named sub db in lmdb - key (bytes): key within sub db's keyspace plus trailing part on - when key is empty then retrieves whole db - on (int): ordinal number at which to initiate retrieval - sep (bytes): separator character for split - """ - for key, on, val in self.getOnItemIter(db=db, key=key, on=on, sep=sep): - val = val[33:] # strip proem - yield (key, on, val) From 84315d6db10a8ce40ec65664c2d261d0c1ac04d3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 1 Sep 2024 20:19:15 -0600 Subject: [PATCH 300/418] more clean up refactoring --- src/keri/db/basing.py | 6 +++--- src/keri/db/dbing.py | 34 ++++++++++++++++++++++++++-------- tests/db/test_dbing.py | 4 ++-- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 0fc69cfcf..c90ef3877 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2585,7 +2585,7 @@ def getKelIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoDupValsAllPreIter(self.kels, pre, on=sn) + return self.getOnIoDupValsAllPreIter(self.kels, pre, on=sn) def getKelBackIter(self, pre, sn=0): @@ -2609,7 +2609,7 @@ def getKelBackIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoDupValsAllPreBackIter(self.kels, pre, sn) + return self.getOnIoDupValsAllPreBackIter(self.kels, pre, sn) def getKelLastIter(self, pre, sn=0): @@ -3209,7 +3209,7 @@ def getDelIter(self, pre): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoDupValsAnyPreIter(self.dels, pre) + return self.getOnIoDupValsAnyPreIter(self.dels, pre) def putLdes(self, key, vals): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index a002bf462..8421303c5 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1986,9 +1986,10 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): -# Should not need this in OnIoDupSuber because can replace with get getIoDupItemIter - def getIoDupValsAllPreIter(self, db, pre, on=0): +# without gaps is used for replay with on to provide partial replay + + def getOnIoDupValsAllPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries with same prefix across all ordinal numbers in increasing order @@ -2024,7 +2025,12 @@ def getIoDupValsAllPreIter(self, db, pre, on=0): key = snKey(pre, cnt:=cnt+1) - def getIoDupValsAllPreBackIter(self, db, pre, on=0): + + # ToDo need unit tests in dbing Used to replay backwards all duplicate + # values starting at on + # need to fix this so it is not stopped by gaps or if gap raises error as + # gap is normally a problem for replay so maybe a parameter to raise error on gap + def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries with same prefix across all sequence numbers in decreasing order without gaps @@ -2061,7 +2067,9 @@ def getIoDupValsAllPreBackIter(self, db, pre, on=0): yield val[33:] key = snKey(pre, cnt:=cnt-1) -# need to fix this so it is not stopped by gaps in on +# Used to replay forward all last duplicate values starting at on +# need to fix this so it is not stopped by gaps or if gap raises error as +# gap is normally a problem for replay so maybe a parameter to raise error on gap def getIoDupValLastAllPreIter(self, db, pre, on=0): """ Returns iterator of last only of dup vals of each key in insertion order @@ -2096,9 +2104,20 @@ def getIoDupValLastAllPreIter(self, db, pre, on=0): yield cursor.value()[33:] # slice off prepended ordering proem key = snKey(pre, cnt:=cnt+1) -# used by .dels should be able to replace with self.getTopIoDupItemIter which -# gets used by IoDupSuber.getIoDupItemIter - def getIoDupValsAnyPreIter(self, db, pre, on=0): + + +# to do do we need a replay last backwards? + + # used by .dels do we need "on" parameter? if not then should be able to + # replace with self.getTopIoDupItemIter which gets used by + # IoDupSuber.getIoDupItemIter which crosses gaps. + # duplicitous items in escrow may have gaps but do we need to have an on start + # of duplicity escrow processing? + + # use this method to inform how to cross gaps in other replays above with parameter + # to raise error if detect gap when should not be one for event logs vs escrows + + def getOnIoDupValsAnyPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for any entries with same prefix across all ordinal numbers in order including gaps @@ -2139,4 +2158,3 @@ def getIoDupValsAnyPreIter(self, db, pre, on=0): yield val[33:] # slice off prepended ordering prefix cnt = int(back, 16) key = snKey(pre, cnt:=cnt+1) - diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 7a8c3afe3..a0ef7043f 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -545,7 +545,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoDupValsAllPreIter(db, pre)] + vals = [bytes(val) for val in dber.getOnIoDupValsAllPreIter(db, pre)] allvals = vals0 + vals1 + vals2 assert vals == allvals @@ -589,7 +589,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoDupValsAnyPreIter(db, pre)] + vals = [bytes(val) for val in dber.getOnIoDupValsAnyPreIter(db, pre)] allvals = vals0 + vals1 + vals2 assert vals == allvals From 047b54704a85485121787d27589d2b22189c48c1 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 09:13:20 -0600 Subject: [PATCH 301/418] some more refactoring simplifying --- src/keri/db/basing.py | 5 +++-- src/keri/db/dbing.py | 17 +++++++++++------ tests/db/test_basing.py | 4 ++-- tests/db/test_dbing.py | 3 ++- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index c90ef3877..fc132f52b 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -3192,7 +3192,7 @@ def delDes(self, key): """ return self.delIoDupVals(self.dels, key) - def getDelIter(self, pre): + def getDelItemIter(self, pre): """ Returns iterator of all dup vals in insertion order for any entries with same prefix across all sequence numbers including gaps. @@ -3209,7 +3209,8 @@ def getDelIter(self, pre): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getOnIoDupValsAnyPreIter(self.dels, pre) + return self.getTopIoDupItemIter(self.dels, pre) + #return self.getOnIoDupValsAnyPreIter(self.dels, pre) def putLdes(self, key, vals): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 8421303c5..9895b5220 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1722,6 +1722,8 @@ def cntIoDupVals(self, db, key): # used in IoDupSuber +# do we really need this? Do we really need to access a subset of duplicates by +# their ion? Is not yet used so now is the time to remove this def getIoDupItemIter(self, db, key=b'', *, ion=0): """ Iterates over branch of db given by key of IoDup items where each value @@ -1988,6 +1990,8 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): # without gaps is used for replay with on to provide partial replay +# Consider using ItemIter instead of just replaying vals since ItemIter already +# works with either dupsort==True or False def getOnIoDupValsAllPreIter(self, db, pre, on=0): """ @@ -2030,6 +2034,10 @@ def getOnIoDupValsAllPreIter(self, db, pre, on=0): # values starting at on # need to fix this so it is not stopped by gaps or if gap raises error as # gap is normally a problem for replay so maybe a parameter to raise error on gap + + # Consider creating BackItemIter instead of just replaying vals since + # ItemIter already works with either dupsort==True or False + # so if we create TopBackItemIter then we can use that everywhere def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries @@ -2106,16 +2114,13 @@ def getIoDupValLastAllPreIter(self, db, pre, on=0): -# to do do we need a replay last backwards? +# ToDo do we need a replay last backwards? + - # used by .dels do we need "on" parameter? if not then should be able to - # replace with self.getTopIoDupItemIter which gets used by - # IoDupSuber.getIoDupItemIter which crosses gaps. - # duplicitous items in escrow may have gaps but do we need to have an on start - # of duplicity escrow processing? # use this method to inform how to cross gaps in other replays above with parameter # to raise error if detect gap when should not be one for event logs vs escrows + # then remove this method since not used otherwise def getOnIoDupValsAnyPreIter(self, db, pre, on=0): """ diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 825c039de..fc02d3ec3 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -1926,7 +1926,7 @@ def test_fetchkeldel(): assert vals == lastvals - # test getDelIter + # test getDelItemIter preb = 'BTmuupUhPx5_yZ-Wk1x4ejhccWzwEHHzq7K0gzQPYGGw'.encode("utf-8") sn = 1 # do not start at zero key = snKey(preb, sn) @@ -1947,8 +1947,8 @@ def test_fetchkeldel(): for val in vals2: assert db.addDe(key, val) == True - vals = [bytes(val) for val in db.getDelIter(preb)] allvals = vals0 + vals1 + vals2 + vals = [bytes(val) for key, val in db.getDelItemIter(preb)] assert vals == allvals assert not os.path.exists(db.path) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index a0ef7043f..609c0894c 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -589,8 +589,9 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getOnIoDupValsAnyPreIter(db, pre)] allvals = vals0 + vals1 + vals2 + vals = [bytes(val) for key, val in dber.getTopIoDupItemIter(db, pre)] + # dber.getTopIoDupItemIter() assert vals == allvals # Setup Tests for getIoItemsNext and getIoItemsNextIter From 030d00633796a926a4192330cd1485155779c852 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 13:15:05 -0600 Subject: [PATCH 302/418] refactoring to getIoSetItemIter and getIoDupItemIter are symmetrical --- src/keri/app/storing.py | 18 ++++-- src/keri/db/dbing.py | 35 +++++----- src/keri/db/koming.py | 16 ++--- src/keri/db/subing.py | 20 +++--- tests/app/test_forwarding.py | 2 +- tests/app/test_storing.py | 20 +++--- tests/db/test_dbing.py | 20 +++--- tests/db/test_koming.py | 21 +----- tests/db/test_subing.py | 121 ++++++++++++++++------------------- 9 files changed, 128 insertions(+), 145 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index b64cf6b67..6b5d54417 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -121,14 +121,24 @@ def cloneTopicIter(self, topic, fn=0): Returns iterator of first seen exn messages with attachments for the identifier prefix pre starting at first seen order number, fn. + ToDo looks like misuse of IoSet this should not be IoSet but simply + Ordinal Numbered db. since should not be using hidden ion has not + hidden. + """ if hasattr(topic, 'encode'): topic = topic.encode("utf-8") - for (key, dig) in self.getIoSetItemIter(self.tpcs, key=topic, ion=fn): - topic, ion = dbing.unsuffix(key) - if msg := self.msgs.get(keys=dig): - yield ion, topic, msg.encode("utf-8") + for ion, (topic, dig) in enumerate(self.getIoSetItemIter(self.tpcs, topic)): + if ion >= fn: + if msg := self.msgs.get(keys=dig): + yield ion, topic, msg.encode("utf-8") + + #for (key, dig) in self.getIoSetItemIter(self.tpcs, key=topic, ion=fn): + #topic, ion = dbing.unsuffix(key) + #if msg := self.msgs.get(keys=dig): + #yield ion, topic, msg.encode("utf-8") + class Respondant(doing.DoDoer): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 9895b5220..aa846fee5 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1197,7 +1197,7 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return False - def delIoSetIokey(self, db, iokey): + def delIoSetIokey(self, db, iokey): # change this to key, ion """ Deletes val at at actual iokey that includes ordinal key suffix. @@ -1217,33 +1217,28 @@ def delIoSetIokey(self, db, iokey): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoSetItemIter(self, db, key, *, ion=0, sep=b'.'): + def getIoSetItemIter(self, db, top=b'', *, sep=b'.'): """ Returns: - items (abc.Iterator): iterator over insertion ordered set of values - at same apparent effective key where each iteration returns tuple - (iokey, val). iokey includes the ordinal key suffix. + items (Iterator[(key,val)]): iterator of tuples (key, val) where + key is apparent key with hidden insertion ordering suffixe removed + from effective key. + Iterates over top branch of insertion ordered set values where each + effective key has trailing hidden suffix of serialization of insertion + ordering ordinal. + Uses hidden ordinal key suffix for insertion ordering. - Does not work with partial effective key. Raises StopIteration Error when empty. Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key. Does not work with partial key - that is not the full effective key sans the ion part. - ion (int): starting ordinal value, default 0 + top (bytes): top key in db. When top is empty then every item in db. + sep (bytes): sep character for attached io suffix """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - iokey = suffix(key, ion, sep=sep) # start ion th value for key zeroth default - cursor = txn.cursor() - if cursor.set_range(iokey): # move to val at key >= iokey if any - for iokey, val in cursor.iternext(): # get key, val at cursor - ckey, cion = unsuffix(iokey, sep=sep) - if ckey != key: # prev entry if any was the last entry for key - break # done - yield (iokey, val) # another entry at key - return # done raises StopIteration + for iokey, val in self.getTopItemIter(db=db, top=top): + key, ion = splitOnKey(iokey, sep=sep) + yield (key, val) @@ -1685,6 +1680,8 @@ def delIoDupVal(self, db, key, val): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return False + #def delIoDupIonVal(self, db, key, ion): # to match delIoSetIoKey + def cntIoDupVals(self, db, key): """ diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index 05ac15a25..e3cda6a55 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -591,27 +591,27 @@ def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def getIoSetItemIter(self, keys: Union[str, Iterable], *, ion=0): + def getIoSetItemIter(self, keys: Union[str, Iterable] = ''): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key Parameters: - keys (Iterable): of key strs to be combined in order to form key - ion (int): starting ordinal value, default 0 + keys (Iterable): of key strs to be combined in order to form apparent + key without insertion ordering suffix. When keys + is empty then retrieves all items in db. Returns: - ioitems (Iterator): each item iterated is tuple (iokeys, val) where - each iokeys is actual keys tuple including hidden suffix and + items (Iterator): each item iterated is tuple (keys, val) where + each keys is apparent keys tuple without hidden suffix and each val is str empty list if no entry at keys. Raises StopIteration when done """ for iokey, val in self.db.getIoSetItemIter(db=self.sdb, - key=self._tokey(keys), - ion=ion, - sep=self.sep): + top=self._tokey(keys), + sep=self.sep.encode()): yield (self._tokeys(iokey), self.deserializer(val)) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 7031796c2..86c76a8af 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -928,28 +928,28 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): sep=self.sep)) - def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): + def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable = ''): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key Parameters: - keys (Iterable): of key strs to be combined in order to form key - ion (int): starting ordinal value, default 0 + keys (Iterable): of key strs to be combined in order to form apparent + key. When key is empty then retrieves all items in db. + Returns: - ioitems (Iterator): each item iterated is tuple (iokeys, val) where - each iokeys is actual keys tuple including hidden suffix and + items (Iterator): each item iterated is tuple (keys, val) where + each keys is apparent keys tuple without hidden suffix and each val is str empty list if no entry at keys. Raises StopIteration when done """ - for iokey, val in self.db.getIoSetItemIter(db=self.sdb, - key=self._tokey(keys), - ion=ion, - sep=self.sep): - yield (self._tokeys(iokey), self._des(val)) + for key, val in self.db.getIoSetItemIter(db=self.sdb, + top=self._tokey(keys), + sep=self.sep.encode()): + yield (self._tokeys(key), self._des(val)) diff --git a/tests/app/test_forwarding.py b/tests/app/test_forwarding.py index 6f3523cdb..0fa1b3c5f 100644 --- a/tests/app/test_forwarding.py +++ b/tests/app/test_forwarding.py @@ -68,7 +68,7 @@ def test_postman(seeder): doist.exit() msgs = [] - for _, topic, msg in mbx.cloneTopicIter(topic=recpHab.pre + "/echo", fn=0): + for _, topic, msg in mbx.cloneTopicIter(topic=recpHab.pre + "/echo"): msgs.append(msg) assert len(msgs) == 1 diff --git a/tests/app/test_storing.py b/tests/app/test_storing.py index d37eeae3c..f0aebdd48 100644 --- a/tests/app/test_storing.py +++ b/tests/app/test_storing.py @@ -85,7 +85,7 @@ def test_mailboxing(): mber.storeMsg(topic=dest.qb64b, msg=exn.raw) msgs = [] - for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=0): + for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b): msgs.append((fn, msg)) assert(len(msgs)) == 10 @@ -95,18 +95,18 @@ def test_mailboxing(): d = exn.ked["a"] assert d["b"] == idx - msgs = [] - for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=10): - msgs.append(msg) + #msgs = [] + #for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=10): + #msgs.append(msg) - assert(len(msgs)) == 0 + #assert(len(msgs)) == 0 - msgs = [] - for tn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=4): - msgs.append((tn, msg)) + #msgs = [] + #for tn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=4): + #msgs.append((tn, msg)) - assert(len(msgs)) == 6 - assert msgs[0][0] == 4 + #assert(len(msgs)) == 6 + #assert msgs[0][0] == 4 diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 609c0894c..e48f706cd 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -930,15 +930,15 @@ def test_lmdber(): assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] - assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemIter(db, key0)] == - [(b'ABC.ZYX.00000000000000000000000000000000', b'z'), - (b'ABC.ZYX.00000000000000000000000000000001', b'm'), - (b'ABC.ZYX.00000000000000000000000000000002', b'x'), - (b'ABC.ZYX.00000000000000000000000000000003', b'a')]) - - for iokey, val in dber.getIoSetItemIter(db, key0): - assert dber.delIoSetIokey(db, iokey) - assert dber.getIoSetVals(db, key0) == [] + #assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemIter(db, key0)] == + #[(b'ABC.ZYX.00000000000000000000000000000000', b'z'), + #(b'ABC.ZYX.00000000000000000000000000000001', b'm'), + #(b'ABC.ZYX.00000000000000000000000000000002', b'x'), + #(b'ABC.ZYX.00000000000000000000000000000003', b'a')]) + + #for iokey, val in dber.getIoSetItemIter(db, key0): + #assert dber.delIoSetIokey(db, iokey) + #assert dber.getIoSetVals(db, key0) == [] vals3 = [b"q", b"e"] assert dber.setIoSetVals(db, key2, vals3) @@ -967,7 +967,7 @@ def test_lmdber(): dber.cntIoSetVals(db, empty_key) dber.delIoSetVals(db, empty_key) dber.delIoSetVal(db, empty_key, some_value) - dber.getIoSetItemIter(db, empty_key) + #dber.getIoSetItemIter(db, empty_key) with pytest.raises(KeyError): dber.delIoSetIokey(db, empty_key) with pytest.raises(KeyError): diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index 1359bb280..86c305a69 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -758,7 +758,7 @@ def __iter__(self): loc = locDB.get(keys=(end.eid, scheme)) assert loc == wit3loc - # test IoItem methods + ## test IoItem methods iokeys0 = [f'{cid0}.witness.00000000000000000000000000000000'.encode("utf-8"), f'{cid0}.witness.00000000000000000000000000000001'.encode("utf-8"), f'{cid0}.witness.00000000000000000000000000000002'.encode("utf-8")] @@ -775,28 +775,13 @@ def __iter__(self): i = 0 - for iokeys, end in endDB.getIoSetItemIter(keys=keys0): + for keys, end in endDB.getIoSetItemIter(keys=keys0): assert end == ends[i] - assert iokeys == iokeys0[i] + assert keys == keys0 i += 1 - i = 0 - for iokeys, end in endDB.getIoSetItemIter(keys=keys0): - assert end == ends[i] - assert iokeys == iokeys0[i] - i += 1 - i = 1 - for iokeys, end in endDB.getIoSetItemIter(keys=keys0, ion=i): - assert end == ends[i] - assert iokeys == iokeys0[i] - i += 1 - i = 1 - for iokeys, end in endDB.getIoSetItemIter(keys=keys0, ion=i): - assert end == ends[i] - assert iokeys == iokeys0[i] - i += 1 # test getAllItemIter ends = ends + [wit3end] diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index a3db90b1d..7d7dc16fe 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -842,21 +842,15 @@ def test_ioset_suber(): (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys1)] - assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), - (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys0)] - assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), - (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] - - items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys1, ion=1)] - assert items ==[(('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + items = [(keys, val) for keys, val in iosuber.getIoSetItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!')] - items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys0, ion=1)] - assert items == [(('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + items = [(keys, val) for keys, val in iosuber.getIoSetItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] # Test with top keys @@ -952,10 +946,10 @@ def test_cesr_ioset_suber(): assert db.name == "test" assert db.opened - sdb = subing.CesrIoSetSuber(db=db, subkey='bags.', klas=coring.Saider) - assert isinstance(sdb, subing.CesrIoSetSuber) - assert issubclass(sdb.klas, coring.Saider) - assert not sdb.sdb.flags()["dupsort"] + cisuber = subing.CesrIoSetSuber(db=db, subkey='bags.', klas=coring.Saider) + assert isinstance(cisuber, subing.CesrIoSetSuber) + assert issubclass(cisuber.klas, coring.Saider) + assert not cisuber.sdb.flags()["dupsort"] seqner0 = coring.Seqner(sn=20) seq0 = seqner0.qb64 @@ -1029,63 +1023,63 @@ def test_cesr_ioset_suber(): said2 = saider2.qb64 assert said2 == 'EJxOaEsBSObrcmrsnlfHOdVAowGhUBKoE2Ce3TZ4Mhgu' - assert sdb.put(keys=keys0, vals=[saider1, saider0]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.put(keys=keys0, vals=[saider1, saider0]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said1, said0] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said0 - assert sdb.rem(keys0) - actuals = sdb.get(keys=keys0) + assert cisuber.rem(keys0) + actuals = cisuber.get(keys=keys0) assert not actuals assert actuals == [] - assert sdb.cnt(keys0) == 0 + assert cisuber.cnt(keys0) == 0 - assert sdb.put(keys=keys0, vals=[saider0, saider1]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.put(keys=keys0, vals=[saider0, saider1]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said0, said1] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said1 - assert sdb.add(keys=keys0, val=saider2) - assert sdb.cnt(keys0) == 3 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.add(keys=keys0, val=saider2) + assert cisuber.cnt(keys0) == 3 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 3 sers = [actual.qb64 for actual in actuals] assert sers == [said0, said1, said2] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said2 - assert sdb.pin(keys=keys0, vals=[saider1, saider2]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.pin(keys=keys0, vals=[saider1, saider2]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said1, said2] - assert sdb.put(keys=keys1, vals=[saider2, saider1, saider0]) - assert sdb.cnt(keys1) == 3 - actuals = sdb.get(keys=keys1) # insertion order not lexicographic + assert cisuber.put(keys=keys1, vals=[saider2, saider1, saider0]) + assert cisuber.cnt(keys1) == 3 + actuals = cisuber.get(keys=keys1) # insertion order not lexicographic assert len(actuals) == 3 sers = [actual.qb64 for actual in actuals] assert sers == [said2, said1, said0] - assert sdb.rem(keys=keys1, val=saider1) - assert sdb.cnt(keys1) == 2 - actuals = sdb.get(keys=keys1) # insertion order not lexicographic + assert cisuber.rem(keys=keys1, val=saider1) + assert cisuber.cnt(keys1) == 2 + actuals = cisuber.get(keys=keys1) # insertion order not lexicographic sers = [actual.qb64 for actual in actuals] assert sers == [said2, said0] - sers = [val.qb64 for val in sdb.getIter(keys=keys1)] + sers = [val.qb64 for val in cisuber.getIter(keys=keys1)] assert sers == [said2, said0] - items = [(keys, val.qb64) for keys, val in sdb.getItemIter()] + items = [(keys, val.qb64) for keys, val in cisuber.getItemIter()] assert items == [ (keys1, said2), (keys1, said0), @@ -1094,7 +1088,7 @@ def test_cesr_ioset_suber(): ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getFullItemIter()] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getFullItemIter()] assert items == [ ((*keys1, '00000000000000000000000000000000'), said2), ((*keys1, '00000000000000000000000000000002'), said0), @@ -1102,38 +1096,35 @@ def test_cesr_ioset_suber(): ((*keys0, '00000000000000000000000000000001'), said2), ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItemIter(keys=keys1)] - assert items == [ - ((*keys1, '00000000000000000000000000000000'), said2), - ((*keys1, '00000000000000000000000000000002'), said0), - ] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getIoSetItemIter(keys=keys0)] + assert items == [(keys0, said1), (keys0, said2)] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getIoSetItemIter(keys=keys1)] assert items == [ - ((*keys0, '00000000000000000000000000000000'), said1), - ((*keys0, '00000000000000000000000000000001'), said2), + (keys1, said2), + (keys1, said0), ] topkeys = (seq1, "") - items = [(keys, val.qb64) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val.qb64) for keys, val in cisuber.getItemIter(keys=topkeys)] assert items == [ (keys1, said2), (keys1, said0), ] topkeys = (seq0, "") - items = [(iokeys, val.qb64) for iokeys, val in sdb.getFullItemIter(keys=topkeys)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getFullItemIter(keys=topkeys)] assert items == [ ((*keys0, '00000000000000000000000000000000'), said1), ((*keys0, '00000000000000000000000000000001'), said2), ] - for iokeys, val in sdb.getFullItemIter(): - assert sdb.remIokey(iokeys=iokeys) + for iokeys, val in cisuber.getFullItemIter(): + assert cisuber.remIokey(iokeys=iokeys) - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 + assert cisuber.cnt(keys=keys0) == 0 + assert cisuber.cnt(keys=keys1) == 0 assert not os.path.exists(db.path) @@ -1774,15 +1765,15 @@ def test_cat_cesr_ioset_suber(): (keys2 + ('00000000000000000000000000000001', ), [sqr2.qb64, dgr2.qb64]) ] - items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoSetItemIter(keys=keys1)] - assert items == [(keys1 + ('00000000000000000000000000000000', ), [sqr2.qb64, dgr2.qb64])] + items = [(keys, [val.qb64 for val in vals]) + for keys, vals in sdb.getIoSetItemIter(keys=keys1)] + assert items == [(keys1, [sqr2.qb64, dgr2.qb64])] - items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoSetItemIter(keys=keys0)] + items = [(keys, [val.qb64 for val in vals]) + for keys, vals in sdb.getIoSetItemIter(keys=keys0)] assert items == [ - (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), - (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), + (keys0, [sqr0.qb64, dgr0.qb64]), + (keys0, [sqr1.qb64, dgr1.qb64]), ] From 6b11ac18ebad28698cc970cada802a1eaed0e24a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 14:29:47 -0600 Subject: [PATCH 303/418] refactored out getIoSetItemIter and getIoDupItemIter they are now same functionality as getItemIter so no longer neede. --- src/keri/app/storing.py | 2 +- src/keri/db/dbing.py | 63 +-------------------------- src/keri/db/koming.py | 96 +++++++++++++++++++++-------------------- src/keri/db/subing.py | 73 +++++++++---------------------- tests/db/test_dbing.py | 58 ++++++++++++------------- tests/db/test_koming.py | 8 +--- tests/db/test_subing.py | 32 ++++++-------- 7 files changed, 116 insertions(+), 216 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index 6b5d54417..c003f39ec 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -129,7 +129,7 @@ def cloneTopicIter(self, topic, fn=0): if hasattr(topic, 'encode'): topic = topic.encode("utf-8") - for ion, (topic, dig) in enumerate(self.getIoSetItemIter(self.tpcs, topic)): + for ion, (topic, dig) in enumerate(self.getTopIoSetItemIter(self.tpcs, topic)): if ion >= fn: if msg := self.msgs.get(keys=dig): yield ion, topic, msg.encode("utf-8") diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index aa846fee5..279621596 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1217,7 +1217,7 @@ def delIoSetIokey(self, db, iokey): # change this to key, ion " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoSetItemIter(self, db, top=b'', *, sep=b'.'): + def getTopIoSetItemIter(self, db, top=b'', *, sep=b'.'): """ Returns: items (Iterator[(key,val)]): iterator of tuples (key, val) where @@ -1718,67 +1718,6 @@ def cntIoDupVals(self, db, key): return count -# used in IoDupSuber -# do we really need this? Do we really need to access a subset of duplicates by -# their ion? Is not yet used so now is the time to remove this - def getIoDupItemIter(self, db, key=b'', *, ion=0): - """ - Iterates over branch of db given by key of IoDup items where each value - has 33 byte insertion ordinal number proem (prefixed) with separator. - Automagically removes (strips) proem before returning items. - - Assumes DB opened with dupsort=True - - Returns: - ioitems (Iterator): each item iterated is tuple (keys, ioval) where - each keys is actual keys tuple and each ioval is dup that - includes prefixed insertion ordering proem - empty list if no entry at keys. - Raises StopIteration when done - - - Because cursor.iternext() advances cursor after returning item its safe - to delete the item within the iteration loop. curson.iternext() works - for both dupsort==False and dupsort==True - - Raises StopIteration Error when empty. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items - from multiple branches of the key space. If top key is - empty then gets all items in database - ion (int): starting ordinal value, default 0 - - - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any - for ckey, cval in cursor.iternext(): # get key, val first dup at cursor - ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch - break # done - cion = int(bytes(cval[:32]), 16) # convert without sep '.' in [33] - if cion < ion: # cion cursor insertion order proem as int - continue # skip dups whose cion is below ion - yield (ckey, cval) # include proem on val - return # done raises StopIteration - - # used in IoDupSuber.getItemIter def getTopIoDupItemIter(self, db, top=b''): """ diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index e3cda6a55..1354163cb 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -73,23 +73,43 @@ def __init__(self, db: dbing.LMDBer, *, self.sep = sep if sep is not None else self.Sep - def _tokey(self, keys: Union[str, bytes, memoryview, Iterable]): + def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], + topive: bool=False): """ - Converts key to key str with proper separators and returns key bytes. - If key is already str then returns. Else If key is iterable (non-str) - of strs then joins with separator converts to bytes and returns + Converts keys to key bytes with proper separators and returns key bytes. + If keys is already str or bytes or memoryview then returns key bytes. + Else If keys is iterable (non-str) of strs or bytes then joins with + separator converts to key bytes and returns. When keys is iterable and + topive is True then enables partial key from top branch of key space given + by partial keys by appending separator to end of partial key + + Returns: + key (bytes): each element of keys is joined by .sep. If top then last + char of key is also .sep Parameters: - keys (Union[str, bytes, Iterable]): str, bytes, or Iterable of str. + keys (str | bytes | memoryview | Iterable[str | bytes]): db key or + Iterable of (str | bytes) to form key. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ + if hasattr(keys, "encode"): # str + return keys.encode("utf-8") if isinstance(keys, memoryview): # memoryview of bytes return bytes(keys) # return bytes - if hasattr(keys, "encode"): # str - return keys.encode("utf-8") # convert to bytes elif hasattr(keys, "decode"): # bytes - return keys # return as is - return (self.sep.join(keys).encode("utf-8")) # iterable so join + return keys + if topive and keys[-1]: # topive and keys is not already partial + keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep + return (self.sep.join(key.decode() if hasattr(key, "decode") else key + for key in keys).encode("utf-8")) + def _tokeys(self, key: Union[str, bytes, memoryview]): @@ -591,33 +611,7 @@ def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def getIoSetItemIter(self, keys: Union[str, Iterable] = ''): - """ - Gets (iokeys, val) ioitems iterator at key made from keys where key is - apparent effective key and items all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form apparent - key without insertion ordering suffix. When keys - is empty then retrieves all items in db. - - Returns: - items (Iterator): each item iterated is tuple (keys, val) where - each keys is apparent keys tuple without hidden suffix and - each val is str - empty list if no entry at keys. - Raises StopIteration when done - - """ - for iokey, val in self.db.getIoSetItemIter(db=self.sdb, - top=self._tokey(keys), - sep=self.sep.encode()): - yield (self._tokeys(iokey), self.deserializer(val)) - - - - - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: Union[str, Iterable]=b"", *, topive=False): """Get items iterator Returns: items (Iterator): of (key, val) tuples over the all the items in @@ -627,17 +621,27 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): Returned key in each item has ordinal suffix removed. Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from + keys (Iterable): tuple of bytes or strs that may be a truncation of + a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) - yield (self._tokeys(key), self.deserializer(val)) - + all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + """ + for iokey, val in self.db.getTopIoSetItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive), + sep=self.sep.encode()): + yield (self._tokeys(iokey), self.deserializer(val)) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 86c76a8af..9379ca3a8 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -95,7 +95,7 @@ def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], topive: bool=False): """ Converts keys to key bytes with proper separators and returns key bytes. - If keys is already str or bytes then returns key bytes. + If keys is already str or bytes or memoryview then returns key bytes. Else If keys is iterable (non-str) of strs or bytes then joins with separator converts to key bytes and returns. When keys is iterable and topive is True then enables partial key from top branch of key space given @@ -928,31 +928,6 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): sep=self.sep)) - def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable = ''): - """ - Gets (iokeys, val) ioitems iterator at key made from keys where key is - apparent effective key and items all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form apparent - key. When key is empty then retrieves all items in db. - - - Returns: - items (Iterator): each item iterated is tuple (keys, val) where - each keys is apparent keys tuple without hidden suffix and - each val is str - empty list if no entry at keys. - Raises StopIteration when done - - """ - for key, val in self.db.getIoSetItemIter(db=self.sdb, - top=self._tokey(keys), - sep=self.sep.encode()): - yield (self._tokeys(key), self._des(val)) - - - def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ @@ -985,14 +960,11 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", partial ending in sep regardless of top value """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, topive=topive)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) + for key, val in self.db.getTopIoSetItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive), sep=self.sep.encode()): yield (self._tokeys(key), self._des(val)) - - class CesrIoSetSuber(CesrSuberBase, IoSetSuber): """ Subclass of CesrSuber and IoSetSuber. @@ -1950,30 +1922,27 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): return (self.db.cntIoDupVals(db=self.sdb, key=self._tokey(keys))) - def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable = '', - *, ion=0): - """ - Gets (iokeys, val) ioitems iterator at key made from keys where key is - apparent effective key and items all have same apparent effective key + #def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable = ''): + #""" + #Gets (iokeys, val) ioitems iterator at key made from keys where key is + #apparent effective key and items all have same apparent effective key - Parameters: - keys (str|bytes|memoryview|Iterable): of key strs to be combined - in order to form key. When keys is empty then retrieves - all items in database. - ion (int): starting ordinal value, default 0 + #Parameters: + #keys (str|bytes|memoryview|Iterable): of key strs to be combined + #in order to form key. When keys is empty then retrieves + #all items in database. - Returns: - ioitems (Iterator): each item iterated is tuple (keys, ioval) where - each keys is actual keys tuple and each ioval is dup that - includes prefixed insertion ordering proem. - Empty list if no entry at keys. - Raises StopIteration when done + #Returns: + #items (Iterator): each item iterated is tuple (keys, val) where + #each keys is actual keys tuple and each val is dup without + #prefixed insertion ordering proem. + #Empty list if no entry at keys. + #Raises StopIteration when done - """ - for key, ioval in self.db.getIoDupItemIter(db=self.sdb, - key=self._tokey(keys), - ion=ion): - yield (self._tokeys(key), self._des(ioval)) + #""" + #for key, val in self.db.getIoDupItemIter(db=self.sdb, + #key=self._tokey(keys)): + #yield (self._tokeys(key), self._des(val)) def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index e48f706cd..caae1bb5e 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -653,46 +653,42 @@ def test_lmdber(): # test getIoDupItemIter - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb)] # default all - assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000000.z'), - (b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), - (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x'), - (b'A.00000000000000000000000000000002', b'00000000000000000000000000000000.o'), - (b'A.00000000000000000000000000000002', b'00000000000000000000000000000001.r'), - (b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z'), - (b'A.00000000000000000000000000000004', b'00000000000000000000000000000000.h'), - (b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n'), - (b'A.00000000000000000000000000000007', b'00000000000000000000000000000000.k'), - (b'A.00000000000000000000000000000007', b'00000000000000000000000000000001.b')] - - # all keys but ion is non zero - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, ion=1)] # default all - assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), - (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x'), - (b'A.00000000000000000000000000000002', b'00000000000000000000000000000001.r'), - (b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z'), - (b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n'), - (b'A.00000000000000000000000000000007', b'00000000000000000000000000000001.b')] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x'), + (b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z'), + (b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n'), + (b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] - # dups at aKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=aKey, ion=1)] - assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), - (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x')] + # dups at aKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=aKey)] + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x')] # dups at bKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=bKey, ion=2)] - assert items == [(b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z')] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=bKey)] + assert items == [(b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z')] # dups at cKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=cKey, ion=1)] - assert items ==[(b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n')] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=cKey)] + assert items == [(b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n')] # dups at dKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=dKey, ion=3)] - assert not items + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=dKey)] + assert items == [(b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] # dups at missing key - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=b"B.")] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=b"B.")] assert not items diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index 86c305a69..76a7e83a8 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -773,17 +773,13 @@ def __iter__(self): 'witness', '00000000000000000000000000000002')] - + # test getItemIter i = 0 - for keys, end in endDB.getIoSetItemIter(keys=keys0): + for keys, end in endDB.getItemIter(keys=keys0): assert end == ends[i] assert keys == keys0 i += 1 - - - - # test getAllItemIter ends = ends + [wit3end] i = 0 for keys, end in endDB.getItemIter(): diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 7d7dc16fe..64b599a87 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -530,21 +530,17 @@ def test_iodup_suber(): (('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] - items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys1)] - assert items == [(('test_key', '0002'), '00000000000000000000000000000000.Not my type.'), - (('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), - (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] + + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!')] - items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys0)] - assert items == [(('test_key', '0001'), '00000000000000000000000000000000.See ya later.'), - (('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!')] - items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys1, ion=1)] - assert items ==[(('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), - (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] - items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys0, ion=1)] - assert items == [(('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!')] # Test with top keys assert ioduber.put(keys=("test", "pop"), vals=[sal, sue, sam]) @@ -843,11 +839,11 @@ def test_ioset_suber(): (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(keys, val) for keys, val in iosuber.getIoSetItemIter(keys=keys0)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys0)] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!')] - items = [(keys, val) for keys, val in iosuber.getIoSetItemIter(keys=keys1)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys1)] assert items == [(('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'Hello sailer!'), (('test_key', '0002'), 'A real charmer!')] @@ -1096,10 +1092,10 @@ def test_cesr_ioset_suber(): ((*keys0, '00000000000000000000000000000001'), said2), ] - items = [(iokeys, val.qb64) for iokeys, val in cisuber.getIoSetItemIter(keys=keys0)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getItemIter(keys=keys0)] assert items == [(keys0, said1), (keys0, said2)] - items = [(iokeys, val.qb64) for iokeys, val in cisuber.getIoSetItemIter(keys=keys1)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getItemIter(keys=keys1)] assert items == [ (keys1, said2), (keys1, said0), @@ -1766,11 +1762,11 @@ def test_cat_cesr_ioset_suber(): ] items = [(keys, [val.qb64 for val in vals]) - for keys, vals in sdb.getIoSetItemIter(keys=keys1)] + for keys, vals in sdb.getItemIter(keys=keys1)] assert items == [(keys1, [sqr2.qb64, dgr2.qb64])] items = [(keys, [val.qb64 for val in vals]) - for keys, vals in sdb.getIoSetItemIter(keys=keys0)] + for keys, vals in sdb.getItemIter(keys=keys0)] assert items == [ (keys0, [sqr0.qb64, dgr0.qb64]), (keys0, [sqr1.qb64, dgr1.qb64]), From d3cd1fccaf21782760dc9269fe2029db2609e18f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 15:21:19 -0600 Subject: [PATCH 304/418] more refactoring. --- src/keri/db/dbing.py | 9 +++++++-- src/keri/db/koming.py | 21 +++++++++++---------- src/keri/db/subing.py | 5 +++-- tests/db/test_koming.py | 13 ++++--------- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 279621596..354bcd999 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -919,7 +919,7 @@ def addIoSetVal(self, db, key, val, *, sep=b'.'): iokey = suffix(key, ion, sep=sep) # ion is at add on amount return cursor.put(iokey, val, dupdata=False, overwrite=False) - + # deprecated since violates set property of each item in a set is unique def appendIoSetVal(self, db, key, val, *, sep=b'.'): """ Append val non-idempotently to insertion ordered set of values all with @@ -1196,10 +1196,15 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return cursor.delete() # delete also moves to next so doubly moved return False - + # deprecated since violates set property of each item in a set is unique + # this is needed to remove non-idempotently appended items which violate + # set property def delIoSetIokey(self, db, iokey): # change this to key, ion """ Deletes val at at actual iokey that includes ordinal key suffix. + Need this to delete value that is non-idempotently added with + appendIoSetVal where multiple of same value exist at same key because + delIoSetVal will only delete the first one in list. Returns: result (bool): True if val was deleted at iokey. False otherwise diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index 1354163cb..54a336965 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -596,19 +596,20 @@ def rem(self, keys: Union[str, Iterable], val=None): key=self._tokey(keys), sep=self.sep) - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): - """ - Removes entry at iokeys + #def remIokey(self, iokeys: str | bytes | memoryview | Iterable): + #""" + #Removes entries at iokeys - Parameters: - iokeys (tuple): of key str or tuple of key strs to be combined in - order to form key + #Parameters: + #iokeys (str | bytes | memoryview | Iterable): of key str or + #tuple of key strs to be combined in order to form key - Returns: - result (bool): True if key exists so delete successful. False otherwise + #Returns: + #result (bool): True if key exists so delete successful. False otherwise + + #""" + #return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) def getItemIter(self, keys: Union[str, Iterable]=b"", *, topive=False): diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 9379ca3a8..87ba41901 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -765,7 +765,7 @@ def add(self, keys: str | bytes | memoryview | Iterable, val=self._ser(val), sep=self.sep)) - + # deprecated since violates set property of each item in a set is unique def append(self, keys: str | bytes | memoryview | Iterable, val: str | bytes | memoryview): """Append val non-idempotently to insertion ordered set of values all with @@ -899,7 +899,8 @@ def rem(self, keys: str | bytes | memoryview | Iterable, key=self._tokey(keys), sep=self.sep) - + # deprecated since violates set property of each item in a set is unique + # this is to be able to remove non idempotent append def remIokey(self, iokeys: str | bytes | memoryview | Iterable): """ Removes entries at iokeys diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index 76a7e83a8..ecc58ff9d 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -827,16 +827,11 @@ def __iter__(self): assert iokeys == iokeysall[i] i += 1 - i = 0 - for iokeys, end in endDB.getFullItemIter(): - assert end == ends[i] - assert iokeys == iokeysall[i] - i += 1 - assert endDB.remIokey(iokeys) - - assert endDB.cnt(keys0) == 0 - assert endDB.cnt(keys1) == 0 + #for iokeys, val in endDB.getFullItemIter(): + #assert endDB.remIokey(iokeys=iokeys) + #assert endDB.cnt(keys=keys0) == 0 + #assert endDB.cnt(keys=keys1) == 0 assert not os.path.exists(db.path) assert not db.opened From 03e26c6cdb258915c858098b86bce8e19de90637 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 16:05:03 -0600 Subject: [PATCH 305/418] created CesrOnSuber subclass with unit test --- src/keri/db/escrowing.py | 2 +- src/keri/db/subing.py | 27 ++++++++++ tests/db/test_subing.py | 112 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index dcf741d07..082833cb8 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -170,7 +170,7 @@ def escrowStateNotice(self, *, typ, pre, aid, serder, saider, dater, cigars=None for cigar in cigars: # process each couple to verify sig and write to db self.cigardb.put(keys=keys, vals=[(cigar.verfer, cigar)]) - return self.escrowdb.put(keys=(typ, pre, aid), vals=[saider]) # overwrite + return self.escrowdb.put(keys=(typ, pre, aid), vals=[saider]) # does not overwrite def updateReply(self, aid, serder, saider, dater): """ diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 87ba41901..5242bf952 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -568,6 +568,33 @@ def __init__(self, *pa, **kwa): super(CesrSuber, self).__init__(*pa, **kwa) +class CesrOnSuber(CesrSuberBase, OnSuberBase, Suber): + """ + Subclass of CesrSuberBase, OnSuberBase, and Suber that adds methods for + keys with ordinal numbered suffixes and values that are Cesr serializations + of Matter subclass ducktypes. + + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(CesrOnSuber, self).__init__(*pa, **kwa) + + class CatCesrSuberBase(CesrSuberBase): """ Base Class whose values stored in db are a concatenation of the .qb64b property diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 64b599a87..a10a7a536 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -619,7 +619,7 @@ def test_iodup_suber(): assert not db.opened -def test_oniodup_suber(): +def test_on_iodup_suber(): """ Test OnIoDupSuber LMDBer sub database class """ @@ -1468,6 +1468,113 @@ def test_cesr_suber(): assert not db.opened """Done Test""" +def test_cesr_on_suber(): + """ + Test CesrOnSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.CesrOnSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.CesrOnSuber) + assert not onsuber.sdb.flags()["dupsort"] + + prew = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + w = coring.Matter(qb64=prew) + prex = "BHHzqZWzwE-Wk7K0gzQPYGGwTmuupUhPx5_y1x4ejhcc" + x = coring.Matter(qb64=prex) + prey = "EHHzqZWzwE-Wk7K0gzQPYGGwTmuupUhPx5_y1x4ejhcc" + y = coring.Matter(qb64=prey) + prez = "EAPYGGwTmuupWzwEHHzq7K0gzUhPx5_yZ-Wk1x4ejhcc" + z = coring.Matter(qb64=prez) + + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val.qb64) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), w.qb64), + (('a', '00000000000000000000000000000001'), x.qb64), + (('a', '00000000000000000000000000000002'), y.qb64), + (('a', '00000000000000000000000000000003'), z.qb64)] + + # test getOnItemIter + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, y.qb64), + (('a',), 3, z.qb64)] + + assert 0 == onsuber.appendOn(keys=("ac",), val=z) + assert 0 == onsuber.appendOn(keys=("b",), val=w) + assert 1 == onsuber.appendOn(keys=("b",), val=x) + assert 0 == onsuber.appendOn(keys=("bc",), val=y) + + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 + + items = [(keys, val.qb64) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), w.qb64), + (('a', '00000000000000000000000000000001'), x.qb64), + (('a', '00000000000000000000000000000002'), y.qb64), + (('a', '00000000000000000000000000000003'), z.qb64), + (('ac', '00000000000000000000000000000000'), z.qb64), + (('b', '00000000000000000000000000000000'), w.qb64), + (('b', '00000000000000000000000000000001'), x.qb64), + (('bc', '00000000000000000000000000000000'), y.qb64)] + + # test getOnItemIter + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, w.qb64), + (('b',), 1, x.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, w.qb64), + (('b',), 1, x.qb64)] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64), + (('ac',), 0, z.qb64), + (('b',), 0, w.qb64), + (('b',), 1, x.qb64), + (('bc',), 0, y.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter()] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64), + (('ac',), 0, z.qb64), + (('b',), 0, w.qb64), + (('b',), 1, x.qb64), + (('bc',), 0, y.qb64)] + + assert not os.path.exists(db.path) + assert not db.opened + + def test_cat_suber(): """ @@ -2280,10 +2387,11 @@ def test_crypt_signer_suber(): test_on_suber() test_dup_suber() test_iodup_suber() - test_oniodup_suber() + test_on_iodup_suber() test_ioset_suber() test_cat_suber() test_cesr_suber() + test_cesr_on_suber() test_cesr_ioset_suber() test_cat_cesr_ioset_suber() test_cesr_dup_suber() From bb4811ea2582d59e36e6aaf2d21a09925ccbb1c4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 17:48:05 -0600 Subject: [PATCH 306/418] more clean up refactoring. Created delOnVal method and corresponding method in OnSuberBase --- src/keri/db/dbing.py | 149 +++++++++++++++++++++++---------------- src/keri/db/escrowing.py | 6 +- src/keri/db/subing.py | 38 +++++++--- tests/db/test_dbing.py | 12 ++++ tests/db/test_subing.py | 25 +++++++ 5 files changed, 156 insertions(+), 74 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 354bcd999..d322eec42 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -693,14 +693,98 @@ def delTopVal(self, db, top=b''): # For subdbs the use keys with trailing part the is monotonically # ordinal number serialized as 32 hex bytes + + + def appendOnVal(self, db, key, val, *, sep=b'.'): + """ + Appends val in order after last previous onkey in db where + onkey has same given key prefix but with different serialized on suffix + attached with sep. + Returns ordinal number on, of appended entry. Appended on is 1 greater + than previous latest on at key. + Uses onKey(key, on) for entries. + + Works with either dupsort==True or False since always creates new full + key. + + Append val to end of db entries with same key but with on incremented by + 1 relative to last preexisting entry at key. + + Returns: + on (int): ordinal number of newly appended val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split + """ + # set key with fn at max and then walk backwards to find last entry at pre + # if any otherwise zeroth entry at pre + onkey = onKey(key, MaxON, sep=sep) + with self.env.begin(db=db, write=True, buffers=True) as txn: + on = 0 # unless other cases match then zeroth entry at pre + cursor = txn.cursor() + if not cursor.set_range(onkey): # max is past end of database + # so either empty database or last is earlier pre or + # last is last entry at same pre + if cursor.last(): # not empty db. last entry earlier than max + ckey = cursor.key() + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last is last entry for same pre + on = cn + 1 # increment + else: # not past end so not empty either later pre or max entry at pre + ckey = cursor.key() + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last entry for pre is already at max + raise ValueError(f"Number part {cn=} for key part {ckey=}" + f"exceeds maximum size.") + else: # later pre so backup one entry + # either no entry before last or earlier pre with entry + if cursor.prev(): # prev entry, maybe same or earlier pre + ckey = cursor.key() + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last entry at pre + on = cn + 1 # increment + + onkey = onKey(key, on, sep=sep) + + if not cursor.put(onkey, val, overwrite=False): + raise ValueError(f"Failed appending {val=} at {key=}.") + return on + + + def delOnVal(self, db, key, on=0, *, sep=b'.'): + """ + Deletes value at onkey consisting of key + sep + serialized on in db. + Returns True If key exists in database Else False + + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which to delete + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=True, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return (txn.delete(onkey)) # when empty deletes whole db + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): """ - Returns (int): count of of all ordinal keyed vals with top key - but different on tail in db starting at ordinal number on of top. + Returns (int): count of of all ordinal keyed vals with key + but different on tail in db starting at ordinal number on of key. Full key is composed of top+sep+ When dupsort==true then duplicates are included in count since .iternext includes duplicates. - when key is empty then retrieves whole db + when key is empty then counts whole db Parameters: db (lmdbsubdb): named sub db of lmdb @@ -771,65 +855,6 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): - def appendOnVal(self, db, key, val, *, sep=b'.'): - """ - Appends val in order after last previous key with same pre in db where - full key has key prefix and serialized on suffix attached with sep. - Returns ordinal number in, on, of appended entry. Appended on is 1 greater - than previous latest on at pre. - Uses onKey(pre, on) for entries. - - Works with either dupsort==True or False since always creates new full - key. - - Append val to end of db entries with same pre but with on incremented by - 1 relative to last preexisting entry at pre. - - Returns: - on (int): ordinal number of newly appended val - - Parameters: - db (subdb): named sub db in lmdb - key (bytes): key within sub db's keyspace plus trailing part on - val (bytes): serialized value to append - sep (bytes): separator character for split - """ - # set key with fn at max and then walk backwards to find last entry at pre - # if any otherwise zeroth entry at pre - onkey = onKey(key, MaxON, sep=sep) - with self.env.begin(db=db, write=True, buffers=True) as txn: - on = 0 # unless other cases match then zeroth entry at pre - cursor = txn.cursor() - if not cursor.set_range(onkey): # max is past end of database - # so either empty database or last is earlier pre or - # last is last entry at same pre - if cursor.last(): # not empty db. last entry earlier than max - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) - if ckey == key: # last is last entry for same pre - on = cn + 1 # increment - else: # not past end so not empty either later pre or max entry at pre - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) - if ckey == key: # last entry for pre is already at max - raise ValueError(f"Number part {cn=} for key part {ckey=}" - f"exceeds maximum size.") - else: # later pre so backup one entry - # either no entry before last or earlier pre with entry - if cursor.prev(): # prev entry, maybe same or earlier pre - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) - if ckey == key: # last entry at pre - on = cn + 1 # increment - - onkey = onKey(key, on, sep=sep) - - if not cursor.put(onkey, val, overwrite=False): - raise ValueError(f"Failed appending {val=} at {key=}.") - return on - - - # IoSet insertion order in val so can have effective dups but with # dupsort = False so val not limited to 511 bytes # For databases that support set of insertion ordered values with apparent diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 082833cb8..3a4267a95 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -117,20 +117,20 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) except Exception as ex: # other error so remove from reply escrow - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: logger.error("Kevery unescrowed due to error: %s", ex.args[0]) else: # unescrow succeded - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow only + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow only self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) logger.info("Kevery unescrow succeeded for txn state=%s", serder.said) logger.debug(f"event=\n{serder.pretty()}\n") except Exception as ex: # log diagnostics errors etc - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) self.removeState(saider) if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 5242bf952..0cefb1be8 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -413,7 +413,7 @@ def appendOn(self, keys: str | bytes | memoryview, keys (str | bytes | memoryview | Iterable): top keys as prefix to be combined with serialized on suffix and sep to form key val (str | bytes | memoryview): serialization - on (int): ordinal number used with onKey(pre,on) to form key. + on (int): ordinal number used with onKey(key,on) to form key. """ return (self.db.appendOnVal(db=self.sdb, key=self._tokey(keys), @@ -421,21 +421,40 @@ def appendOn(self, keys: str | bytes | memoryview, sep=self.sep.encode())) + def remOn(self, keys: str | bytes | memoryview, on: int=0): + """ + Returns + result (bool): True if onkey made from key+sep+serialized on is + found in database so value is removed + Removes all duplicates if any at onkey. + False otherwise + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + """ + return (self.db.delOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode())) + def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): """ Returns - cnt (int): count of of all ordinal suffix keyed vals with same top - in key but different on in key in db starting at ordinal number - on of pre where key is formed with onKey(pre,on) - Does not count dups at same on for a given pre, only - unique on at a given pre. + cnt (int): count of of all ordinal suffix keyed vals with same + key prefix but different on in onkey in db starting at ordinal + number on where key is formed with onKey(key,on). Count at + each onkey includes duplicates if any. + Parameters: keys (str | bytes | memoryview | Iterable): top keys as prefix to be combined with serialized on suffix and sep to form top key - When keys is empty then counts whole database including duplicates - on (int): ordinal number used with onKey(pre,on) to form key. + When keys is empty then counts whole database including + duplicates if any. + on (int): ordinal number used with onKey(key,on) to form key. """ return (self.db.cntOnVals(db=self.sdb, key=self._tokey(keys), @@ -451,7 +470,8 @@ def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): Parameters: keys (str | bytes | memoryview | iterator): top keys as prefix to be combined with serialized on suffix and sep to form key - When keys is empty then retrieves whole database including duplicates + When keys is empty then retrieves whole database including + duplicates if any on (int): ordinal number used with onKey(pre,on) to form key. sep (bytes): separator character for split """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index caae1bb5e..b75cf6b32 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -459,6 +459,18 @@ def test_lmdber(): items = [item for item in dber.getOnItemIter(db, key=preC, on=1)] assert items == [] + # test delOnVal + assert dber.delOnVal(db, key=preB) # default on=0 + assert not dber.delOnVal(db, key=preB, on=0) + assert dber.delOnVal(db, key=preB, on=1) + assert not dber.delOnVal(db, key=preB, on=1) + + items = [item for item in dber.getOnItemIter(db, key=preB)] + assert items == [(top, 2, digW), (top, 3, digX), (top, 4, digY)] + + with pytest.raises(KeyError): + assert dber.delOnVal(db, key=b'') # empty key + # test Vals dup methods. dup vals are lexocographic key = b'A' diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index a10a7a536..698bc7f2b 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -317,6 +317,17 @@ def test_on_suber(): (('b',), 1, 'Green tree'), (('bc',), 0, 'Red apple')] + assert onsuber.remOn(keys='a', on=1) + assert not onsuber.remOn(keys='a', on=1) + assert onsuber.remOn(keys='a', on=3) + assert not onsuber.remOn(keys='a', on=3) + + assert onsuber.cntOn(keys=("a",)) == 2 + + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 2, 'Red apple')] + assert not os.path.exists(db.path) assert not db.opened @@ -759,6 +770,20 @@ def test_on_iodup_suber(): assert 4 == onsuber.appendOn(keys=("a",), val=x) assert onsuber.cntOn(keys=("a",)) == 9 + assert onsuber.remOn(keys='a', on=1) + assert not onsuber.remOn(keys='a', on=1) + assert onsuber.remOn(keys='a', on=3) + assert not onsuber.remOn(keys='a', on=3) + + assert onsuber.cntOn(keys=("a",)) == 5 + + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 0, 'White snow'), + (('a',), 2, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 4, 'Green tree')] + assert not os.path.exists(db.path) assert not db.opened From 6e67288dd0ff571e9fed0df8cd7595e88b2e2781 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 17:56:39 -0600 Subject: [PATCH 307/418] final refactor of broker escrowing --- src/keri/db/escrowing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 3a4267a95..36cdbc0e9 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -79,7 +79,7 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): extype (Type[Exception]): the expected exception type if the message should remain in escrow """ - for (typ, pre, aid, ion), saider in self.escrowdb.getFullItemIter(keys=(typ, '')): + for (typ, pre, aid), saider in self.escrowdb.getItemIter(keys=(typ, '')): try: tsgs = eventing.fetchTsgs(db=self.tigerdb, saider=saider) @@ -117,20 +117,20 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) except Exception as ex: # other error so remove from reply escrow - self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: logger.error("Kevery unescrowed due to error: %s", ex.args[0]) else: # unescrow succeded - self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow only self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow logger.info("Kevery unescrow succeeded for txn state=%s", serder.said) logger.debug(f"event=\n{serder.pretty()}\n") except Exception as ex: # log diagnostics errors etc - self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.removeState(saider) if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) From caa534f02a26c0714fc4f1571e721564c45f1316 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 14:20:19 -0600 Subject: [PATCH 308/418] refactored mailbox to use OnSuber --- src/keri/app/storing.py | 135 ++++++++++++++++++++++++++------------ src/keri/db/dbing.py | 9 ++- src/keri/db/subing.py | 13 ++-- tests/app/test_storing.py | 5 +- 4 files changed, 109 insertions(+), 53 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index c003f39ec..073b365c5 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -34,6 +34,17 @@ def __init__(self, name="mbx", headDirPath=None, reopen=True, **kwa): perm: reopen: kwa: + + Mailboxer uses two dbs for mailbox messages these are .tpcs and .msgs. + The message index is in .tpcs (topics). + Each .tpcs index key consists of topic.on where topic is bytes + identifier or prefix/topic for message and on is serialized + ordinal number to orders the appearance of a topic message. + Eash .tpcs val is the digest of the message. + The message itself is stored in .msgs where the key is the msg digest + and the value is the serialized messag itself. + Multiple messages can share the same topic but with a different ordinal. + """ self.tpcs = None self.msgs = None @@ -48,91 +59,131 @@ def reopen(self, **kwa): """ super(Mailboxer, self).reopen(**kwa) - self.tpcs = self.env.open_db(key=b'tpcs.', dupsort=True) + #self.tpcs = self.env.open_db(key=b'tpcs.', dupsort=True) + self.tpcs = subing.OnSuber(db=self, subkey='tpcs.') self.msgs = subing.Suber(db=self, subkey='msgs.') # key states return self.env - def delTopic(self, key): - """ - Use snKey() - Deletes value at key. - Returns True If key exists in database Else False + def delTopic(self, key, on=0): + """Removes topic index from .tpcs without deleting message from .msgs + + Returns: + result (boo): True if full key consisting of key and serialized on + exists in database so removed + False otherwise (not removed) """ - return self.delIoSetVals(self.tpcs, key) + #return self.delIoSetVals(self.tpcs, key) + return self.tpcs.remOn(keys=key, on=on) def appendToTopic(self, topic, val): - """ - Return first seen order number int, fn, of appended entry. - Computes fn as next fn after last entry. + """Appends val to end of db entries with same topic but with on + incremented by 1 relative to last preexisting entry at topic. - Append val to end of db entries with same topic but with fn incremented by - 1 relative to last preexisting entry at pre. + Returns: + on (int): order number int, on, of appended entry. + Computes on as next on after last entry. Parameters: - topic is bytes identifier prefix/topic for message - val is event digest + topic (bytes): topic identifier for message + val (bytes): msg digest """ - return self.appendIoSetVal(db=self.tpcs, key=topic, val=val) + #return self.appendIoSetVal(db=self.tpcs, key=topic, val=val) + return self.tpcs.appendOn(key=topic, val=val) + def getTopicMsgs(self, topic, fn=0): """ Returns: - ioset (oset): the insertion ordered set of values at same apparent - effective key. - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. + msgs (Iterable[bytes]): belonging to topic indices with same topic but all + on >= fn i.e. all topic.on beginning with fn Parameters: - topic (Option(bytes|str)): Apparent effective key - fn (int) starting index + topic (Option(bytes|str)): key prefix combined with serialized on + to form full actual key. When key is empty then retrieves + whole database. + fn (int): starting index ordinal number used with onKey(pre,on) + to form key at at which to initiate retrieval """ - if hasattr(topic, "encode"): - topic = topic.encode("utf-8") - - digs = self.getIoSetVals(db=self.tpcs, key=topic, ion=fn) msgs = [] - for dig in digs: + for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn): if msg := self.msgs.get(keys=dig): - msgs.append(msg.encode("utf-8")) + msgs.append(msg.encode()) # want bytes not str return msgs + #if hasattr(topic, "encode"): + #topic = topic.encode("utf-8") + + #digs = self.getIoSetVals(db=self.tpcs, key=topic, ion=fn) + #msgs = [] + #for dig in digs: + #if msg := self.msgs.get(keys=dig): + #msgs.append(msg.encode("utf-8")) + #return msgs + def storeMsg(self, topic, msg): """ - Add exn event to mailbox of dest identifier + Add exn event to mailbox topic and on that is 1 greater than last msg + at topic. + + Returns: + result (bool): True if msg successfully stored and indexed at topic + False otherwise Parameters: - msg (bytes): - topic (qb64b): + topic (str | bytes): topic (Option(bytes|str)): key prefix combined + with serialized on to form full actual key. + msg (bytes): serialized message """ - if hasattr(topic, "encode"): - topic = topic.encode("utf-8") - if hasattr(msg, "encode"): msg = msg.encode("utf-8") digb = coring.Diger(ser=msg, code=MtrDex.Blake3_256).qb64b - self.appendToTopic(topic=topic, val=digb) + on = self.tpcs.appendOn(keys=topic, val=digb) return self.msgs.pin(keys=digb, val=msg) + #if hasattr(topic, "encode"): + #topic = topic.encode("utf-8") + + #if hasattr(msg, "encode"): + #msg = msg.encode("utf-8") + + #digb = coring.Diger(ser=msg, code=MtrDex.Blake3_256).qb64b + #self.appendToTopic(topic=topic, val=digb) + #return self.msgs.pin(keys=digb, val=msg) + + def cloneTopicIter(self, topic, fn=0): """ - Returns iterator of first seen exn messages with attachments for the - identifier prefix pre starting at first seen order number, fn. + Returns: + triple (Iterator[(on, topic, msg): iterator of messages at topic + beginning with ordinal fn. + + topic (Option(bytes|str)): key prefix combined with serialized on + to form full actual key. When key is empty then retrieves + whole database. + fn (int): starting index ordinal number used with onKey(pre,on) + to form key at at which to initiate retrieval + + ToDo looks like misuse of IoSet this should not be IoSet but simply Ordinal Numbered db. since should not be using hidden ion has not hidden. """ - if hasattr(topic, 'encode'): - topic = topic.encode("utf-8") + for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn): + if msg := self.msgs.get(keys=dig): + yield (on, topic, msg.encode("utf-8")) + + #if hasattr(topic, 'encode'): + #topic = topic.encode("utf-8") - for ion, (topic, dig) in enumerate(self.getTopIoSetItemIter(self.tpcs, topic)): - if ion >= fn: - if msg := self.msgs.get(keys=dig): - yield ion, topic, msg.encode("utf-8") + #for ion, (topic, dig) in enumerate(self.getTopIoSetItemIter(self.tpcs, topic)): + #if ion >= fn: + #if msg := self.msgs.get(keys=dig): + #yield ion, topic, msg.encode("utf-8") #for (key, dig) in self.getIoSetItemIter(self.tpcs, key=topic, ion=fn): #topic, ion = dbing.unsuffix(key) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index d322eec42..fece6423c 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -522,7 +522,9 @@ def setVal(self, db, key, val): """ Write serialized bytes val to location key in db Overwrites existing val if any - Returns True If val successfully written Else False + Returns: + result (bool): True If val successfully written + False otherwise Parameters: db is opened named sub db with dupsort=False @@ -829,7 +831,8 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): Raises StopIteration Error when empty. Returns: - items (Iterator[(key, on, val)]): triples of key, on, val + items (Iterator[(key, on, val)]): triples of key, on, val with same + key but increments of on beginning with on Parameters: db (subdb): named sub db in lmdb @@ -856,7 +859,7 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): # IoSet insertion order in val so can have effective dups but with - # dupsort = False so val not limited to 511 bytes + # dupsort==False so val not limited to 511 bytes # For databases that support set of insertion ordered values with apparent # effective duplicate key but with (dupsort==False). Actual key uses hidden # key suffix ordinal to provide insertion ordering of value members of set diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 0cefb1be8..fa36ef8bf 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -463,16 +463,17 @@ def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ - Returns - items (Iterator[(top keys, on, val)]): triples of (top keys, on int, - deserialized val) + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val with same + key but increments of on >= on i.e. all key.on beginning with on Parameters: - keys (str | bytes | memoryview | iterator): top keys as prefix to be - combined with serialized on suffix and sep to form key + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form actual key When keys is empty then retrieves whole database including duplicates if any - on (int): ordinal number used with onKey(pre,on) to form key. + on (int): ordinal number used with onKey(pre,on) to form key at at + which to initiate retrieval sep (bytes): separator character for split """ for keys, on, val in (self.db.getOnItemIter(db=self.sdb, diff --git a/tests/app/test_storing.py b/tests/app/test_storing.py index f0aebdd48..2abafbc7d 100644 --- a/tests/app/test_storing.py +++ b/tests/app/test_storing.py @@ -9,7 +9,7 @@ from keri.app import keeping from keri.core import coring, serdering -from keri.db import dbing, basing +from keri.db import dbing, basing, subing from keri.peer import exchanging from keri.app.storing import Mailboxer @@ -28,7 +28,8 @@ def test_mailboxing(): assert mber.env.path() == mber.path assert os.path.exists(mber.path) - assert isinstance(mber.tpcs, lmdb._Database) + #assert isinstance(mber.tpcs, lmdb._Database) + assert isinstance(mber.tpcs, subing.OnSuber) mber.close(clear=True) assert not os.path.exists(mber.path) From 00ad0c26f290005abf4168c60d44f2dfb6f1ff4e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 14:53:51 -0600 Subject: [PATCH 309/418] Removed obsolete methods Comments on todo things --- src/keri/db/dbing.py | 109 +++++++--------------------------------- src/keri/db/subing.py | 45 ----------------- tests/db/test_dbing.py | 17 ------- tests/db/test_subing.py | 18 ------- 4 files changed, 17 insertions(+), 172 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index fece6423c..8094c0862 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -589,6 +589,7 @@ def cnt(self, db): return count # Suber subclasses don't need this since they always split keys int tuple to return + # replace this everywhere with getTopItemIter def getAllItemIter(self, db, key=b'', split=True, sep=b'.'): """ Returns iterator of item duple (key, val), at each key over all @@ -947,65 +948,6 @@ def addIoSetVal(self, db, key, val, *, sep=b'.'): iokey = suffix(key, ion, sep=sep) # ion is at add on amount return cursor.put(iokey, val, dupdata=False, overwrite=False) - # deprecated since violates set property of each item in a set is unique - def appendIoSetVal(self, db, key, val, *, sep=b'.'): - """ - Append val non-idempotently to insertion ordered set of values all with - the same apparent effective key. If val already in set at key then - after append there will be multiple entries in database with val at key - each with different insertion order (iokey). - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. - - Works by walking backward to find last iokey for key instead of reading - all vals for ioky. - - Returns: - ion (int): hidden insertion ordering ordinal of appended val - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - val (bytes): value to append - """ - ion = 0 # default is zeroth insertion at key - iokey = suffix(key, ion=MaxSuffix, sep=sep) # make iokey at max and walk back - with self.env.begin(db=db, write=True, buffers=True) as txn: - cursor = txn.cursor() # create cursor to walk back - if not cursor.set_range(iokey): # max is past end of database - # Three possibilities for max past end of database - # 1. last entry in db is for same key - # 2. last entry in db is for other key before key - # 3. database is empty - if cursor.last(): # not 3. empty db, so either 1. or 2. - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # 1. last is last entry for same key - ion = cion + 1 # so set ion to the increment of cion - else: # max is not past end of database - # Two possibilities for max not past end of databseso - # 1. cursor at max entry at key - # 2. other key after key with entry in database - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # 1. last entry for key is already at max - raise ValueError("Number part of key {} at maximum" - " size.".format(ckey)) - else: # 2. other key after key so backup one entry - # Two possibilities: 1. no prior entry 2. prior entry - if cursor.prev(): # prev entry, maybe same or earlier pre - # 2. prior entry with two possiblities: - # 1. same key - # 2. other key before key - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # prior (last) entry at key - ion = cion + 1 # so set ion to the increment of cion - - iokey = suffix(key, ion=ion, sep=sep) - if not cursor.put(iokey, val, overwrite=False): - raise ValueError("Failed appending {} at {}.".format(val, key)) - - return ion - - def setIoSetVals(self, db, key, vals, *, sep=b'.'): """ @@ -1224,31 +1166,6 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return cursor.delete() # delete also moves to next so doubly moved return False - # deprecated since violates set property of each item in a set is unique - # this is needed to remove non-idempotently appended items which violate - # set property - def delIoSetIokey(self, db, iokey): # change this to key, ion - """ - Deletes val at at actual iokey that includes ordinal key suffix. - Need this to delete value that is non-idempotently added with - appendIoSetVal where multiple of same value exist at same key because - delIoSetVal will only delete the first one in list. - - Returns: - result (bool): True if val was deleted at iokey. False otherwise - if no val at iokey - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - iokey (bytes): actual key with ordinal key suffix - """ - with self.env.begin(db=db, write=True, buffers=True) as txn: - try: - return txn.delete(iokey) - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getTopIoSetItemIter(self, db, top=b'', *, sep=b'.'): """ @@ -1713,8 +1630,6 @@ def delIoDupVal(self, db, key, val): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return False - #def delIoDupIonVal(self, db, key, ion): # to match delIoSetIoKey - def cntIoDupVals(self, db, key): """ @@ -1960,7 +1875,7 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): # without gaps is used for replay with on to provide partial replay # Consider using ItemIter instead of just replaying vals since ItemIter already -# works with either dupsort==True or False +# works with either dupsort==True or False. also skips gaps. def getOnIoDupValsAllPreIter(self, db, pre, on=0): """ @@ -1998,15 +1913,23 @@ def getOnIoDupValsAllPreIter(self, db, pre, on=0): key = snKey(pre, cnt:=cnt+1) + # Create multiple methods + # getTopItemBackIter symmetric with getTopItemIter + # getTopIoSetItemBackIter symmetric getTopIoSetItemIter + # getTopIoDupItemBackIter symmetric with getTopIoDupItemIter + # getOnItemBackIter symmetric with getOnItemIter + + + # instead of just replaying vals replay items since + # getTopItemIter already works with either dupsort==True or False + # so if we create getTopBackItemIter then we can use that everywhere # ToDo need unit tests in dbing Used to replay backwards all duplicate # values starting at on # need to fix this so it is not stopped by gaps or if gap raises error as # gap is normally a problem for replay so maybe a parameter to raise error on gap - # Consider creating BackItemIter instead of just replaying vals since - # ItemIter already works with either dupsort==True or False - # so if we create TopBackItemIter then we can use that everywhere + def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries @@ -2044,6 +1967,7 @@ def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): yield val[33:] key = snKey(pre, cnt:=cnt-1) +# Last is special so need method. # Used to replay forward all last duplicate values starting at on # need to fix this so it is not stopped by gaps or if gap raises error as # gap is normally a problem for replay so maybe a parameter to raise error on gap @@ -2086,8 +2010,9 @@ def getIoDupValLastAllPreIter(self, db, pre, on=0): # ToDo do we need a replay last backwards? - - # use this method to inform how to cross gaps in other replays above with parameter + # not used anymore. + # before deleting this method use it inform how to cross gaps in other + # replays above with parameter such as replay last above. # to raise error if detect gap when should not be one for event logs vs escrows # then remove this method since not used otherwise diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index fa36ef8bf..a136701b6 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -813,35 +813,6 @@ def add(self, keys: str | bytes | memoryview | Iterable, val=self._ser(val), sep=self.sep)) - # deprecated since violates set property of each item in a set is unique - def append(self, keys: str | bytes | memoryview | Iterable, - val: str | bytes | memoryview): - """Append val non-idempotently to insertion ordered set of values all with - the same apparent effective key. If val already in set at key then - after append there will be multiple entries in database with val at key - each with different insertion order (iokey). - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. - - Works by walking backward to find last iokey for key instead of reading - all vals for iokey. - - Returns: - ion (int): hidden insertion ordering ordinal of appended val - - Parameters: - keys (str | bytes | memoryview | Iterable): of key parts to be - combined in order to form key - val (str | bytes | memoryview): serialization - - - """ - return (self.db.appendIoSetVal(db=self.sdb, - key=self._tokey(keys), - val=self._ser(val), - sep=self.sep)) - - def pin(self, keys: str | bytes | memoryview | Iterable, vals: str | bytes | memoryview | Iterable): @@ -947,22 +918,6 @@ def rem(self, keys: str | bytes | memoryview | Iterable, key=self._tokey(keys), sep=self.sep) - # deprecated since violates set property of each item in a set is unique - # this is to be able to remove non idempotent append - def remIokey(self, iokeys: str | bytes | memoryview | Iterable): - """ - Removes entries at iokeys - - Parameters: - iokeys (str | bytes | memoryview | Iterable): of key str or - tuple of key strs to be combined in order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - - """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def cnt(self, keys: str | bytes | memoryview | Iterable): """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index b75cf6b32..b52e980e5 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -934,19 +934,6 @@ def test_lmdber(): assert dber.getIoSetVals(db, key1) == vals1 assert dber.getIoSetVals(db, key2) == vals2 - assert dber.appendIoSetVal(db, key1, val=b"k") == 4 - assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] - - - #assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemIter(db, key0)] == - #[(b'ABC.ZYX.00000000000000000000000000000000', b'z'), - #(b'ABC.ZYX.00000000000000000000000000000001', b'm'), - #(b'ABC.ZYX.00000000000000000000000000000002', b'x'), - #(b'ABC.ZYX.00000000000000000000000000000003', b'a')]) - - #for iokey, val in dber.getIoSetItemIter(db, key0): - #assert dber.delIoSetIokey(db, iokey) - #assert dber.getIoSetVals(db, key0) == [] vals3 = [b"q", b"e"] assert dber.setIoSetVals(db, key2, vals3) @@ -968,16 +955,12 @@ def test_lmdber(): dber.putIoSetVals(db, empty_key, [some_value]) dber.addIoSetVal(db, empty_key, some_value) dber.setIoSetVals(db, empty_key, [some_value]) - dber.appendIoSetVal(db, empty_key, some_value) dber.getIoSetVals(db, empty_key) [_ for _ in dber.getIoSetValsIter(db, empty_key)] dber.getIoSetValLast(db, empty_key) dber.cntIoSetVals(db, empty_key) dber.delIoSetVals(db, empty_key) dber.delIoSetVal(db, empty_key, some_value) - #dber.getIoSetItemIter(db, empty_key) - with pytest.raises(KeyError): - dber.delIoSetIokey(db, empty_key) with pytest.raises(KeyError): dber.putVals(db, empty_key, [some_value]) with pytest.raises(KeyError): diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 698bc7f2b..1a0424623 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -913,11 +913,6 @@ def test_ioset_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in iosuber.getFullItemIter(): - assert iosuber.remIokey(iokeys=iokeys) - - assert iosuber.cnt(keys=keys0) == 0 - assert iosuber.cnt(keys=keys1) == 0 # test with keys as string not tuple @@ -949,8 +944,6 @@ def test_ioset_suber(): assert iosuber.trim() # default trims whole database assert iosuber.put(keys=keys1, vals=[bob, bil]) assert iosuber.get(keys=keys1) == [bob, bil] - assert iosuber.append(keys=keys1, val=bob) == 2 - assert iosuber.get(keys=keys1) == [bob, bil, bob] assert not os.path.exists(db.path) assert not db.opened @@ -1141,11 +1134,6 @@ def test_cesr_ioset_suber(): ((*keys0, '00000000000000000000000000000001'), said2), ] - for iokeys, val in cisuber.getFullItemIter(): - assert cisuber.remIokey(iokeys=iokeys) - - assert cisuber.cnt(keys=keys0) == 0 - assert cisuber.cnt(keys=keys1) == 0 assert not os.path.exists(db.path) @@ -1921,12 +1909,6 @@ def test_cat_cesr_ioset_suber(): (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), ] - for iokeys, val in sdb.getFullItemIter(): - assert sdb.remIokey(iokeys=iokeys) - - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 - assert sdb.cnt(keys=keys2) == 0 # Try Siger Indexer Subclass sdb = subing.CatCesrIoSetSuber(db=db, subkey='pigs.', klas=(indexing.Siger, )) From cf0407ed0b7c75845d8eeedc6df03f05913551a0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 16:10:53 -0600 Subject: [PATCH 310/418] more refactoring removing unused methods dbing basing --- src/keri/app/notifying.py | 3 +- src/keri/db/basing.py | 103 +-------- src/keri/db/dbing.py | 123 ++++------- src/keri/db/subing.py | 31 +++ src/keri/vdr/eventing.py | 12 +- src/keri/vdr/viring.py | 6 +- tests/db/test_basing.py | 441 +------------------------------------- tests/db/test_dbing.py | 83 +------ 8 files changed, 96 insertions(+), 706 deletions(-) diff --git a/src/keri/app/notifying.py b/src/keri/app/notifying.py index 4ad03d22f..09ac24db7 100644 --- a/src/keri/app/notifying.py +++ b/src/keri/app/notifying.py @@ -184,7 +184,8 @@ def getItemIter(self, keys: Union[str, Iterable] = b""): each entry in db """ - for key, val in self.db.getAllItemIter(db=self.sdb, key=self._tokey(keys), split=False): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys)): yield self._tokeys(key), self.klas(raw=bytes(val)) def cntAll(self): diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index fc132f52b..750f6211a 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1467,7 +1467,7 @@ def clean(self): cpydb.add(keys=keys, val=val) # Insecure raw imgs database copy. - for (key, val) in self.getAllItemIter(self.imgs): + for (key, val) in self.getTopItemIter(self.imgs): copy.imgs.setVal(key=key, val=val) # clone .habs habitat name prefix Komer subdb @@ -2286,19 +2286,6 @@ def getUreLast(self, key): """ return self.getIoDupValLast(self.ures, key) - def getUreItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event triple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is triple dig+pre+cig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.ures, key, skip) def getUreItemsNextIter(self, key=b'', skip=True): """ @@ -2453,20 +2440,6 @@ def getVreLast(self, key): """ return self.getIoDupValLast(self.vres, key) - def getVreItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event quintuple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is Quinlet is edig + spre + ssnu + sdig +sig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.vres, key, skip) - def getVreItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -2683,18 +2656,6 @@ def getPseLast(self, key): """ return self.getIoDupValLast(self.pses, key) - def getPseItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.pses, key, skip) - def getPseItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -2783,18 +2744,6 @@ def getPweLast(self, key): """ return self.getIoDupValLast(self.pwes, key) - def getPweItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial witnessed escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.pwes, key, skip) - def getPweItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -2899,20 +2848,6 @@ def getUweLast(self, key): """ return self.getIoDupValLast(self.uwes, key) - def getUweItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed receipt couple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is couple edig+wig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.uwes, key, skip) - def getUweItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -2993,18 +2928,6 @@ def getOoeLast(self, key): """ return self.getIoDupValLast(self.ooes, key) - def getOoeItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of out of order escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.ooes, key, skip) - def getOoeItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -3084,18 +3007,6 @@ def getQnfLast(self, key): """ return self.getIoDupValLast(self.qnfs, key) - def getQnfItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of out of order escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.qnfs, key, skip) - def getQnfItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -3250,18 +3161,6 @@ def getLdeLast(self, key): """ return self.getIoDupValLast(self.ldes, key) - def getLdeItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of likely duplicitous escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.ldes, key, skip) - def getLdeItemsNextIter(self, key=b'', skip=True): """ Use sgKey() diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 8094c0862..5f3a17045 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -588,39 +588,6 @@ def cnt(self, db): count += 1 return count - # Suber subclasses don't need this since they always split keys int tuple to return - # replace this everywhere with getTopItemIter - def getAllItemIter(self, db, key=b'', split=True, sep=b'.'): - """ - Returns iterator of item duple (key, val), at each key over all - keys in db. If split is true then the key is split at sep and instead - of returing duple it results tuple with one entry for each key split - as well as the value. - - Works for both dupsort==False and dupsort==True - - Raises StopIteration Error when empty. - - Parameters: - db is opened named sub db with dupsort=False - key is key location in db to resume replay, - If empty then start at first key in database - split (bool): True means split key at sep before returning - sep (bytes): separator char for key - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if not cursor.set_range(key): # moves to val at key >= key, first if empty - return # no values end of db - - for key, val in cursor.iternext(): # return key, val at cursor - if split: - splits = bytes(key).split(sep) - splits.append(val) - else: - splits = (bytes(key), val) - yield tuple(splits) - def getTopItemIter(self, db, top=b''): """ @@ -1714,51 +1681,51 @@ def getTopIoDupItemIter(self, db, top=b''): # not needed in IoDupSuber since not used anywhere always uses the Iter version # below. confirm this in general and refactor any that do use to use the iter verio - def getIoDupItemsNext(self, db, key=b"", skip=True): - """ - Return list of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key == b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Returns empty list if no entries at next key after key - - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database - - Assumes DB opened with dupsort=True - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty string - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass - """ - - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - items = [] - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - # slice off prepended ordering prefix on value in item - items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] - return items + #def getIoDupItemsNext(self, db, key=b"", skip=True): + #""" + #Return list of all dup items at next key after key in db in insertion order. + #Item is (key, val) with proem stripped from val stored in db. + #If key == b'' then returns list of dup items at first key in db. + #If skip is False and key is not empty then returns dup items at key + #Returns empty list if no entries at next key after key + + #If key is empty then gets io items (key, io value) at first key in db + #Use the return key from items as next key for next call to function in + #order to iterate through the database + + #Assumes DB opened with dupsort=True + + #Duplicates at a given key preserve insertion order of duplicate. + #Because lmdb is lexocographic an insertion ordering proem is prepended to + #all values that makes lexocographic order that same as insertion order. + + #Duplicates are ordered as a pair of key plus value so prepending proem + #to each value changes duplicate ordering. Proem is 33 characters long. + #With 32 character hex string followed by '.' for essentiall unlimited + #number of values which will be limited by memory. + + #With prepended proem ordinal must explicity check for duplicate values + #before insertion. Uses a python set for the duplicate inclusion test. + #Set inclusion scales with O(1) whereas list inclusion scales with O(n). + + #Parameters: + #db is opened named sub db with dupsort=True + #key is bytes of key within sub db's keyspace or empty string + #skip is Boolean If True skips to next key if key is not empty string + #Othewise don't skip for first pass + #""" + + #with self.env.begin(db=db, write=False, buffers=True) as txn: + #cursor = txn.cursor() + #items = [] + #if cursor.set_range(key): # moves to first_dup at key + #found = True + #if skip and key and cursor.key() == key: # skip to next key + #found = cursor.next_nodup() # skip to next key not dup if any + #if found: + ## slice off prepended ordering prefix on value in item + #items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] + #return items # don't need this in IoDupSuber because can replace with self.getTopIoDupItemIter # which in turen can be replaced with IoDupSuber.getItemIter diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index a136701b6..09c1bb8e6 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -36,6 +36,37 @@ DupSuber CesrDupSuber +Class Architecture + +Suber is simple lexographic database with only one value per key +OnSuber is simple lexographic database where trailing part of key is serialized + ordinal number so that the ordering within each key prefix is monotonically + increasing numeric + +The term 'set' of values means that no value may appear more than once in the set. + +DupSuber provides set of lexicographic ordered values at each key. Each value has + a limited size (key + value <= 511 byes). The set is performant. Good for indices. + +IoDupSuber provides set of insertion ordered values at each key. Each value has + a limited size (key + value <= 511 byes). The set is less perfromant than DupSuber + but more performant than IoSetSuber. Good for insertion ordered indices + +IoSetSuber proves set of insertion ordered values at each key. Value size is not limited + Good for any insertion ordered set where size may be too large for IoDupSuber + +OnIoDupSuber provides set of insertion ordered values where the where trailing + part of key is serialized ordinal number so that the ordering within each + key prefix is monotonically increasing numeric. + +Each of these base types for managing the key space may be mixed with other +Classes that provide different types of values these include. + +Cesr +CatCesr +Serder +etc. + """ from typing import Type, Union diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 2cc89a235..e696babac 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -23,7 +23,7 @@ from ..core.signing import (Salter,) from ..core.eventing import SealEvent, ample, TraitDex, verifySigs from ..db import basing, dbing -from ..db.dbing import dgKey, snKey +from ..db.dbing import dgKey, snKey, splitSnKey from ..help import helping from ..kering import (MissingWitnessSignatureError, Version, MissingAnchorError, ValidationError, OutOfOrderError, LikelyDuplicitousError) @@ -2031,9 +2031,10 @@ def processEscrowOutOfOrders(self): 5. Remove event digest from oots if processed successfully or a non-out-of-order event occurs. """ - for (pre, snb, digb) in self.reger.getOotItemIter(): + for key, digb in self.reger.getOotItemIter(): # (pre, snb, digb) in self.reger.getOotItemIter() try: - sn = int(snb, 16) + #sn = int(snb, 16) + pre, sn = splitSnKey(key) dgkey = dgKey(pre, digb) traw = self.reger.getTvt(dgkey) if traw is None: @@ -2099,8 +2100,9 @@ def processEscrowAnchorless(self): 6. Remove event digest from oots if processed successfully or a non-anchorless event occurs. """ - for (pre, snb, digb) in self.reger.getTaeItemIter(): - sn = int(snb, 16) + for key, digb in self.reger.getTaeItemIter(): #(pre, snb, digb) in self.reger.getTaeItemIter() + pre, sn = splitSnKey(key) + #sn = int(snb, 16) try: dgkey = dgKey(pre, digb) traw = self.reger.getTvt(dgkey) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 11123c57b..c2fccfdab 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -809,7 +809,8 @@ def getTaeItemIter(self): Return iterator of all items in .taes """ - return self.getAllItemIter(self.taes, split=True) + return self.getTopItemIter(self.taes) + def delTae(self, key): """ @@ -852,7 +853,8 @@ def getOotItemIter(self): Return iterator of all items in .taes """ - return self.getAllItemIter(self.oots, split=True) + return self.getTopItemIter(self.oots) + def delOot(self, key): """ diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index fc02d3ec3..29a498a41 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -491,64 +491,6 @@ def test_baser(): assert db.putUres(key=cKey, vals=cVals) assert db.putUres(key=dKey, vals=dVals) - # Test getUreItemsNext( key=b"") - # aVals - items = db.getUreItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUreItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUreItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getUreItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getUreItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getUreItemsNextIter(key=b"") # get dups at first key in database @@ -613,10 +555,6 @@ def test_baser(): for key, val in items: assert db.delUre(ikey, val) == True - # none - items = [item for item in db.getUreItemsNext(key=ikey)] - assert items == [] # empty - assert not items # Validator (transferable) Receipts # test .vrcs sub db methods dgkey @@ -698,64 +636,6 @@ def test_baser(): assert db.putVres(key=cKey, vals=cVals) assert db.putVres(key=dKey, vals=dVals) - # Test getVreItemsNext( key=b"") - # aVals - items = db.getVreItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getVreItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getVreItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getVreItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getVreItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getVreItemsNextIter(key=b"") # get dups at first key in database @@ -820,10 +700,6 @@ def test_baser(): for key, val in items: assert db.delVre(ikey, val) == True - # none - items = [item for item in db.getVreItemsNext(key=ikey)] - assert items == [] # empty - assert not items # test .kels insertion order dup methods. dup vals are insertion order @@ -883,64 +759,6 @@ def test_baser(): assert db.putPses(key=cKey, vals=cVals) assert db.putPses(key=dKey, vals=dVals) - # Test getPseItemsNext( key=b"") - # aVals - items = db.getPseItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPseItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPseItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getPseItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getPseItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getPseItemsNextIter(key=b"") # get dups at first key in database @@ -1005,10 +823,6 @@ def test_baser(): for key, val in items: assert db.delPse(ikey, val) == True - # none - items = [item for item in db.getPseItemsNext(key=ikey)] - assert items == [] # empty - assert not items # Test .udes partial delegated escrow seal source couples key = dgKey(preb, digb) @@ -1085,64 +899,6 @@ def test_baser(): assert db.putPwes(key=cKey, vals=cVals) assert db.putPwes(key=dKey, vals=dVals) - # Test getPweItemsNext( key=b"") - # aVals - items = db.getPweItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPweItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPweItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getPweItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getPweItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getPweItemsNextIter(key=b"") # get dups at first key in database @@ -1207,10 +963,6 @@ def test_baser(): for key, val in items: assert db.delPwe(ikey, val) == True - # none - items = [item for item in db.getPweItemsNext(key=ikey)] - assert items == [] # empty - assert not items # Unverified Witness Receipt Escrows # test .uwes insertion order dup methods. dup vals are insertion order @@ -1249,64 +1001,6 @@ def test_baser(): assert db.putUwes(key=cKey, vals=cVals) assert db.putUwes(key=dKey, vals=dVals) - # Test getUweItemsNext( key=b"") - # aVals - items = db.getUweItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUweItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUweItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getUweItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getUweItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getUweItemsNextIter(key=b"") # get dups at first key in database @@ -1371,10 +1065,7 @@ def test_baser(): for key, val in items: assert db.delUwe(ikey, val) == True - # none - items = [item for item in db.getUweItemsNext(key=ikey)] - assert items == [] # empty - assert not items + # test .ooes insertion order dup methods. dup vals are insertion order key = b'A' @@ -1411,65 +1102,6 @@ def test_baser(): assert db.putOoes(key=cKey, vals=cVals) assert db.putOoes(key=dKey, vals=dVals) - # Test getOoeItemsNext( key=b"") - # aVals - items = db.getOoeItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getOoeItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getOoeItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getOoeItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getOoeItemsNext(key=ikey) - assert items == [] # empty - assert not items - # Test getOoeItemsNextIter(key=b"") # get dups at first key in database # aVals @@ -1533,12 +1165,6 @@ def test_baser(): for key, val in items: assert db.delOoe(ikey, val) == True - # none - items = [item for item in db.getOoeItemsNext(key=ikey)] - assert items == [] # empty - assert not items - - # test .dels insertion order dup methods. dup vals are insertion order key = b'A' vals = [b"z", b"m", b"x", b"a"] @@ -1591,64 +1217,6 @@ def test_baser(): assert db.putLdes(key=cKey, vals=cVals) assert db.putLdes(key=dKey, vals=dVals) - # Test getOoeItemsNext( key=b"") - # aVals - items = db.getLdeItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getLdeItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getLdeItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getLdeItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getLdeItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getLdeItemsNextIter(key=b"") # get dups at first key in database @@ -1713,13 +1281,6 @@ def test_baser(): for key, val in items: assert db.delLde(ikey, val) == True - # none - items = [item for item in db.getLdeItemsNext(key=ikey)] - assert items == [] # empty - assert not items - - - assert not os.path.exists(db.path) """ End Test """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index b52e980e5..e7f14a882 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -292,7 +292,7 @@ def test_lmdber(): assert dber.delVal(db, key) == True assert dber.getVal(db, key) == None - # Test getAllItemIter(self, db, key=b'', split=True, sep=b'.') + # Test getTopItemIter key = b"a.1" val = b"wow" assert dber.putVal(db, key, val) == True @@ -302,10 +302,10 @@ def test_lmdber(): key = b"b.1" val = b"woo" assert dber.putVal(db, key, val) == True - assert [(bytes(pre), bytes(num), bytes(val)) for pre, num, val - in dber.getAllItemIter(db=db)] == [(b'a', b'1', b'wow'), - (b'a', b'2', b'wee'), - (b'b', b'1', b'woo')] + assert [(bytes(key), bytes(val)) for key, val + in dber.getTopItemIter(db=db)] == [(b'a.1', b'wow'), + (b'a.2', b'wee'), + (b'b.1', b'woo')] assert dber.delTopVal(db, top=b"a.") items = [ (key, bytes(val)) for key, val in dber.getTopItemIter(db=db )] @@ -704,65 +704,6 @@ def test_lmdber(): assert not items - # Test getIoItemsNext(self, db, key=b"") - # aVals - items = dber.getIoDupItemsNext(edb) # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = dber.getIoDupItemsNext(edb, key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = dber.getIoDupItemsNext(edb, key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = dber.getIoDupItemsNext(edb, key=b'', skip=False) # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = dber.getIoDupItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = dber.getIoDupItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = dber.getIoDupItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = dber.getIoDupItemsNext(edb, key=ikey) - assert items == [] # empty - assert not items - # Test getIoItemsNextIter(self, db, key=b"") # get dups at first key in database # aVals @@ -816,20 +757,6 @@ def test_lmdber(): for key, val in items: assert dber.delIoDupVal(edb, ikey, val) == True - # dVals - items = [item for item in dber.getIoDupItemsNext(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - for key, val in items: - assert dber.delIoDupVal(edb, ikey, val) == True - - # none - items = [item for item in dber.getIoDupItemsNext(edb, key=ikey)] - assert items == [] # empty - assert not items # test OnIoDup methods key = b'Z' From a460caa87f4cbed669816ac79c368364af8d2b79 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 17:01:10 -0600 Subject: [PATCH 311/418] more refactoring --- src/keri/core/eventing.py | 2 +- src/keri/db/basing.py | 5 +++-- src/keri/db/dbing.py | 47 --------------------------------------- tests/db/test_basing.py | 45 ++++++------------------------------- 4 files changed, 11 insertions(+), 88 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index e7c597683..1200b7e31 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5873,7 +5873,7 @@ def processEscrowUnverNonTrans(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, etriplet in self.db.getUreItemsNextIter(key=key): + for ekey, etriplet in self.db.getUreItemIter(key=key): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item rsaider, sprefixer, cigar = deReceiptTriple(etriplet) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 750f6211a..27f363776 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2287,7 +2287,7 @@ def getUreLast(self, key): return self.getIoDupValLast(self.ures, key) - def getUreItemsNextIter(self, key=b'', skip=True): + def getUreItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed event triple items at next @@ -2299,7 +2299,8 @@ def getUreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.ures, key, skip) + return self.getTopIoDupItemIter(self.ures, key) + #return self.getIoDupItemsNextIter(self.ures, key, skip) def cntUres(self, key): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 5f3a17045..12477384c 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1679,53 +1679,6 @@ def getTopIoDupItemIter(self, db, top=b''): val = val[33:] # strip proem yield (top, val) -# not needed in IoDupSuber since not used anywhere always uses the Iter version -# below. confirm this in general and refactor any that do use to use the iter verio - #def getIoDupItemsNext(self, db, key=b"", skip=True): - #""" - #Return list of all dup items at next key after key in db in insertion order. - #Item is (key, val) with proem stripped from val stored in db. - #If key == b'' then returns list of dup items at first key in db. - #If skip is False and key is not empty then returns dup items at key - #Returns empty list if no entries at next key after key - - #If key is empty then gets io items (key, io value) at first key in db - #Use the return key from items as next key for next call to function in - #order to iterate through the database - - #Assumes DB opened with dupsort=True - - #Duplicates at a given key preserve insertion order of duplicate. - #Because lmdb is lexocographic an insertion ordering proem is prepended to - #all values that makes lexocographic order that same as insertion order. - - #Duplicates are ordered as a pair of key plus value so prepending proem - #to each value changes duplicate ordering. Proem is 33 characters long. - #With 32 character hex string followed by '.' for essentiall unlimited - #number of values which will be limited by memory. - - #With prepended proem ordinal must explicity check for duplicate values - #before insertion. Uses a python set for the duplicate inclusion test. - #Set inclusion scales with O(1) whereas list inclusion scales with O(n). - - #Parameters: - #db is opened named sub db with dupsort=True - #key is bytes of key within sub db's keyspace or empty string - #skip is Boolean If True skips to next key if key is not empty string - #Othewise don't skip for first pass - #""" - - #with self.env.begin(db=db, write=False, buffers=True) as txn: - #cursor = txn.cursor() - #items = [] - #if cursor.set_range(key): # moves to first_dup at key - #found = True - #if skip and key and cursor.key() == key: # skip to next key - #found = cursor.next_nodup() # skip to next key not dup if any - #if found: - ## slice off prepended ordering prefix on value in item - #items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] - #return items # don't need this in IoDupSuber because can replace with self.getTopIoDupItemIter # which in turen can be replaced with IoDupSuber.getItemIter diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 29a498a41..5655838e6 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -495,38 +495,23 @@ def test_baser(): # Test getUreItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getUreItemsNextIter()] + items = [item for item in db.getUreItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals + vals = [bytes(val) for key, val in items] + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getUreItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - items = [item for item in db.getUreItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getUreItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getUreItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delUre(ikey, val) == True # bVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -536,7 +521,7 @@ def test_baser(): assert db.delUre(ikey, val) == True # cVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -546,7 +531,7 @@ def test_baser(): assert db.delUre(ikey, val) == True # dVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -647,13 +632,6 @@ def test_baser(): vals = [val for key, val in items] assert vals == aVals - items = [item for item in db.getVreItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - items = [item for item in db.getVreItemsNextIter(key=aKey)] assert items # not empty ikey = items[0][0] @@ -661,15 +639,6 @@ def test_baser(): vals = [val for key, val in items] assert vals == bVals - items = [item for item in db.getVreItemsNextIter(key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert db.delVre(ikey, val) == True - # bVals items = [item for item in db.getVreItemsNextIter(key=ikey)] assert items # not empty From 752614a4c6edfc113f1e7141669ebf401653726c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 18:57:13 -0600 Subject: [PATCH 312/418] refactored out getIoDupItemsNextIter --- src/keri/app/cli/commands/escrow.py | 8 +- src/keri/core/eventing.py | 14 +-- src/keri/db/basing.py | 57 +++++----- src/keri/db/dbing.py | 86 +++++++-------- tests/db/test_basing.py | 156 +++++++--------------------- tests/db/test_dbing.py | 104 +++++++++---------- 6 files changed, 177 insertions(+), 248 deletions(-) diff --git a/src/keri/app/cli/commands/escrow.py b/src/keri/app/cli/commands/escrow.py index 014daf541..277ad4b7b 100644 --- a/src/keri/app/cli/commands/escrow.py +++ b/src/keri/app/cli/commands/escrow.py @@ -56,7 +56,7 @@ def escrows(tymth, tock=0.0, **opts): oots = list() key = ekey = b'' # both start same. when not same means escrows found while True: - for ekey, edig in hby.db.getOoeItemsNextIter(key=key): + for ekey, edig in hby.db.getOoeItemIter(key=key): pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: @@ -74,7 +74,7 @@ def escrows(tymth, tock=0.0, **opts): pwes = list() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in hby.db.getPweItemsNextIter(key=key): + for ekey, edig in hby.db.getPweItemIter(key=key): pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: @@ -92,7 +92,7 @@ def escrows(tymth, tock=0.0, **opts): pses = list() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in hby.db.getPseItemsNextIter(key=key): + for ekey, edig in hby.db.getPseItemIter(key=key): pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: @@ -110,7 +110,7 @@ def escrows(tymth, tock=0.0, **opts): ldes = list() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in hby.db.getLdeItemsNextIter(key=key): + for ekey, edig in hby.db.getLdeItemIter(key=key): pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 1200b7e31..580bc1f9a 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5209,7 +5209,7 @@ def processEscrowOutOfOrders(self): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in self.db.getOoeItemsNextIter(key=key): + for ekey, edig in self.db.getOoeItemIter(key=key): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) @@ -5347,7 +5347,7 @@ def processEscrowPartialSigs(self): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in self.db.getPseItemsNextIter(key=key): + for ekey, edig in self.db.getPseItemIter(key=key): eserder = None try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item @@ -5528,7 +5528,7 @@ def processEscrowPartialWigs(self): #key = ekey = b'' # both start same. when not same means escrows found #while True: # break when done - for ekey, edig in self.db.getPweIoDupItemIter(key=b''): #self.db.getPweItemsNextIter(key=key) + for ekey, edig in self.db.getPweItemIter(key=b''): #self.db.getPweItemsNextIter(key=key) try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) @@ -5753,7 +5753,7 @@ def processEscrowUnverWitness(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, ecouple in self.db.getUweItemsNextIter(key=key): + for ekey, ecouple in self.db.getUweItemIter(key=key): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key @@ -6139,7 +6139,7 @@ def processQueryNotFound(self): pre = b'' sn = 0 while True: # break when done - for ekey, edig in self.db.getQnfItemsNextIter(key=key): + for ekey, edig in self.db.getQnfItemIter(key=key): try: pre, _ = splitKey(ekey) # get pre and sn from escrow item # check date if expired then remove escrow. @@ -6386,7 +6386,7 @@ def processEscrowUnverTrans(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, equinlet in self.db.getVreItemsNextIter(key=key): + for ekey, equinlet in self.db.getVreItemIter(key=key): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) @@ -6556,7 +6556,7 @@ def processEscrowDuplicitous(self): """ key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in self.db.getLdeItemsNextIter(key=key): + for ekey, edig in self.db.getLdeItemIter(key=key): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 27f363776..0aff8f1a7 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2441,7 +2441,7 @@ def getVreLast(self, key): """ return self.getIoDupValLast(self.vres, key) - def getVreItemsNextIter(self, key=b'', skip=True): + def getVreItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed event quintuple items at next @@ -2453,7 +2453,8 @@ def getVreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.vres, key, skip) + return self.getTopIoDupItemIter(self.vres, key) + #return self.getIoDupItemsNextIter(self.vres, key, skip) def cntVres(self, key): """ @@ -2657,7 +2658,7 @@ def getPseLast(self, key): """ return self.getIoDupValLast(self.pses, key) - def getPseItemsNextIter(self, key=b'', skip=True): + def getPseItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed event dig items at next key after key. @@ -2667,7 +2668,8 @@ def getPseItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.pses, key, skip) + return self.getTopIoDupItemIter(self.pses, key) + #return self.getIoDupItemsNextIter(self.pses, key, skip) def cntPses(self, key): """ @@ -2745,19 +2747,7 @@ def getPweLast(self, key): """ return self.getIoDupValLast(self.pwes, key) - def getPweItemsNextIter(self, key=b'', skip=True): - """ - Use sgKey() - Return iterator of partial witnessed escrowed event dig items at next key after key. - Items is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNextIter(self.pwes, key, skip) - - def getPweIoDupItemIter(self, key=b''): + def getPweItemIter(self, key=b''): """ Use sgKey() Return iterator of partial witnessed escrowed event dig items at next key after key. @@ -2768,6 +2758,19 @@ def getPweIoDupItemIter(self, key=b''): Duplicates are retrieved in insertion order. """ return self.getTopIoDupItemIter(self.pwes, key) + #return self.getIoDupItemsNextIter(self.pwes, key, skip) + + #def getPweIoDupItemIter(self, key=b''): + #""" + #Use sgKey() + #Return iterator of partial witnessed escrowed event dig items at next key after key. + #Items is (key, val) where proem has already been stripped from val + #If key is b'' empty then returns dup items at first key. + #If skip is False and key is not b'' empty then returns dup items at key + #Raises StopIteration Error when empty + #Duplicates are retrieved in insertion order. + #""" + #return self.getTopIoDupItemIter(self.pwes, key) def cntPwes(self, key): """ @@ -2849,7 +2852,7 @@ def getUweLast(self, key): """ return self.getIoDupValLast(self.uwes, key) - def getUweItemsNextIter(self, key=b'', skip=True): + def getUweItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed receipt couple items at next @@ -2861,7 +2864,8 @@ def getUweItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.uwes, key, skip) + return self.getTopIoDupItemIter(self.uwes, key) + #return self.getIoDupItemsNextIter(self.uwes, key, skip) def cntUwes(self, key): """ @@ -2929,7 +2933,7 @@ def getOoeLast(self, key): """ return self.getIoDupValLast(self.ooes, key) - def getOoeItemsNextIter(self, key=b'', skip=True): + def getOoeItemIter(self, key=b''): """ Use sgKey() Return iterator of out of order escrowed event dig items at next key after key. @@ -2939,7 +2943,8 @@ def getOoeItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.ooes, key, skip) + return self.getTopIoDupItemIter(self.ooes, key) + #return self.getIoDupItemsNextIter(self.ooes, key, skip) def cntOoes(self, key): """ @@ -3008,7 +3013,7 @@ def getQnfLast(self, key): """ return self.getIoDupValLast(self.qnfs, key) - def getQnfItemsNextIter(self, key=b'', skip=True): + def getQnfItemIter(self, key=b''): """ Use sgKey() Return iterator of out of order escrowed event dig items at next key after key. @@ -3018,7 +3023,8 @@ def getQnfItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.qnfs, key, skip) + return self.getTopIoDupItemIter(self.qnfs, key) + #return self.getIoDupItemsNextIter(self.qnfs, key, skip) def cntQnfs(self, key): """ @@ -3162,7 +3168,7 @@ def getLdeLast(self, key): """ return self.getIoDupValLast(self.ldes, key) - def getLdeItemsNextIter(self, key=b'', skip=True): + def getLdeItemIter(self, key=b''): """ Use sgKey() Return iterator of likely duplicitous escrowed event dig items at next key after key. @@ -3172,7 +3178,8 @@ def getLdeItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.ldes, key, skip) + return self.getTopIoDupItemIter(self.ldes, key) + #return self.getIoDupItemsNextIter(self.ldes, key, skip) def cntLdes(self, key): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 12477384c..7786285a3 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1683,49 +1683,49 @@ def getTopIoDupItemIter(self, db, top=b''): # don't need this in IoDupSuber because can replace with self.getTopIoDupItemIter # which in turen can be replaced with IoDupSuber.getItemIter - def getIoDupItemsNextIter(self, db, key=b"", skip=True): - """ - Return iterator of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key = b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Raises StopIteration Error when no remaining dup items = empty. - - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database - - Assumes DB opened with dupsort=True - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass - """ - - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - for key, val in cursor.iternext_dup(keys=True): - yield (key, val[33:]) # slice off prepended ordering prefix + #def getIoDupItemsNextIter(self, db, key=b"", skip=True): + #""" + #Return iterator of all dup items at next key after key in db in insertion order. + #Item is (key, val) with proem stripped from val stored in db. + #If key = b'' then returns list of dup items at first key in db. + #If skip is False and key is not empty then returns dup items at key + #Raises StopIteration Error when no remaining dup items = empty. + + #If key is empty then gets io items (key, io value) at first key in db + #Use the return key from items as next key for next call to function in + #order to iterate through the database + + #Assumes DB opened with dupsort=True + + #Duplicates at a given key preserve insertion order of duplicate. + #Because lmdb is lexocographic an insertion ordering proem is prepended to + #all values that makes lexocographic order that same as insertion order. + + #Duplicates are ordered as a pair of key plus value so prepending proem + #to each value changes duplicate ordering. Proem is 33 characters long. + #With 32 character hex string followed by '.' for essentiall unlimited + #number of values which will be limited by memory. + + #With prepended proem ordinal must explicity check for duplicate values + #before insertion. Uses a python set for the duplicate inclusion test. + #Set inclusion scales with O(1) whereas list inclusion scales with O(n). + + #Parameters: + #db is opened named sub db with dupsort=True + #key is bytes of key within sub db's keyspace or empty + #skip is Boolean If True skips to next key if key is not empty string + #Othewise don't skip for first pass + #""" + + #with self.env.begin(db=db, write=False, buffers=True) as txn: + #cursor = txn.cursor() + #if cursor.set_range(key): # moves to first_dup at key + #found = True + #if skip and key and cursor.key() == key: # skip to next key + #found = cursor.next_nodup() # skip to next key not dup if any + #if found: + #for key, val in cursor.iternext_dup(keys=True): + #yield (key, val[33:]) # slice off prepended ordering prefix # methods for OnIoDup that combines IoDup value proem with On ordinal numbered diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 5655838e6..f8971f7ba 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -625,22 +625,22 @@ def test_baser(): # Test getVreItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getVreItemsNextIter()] + items = [item for item in db.getVreItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getVreItemsNextIter(key=aKey)] + items = [item for item in db.getVreItemIter(key=aKey)] assert items # not empty ikey = items[0][0] - assert ikey == bKey + assert ikey == aKey vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals # bVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -650,7 +650,7 @@ def test_baser(): assert db.delVre(ikey, val) == True # cVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -660,7 +660,7 @@ def test_baser(): assert db.delVre(ikey, val) == True # dVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -732,38 +732,24 @@ def test_baser(): # Test getPseItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getPseItemsNextIter()] + items = [item for item in db.getPseItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getPseItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getPseItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - items = [item for item in db.getPseItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getPseItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delPse(ikey, val) == True + # bVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -773,7 +759,7 @@ def test_baser(): assert db.delPse(ikey, val) == True # cVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -783,7 +769,7 @@ def test_baser(): assert db.delPse(ikey, val) == True # dVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -872,38 +858,22 @@ def test_baser(): # Test getPweItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getPweItemsNextIter()] + items = [item for item in db.getPweItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getPweItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getPweItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getPweItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getPweItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delPwe(ikey, val) == True # bVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -913,7 +883,7 @@ def test_baser(): assert db.delPwe(ikey, val) == True # cVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -923,7 +893,7 @@ def test_baser(): assert db.delPwe(ikey, val) == True # dVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -974,38 +944,22 @@ def test_baser(): # Test getUweItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getUweItemsNextIter()] + items = [item for item in db.getUweItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getUweItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getUweItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in db.getUweItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getUweItemsNextIter(key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert db.delUwe(ikey, val) == True - # bVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1015,7 +969,7 @@ def test_baser(): assert db.delUwe(ikey, val) == True # cVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1025,7 +979,7 @@ def test_baser(): assert db.delUwe(ikey, val) == True # dVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1074,38 +1028,22 @@ def test_baser(): # Test getOoeItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getOoeItemsNextIter()] + items = [item for item in db.getOoeItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getOoeItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getOoeItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getOoeItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getOoeItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delOoe(ikey, val) == True # bVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1115,7 +1053,7 @@ def test_baser(): assert db.delOoe(ikey, val) == True # cVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1125,7 +1063,7 @@ def test_baser(): assert db.delOoe(ikey, val) == True # dVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1190,38 +1128,22 @@ def test_baser(): # Test getLdeItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getLdeItemsNextIter()] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getLdeItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getLdeItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getLdeItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getLdeItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getLdeItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delLde(ikey, val) == True # bVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1231,7 +1153,7 @@ def test_baser(): assert db.delLde(ikey, val) == True # cVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1241,7 +1163,7 @@ def test_baser(): assert db.delLde(ikey, val) == True # dVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index e7f14a882..48be99dec 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -704,58 +704,58 @@ def test_lmdber(): assert not items - # Test getIoItemsNextIter(self, db, key=b"") - # get dups at first key in database - # aVals - items = [item for item in dber.getIoDupItemsNextIter(edb)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in dber.getIoDupItemsNextIter(edb, key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert dber.delIoDupVal(edb, ikey, val) == True - - # bVals - items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - for key, val in items: - assert dber.delIoDupVal(edb, ikey, val) == True - - # cVals - items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - for key, val in items: - assert dber.delIoDupVal(edb, ikey, val) == True + ## Test getIoItemsNextIter(self, db, key=b"") + ## get dups at first key in database + ## aVals + #items = [item for item in dber.getIoDupItemsNextIter(edb)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == aKey + #vals = [val for key, val in items] + #assert vals == aVals + + #items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey, skip=False)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == aKey + #vals = [val for key, val in items] + #assert vals == aVals + + #items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == bKey + #vals = [val for key, val in items] + #assert vals == bVals + + #items = [item for item in dber.getIoDupItemsNextIter(edb, key=b'', skip=False)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == aKey + #vals = [val for key, val in items] + #assert vals == aVals + #for key, val in items: + #assert dber.delIoDupVal(edb, ikey, val) == True + + ## bVals + #items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == bKey + #vals = [val for key, val in items] + #assert vals == bVals + #for key, val in items: + #assert dber.delIoDupVal(edb, ikey, val) == True + + ## cVals + #items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == cKey + #vals = [val for key, val in items] + #assert vals == cVals + #for key, val in items: + #assert dber.delIoDupVal(edb, ikey, val) == True # test OnIoDup methods From 502a7b7841adafb6c7fbd6b14683f0e15c19aac6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 18:58:00 -0600 Subject: [PATCH 313/418] clean up --- src/keri/db/dbing.py | 48 -------------------------------------------- 1 file changed, 48 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 7786285a3..21b6380e4 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1680,54 +1680,6 @@ def getTopIoDupItemIter(self, db, top=b''): yield (top, val) -# don't need this in IoDupSuber because can replace with self.getTopIoDupItemIter -# which in turen can be replaced with IoDupSuber.getItemIter - - #def getIoDupItemsNextIter(self, db, key=b"", skip=True): - #""" - #Return iterator of all dup items at next key after key in db in insertion order. - #Item is (key, val) with proem stripped from val stored in db. - #If key = b'' then returns list of dup items at first key in db. - #If skip is False and key is not empty then returns dup items at key - #Raises StopIteration Error when no remaining dup items = empty. - - #If key is empty then gets io items (key, io value) at first key in db - #Use the return key from items as next key for next call to function in - #order to iterate through the database - - #Assumes DB opened with dupsort=True - - #Duplicates at a given key preserve insertion order of duplicate. - #Because lmdb is lexocographic an insertion ordering proem is prepended to - #all values that makes lexocographic order that same as insertion order. - - #Duplicates are ordered as a pair of key plus value so prepending proem - #to each value changes duplicate ordering. Proem is 33 characters long. - #With 32 character hex string followed by '.' for essentiall unlimited - #number of values which will be limited by memory. - - #With prepended proem ordinal must explicity check for duplicate values - #before insertion. Uses a python set for the duplicate inclusion test. - #Set inclusion scales with O(1) whereas list inclusion scales with O(n). - - #Parameters: - #db is opened named sub db with dupsort=True - #key is bytes of key within sub db's keyspace or empty - #skip is Boolean If True skips to next key if key is not empty string - #Othewise don't skip for first pass - #""" - - #with self.env.begin(db=db, write=False, buffers=True) as txn: - #cursor = txn.cursor() - #if cursor.set_range(key): # moves to first_dup at key - #found = True - #if skip and key and cursor.key() == key: # skip to next key - #found = cursor.next_nodup() # skip to next key not dup if any - #if found: - #for key, val in cursor.iternext_dup(keys=True): - #yield (key, val[33:]) # slice off prepended ordering prefix - - # methods for OnIoDup that combines IoDup value proem with On ordinal numbered # trailing prefix # this is so we do the proem add and strip here not in some higher level class From 180e4db83526f85da04e7e6397aaa07517340ee8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 10:16:17 -0600 Subject: [PATCH 314/418] created methods for OnValIterators to better support event logs --- src/keri/db/basing.py | 5 ++- src/keri/db/dbing.py | 67 ++++++++++++++++++++++++++++++++++++++--- src/keri/db/subing.py | 40 ++++++++++++++++++++++++ tests/db/test_dbing.py | 52 +++++++++++++++++++++++++++++++- tests/db/test_subing.py | 23 ++++++++++++++ 5 files changed, 180 insertions(+), 7 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 0aff8f1a7..983d8c8b8 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2560,7 +2560,10 @@ def getKelIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getOnIoDupValsAllPreIter(self.kels, pre, on=sn) + for key, on, val in self.getOnIoDupItemIter(self.kels, pre, on=sn): + yield val + + #return self.getOnIoDupValsAllPreIter(self.kels, pre, on=sn) def getKelBackIter(self, pre, sn=0): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 21b6380e4..47a84655c 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -662,9 +662,7 @@ def delTopVal(self, db, top=b''): # For subdbs the use keys with trailing part the is monotonically # ordinal number serialized as 32 hex bytes - - - + # used in OnSuberBase def appendOnVal(self, db, key, val, *, sep=b'.'): """ Appends val in order after last previous onkey in db where @@ -723,7 +721,7 @@ def appendOnVal(self, db, key, val, *, sep=b'.'): raise ValueError(f"Failed appending {val=} at {key=}.") return on - + # used in OnSuberBase def delOnVal(self, db, key, on=0, *, sep=b'.'): """ Deletes value at onkey consisting of key + sep + serialized on in db. @@ -746,7 +744,7 @@ def delOnVal(self, db, key, on=0, *, sep=b'.'): raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - + # used in OnSuberBase def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): """ Returns (int): count of of all ordinal keyed vals with key @@ -785,7 +783,34 @@ def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): return count + # used in OnSuberBase + def getOnValIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of the val at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + Returns: + items (Iterator[bytes]): val with same + key but increments of on beginning with on + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for (key, on, val) in self.getOnItemIter(db=db, key=key, on=on, sep=sep): + yield (val) + + # used in OnSuberBase def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): """ Returns iterator of triples (key, on, val), at each key over all ordinal @@ -1713,6 +1738,36 @@ def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): val = (b'%032x.' % (0)) + val # prepend ordering proem return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) + + # used in OnIoDupSuber + def getOnIoDupValIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnIoDupItemIter(db=db, key=key, on=on, sep=sep): + yield (val) + + # used in OnIoDupSuber def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): """ @@ -1749,6 +1804,8 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): # Consider using ItemIter instead of just replaying vals since ItemIter already # works with either dupsort==True or False. also skips gaps. +# replace with getOnIoDupItemIter which iterates over all pre starting at given on + def getOnIoDupValsAllPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 09c1bb8e6..b32cf9842 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -492,6 +492,27 @@ def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): on=on, sep=self.sep.encode())) + + def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[bytes]): of val with same key but increments of + on >= on i.e. all key.on beginning with on + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form actual key + When keys is empty then retrieves whole database including + duplicates if any + on (int): ordinal number used with onKey(pre,on) to form key at at + which to initiate retrieval + sep (bytes): separator character for split + """ + for val in (self.db.getOnValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns: @@ -2098,6 +2119,25 @@ def appendOn(self, keys: str | bytes | memoryview, sep=self.sep.encode())) + def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + items (Iterator[(top keys, on, val)]): triples of (top keys, on int, + deserialized val) + + Parameters: + keys (str | bytes | memoryview | iterator): top keys as prefix to be + combined with serialized on suffix and sep to form key + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 48be99dec..8603eeb4a 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -413,7 +413,8 @@ def test_lmdber(): assert dber.cntOnVals(db, key=b'') == 6 # all keys assert dber.cntOnVals(db) == 6 # all keys - # replay preB events in database + # iter replay + # replay preB event items in database items = [item for item in dber.getOnItemIter(db, preB)] assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY)] @@ -459,6 +460,49 @@ def test_lmdber(): items = [item for item in dber.getOnItemIter(db, key=preC, on=1)] assert items == [] + # val replay + # replay preB event vals in database + vals = [val for val in dber.getOnValIter(db, preB)] + assert vals == [digU, digV, digW, digX, digY] + + # resume replay preB events at on = 3 + vals = [val for val in dber.getOnValIter(db, preB, on=3)] + assert vals == [digX, digY] + + # resume replay preB events at on = 5 + vals = [val for val in dber.getOnValIter(db, preB, on=5)] + assert vals == [] + + vals = [val for val in dber.getOnValIter(db, key=b'')] + assert vals == [digA, + digY, + digU, + digV, + digW, + digX, + digY, + digC] + + vals = [val for val in dber.getOnValIter(db)] + assert vals == [digA, + digY, + digU, + digV, + digW, + digX, + digY, + digC] + + # resume replay all starting at preB on=2 + top, on = splitOnKey(keyB2) + vals = [val for val in dber.getOnValIter(db, key=top, on=on)] + assert vals == [digW, digX, digY] + + # resume replay all starting at preC on=1 + vals = [val for val in dber.getOnValIter(db, key=preC, on=1)] + assert vals == [] + + # test delOnVal assert dber.delOnVal(db, key=preB) # default on=0 assert not dber.delOnVal(db, key=preB, on=0) @@ -779,6 +823,12 @@ def test_lmdber(): (b'Z', 3, b'n')] + vals = [ bytes(val) for val in dber.getOnIoDupValIter(edb, key=key)] + assert vals == [b'k', b'l', b'm', b'n'] + + vals = [ bytes(val) for val in dber.getOnIoDupValIter(edb, key=key, on=2)] + assert vals == [ b'm', b'n'] + # test IoSetVals insertion order set of vals methods. key0 = b'ABC.ZYX' diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 1a0424623..7a5081132 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -255,6 +255,12 @@ def test_on_suber(): (('a', '00000000000000000000000000000003'), 'White snow')] # test getOnItemIter + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + items = [item for item in onsuber.getOnItemIter(keys='a')] assert items == [(('a',), 0, 'Blue dog'), (('a',), 1, 'Green tree'), @@ -265,6 +271,23 @@ def test_on_suber(): assert items == [(('a',), 2, 'Red apple'), (('a',), 3, 'White snow')] + # test getOnIter + vals = [val for val in onsuber.getOnIter()] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == ['Red apple', + 'White snow'] + assert 0 == onsuber.appendOn(keys=("b",), val=w) assert 1 == onsuber.appendOn(keys=("b",), val=x) From 84751b3bbb1cec06b661cef7cc043e6c562f576d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 10:28:36 -0600 Subject: [PATCH 315/418] refactor getKelIter to use new methods --- src/keri/db/basing.py | 4 ++-- src/keri/db/dbing.py | 43 ------------------------------------------ tests/db/test_dbing.py | 4 ---- 3 files changed, 2 insertions(+), 49 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 983d8c8b8..3e560cb54 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2560,8 +2560,8 @@ def getKelIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - for key, on, val in self.getOnIoDupItemIter(self.kels, pre, on=sn): - yield val + + return (self.getOnIoDupValIter(self.kels, pre, on=sn)) #return self.getOnIoDupValsAllPreIter(self.kels, pre, on=sn) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 47a84655c..a106c7bff 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1799,49 +1799,6 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): - -# without gaps is used for replay with on to provide partial replay -# Consider using ItemIter instead of just replaying vals since ItemIter already -# works with either dupsort==True or False. also skips gaps. - -# replace with getOnIoDupItemIter which iterates over all pre starting at given on - - def getOnIoDupValsAllPreIter(self, db, pre, on=0): - """ - Returns iterator of all dup vals in insertion order for all entries - with same prefix across all ordinal numbers in increasing order - without gaps between ordinal numbers - starting with on, default 0. Stops if gap or different pre. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning - - Raises StopIteration Error when empty. - - Duplicates are retrieved in insertion order. - - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. - - Parameters: - db is opened named sub db with dupsort=True - pre (bytes | str): of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): ordinal number to begin iteration at - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - # slice off prepended ordering prefix - yield val[33:] - key = snKey(pre, cnt:=cnt+1) - - # Create multiple methods # getTopItemBackIter symmetric with getTopItemIter # getTopIoSetItemBackIter symmetric getTopIoSetItemIter diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 8603eeb4a..b84bff026 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -601,10 +601,6 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getOnIoDupValsAllPreIter(db, pre)] - allvals = vals0 + vals1 + vals2 - assert vals == allvals - # Test getIoValsLastAllPreIter(self, db, pre) pre = b'BAejWzwQPYGGwTmuupUhPx5_yZ-Wk1xEHHzq7K0gzhcc' vals0 = [b"gamma", b"beta"] From 6bc1f8a4bcac9409199c06e4b00956c035a24d1d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 10:30:28 -0600 Subject: [PATCH 316/418] some clean up --- src/keri/db/dbing.py | 83 +++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index a106c7bff..a7242b40f 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1798,12 +1798,53 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): yield (key, on, val) + # Last is special so need method. + # Used to replay forward all last duplicate values starting at on + # need to fix this so it is not stopped by gaps or if gap raises error as + # gap is normally a problem for replay so maybe a parameter to raise error on gap + def getIoDupValLastAllPreIter(self, db, pre, on=0): + """ + Returns iterator of last only of dup vals of each key in insertion order + for all entries with same key across all sequence numbers in increasing order + without gaps starting with on (default = 0). Stops if gap or different key. + Assumes that key is combination of prefix and sequence number given + by .snKey(). + Removes prepended proem ordinal from each val before returning + + Raises StopIteration Error when empty. + + Duplicates are retrieved in insertion order. + + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order + Duplicates are ordered as a pair of key plus value so prepending prefix + to each value changes duplicate ordering. Proem is 17 characters long. + With 16 character hex string followed by '.'. + + + Parameters: + db is opened named sub db with dupsort=True + pre is bytes of itdentifier prefix prepended to sn in key + within sub db's keyspace + on (int): ordinal number to being iteration + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + key = snKey(pre, cnt:=on) + while cursor.set_key(key): # moves to first_dup + if cursor.last_dup(): # move to last_dup + yield cursor.value()[33:] # slice off prepended ordering proem + key = snKey(pre, cnt:=cnt+1) + + + # Create Two methods + # getOnItemBackIter symmetric with getOnItemIter + # getOnIoDupItemBackIter symmetric with getOnIoDupItemIter - # Create multiple methods # getTopItemBackIter symmetric with getTopItemIter # getTopIoSetItemBackIter symmetric getTopIoSetItemIter # getTopIoDupItemBackIter symmetric with getTopIoDupItemIter - # getOnItemBackIter symmetric with getOnItemIter + # instead of just replaying vals replay items since @@ -1853,44 +1894,6 @@ def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): yield val[33:] key = snKey(pre, cnt:=cnt-1) -# Last is special so need method. -# Used to replay forward all last duplicate values starting at on -# need to fix this so it is not stopped by gaps or if gap raises error as -# gap is normally a problem for replay so maybe a parameter to raise error on gap - def getIoDupValLastAllPreIter(self, db, pre, on=0): - """ - Returns iterator of last only of dup vals of each key in insertion order - for all entries with same key across all sequence numbers in increasing order - without gaps starting with on (default = 0). Stops if gap or different key. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning - - Raises StopIteration Error when empty. - - Duplicates are retrieved in insertion order. - - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. - - - Parameters: - db is opened named sub db with dupsort=True - pre is bytes of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): ordinal number to being iteration - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_key(key): # moves to first_dup - if cursor.last_dup(): # move to last_dup - yield cursor.value()[33:] # slice off prepended ordering proem - key = snKey(pre, cnt:=cnt+1) - # ToDo do we need a replay last backwards? From f2b30337c9433aae2c79fb756f936658ff7be3a6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 11:53:44 -0600 Subject: [PATCH 317/418] remove commented out code --- src/keri/app/storing.py | 36 ------------------------------------ src/keri/db/subing.py | 23 ----------------------- 2 files changed, 59 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index 073b365c5..80cf0f2fd 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -58,8 +58,6 @@ def reopen(self, **kwa): :return: """ super(Mailboxer, self).reopen(**kwa) - - #self.tpcs = self.env.open_db(key=b'tpcs.', dupsort=True) self.tpcs = subing.OnSuber(db=self, subkey='tpcs.') self.msgs = subing.Suber(db=self, subkey='msgs.') # key states @@ -73,7 +71,6 @@ def delTopic(self, key, on=0): exists in database so removed False otherwise (not removed) """ - #return self.delIoSetVals(self.tpcs, key) return self.tpcs.remOn(keys=key, on=on) def appendToTopic(self, topic, val): @@ -88,7 +85,6 @@ def appendToTopic(self, topic, val): topic (bytes): topic identifier for message val (bytes): msg digest """ - #return self.appendIoSetVal(db=self.tpcs, key=topic, val=val) return self.tpcs.appendOn(key=topic, val=val) @@ -111,15 +107,6 @@ def getTopicMsgs(self, topic, fn=0): msgs.append(msg.encode()) # want bytes not str return msgs - #if hasattr(topic, "encode"): - #topic = topic.encode("utf-8") - - #digs = self.getIoSetVals(db=self.tpcs, key=topic, ion=fn) - #msgs = [] - #for dig in digs: - #if msg := self.msgs.get(keys=dig): - #msgs.append(msg.encode("utf-8")) - #return msgs def storeMsg(self, topic, msg): """ @@ -143,16 +130,6 @@ def storeMsg(self, topic, msg): on = self.tpcs.appendOn(keys=topic, val=digb) return self.msgs.pin(keys=digb, val=msg) - #if hasattr(topic, "encode"): - #topic = topic.encode("utf-8") - - #if hasattr(msg, "encode"): - #msg = msg.encode("utf-8") - - #digb = coring.Diger(ser=msg, code=MtrDex.Blake3_256).qb64b - #self.appendToTopic(topic=topic, val=digb) - #return self.msgs.pin(keys=digb, val=msg) - def cloneTopicIter(self, topic, fn=0): """ @@ -177,19 +154,6 @@ def cloneTopicIter(self, topic, fn=0): if msg := self.msgs.get(keys=dig): yield (on, topic, msg.encode("utf-8")) - #if hasattr(topic, 'encode'): - #topic = topic.encode("utf-8") - - #for ion, (topic, dig) in enumerate(self.getTopIoSetItemIter(self.tpcs, topic)): - #if ion >= fn: - #if msg := self.msgs.get(keys=dig): - #yield ion, topic, msg.encode("utf-8") - - #for (key, dig) in self.getIoSetItemIter(self.tpcs, key=topic, ion=fn): - #topic, ion = dbing.unsuffix(key) - #if msg := self.msgs.get(keys=dig): - #yield ion, topic, msg.encode("utf-8") - class Respondant(doing.DoDoer): diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index b32cf9842..30d5d107e 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1978,29 +1978,6 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): return (self.db.cntIoDupVals(db=self.sdb, key=self._tokey(keys))) - #def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable = ''): - #""" - #Gets (iokeys, val) ioitems iterator at key made from keys where key is - #apparent effective key and items all have same apparent effective key - - #Parameters: - #keys (str|bytes|memoryview|Iterable): of key strs to be combined - #in order to form key. When keys is empty then retrieves - #all items in database. - - #Returns: - #items (Iterator): each item iterated is tuple (keys, val) where - #each keys is actual keys tuple and each val is dup without - #prefixed insertion ordering proem. - #Empty list if no entry at keys. - #Raises StopIteration when done - - #""" - #for key, val in self.db.getIoDupItemIter(db=self.sdb, - #key=self._tokey(keys)): - #yield (self._tokeys(key), self._des(val)) - - def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ From 624da64f20ed1bd1a3f6dee77174f1ad39b7e7b9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 11:54:21 -0600 Subject: [PATCH 318/418] clean up --- src/keri/db/basing.py | 2 +- src/keri/db/dbing.py | 2 +- tests/db/test_dbing.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 3e560cb54..4f6895bd0 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2611,7 +2611,7 @@ def getKelLastIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoDupValLastAllPreIter(self.kels, pre, on=sn) + return self.getOnIoDupValLastAllPreIter(self.kels, pre, on=sn) def putPses(self, key, vals): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index a7242b40f..52fe068c1 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1802,7 +1802,7 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): # Used to replay forward all last duplicate values starting at on # need to fix this so it is not stopped by gaps or if gap raises error as # gap is normally a problem for replay so maybe a parameter to raise error on gap - def getIoDupValLastAllPreIter(self, db, pre, on=0): + def getOnIoDupValLastAllPreIter(self, db, pre, on=0): """ Returns iterator of last only of dup vals of each key in insertion order for all entries with same key across all sequence numbers in increasing order diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index b84bff026..8ea3cfc7b 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -619,7 +619,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoDupValLastAllPreIter(db, pre)] + vals = [bytes(val) for val in dber.getOnIoDupValLastAllPreIter(db, pre)] lastvals = [vals0[-1], vals1[-1], vals2[-1]] assert vals == lastvals From 6c5c5cccbab13c31d9bb0cc3a6c766dc24770d40 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 17:15:47 -0600 Subject: [PATCH 319/418] refactored backIter methods fixed bug --- src/keri/core/eventing.py | 2 + src/keri/db/basing.py | 4 +- src/keri/db/dbing.py | 254 +++++++++++++------------ src/keri/db/subing.py | 67 ++++--- tests/core/test_eventing.py | 2 +- tests/db/test_dbing.py | 367 +++++++++++++++++++++++++----------- tests/db/test_subing.py | 43 +++++ 7 files changed, 490 insertions(+), 249 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 580bc1f9a..d07caf2bb 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -3245,6 +3245,8 @@ def fetchPriorDigers(self, sn: int | None = None) -> list | None: pre = self.prefixer.qb64 if sn is None: sn = self.lastEst.s - 1 + if sn < 0: + return None for digb in self.db.getKelBackIter(pre, sn): dgkey = dgKey(pre, digb) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 4f6895bd0..69557fedc 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2587,7 +2587,7 @@ def getKelBackIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getOnIoDupValsAllPreBackIter(self.kels, pre, sn) + return self.getOnIoDupValBackIter(self.kels, pre, sn) def getKelLastIter(self, pre, sn=0): @@ -2611,7 +2611,7 @@ def getKelLastIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getOnIoDupValLastAllPreIter(self.kels, pre, on=sn) + return self.getOnIoDupLastValIter(self.kels, pre, on=sn) def putPses(self, key, vals): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 52fe068c1..57a3f491c 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -697,21 +697,21 @@ def appendOnVal(self, db, key, val, *, sep=b'.'): # so either empty database or last is earlier pre or # last is last entry at same pre if cursor.last(): # not empty db. last entry earlier than max - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) if ckey == key: # last is last entry for same pre on = cn + 1 # increment else: # not past end so not empty either later pre or max entry at pre - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) if ckey == key: # last entry for pre is already at max raise ValueError(f"Number part {cn=} for key part {ckey=}" f"exceeds maximum size.") else: # later pre so backup one entry # either no entry before last or earlier pre with entry if cursor.prev(): # prev entry, maybe same or earlier pre - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) if ckey == key: # last entry at pre on = cn + 1 # increment @@ -849,6 +849,9 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): break yield (ckey, cn, cval) + # ToDo + # getOnItemBackIter symmetric with getOnItemIter + # getOnValBackIter symmetric with getOnValIter # IoSet insertion order in val so can have effective dups but with @@ -1777,8 +1780,6 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): Values duplicates are sorted internally by hidden prefixed insertion order proem ordinal Returned items are triples of (key, on, val) - When dupsort==true then duplicates are included in items since .iternext - includes duplicates. when key is empty then retrieves whole db Raises StopIteration Error when empty. @@ -1798,151 +1799,166 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): yield (key, on, val) - # Last is special so need method. - # Used to replay forward all last duplicate values starting at on - # need to fix this so it is not stopped by gaps or if gap raises error as - # gap is normally a problem for replay so maybe a parameter to raise error on gap - def getOnIoDupValLastAllPreIter(self, db, pre, on=0): - """ - Returns iterator of last only of dup vals of each key in insertion order - for all entries with same key across all sequence numbers in increasing order - without gaps starting with on (default = 0). Stops if gap or different key. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + def getOnIoDupLastValIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator of val of last insertion ordered duplicate at each + key over all ordinal numbered keys with same full key + of key + sep + on in db. Values are sorted by onKey(key, on) where on + is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + + when key is empty then retrieves whole db Raises StopIteration Error when empty. + Returns: + val (Iterator[bytes]): last dup val at each onkey - Duplicates are retrieved in insertion order. + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnIoDupLastItemIter(db=db, key=key, on=on, sep=sep): + yield (val) - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + def getOnIoDupLastItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator of triples (key, on, val), of last insertion ordered + duplicate at each key over all ordinal numbered keys with same full key + of key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: - db is opened named sub db with dupsort=True - pre is bytes of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): ordinal number to being iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_key(key): # moves to first_dup - if cursor.last_dup(): # move to last_dup - yield cursor.value()[33:] # slice off prepended ordering proem - key = snKey(pre, cnt:=cnt+1) + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: # empty + onkey = key + if not cursor.set_range(onkey): # # moves to first_dup at key>=onkey + return # no values end of db raises StopIteration - # Create Two methods - # getOnItemBackIter symmetric with getOnItemIter - # getOnIoDupItemBackIter symmetric with getOnIoDupItemIter + while cursor.last_dup(): # move to last_dup at current ckey + onkey, cval = cursor.item() # get ckey cval of last dup + ckey, on = splitOnKey(onkey, sep=sep) # get key on + if key and not ckey == key: + break - # getTopItemBackIter symmetric with getTopItemIter - # getTopIoSetItemBackIter symmetric getTopIoSetItemIter - # getTopIoDupItemBackIter symmetric with getTopIoDupItemIter + yield (ckey, on, cval[33:]) # slice off prepended ordering proem + onkey = onKey(ckey, on+1) + if not cursor.set_range(onkey): # # moves to first_dup at key>=onkey + return # no values end of db raises StopIteration - # instead of just replaying vals replay items since - # getTopItemIter already works with either dupsort==True or False - # so if we create getTopBackItemIter then we can use that everywhere - # ToDo need unit tests in dbing Used to replay backwards all duplicate - # values starting at on - # need to fix this so it is not stopped by gaps or if gap raises error as - # gap is normally a problem for replay so maybe a parameter to raise error on gap + # getOnIoDupItemBackIter symmetric with getOnIoDupItemIter + # getOnIoDupValBackIter symmetric with getOnIoDupValIter - def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): - """ - Returns iterator of all dup vals in insertion order for all entries - with same prefix across all sequence numbers in decreasing order without gaps - between ordinals at a given pre. - Starting with on (default = 0) as begining ordinal number or sequence number. - Stops if gap or different pre. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + def getOnIoDupValBackIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator going backwards of values, + of insertion ordered item at each key over all ordinal numbered keys + with same full key of key + sep + on in db. + Values are sorted by onKey(key, on) where on is ordinal number int and + key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Backwards means decreasing numerical value of duplicate proem, for each on, + decreasing numerical value on for each key and decresing lexocogrphic + order of each key prefix. - Raises StopIteration Error when empty. + Returned items are vals + + when key is empty then retrieves whole db - Duplicates are retrieved in insertion order. + Raises StopIteration Error when empty. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + Returns: + val (Iterator[bytes]): at key including duplicates in backwards order Parameters: - db is opened named sub db with dupsort=True - pre is bytes of identifier prefix prepended to sn in key - within sub db's keyspace - on (int): is ordinal number to begin iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = snKey(pre, cnt := on) - # set_key returns True if exact key else false - while cursor.set_key(key): # moves to first_dup if valid key - for val in cursor.iternext_dup(): - # slice off prepended ordering prefix - yield val[33:] - key = snKey(pre, cnt:=cnt-1) - - - -# ToDo do we need a replay last backwards? - + for key, on, val in self.getOnIoDupItemBackIter(db=db, key=key, on=on, sep=sep): + yield (val) - # not used anymore. - # before deleting this method use it inform how to cross gaps in other - # replays above with parameter such as replay last above. - # to raise error if detect gap when should not be one for event logs vs escrows - # then remove this method since not used otherwise - def getOnIoDupValsAnyPreIter(self, db, pre, on=0): - """ - Returns iterator of all dup vals in insertion order for any entries - with same prefix across all ordinal numbers in order including gaps - between ordinals at a given pre. Staring with on (default = 0). - Stops when pre is different. + def getOnIoDupItemBackIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator going backwards of triples (key, on, val), + of insertion ordered item at each key over all ordinal numbered keys + with same full key of key + sep + on in db. + Values are sorted by onKey(key, on) where on is ordinal number int and + key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Backwards means decreasing numerical value of duplicate proem, for each on, + decreasing numerical value on for each key and decresing lexocogrphic + order of each key prefix. - Duplicates that may be deleted such as duplicitous event logs need - to be able to iterate across gaps in ordinal number. + Returned items are triples of (key, on, val) - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + when key is empty then retrieves whole db Raises StopIteration Error when empty. - Duplicates are retrieved in insertion order. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: - db is opened named sub db with dupsort=True - pre is bytes of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): beginning ordinal number to start iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_range(key): # moves to first dup of key >= key - key = cursor.key() # actual key - front, back = bytes(key).split(sep=b'.', maxsplit=1) - if front != pre: # set range may skip pre if none - break - for val in cursor.iternext_dup(): - yield val[33:] # slice off prepended ordering prefix - cnt = int(back, 16) - key = snKey(pre, cnt:=cnt+1) + if not cursor.last(): # pre-position cursor at last dup of last key + return # empty database so raise StopIteration + + if key: # not empty so attempt to position at starting key not last + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + if cursor.set_range(onkey): # found key >= onkey + ckey, cn = splitOnKey(cursor.key(), sep=sep) + if ckey == key: # onkey in db + cursor.last_dup() # start at its last dup + else: # get closest key < onkey + if not cursor.prev(): # last dup of previous key + return # no earlier keys to designated start + + # cursor should now be correctly positioned for start either at + # last dup of either last key or onkey + for onkey, cval in cursor.iterprev(): # iterate backwards + ckey, on = splitOnKey(onkey, sep=sep) + if key and ckey != key: + return + yield (ckey, on, cval[33:]) + + + + # ToDo do we need a replay last backwards? + diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 30d5d107e..48ec914e4 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -2015,21 +2015,6 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", top=self._tokey(keys, topive=topive)): yield (self._tokeys(key), self._des(val)) -# ToDo -# OnIoDupSuber methods instead of pre just use key prefix parts top val so -# can work with any key space that has last part has ordinal -# try refactoring these to use cursor.iternext cursor.iterpre cursor.next cursor.prev -# so they work across prefixes without having the extra burden doing repeated -# iter within a given pre dups. This way they could work with or without duplicates -# and then the IoDupIter could just strip the proem whereas OrdSuber would not care - -# used by .kels - # getIoDupValsAllPreIter(self, db, pre, on=0): - # getIoDupValsAllPreBackIter(self, db, pre, on=0): - # getIoDupValLastAllPreIter(self.kels, pre, on=sn) -# used by .dels - # getIoDupValsAnyPreIter(self, db, pre, on=0) - class OnIoDupSuber(OnSuberBase, IoDupSuber): """ @@ -2099,9 +2084,45 @@ def appendOn(self, keys: str | bytes | memoryview, def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns - items (Iterator[(top keys, on, val)]): triples of (top keys, on int, + val (Iterator[bytes]): deserialized val of of each + onkey + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[(top keys, on, val)]): triples of (onkeys, on int, deserialized val) + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnIoDupItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) + + + def getOnLastIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + last (Iterator[bytes]): deserialized last duplicate val of of each + onkey + Parameters: keys (str | bytes | memoryview | iterator): top keys as prefix to be combined with serialized on suffix and sep to form key @@ -2109,25 +2130,27 @@ def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): on (int): ordinal number used with onKey(pre,on) to form key. sep (bytes): separator character for split """ - for val in (self.db.getOnIoDupValIter(db=self.sdb, + for val in (self.db.getOnIoDupLastValIter(db=self.sdb, key=self._tokey(keys), on=on, sep=self.sep.encode())): yield (self._des(val)) - def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + def getOnLastItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns - items (Iterator[(top keys, on, val)]): triples of (top keys, on int, - deserialized val) + items (Iterator[(top keys, on, val)]): triples of (keys, on int, + deserialized val) last duplicate item as each onkey where onkey + is the key+serialized on Parameters: - keys (str | bytes | memoryview | iterator): top keys as prefix to be + keys (str | bytes | memoryview | iterator): keys as prefix to be combined with serialized on suffix and sep to form key When keys is empty then retrieves whole database including duplicates on (int): ordinal number used with onKey(pre,on) to form key. sep (bytes): separator character for split """ - for keys, on, val in (self.db.getOnIoDupItemIter(db=self.sdb, + for keys, on, val in (self.db.getOnIoDupLastItemIter(db=self.sdb, key=self._tokey(keys), on=on, sep=self.sep.encode())): yield (self._tokeys(keys), on, self._des(val)) + diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index c50cee10a..a4474d54a 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -4947,7 +4947,7 @@ def test_load_event(mockHelpingNowUTC): if __name__ == "__main__": # pytest.main(['-vv', 'test_eventing.py::test_keyeventfuncs']) #test_process_manual() - #test_keyeventsequence_0() + test_keyeventsequence_0() #test_process_transferable() #test_messagize() test_direct_mode() diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 8ea3cfc7b..9c2348c56 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -583,47 +583,7 @@ def test_lmdber(): assert dber.getIoDupVals(db, key) == [b'm', b'a', b'w', b'e'] - - # Test getIoValsAllPreIter(self, db, pre) - vals0 = [b"gamma", b"beta"] - sn = 0 - key = snKey(pre, sn) - assert dber.addIoDupVal(db, key, vals0[0]) == True - assert dber.addIoDupVal(db, key, vals0[1]) == True - - vals1 = [b"mary", b"peter", b"john", b"paul"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoDupVals(db, key, vals1) == True - - vals2 = [b"dog", b"cat", b"bird"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoDupVals(db, key, vals2) == True - - # Test getIoValsLastAllPreIter(self, db, pre) - pre = b'BAejWzwQPYGGwTmuupUhPx5_yZ-Wk1xEHHzq7K0gzhcc' - vals0 = [b"gamma", b"beta"] - sn = 0 - key = snKey(pre, sn) - assert dber.addIoDupVal(db, key, vals0[0]) == True - assert dber.addIoDupVal(db, key, vals0[1]) == True - - vals1 = [b"mary", b"peter", b"john", b"paul"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoDupVals(db, key, vals1) == True - - vals2 = [b"dog", b"cat", b"bird"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoDupVals(db, key, vals2) == True - - vals = [bytes(val) for val in dber.getOnIoDupValLastAllPreIter(db, pre)] - lastvals = [vals0[-1], vals1[-1], vals2[-1]] - assert vals == lastvals - - # Test getIoValsAnyPreIter(self, db, pre) + # Test TopIoDupItemIter(self, db, pre) pre = b'BBPYGGwTmuupUhPx5_yZ-Wk1x4ejWzwEHHzq7K0gzhcc' vals0 = [b"gamma", b"beta"] sn = 1 # not start at zero @@ -646,7 +606,9 @@ def test_lmdber(): # dber.getTopIoDupItemIter() assert vals == allvals - # Setup Tests for getIoItemsNext and getIoItemsNextIter + + + # Setup Tests for TopIoDupItemsIter edb = dber.env.open_db(key=b'escrow.', dupsort=True) aKey = snKey(pre=b'A', sn=1) aVals = [b"z", b"m", b"x"] @@ -744,86 +706,281 @@ def test_lmdber(): assert not items - ## Test getIoItemsNextIter(self, db, key=b"") - ## get dups at first key in database - ## aVals - #items = [item for item in dber.getIoDupItemsNextIter(edb)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == aKey - #vals = [val for key, val in items] - #assert vals == aVals - - #items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey, skip=False)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == aKey - #vals = [val for key, val in items] - #assert vals == aVals - - #items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == bKey - #vals = [val for key, val in items] - #assert vals == bVals - - #items = [item for item in dber.getIoDupItemsNextIter(edb, key=b'', skip=False)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == aKey - #vals = [val for key, val in items] - #assert vals == aVals - #for key, val in items: - #assert dber.delIoDupVal(edb, ikey, val) == True - - ## bVals - #items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == bKey - #vals = [val for key, val in items] - #assert vals == bVals - #for key, val in items: - #assert dber.delIoDupVal(edb, ikey, val) == True - - ## cVals - #items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == cKey - #vals = [val for key, val in items] - #assert vals == cVals - #for key, val in items: - #assert dber.delIoDupVal(edb, ikey, val) == True + # test OnIoDup methods + ldb = dber.env.open_db(key=b'log.', dupsort=True) + # first pre + sn = 0 + key = snKey(preA, sn) + valsA0 = [b"echo", b"bravo"] + itemsA0 = [ + (preA, sn, valsA0[0]), + (preA, sn, valsA0[1]) + ] + assert dber.addIoDupVal(ldb, key, valsA0[0]) == True + assert dber.addIoDupVal(ldb, key, valsA0[1]) == True + + sn += 1 + key = snKey(preA, sn) + valsA1 = [b"sue", b"bob", b"val", b"zoe"] + itemsA1 = [ + (preA, sn, valsA1[0]), + (preA, sn, valsA1[1]), + (preA, sn, valsA1[2]), + (preA, sn, valsA1[3]), + ] + assert dber.putIoDupVals(ldb, key, valsA1) == True + + sn += 1 + key = snKey(preA, sn) + valsA2 = [b"fish", b"bat", b"snail"] + itemsA2 = [ + (preA, sn, valsA2[0]), + (preA, sn, valsA2[1]), + (preA, sn, valsA2[2]), + ] + assert dber.putIoDupVals(ldb, key, valsA2) == True + + # second pre + sn = 0 + key = snKey(preB, sn) + valsB0 = [b"gamma", b"beta"] + itemsB0 = [ + (preB, sn, valsB0[0]), + (preB, sn, valsB0[1]) + ] + assert dber.addIoDupVal(ldb, key, valsB0[0]) == True + assert dber.addIoDupVal(ldb, key, valsB0[1]) == True + + sn += 1 + key = snKey(preB, sn) + valsB1 = [b"mary", b"peter", b"john", b"paul"] + itemsB1 = [ + (preB, sn, valsB1[0]), + (preB, sn, valsB1[1]), + (preB, sn, valsB1[2]), + (preB, sn, valsB1[3]), + ] + assert dber.putIoDupVals(ldb, key, valsB1) == True + + + sn += 1 + key = snKey(preB, sn) + valsB2 = [b"dog", b"cat", b"bird"] + itemsB2 = [ + (preB, sn, valsB2[0]), + (preB, sn, valsB2[1]), + (preB, sn, valsB2[2]), + ] + assert dber.putIoDupVals(ldb, key, valsB2) == True + + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preA)] + lastitems = [itemsA0[-1], itemsA1[-1], itemsA2[-1]] + assert items == lastitems + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preA, on=1)] + lastitems = [itemsA1[-1], itemsA2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preA)] + lastvals = [valsA0[-1], valsA1[-1], valsA2[-1]] + assert vals == lastvals + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preA, on=1)] + lastvals = [valsA1[-1], valsA2[-1]] + assert vals == lastvals + + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preB)] + lastitems = [itemsB0[-1], itemsB1[-1], itemsB2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preB)] + lastvals = [valsB0[-1], valsB1[-1], valsB2[-1]] + assert vals == lastvals + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb)] + lastitems = [itemsA0[-1], itemsA1[-1], itemsA2[-1], itemsB0[-1], itemsB1[-1], itemsB2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb)] + lastvals = [valsA0[-1], valsA1[-1], valsA2[-1], valsB0[-1], valsB1[-1], valsB2[-1]] + assert vals == lastvals + + # test back iter + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preB, on=3)] + assert items ==[(preB, 2, b'bird'), + (preB, 2, b'cat'), + (preB, 2, b'dog'), + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma')] + + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preB, on=3)] + assert vals ==[ + b'bird', + b'cat', + b'dog', + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preB, on=1)] + assert items ==[ + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma') + ] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preB, on=1)] + assert vals ==[ + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preA, on=5)] + assert items ==[(preA, 2, b'snail'), + (preA, 2, b'bat'), + (preA, 2, b'fish'), + (preA, 1, b'zoe'), + (preA, 1, b'val'), + (preA, 1, b'bob'), + (preA, 1, b'sue'), + (preA, 0, b'bravo'), + (preA, 0, b'echo')] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preA, on=5)] + assert vals ==[ + b'snail', + b'bat', + b'fish', + b'zoe', + b'val', + b'bob', + b'sue', + b'bravo', + b'echo' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preA, on=0)] + assert items ==[ + (preA, 0, b'bravo'), + (preA, 0, b'echo')] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preA, on=0)] + assert vals ==[ + b'bravo', + b'echo' + ] + + # all items from last to first + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb)] + assert items ==[ + (preB, 2, b'bird'), + (preB, 2, b'cat'), + (preB, 2, b'dog'), + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma'), + (preA, 2, b'snail'), + (preA, 2, b'bat'), + (preA, 2, b'fish'), + (preA, 1, b'zoe'), + (preA, 1, b'val'), + (preA, 1, b'bob'), + (preA, 1, b'sue'), + (preA, 0, b'bravo'), + (preA, 0, b'echo'), + ] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb)] + assert vals ==[ + b'bird', + b'cat', + b'dog', + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma', + b'snail', + b'bat', + b'fish', + b'zoe', + b'val', + b'bob', + b'sue', + b'bravo', + b'echo' + ] # test OnIoDup methods key = b'Z' - assert 0 == dber.appendOnIoDupVal(edb, key, val=b'k') - assert 1 == dber.appendOnIoDupVal(edb, key, val=b'l') - assert 2 == dber.appendOnIoDupVal(edb, key, val=b'm') - assert 3 == dber.appendOnIoDupVal(edb, key, val=b'n') + assert 0 == dber.appendOnIoDupVal(ldb, key, val=b'k') + assert 1 == dber.appendOnIoDupVal(ldb, key, val=b'l') + assert 2 == dber.appendOnIoDupVal(ldb, key, val=b'm') + assert 3 == dber.appendOnIoDupVal(ldb, key, val=b'n') - assert dber.cntOnVals(edb, key) == 4 + assert dber.cntOnVals(ldb, key) == 4 + + vals = [ bytes(val) for val in dber.getOnIoDupValIter(ldb, key=key)] + assert vals == [b'k', b'l', b'm', b'n'] - items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(edb, key=key)] + vals = [ bytes(val) for val in dber.getOnIoDupValIter(ldb, key=key, on=2)] + assert vals == [ b'm', b'n'] + + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key)] assert items == [(b'Z', 0, b'k'), (b'Z', 1, b'l'), (b'Z', 2, b'm'), (b'Z', 3, b'n')] - items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(edb, key=key, on=2)] + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key, on=2)] assert items == [ (b'Z', 2, b'm'), (b'Z', 3, b'n')] + # test back iter - vals = [ bytes(val) for val in dber.getOnIoDupValIter(edb, key=key)] - assert vals == [b'k', b'l', b'm', b'n'] + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=3)] + assert items ==[(b'Z', 3, b'n'), + (b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=4)] + assert items ==[(b'Z', 3, b'n'), + (b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=2)] + assert items == [(b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] - vals = [ bytes(val) for val in dber.getOnIoDupValIter(edb, key=key, on=2)] - assert vals == [ b'm', b'n'] # test IoSetVals insertion order set of vals methods. diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 7a5081132..f70440ff9 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -688,6 +688,18 @@ def test_on_iodup_suber(): (('a', '00000000000000000000000000000002'), 'Red apple'), (('a', '00000000000000000000000000000003'), 'White snow')] + + # test getOnIter + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == ['Red apple', + 'White snow'] + # test getOnItemIter items = [item for item in onsuber.getOnItemIter(keys='a')] assert items == [(('a',), 0, 'Blue dog'), @@ -719,6 +731,37 @@ def test_on_iodup_suber(): (('b', '00000000000000000000000000000001'), 'Green tree'), (('bc', '00000000000000000000000000000000'), 'Red apple')] + # test getOnIter + vals = [val for val in onsuber.getOnIter(keys='b')] + assert vals == ['Blue dog', 'Green tree'] + + vals = [val for val in onsuber.getOnIter(keys=('b', ))] + assert vals == ['Blue dog', 'Green tree'] + + vals = [val for val in onsuber.getOnIter(keys=('b', ""))] + assert vals == [] + + vals = [val for val in onsuber.getOnIter(keys='')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + + vals = [val for val in onsuber.getOnIter()] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + + # test getOnItemIter items = [item for item in onsuber.getOnItemIter(keys='b')] assert items == [(('b',), 0, 'Blue dog'), From 56c47188761dae9769a8244dae6486fc783a073a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 17:59:26 -0600 Subject: [PATCH 320/418] Added OnIoDupSuber support for BackIter methods --- src/keri/db/subing.py | 35 +++++++++++ tests/db/test_subing.py | 125 ++++++++++++++++++++++++++++++---------- 2 files changed, 129 insertions(+), 31 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 48ec914e4..efec02a3e 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -2154,3 +2154,38 @@ def getOnLastItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0) key=self._tokey(keys), on=on, sep=self.sep.encode())): yield (self._tokeys(keys), on, self._des(val)) + + def getOnBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + val (Iterator[bytes]): deserialized val of of each + onkey in reverse order + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupValBackIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[(top keys, on, val)]): triples of (onkeys, on int, + deserialized val) in reverse order + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnIoDupItemBackIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index f70440ff9..c9db97498 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -731,42 +731,18 @@ def test_on_iodup_suber(): (('b', '00000000000000000000000000000001'), 'Green tree'), (('bc', '00000000000000000000000000000000'), 'Red apple')] - # test getOnIter + + # test getOnItemIter getOnIter + items = [item for item in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + vals = [val for val in onsuber.getOnIter(keys='b')] assert vals == ['Blue dog', 'Green tree'] vals = [val for val in onsuber.getOnIter(keys=('b', ))] assert vals == ['Blue dog', 'Green tree'] - vals = [val for val in onsuber.getOnIter(keys=('b', ""))] - assert vals == [] - - vals = [val for val in onsuber.getOnIter(keys='')] - assert vals == ['Blue dog', - 'Green tree', - 'Red apple', - 'White snow', - 'White snow', - 'Blue dog', - 'Green tree', - 'Red apple'] - - vals = [val for val in onsuber.getOnIter()] - assert vals == ['Blue dog', - 'Green tree', - 'Red apple', - 'White snow', - 'White snow', - 'Blue dog', - 'Green tree', - 'Red apple'] - - - # test getOnItemIter - items = [item for item in onsuber.getOnItemIter(keys='b')] - assert items == [(('b',), 0, 'Blue dog'), - (('b',), 1, 'Green tree')] - items = [item for item in onsuber.getOnItemIter(keys=('b', ))] assert items == [(('b',), 0, 'Blue dog'), (('b',), 1, 'Green tree')] @@ -774,6 +750,9 @@ def test_on_iodup_suber(): items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] assert items == [] + vals = [val for val in onsuber.getOnIter(keys=('b', ""))] + assert vals == [] + items = [item for item in onsuber.getOnItemIter(keys='')] assert items == [(('a',), 0, 'Blue dog'), (('a',), 1, 'Green tree'), @@ -784,6 +763,16 @@ def test_on_iodup_suber(): (('b',), 1, 'Green tree'), (('bc',), 0, 'Red apple')] + vals = [val for val in onsuber.getOnIter(keys='')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + items = [item for item in onsuber.getOnItemIter()] assert items == [(('a',), 0, 'Blue dog'), (('a',), 1, 'Green tree'), @@ -794,6 +783,16 @@ def test_on_iodup_suber(): (('b',), 1, 'Green tree'), (('bc',), 0, 'Red apple')] + vals = [val for val in onsuber.getOnIter()] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + # test with duplicates assert onsuber.add(keys=dbing.onKey("a", 0), val=z) assert onsuber.add(keys=dbing.onKey("a", 1), val=y) @@ -814,7 +813,7 @@ def test_on_iodup_suber(): (('a', '00000000000000000000000000000003'), 'White snow'), (('a', '00000000000000000000000000000003'), 'Blue dog')] - # test getOnItemIter + # test getOnItemIter getOnIter getOnItemBackIter getOnBackIter items = [item for item in onsuber.getOnItemIter(keys='a')] assert items == [(('a',), 0, 'Blue dog'), (('a',), 0, 'White snow'), @@ -825,17 +824,71 @@ def test_on_iodup_suber(): (('a',), 3, 'White snow'), (('a',), 3, 'Blue dog')] + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'White snow', + 'Green tree', + 'Red apple', + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog', + ] + + + items = [item for item in onsuber.getOnItemBackIter(keys='a', on=4)] + assert items == [(('a',), 3, 'Blue dog'), + (('a',), 3, 'White snow'), + (('a',), 2, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 1, 'Red apple'), + (('a',), 1, 'Green tree'), + (('a',), 0, 'White snow'), + (('a',), 0, 'Blue dog')] + + + vals = [val for val in onsuber.getOnBackIter(keys='a', on=4)] + assert vals == ['Blue dog', + 'White snow', + 'Green tree', + 'Red apple', + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog'] + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] assert items ==[(('a',), 2, 'Red apple'), (('a',), 2, 'Green tree'), (('a',), 3, 'White snow'), (('a',), 3, 'Blue dog')] + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == [ + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog', + ] + + items = [item for item in onsuber.getOnItemBackIter(keys='a', on=1)] + assert items ==[(('a',), 1, 'Red apple'), + (('a',), 1, 'Green tree'), + (('a',), 0, 'White snow'), + (('a',), 0, 'Blue dog')] + + + vals = [val for val in onsuber.getOnBackIter(keys='a', on=1)] + assert vals == ['Red apple', + 'Green tree', + 'White snow', + 'Blue dog'] # test append with duplicates assert 4 == onsuber.appendOn(keys=("a",), val=x) assert onsuber.cntOn(keys=("a",)) == 9 + # test remove assert onsuber.remOn(keys='a', on=1) assert not onsuber.remOn(keys='a', on=1) assert onsuber.remOn(keys='a', on=3) @@ -850,6 +903,16 @@ def test_on_iodup_suber(): (('a',), 2, 'Green tree'), (('a',), 4, 'Green tree')] + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == [ + 'Blue dog', + 'White snow', + 'Red apple', + 'Green tree', + 'Green tree', + ] + + assert not os.path.exists(db.path) assert not db.opened From d552f795688d8a310e9bb30747aa7305cd7e5637 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 5 Sep 2024 10:22:12 -0600 Subject: [PATCH 321/418] refactored SchemerSuber to be ducktyped subclass of SerderSuberBase. Added verify parameter to Schemer.__init__ to be compatible with Suber classes that expensive verification when deserializing like Serder. Added missing unit tests. --- src/keri/core/scheming.py | 26 +++++--- src/keri/core/serdering.py | 7 ++- src/keri/db/subing.py | 121 ++++++------------------------------ tests/core/test_scheming.py | 1 + tests/db/test_subing.py | 99 ++++++++++++++++++++++++++++- 5 files changed, 140 insertions(+), 114 deletions(-) diff --git a/src/keri/core/scheming.py b/src/keri/core/scheming.py index 919d8e035..203ec9500 100644 --- a/src/keri/core/scheming.py +++ b/src/keri/core/scheming.py @@ -266,7 +266,8 @@ class Schemer: """ - def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.Blake3_256): + def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), + code=MtrDex.Blake3_256, verify=True): """ Initialize instance of Schemer Deserialize if raw provided @@ -274,14 +275,19 @@ def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.B When serializing if kind provided then use kind instead of field in sed Parameters: - raw (bytes): of serialized schema - sed (dict): dict or None - if None its deserialized from raw - typ (JSONSchema): type of schema - kind (serialization): kind string value or None (see namedtuple coring.Serials) - supported kinds are 'json', 'cbor', 'msgpack', 'binary' - if kind (None): then its extracted from ked or raw - code (MtrDex): default digest code + raw (bytes): of serialized schema + sed (dict): dict or None + if None its deserialized from raw + typ (JSONSchema): type of schema + kind (serialization): kind string value or None (see namedtuple coring.Serials) + supported kinds are 'json', 'cbor', 'msgpack', 'binary' + if kind (None): then its extracted from ked or raw + code (MtrDex): default digest code + verify (bool): True means verify said(s) of given raw or sad. + Raises ValidationError if verification fails + False means don't verify. Useful to avoid unnecessary + reverification when deserializing from database + as opposed to over the wire reception. """ @@ -295,7 +301,7 @@ def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.B else: raise ValueError("Improper initialization need raw or sed.") - if not self._verify_schema(): + if verify and not self._verify_schema(): raise ValidationError("invalid kind {} for schema {}" "".format(self.kind, self.sed)) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index abb74a9b8..f28ca035c 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -561,8 +561,11 @@ def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, Either provided by parser from stream genus version or desired when generating Serder instance to stream verify (bool): True means verify said(s) of given raw or sad. - Raises ValidationError if verification fails - Ignore when raw not provided or when raw and saidify is True + False means don't verify. Useful to avoid unnecessary + reverification when deserializing from database + as opposed to over the wire reception. + Raises ValidationError if verification fails + Ignore when raw empty or when raw and saidify is True makify (bool): True means compute fields for sad including size and saids. proto (str | None): desired protocol type str value of Protocols diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index efec02a3e..1c68056e8 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1470,116 +1470,35 @@ def __init__(self, *pa, **kwa): -class SchemerSuber(Suber): +class SchemerSuber(SerderSuberBase, Suber): """ - Sub class of Suber where data is serialized Schemer instance + Sub class of SerderSuberBase and Suber where data is serialized Schemer instance + Schemer ser/des is ducktype of Serder using .raw Automatically serializes and deserializes using Schemer methods - - ToDo XXXX make this a subclass of SerderSuber since from a ser des interface - Schemer is duck type of Serder, Then can get rid of the redundant put, add, - pin, get etc definitions - """ - def __init__(self, *pa, **kwa): + def __init__(self, *pa, + klas: Type[ scheming.Schemer] = scheming.Schemer, + **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key - """ - super(SchemerSuber, self).__init__(*pa, **kwa) - - def put(self, keys: Union[str, Iterable], val: scheming.Schemer): - """ - Puts val at key made from keys. Does not overwrite - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Schemer): instance - - Returns: - result (bool): True If successful, False otherwise, such as key - already in database. - """ - return (self.db.putVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) - - def pin(self, keys: Union[str, Iterable], val: scheming.Schemer): - """ - Pins (sets) val at key made from keys. Overwrites. - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Schemer): instance - - Returns: - result (bool): True If successful. False otherwise. - """ - return (self.db.setVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) - - def get(self, keys: Union[str, Iterable]): - """ - Gets Serder at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - Schemer: - None: if no entry at keys - - Usage: - Use walrus operator to catch and raise missing entry - if (srder := mydb.get(keys)) is None: - raise ExceptionHere - use srdr here - - """ - val = self.db.getVal(db=self.sdb, key=self._tokey(keys)) - return scheming.Schemer(raw=bytes(val)) if val is not None else None - - def rem(self, keys: Union[str, Iterable]): - """ - Removes entry at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - """ - return self.db.delVal(db=self.sdb, key=self._tokey(keys)) - - def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", - *, topive=False): - """ - Returns: - iterator (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb - - Parameters: - keys (str | bytes | memoryview | Iterable): tuple of bytes or - strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. - topive (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep - character. - False means treat as full branch in key space. Resultant key - does not end in .sep character. - When last item in keys is empty str then will treat as - partial ending in sep regardless of top value + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[scheming.Schemer]): Class reference to ducktyped subclass + of Serder + Overridden Parameters: + klas (Type[scheming.Schemer]): Class reference to ducktyped subclass + of Serder intercepts passed in klas and forces it to Schemer """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, topive=topive)): - yield self._tokeys(iokey), scheming.Schemer(raw=bytes(val)) + super(SchemerSuber, self).__init__(*pa, klas=scheming.Schemer, **kwa) class DupSuber(SuberBase): diff --git a/tests/core/test_scheming.py b/tests/core/test_scheming.py index a79bac996..09c4df8e2 100644 --- a/tests/core/test_scheming.py +++ b/tests/core/test_scheming.py @@ -172,6 +172,7 @@ def test_json_schema_dict(): sce = Schemer(sed=sed, code=MtrDex.Blake3_256) said = 'ENQKl3r1Z6HiLXOD-050aVvKziCWJtXWg3vY2FWUGSxG' assert sce.said == said + payload = b'{"a": {"b": 123, "c": "2018-11-13T20:20:39+00:00"}}' assert sce.verify(payload) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index c9db97498..dc236ebca 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -12,7 +12,7 @@ from keri import help from keri import core -from keri.core import coring, eventing, serdering, indexing +from keri.core import coring, eventing, serdering, indexing, scheming from keri.db import dbing, subing from keri.app import keeping @@ -1468,6 +1468,102 @@ def test_serder_ioset_suber(): assert not db.opened +def test_schemer_suber(): + """ + Test SchemerSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + scmber = subing.SchemerSuber(db=db, subkey='bags.') + assert isinstance(scmber, subing.SchemerSuber) + assert not scmber.sdb.flags()["dupsort"] + assert scmber.klas == scheming.Schemer + + pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + + raw0 = (b'{"$id":"EMRvS7lGxc1eDleXBkvSHkFs8vUrslRcla6UXOJdcczw","$schema":"http://json' + b'-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"str' + b'ing"},"b":{"type":"number"},"c":{"type":"string","format":"date-time"}}}') + + scmr0 = scheming.Schemer(raw=raw0) + + keys = (pre, scmr0.said) + scmber.put(keys=keys, val=scmr0) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + assert scmber.rem(keys) + actual = scmber.get(keys=keys) + assert actual is None + + scmber.put(keys=keys, val=scmr0) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + raw1 = (b'{"$id":"ENQKl3r1Z6HiLXOD-050aVvKziCWJtXWg3vY2FWUGSxG","$schema":"http://json' + b'-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"obj' + b'ect","properties":{"b":{"type":"number"},"c":{"type":"string","format":"date' + b'-time"}}}}}') + scmr1 = scheming.Schemer(raw=raw1) + + result = scmber.put(keys=keys, val=scmr1) + assert not result + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + result = scmber.pin(keys=keys, val=scmr1) + assert result + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr1.said + + # test with keys as string not tuple + keys = "{}.{}".format(pre, scmr1.said) + + scmber.put(keys=keys, val=scmr1) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr1.said + + assert scmber.rem(keys) + actual = scmber.get(keys=keys) + assert actual is None + + # test missing entry at keys + badkey = "badkey" + actual = scmber.get(badkey) + assert actual is None + + # test iteritems + scmber = subing.SchemerSuber(db=db, subkey='pugs.') + assert isinstance(scmber, subing.SchemerSuber) + scmber.put(keys=("a","1"), val=scmr0) + scmber.put(keys=("a","2"), val=scmr1) + + items = [(keys, srdr.said) for keys, srdr in scmber.getItemIter()] + assert items == [(('a', '1'), scmr0.said), + (('a', '2'), scmr1.said)] + + assert scmber.put(keys=("b","1"), val=scmr0) + assert scmber.put(keys=("b","2"), val=scmr1) + assert scmber.put(keys=("bc","1"), val=scmr0) + + topkeys = ("b", "") # append empty str to force trailing .sep + items = [(keys, srdr.said) for keys, srdr in scmber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), scmr0.said), + (('b', '2'), scmr1.said)] + + assert not os.path.exists(db.path) + assert not db.opened + + def test_cesr_suber(): """ Test CesrSuber LMDBer sub database class @@ -2533,5 +2629,6 @@ def test_crypt_signer_suber(): test_cesr_dup_suber() test_serder_suber() test_serder_ioset_suber() + test_schemer_suber() test_signer_suber() test_crypt_signer_suber() From 6a8ec7f594a4411515134104ef8ada89aebea827 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 5 Sep 2024 10:45:20 -0600 Subject: [PATCH 322/418] Added test for subclass of Schemer to SchemerSuber init --- src/keri/db/subing.py | 4 +++- tests/db/test_subing.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 1c68056e8..1b447d320 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1498,7 +1498,9 @@ def __init__(self, *pa, klas (Type[scheming.Schemer]): Class reference to ducktyped subclass of Serder intercepts passed in klas and forces it to Schemer """ - super(SchemerSuber, self).__init__(*pa, klas=scheming.Schemer, **kwa) + if not issubclass(klas, scheming.Schemer): + raise TypeError(f"Invalid {klas=}, not subclass of {scheming.Schemer}.") + super(SchemerSuber, self).__init__(*pa, klas=klas, **kwa) class DupSuber(SuberBase): diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index dc236ebca..c7f62fdfb 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -1351,6 +1351,7 @@ def test_serder_suber(): assert items == [(('b', '1'), srdr0.said), (('b', '2'), srdr1.said)] + assert not os.path.exists(db.path) assert not db.opened @@ -1560,6 +1561,10 @@ def test_schemer_suber(): assert items == [(('b', '1'), scmr0.said), (('b', '2'), scmr1.said)] + with pytest.raises(TypeError): + scmber = subing.SchemerSuber(db=db, subkey='bags.', klas=coring.Matter) + + assert not os.path.exists(db.path) assert not db.opened From 49c9393f4bf0c8759ed91b99202e5b5bd43c752f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 5 Sep 2024 10:52:59 -0600 Subject: [PATCH 323/418] some clean up --- tests/db/test_subing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index c7f62fdfb..f11e229c9 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -1351,7 +1351,6 @@ def test_serder_suber(): assert items == [(('b', '1'), srdr0.said), (('b', '2'), srdr1.said)] - assert not os.path.exists(db.path) assert not db.opened From 4e602efa1aa3ee526c516ef4868241bf8e68aad7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 5 Sep 2024 13:55:22 -0600 Subject: [PATCH 324/418] added B64Suber subclass with unit test B64SuberBase can be mixed with IoSetSuber or OnSuber or combinations to create subclasses where the values are separated strings of Base64 charcters when the values are just groups of qb64 identifiers that don't need to be serialized and reserialized from Matter subclasses as would be the case with CatCesrSuber --- src/keri/db/subing.py | 175 +++++++++++++++++++++++++++++++++++++--- tests/db/test_subing.py | 92 ++++++++++++++++++++- 2 files changed, 251 insertions(+), 16 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 1b447d320..75b94abeb 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -73,7 +73,7 @@ from collections.abc import Iterable, Iterator from .. import help -from ..help.helping import nonStringIterable +from ..help.helping import nonStringIterable, Reb64 from .. import core from ..core import coring, scheming, serdering from . import dbing @@ -133,8 +133,8 @@ def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], by partial keys by appending separator to end of partial key Returns: - key (bytes): each element of keys is joined by .sep. If top then last - char of key is also .sep + key (bytes): each element of keys is joined by .sep. If topive then + last char of key is .sep Parameters: keys (str | bytes | memoryview | Iterable[str | bytes]): db key or @@ -145,7 +145,7 @@ def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], False means treat as full branch in key space. Resultant key does not end in .sep character. When last item in keys is empty str then will treat as - partial ending in sep regardless of top value + partial ending in sep regardless of topive value """ if hasattr(keys, "encode"): # str @@ -190,11 +190,11 @@ def _ser(self, val: str | bytes | memoryview): return (val.encode("utf-8") if hasattr(val, "encode") else val) - def _des(self, val: str | bytes | memoryview): + def _des(self, val: bytes | memoryview): """ Deserialize val to str Parameters: - val (str | bytes | memoryview): decodable as str + val (bytes | memoryview): decodable as str """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes @@ -536,7 +536,8 @@ def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): class OnSuber(OnSuberBase, Suber): """ - Subclass of Suber that adds methods for keys with ordinal numbered suffixes. + Subclass of OnSuberBase andSuber that adds methods for keys with ordinal + numbered suffixes. Each key consistes of pre joined with .sep to ordinal suffix Assumes dupsort==False @@ -559,6 +560,154 @@ def __init__(self, *pa, **kwa): super(OnSuber, self).__init__(*pa, **kwa) +class B64SuberBase(SuberBase): + """ + Base Class whose values are Iterables of Base64 str or bytes that are stored + in db as .sep joined Base64 bytes. Separator character must not be valid + Base64 character so the split will work unambiguously. + + Automatically joins and splits along separator to Iterable (tuple) of Base64 + + Attributes: + db (dbing.LMDBer): base LMDB db + sdb (lmdb._Database): instance of lmdb named sub db for this Suber + sep (str): separator for combining keys tuple of strs into key bytes + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + Must not be Base64 character. + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + + """ + super(B64SuberBase, self).__init__(*pa, **kwa) + if Reb64.match(self.sep.encode()): + raise ValueError("Invalid sep={self.sep}, must not be Base64 char.") + + + def _toval(self, vals: str|bytes|memoryview|Iterable[str|bytes|memoryview]): + """ + Converts vals to val bytes with proper separators and returns val bytes. + If vals is already str or bytes or memoryview then returns val bytes. + Else If vals is iterable (non-str) of strs or bytes or memoryview then + joins with .sep and converts to val bytes and returns. + + Returns: + val (bytes): each element of vals is joined by .sep. + + Parameters: + vals (str | bytes | memoryview | Iterable[str | bytes]): db val or + Iterable of (str | bytes | memoryview) to form val. + Note, join of bytes sep works with memoryview. + + """ + if hasattr(vals, "encode"): # str + val = vals.encode("utf-8") + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + if isinstance(vals, memoryview): # memoryview of bytes + val = bytes(vals) # return bytes + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + elif hasattr(vals, "decode"): # bytes + val = vals + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + return (self.sep.join(val.decode() if hasattr(val, "decode") else val + for val in vals).encode("utf-8")) + + + def _tovals(self, val: bytes | memoryview): + """ + Converts val bytes to vals tuple of strs by decoding and then splitting + at separator .sep. + + Returns: + vals (tuple[str]): makes tuple by splitting val at .sep + + Parameters: + val (bytes | memoryview): db Base64 val. + + """ + if isinstance(val, memoryview): # memoryview of bytes + val = bytes(val) + if hasattr(val, "decode"): # bytes + val = val.decode("utf-8") # convert to str + return tuple(val.split(self.sep)) + + + def _ser(self, val: Union[Iterable, str, bytes]): + """ + Serialize val to bytes to store in db + When val is Iterable then joins each elements with .sep returns val bytes + + Returns: + val (bytes): .sep join of each Base64 bytes in val + + Parameters: + val (Union[Iterable, bytes]): of Base64 bytes + + """ + if not nonStringIterable(val): # not iterable + val = (val, ) # make iterable + return (self._toval(val)) + + + def _des(self, val: memoryview | bytes): + """ + Converts val bytes to vals tuple of subclass instances by deserializing + .qb64b concatenation in order of each instance in .klas + + Returns: + vals (tuple): subclass instances + + Parameters: + val (Union[bytes, memoryview]): of concatenation of .qb64b + + """ + return self._tovals(val) + + +class B64Suber(B64SuberBase, Suber): + """ + Subclass of B64SuberBase and Suber that serializes and deserializes values + as .sep joined strings of Base64 components. + + .sep must not be Base64 character. + + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + Must not be Base64 character. + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(B64Suber, self).__init__(*pa, **kwa) @@ -601,11 +750,11 @@ def _ser(self, val: coring.Matter): return val.qb64b - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: memoryview | bytes): """ Deserialize val to str Parameters: - val (Union[str, memoryview, bytes]): convertable to coring.matter + val (memoryview | bytes): convertable to coring.matter """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes @@ -710,13 +859,13 @@ def __init__(self, *pa, klas: Iterable = None, **kwa): def _ser(self, val: Union[Iterable, coring.Matter]): """ Serialize val to bytes to store in db - Concatenates .qb64b of each instance in objs and returns val bytes + Concatenates .qb64b of each instance in val and returns val bytes Returns: - val (bytes): concatenation of .qb64b of each object instance in vals + cat (bytes): concatenation of .qb64b of each object instance in vals Parameters: - subs (Union[Iterable, coring.Matter]): of subclass instances. + val (Union[Iterable, coring.Matter]): of subclass instances. """ if not nonStringIterable(val): # not iterable @@ -724,7 +873,7 @@ def _ser(self, val: Union[Iterable, coring.Matter]): return (b''.join(obj.qb64b for obj in val)) - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: memoryview | bytes | bytearray): """ Converts val bytes to vals tuple of subclass instances by deserializing .qb64b concatenation in order of each instance in .klas diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index f11e229c9..e84d5f5b3 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -356,6 +356,91 @@ def test_on_suber(): assert not db.opened +def test_B64_suber(): + """ + Test B64Suber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + # Test Single klas + buber = subing.B64Suber(db=db, subkey='bags.') # default klas is [Matter] + assert isinstance(buber, subing.B64Suber) + assert not buber.sdb.flags()["dupsort"] + + vals0 = ("alpha", "beta") + vals1 = ("gamma", ) + + keys0 = ("cat", "dog") + assert buber.put(keys=keys0, val=vals0) + actuals = buber.get(keys=keys0) + assert actuals == vals0 + + assert buber.rem(keys0) + assert not buber.get(keys=keys0) + + assert buber.put(keys=keys0, val=vals0) + actuals = buber.get(keys=keys0) + assert actuals == vals0 + + assert not buber.put(keys=keys0, val=vals1) + + assert buber.pin(keys=keys0, val=vals1) + actuals = buber.get(keys=keys0) + assert actuals == vals1 + + assert buber.rem(keys0) + assert not buber.get(keys=keys0) + + # test with vals as non Iterable or put but Iterable on get + assert buber.put(keys=keys0, val="gamma") + actuals = buber.get(keys=keys0) + assert actuals == vals1 + + assert buber.rem(keys0) + assert buber.put(keys=keys0, val="alpha.beta") + actuals = buber.get(keys=keys0) + assert actuals == vals0 + assert buber.rem(keys0) + + # test with keys as string not tuple + keys1 = "{}.{}".format("bird", "fish") + + assert buber.put(keys=keys1, val=vals1) + actuals = buber.get(keys=keys1) + assert actuals == vals1 + + assert buber.rem(keys1) + assert not buber.get(keys=keys1) + + # test missing entry at keys + badkey = "badkey" + assert not buber.get(badkey) + + # test iteritems + assert buber.put(keys0, vals0) + assert buber.put(keys1, vals1) + + items = [ items for items in buber.getItemIter()] + assert items == [(('bird', 'fish'), ('gamma',)), (('cat', 'dog'), ('alpha', 'beta'))] + + buber.put(keys=("b","1"), val=vals0) + buber.put(keys=("b","2"), val=vals1) + buber.put(keys=("c","1"), val=vals0) + buber.put(keys=("c","2"), val=vals1) + + topkeys = ("b","") # last element empty to force trailing separator + items = [items for items in buber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), ('alpha', 'beta')), + (('b', '2'), ('gamma',))] + + assert not os.path.exists(db.path) + assert not db.opened + """Done Test""" + def test_dup_suber(): """ @@ -1818,9 +1903,9 @@ def test_cesr_on_suber(): -def test_cat_suber(): +def test_cat_cesr_suber(): """ - Test CatSuber LMDBer sub database class + Test CatCesrSuber LMDBer sub database class """ with dbing.openLMDB() as db: @@ -2621,11 +2706,12 @@ def test_crypt_signer_suber(): if __name__ == "__main__": test_suber() test_on_suber() + test_B64_suber() test_dup_suber() test_iodup_suber() test_on_iodup_suber() test_ioset_suber() - test_cat_suber() + test_cat_cesr_suber() test_cesr_suber() test_cesr_on_suber() test_cesr_ioset_suber() From d78054d976788b0433766a4cda46c029ece712f0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 5 Sep 2024 17:05:10 -0600 Subject: [PATCH 325/418] some refactoring and preliminary support for partial delegation escrow --- src/keri/core/eventing.py | 504 ++++++++++++++++++++++++++------------ src/keri/db/basing.py | 16 +- src/keri/db/subing.py | 2 + tests/db/test_basing.py | 3 +- 4 files changed, 354 insertions(+), 171 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index d07caf2bb..f8f1fd5c1 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -3111,39 +3111,61 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): "event = %s\n", serder.ked) - def escrowPACouple(self, serder, seqner, saider, local=True): + def escrowPWEvent(self, serder, wigers, sigers=None, + seqner=None, saider=None, local=True): """ - Update associated logs for escrow of partially authenticated issued event. - Assuming signatures are provided elsewhere. Partial authentication results - from either a partially signed event or a fully signed delegated event - but whose delegation is not yet verified. - - Escrow allows escrow processor to retrieve serder from key and source - couple from val in order to to re-verify authentication status. Sigs - are escrowed elsewhere. + Update associated logs for escrow of partially witnessed event Parameters: - serder is SerderKERI instance of delegated or issued event + serder is SerderKERI instance of event + wigers is list of Siger instance of indexed witness sigs + sigers is optional list of Siger instances of indexed controller sigs seqner is Seqner instance of sn of seal source event of delegator/issuer - saider is Saider instance of said of delegator/issuer + saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). Event validation logic is a function of local or remote + """ - local = True if local else False # ignored since not escrowing serder here + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) - #couple = seqner.qb64b + saider.qb64b - #self.db.putUde(dgkey, couple) # idempotent - self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent - logger.debug("Kever state: Escrowed source couple for partially signed " - "or delegated event = %s\n", serder.ked) + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if sigers: + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if seqner and saider: + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + self.db.putEvt(dgkey, serder.raw) + # update event source + if (esr := self.db.esrs.get(keys=dgkey)): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) - def escrowPWEvent(self, serder, wigers, sigers=None, + logger.debug("Kever state: Escrowed partially witnessed " + "event = %s\n", serder.ked) + return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) + + + def escrowPDEvent(self, serder, sigers=None, wigers=None, seqner=None, saider=None, local=True): """ - Update associated logs for escrow of partially witnessed event + Update associated logs for escrow of partially delegated or otherwise + authorized issued event. + Assumes controller signatures, sigs, and witness signatures, wigs are + provided elsewhere. Partial authentication occurs once an event is + fully signed and witnessed but the authorizing (delegating) source + seal in the authorizer's (delegator's) key has not yet been verified. + + Escrow allows escrow processor to retrieve serder from key and source + couple from val in order to to re-verify authentication status. Parameters: serder is SerderKERI instance of event @@ -3160,17 +3182,20 @@ def escrowPWEvent(self, serder, wigers, sigers=None, local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent - if wigers: - self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) - if sigers: + + if sigers: # idempotent self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) - if seqner and saider: - #couple = seqner.qb64b + saider.qb64b - #self.db.putUde(dgkey, couple) + if wigers: # idempotent + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: # idempotent self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + logger.debug(f"Kever state: Escrowed source couple sn={seqner.sn}, " + f"said={saider.said} for partially delegated/authorized" + f" event said={serder.said}.") - self.db.putEvt(dgkey, serder.raw) - # update event source + self.db.putEvt(dgkey, serder.raw) # idempotent + + # update event source local or remote if (esr := self.db.esrs.get(keys=dgkey)): # preexisting esr if local and not esr.local: # local overwrites prexisting remote esr.local = local @@ -3180,9 +3205,38 @@ def escrowPWEvent(self, serder, wigers, sigers=None, esr = basing.EventSourceRecord(local=local) self.db.esrs.put(keys=dgkey, val=esr) - logger.debug("Kever state: Escrowed partially witnessed " - "event = %s\n", serder.ked) - return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) + logger.debug(f"Kever state: Escrowed partially delegated event=\n" + f"{serder.ked}\n.") + return self.db.pdes.add(keys =(serder.preb, serder.sn), val=serder.saidb) + + + def escrowPACouple(self, serder, seqner, saider, local=True): + """ + Update associated logs for escrow of partially authenticated issued event. + Assuming signatures are provided elsewhere. Partial authentication results + from either a partially signed event or a fully signed delegated event + but whose delegation is not yet verified. + + Escrow allows escrow processor to retrieve serder from key and source + couple from val in order to to re-verify authentication status. Sigs + are escrowed elsewhere. + + Parameters: + serder is SerderKERI instance of delegated or issued event + seqner is Seqner instance of sn of seal source event of delegator/issuer + saider is Saider instance of said of delegator/issuer + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + """ + local = True if local else False # ignored since not escrowing serder here + dgkey = dgKey(serder.preb, serder.saidb) + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + logger.debug("Kever state: Escrowed source couple for partially signed " + "or delegated event = %s\n", serder.ked) + + def state(self): @@ -5347,149 +5401,150 @@ def processEscrowPartialSigs(self): If successful then remove from escrow table """ - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in self.db.getPseItemIter(key=key): - eserder = None - try: - pre, sn = splitSnKey(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) - if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) + #key = ekey = b'' # both start same. when not same means escrows found + #while True: # break when done + for ekey, edig in self.db.getPseItemIter(key=b''): + eserder = None + try: + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) - # check date if expired then remove escrow. - dtb = self.db.getDts(dgkey) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): - # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) - # get the escrowed event using edig - eraw = self.db.getEvt(dgkey) - if eraw is None: - # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) + # get the escrowed event using edig + eraw = self.db.getEvt(dgkey) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) - eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - # get sigs and attach - sigs = self.db.getSigs(dgkey) - if not sigs: # otherwise its a list of sigs - # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + # get sigs and attach + sigs = self.db.getSigs(dgkey) + if not sigs: # otherwise its a list of sigs + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - if not wigs: # empty list wigs witness sigs not wits - # wigs maybe empty if not wits or if wits while waiting - # for first witness signature - # which may not arrive until some time after event is fully signed - # so just log for debugging but do not unescrow by raising - # ValidationError - logger.debug("Kevery unescrow wigs: No event wigs yet at." - "dig = %s", bytes(edig)) - - # seal source (delegator issuer if any) - delseqner = delsaider = None - if (couple := self.db.udes.get(keys=dgkey)): - delseqner, delsaider = couple - #couple = self.db.getUde(dgkey) - #if couple is not None: - #delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - #couple = delseqner.qb64b + delsaider.qb64b - #self.db.putUde(dgkey, couple) - # idempotent - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + if not wigs: # empty list wigs witness sigs not wits + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) - # process event - sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=dgkey)): + delseqner, delsaider = couple + #couple = self.db.getUde(dgkey) + #if couple is not None: + #delseqner, delsaider = deSourceCouple(couple) + elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + if eserder.pre in self.kevers: + delpre = self.kevers[eserder.pre].delpre + else: + delpre = eserder.ked["di"] + seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + #couple = delseqner.qb64b + delsaider.qb64b + #self.db.putUde(dgkey, couple) + # idempotent + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) - # If process does NOT validate sigs or delegation seal (when delegated), - # but there is still one valid signature then process will - # attempt to re-escrow and then raise MissingSignatureError - # or MissingDelegationSealError (subclass of ValidationError) - # so we can distinquish between ValidationErrors that are - # re-escrow vs non re-escrow. We want process to be idempotent - # with respect to processing events that result in escrow items. - # On re-escrow attempt by process, Pse escrow is called by - # Kever.self.escrowPSEvent Which calls - # self.db.addPse(snKey(pre, sn), serder.digb) - # which in turn will not enter dig as dup if one already exists. - # So re-escrow attempt will not change the escrowed pse db. - # Non re-escrow ValidationError means some other issue so unescrow. - # No error at all means processed successfully so also unescrow. + # process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, local=esr.local) - except (MissingSignatureError, MissingDelegationError) as ex: - # still waiting on missing sigs or missing seal to validate - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + # If process does NOT validate sigs or delegation seal (when delegated), + # but there is still one valid signature then process will + # attempt to re-escrow and then raise MissingSignatureError + # or MissingDelegationSealError (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pse escrow is called by + # Kever.self.escrowPSEvent Which calls + # self.db.addPse(snKey(pre, sn), serder.digb) + # which in turn will not enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrowed pse db. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. - except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow - self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any + except (MissingSignatureError, MissingDelegationError) as ex: + # still waiting on missing sigs or missing seal to validate + # processEvent idempotently reescrowed + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) - if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - self.cues.push(dict(kin="psUnescrow", serder=eserder)) + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs or seal so remove from escrow + self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + self.cues.push(dict(kin="psUnescrow", serder=eserder)) - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) - if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - self.cues.push(dict(kin="psUnescrow", serder=eserder)) + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + self.cues.push(dict(kin="psUnescrow", serder=eserder)) - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") + + #if ekey == key: # still same so no escrows found on last while iteration + #break + #key = ekey # setup next while iteration, with key after ekey def processEscrowPartialWigs(self): """ @@ -5527,10 +5582,7 @@ def processEscrowPartialWigs(self): Process event as if it came in over the wire If successful then remove from escrow table """ - - #key = ekey = b'' # both start same. when not same means escrows found - #while True: # break when done - for ekey, edig in self.db.getPweItemIter(key=b''): #self.db.getPweItemsNextIter(key=key) + for ekey, edig in self.db.getPweItemIter(key=b''): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) @@ -5645,6 +5697,7 @@ def processEscrowPartialWigs(self): except (MissingWitnessSignatureError, MissingDelegationError) as ex: # still waiting on missing witness sigs or delegation + # processEvent idempotently reescrowed if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) @@ -5669,9 +5722,6 @@ def processEscrowPartialWigs(self): "event=%s", eserder.said) logger.debug(f"event=\n{eserder.pretty()}\n") - #if ekey == key: # still same so no escrows found on last while iteration - #break - #key = ekey # setup next while iteration, with key after ekey def processEscrowPartialDels(self): """ @@ -5700,7 +5750,135 @@ def processEscrowPartialDels(self): If successful then remove from escrow table """ - pass + for ekey, edig in self.db.pdes.getItemIter(key=b''): + try: + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) + + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) + + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) + + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) + + # get the escrowed event using edig + eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) + + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + + # get sigs + sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs + if not sigs: # empty list + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) + + # get witness signatures (wigs not wits) assumes wont be in this + # escrow if wigs not needed because no wits + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + if not wigs: # empty list + # no wigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event wigs at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt wigs at " + "dig = {}.".format(bytes(edig))) + + # setup parameters to process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + delseqner, delsaider = couple # provided + elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): # walk kel to find + if eserder.pre in self.kevers: + delpre = self.kevers[eserder.pre].delpre + else: + delpre = eserder.ked["di"] + seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: # found seal in srdr + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, local=esr.local) + + # If process does NOT validate delegation then process will attempt + # to re-escrow and then raise MissingDelegationError + # (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pses escrow is called by + # Kever.self.escrowPDEvent Which calls + # self.db.pdes.add(snKey(pre, sn), serder.digb) + # which in turn will NOT enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrows in db.pdes. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + # Assumes that controller signature validation and delegation + # validation will be successful as event would not be in + # partially witnessed escrow unless they had already validated + + except MissingDelegationError as ex: + # still waiting on missing delegation source seal + # processEvent idempotently reescrowed + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) + + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs or seal so remove from escrow + # removes one event escrow at key val + self.db.pdes.rem(keys=(pre, sn), val=edig) # event idx escrow + self.db.udes.rem(keys=dgkey) # remove source seal escrow if any + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) + + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + # removes one event escrow at key val + self.db.pdes.rem(keys=(pre, sn), val=edig) # event idx escrow + self.db.udes.rem(keys=dgkey) # remove source seal escrow if any + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") def processEscrowUnverWitness(self): diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 69557fedc..219805589 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -764,22 +764,23 @@ class Baser(dbing.LMDBer): More than one value per DB key is allowed .pses is named sub DB of partially signed key event escrows - that map sequence numbers to serialized event digests. + that each map pre + sequence number to serialized event digest. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .pwes is named sub DB of partially witnessed key event escrowes - that map sequence numbers to serialized event digests. + that each map pre + sequence number to serialized event digest. + these are for escrows of events with verified signatures but not + yet verified witness reciepts. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed .pdes is named sub DB of partially delegated key event escrows - that map sequence numbers to serialized event digests. This is + that each map pre + sequence number to serialized event digest. This is used in conjunction with .udes which escrows the associated seal source couple. snKey @@ -798,6 +799,8 @@ class Baser(dbing.LMDBer): KEL. DB is keyed by identifier prefix plus digest of key event Only one value per DB key is allowed + Once escrow is accepted then delegation approval source seal couples + go into .aess database of authorizing event source seal couples .uwes is named sub DB of unverified event indexed escrowed couples from witness signers. Witnesses are from witness list of latest establishment @@ -1012,11 +1015,10 @@ def reopen(self, **kwa): self.vrcs = self.env.open_db(key=b'vrcs.', dupsort=True) self.vres = self.env.open_db(key=b'vres.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) - #self.pdes = self.env.open_db(key=b'pdes.') - self.pdes = subing.IoSetSuber(db=self, subkey='pdes.') + self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) + self.pdes = subing.IoDupSuber(db=self, subkey='pdes.') self.udes = subing.CatCesrSuber(db=self, subkey='udes.', klas=(coring.Seqner, coring.Saider)) - self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) self.ooes = self.env.open_db(key=b'ooes.', dupsort=True) self.dels = self.env.open_db(key=b'dels.', dupsort=True) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 75b94abeb..dcba2e6a9 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -44,6 +44,8 @@ increasing numeric The term 'set' of values means that no value may appear more than once in the set. +Sets support idempotent adds and puts to db. This means one can add or put the same +(key, val) pair multiple times and not change the db. DupSuber provides set of lexicographic ordered values at each key. Each value has a limited size (key + value <= 511 byes). The set is performant. Good for indices. diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index f8971f7ba..b4de48322 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -784,7 +784,8 @@ def test_baser(): assert key == f'{preb.decode("utf-8")}.{digb.decode("utf-8")}'.encode("utf-8") # test .pdes SerderIoSetSuber methods - assert isinstance(db.pdes, subing.IoSetSuber) + assert isinstance(db.pdes, subing.IoDupSuber) + # test .udes CatCesrSuber sub db methods assert isinstance(db.udes, subing.CatCesrSuber) From 7ea9290c9106658e19f94168dd92f0c351d5302e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 08:35:05 -0600 Subject: [PATCH 326/418] fixed bug --- src/keri/db/subing.py | 10 ++++++++-- tests/db/test_subing.py | 8 +++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index dcba2e6a9..214cf5ed1 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -627,8 +627,14 @@ def _toval(self, vals: str|bytes|memoryview|Iterable[str|bytes|memoryview]): if not (Reb64.match(val)): raise ValueError(f"Non Base64 {val=}.") return val - return (self.sep.join(val.decode() if hasattr(val, "decode") else val - for val in vals).encode("utf-8")) + vals = tuple(v.encode() if hasattr(v, "encode") else v for v in vals) # make bytes + for val in vals: + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return (self.sep.encode().join(vals)) + + #return (self.sep.join(val.decode() if hasattr(val, "decode") else val + #for val in vals).encode("utf-8")) def _tovals(self, val: bytes | memoryview): diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index e84d5f5b3..6b97d3c55 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -401,7 +401,13 @@ def test_B64_suber(): assert actuals == vals1 assert buber.rem(keys0) - assert buber.put(keys=keys0, val="alpha.beta") + + # test val as already joined string + with pytest.raises(ValueError): + assert buber.put(keys=keys0, val="alpha.beta") + + # test bytes for val + assert buber.put(keys=keys0, val=(b"alpha", b"beta")) actuals = buber.get(keys=keys0) assert actuals == vals0 assert buber.rem(keys0) From d246047d804afde1961f35d24411eb9b4493f4a1 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 10:10:43 -0600 Subject: [PATCH 327/418] updated Kever.escrows superseded EscrowPACouple so not needed anymore. --- src/keri/core/eventing.py | 55 +++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index f8f1fd5c1..a44d9e8a2 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2253,9 +2253,10 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # escrow if not fully signed vs signing threshold if not tholder.satisfy(indices): # at least one but not enough - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - if delseqner and delsaider: - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + #if delseqner and delsaider: + #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" f" on sigs for {[siger.qb64 for siger in sigers]}" f" for evt = {serder.ked}.") @@ -2266,9 +2267,10 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # prior next threshold in .ntholder and digers in .ndigers ondices = self.exposeds(sigers) if not self.ntholder.satisfy(indices=ondices): - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - if delseqner and delsaider: # save in case not attached later - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider,local=local) + #if delseqner and delsaider: # save in case not attached later + #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingSignatureError(f"Failure satisfying prior nsith=" f"{self.ntholder.sith} with exposed " f"sigs= {[siger.qb64 for siger in sigers]}" @@ -2628,9 +2630,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # The processPSEvent should also cue a trigger to get KEL # of delegator if still missing when processing escrow later. self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, - local=local) - if delseqner and delsaider: # save in case not attached later - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + seqner=delseqner, saider=delsaider, local=local) + #if delseqner and delsaider: # save in case not attached later + #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError(f"Missing KEL of delegator " f"{delpre} of evt = {serder.ked}.") @@ -2660,7 +2662,8 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, f"event = {serder.ked}.") else: # not local delegator so escrow PSEvent - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) # since delseqner or delsaider is None there is no PACouple to escrow here #if delseqner and delsaider: # save in case not attached later #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) @@ -2687,9 +2690,10 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False #sn = validateSN(sn=serder.snh, inceptive=inceptive) sn = Number(num=serder.sn).validate(inceptive=inceptive).sn - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - if delseqner and delsaider: # save in case not attached later - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + #if delseqner and delsaider: # save in case not attached later + #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError("No delegating event from {} at {} for " "evt = {}.".format(delpre, delsaider.qb64, @@ -3073,7 +3077,8 @@ def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): logger.debug("Kever state: escrowed delegable event=\n%s\n", json.dumps(serder.ked, indent=1)) - def escrowPSEvent(self, serder, sigers, wigers=None, local=True): + def escrowPSEvent(self, serder, *, sigers=None, wigers=None, + seqner=None, saider=None, local=True): """ Update associated logs for escrow of partially signed event or fully signed delegated event but not yet verified delegation. @@ -3082,6 +3087,8 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): serder is SerderKERI instance of event sigers is list of Siger instances of indexed controller sigs wigers is optional list of Siger instance of indexed witness sigs + seqner is Seqner instance of sn of seal source event of delegator/issuer + saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -3090,9 +3097,12 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent - self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if sigers: + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.putEvt(dgkey, serder.raw) # update event source @@ -3111,15 +3121,15 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): "event = %s\n", serder.ked) - def escrowPWEvent(self, serder, wigers, sigers=None, + def escrowPWEvent(self, serder, *, sigers=None, wigers=None, seqner=None, saider=None, local=True): """ Update associated logs for escrow of partially witnessed event Parameters: serder is SerderKERI instance of event - wigers is list of Siger instance of indexed witness sigs sigers is optional list of Siger instances of indexed controller sigs + wigers is list of Siger instance of indexed witness sigs seqner is Seqner instance of sn of seal source event of delegator/issuer saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic @@ -3131,10 +3141,11 @@ def escrowPWEvent(self, serder, wigers, sigers=None, local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent - if wigers: - self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if sigers: self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent @@ -3154,7 +3165,7 @@ def escrowPWEvent(self, serder, wigers, sigers=None, return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) - def escrowPDEvent(self, serder, sigers=None, wigers=None, + def escrowPDEvent(self, serder, *, sigers=None, wigers=None, seqner=None, saider=None, local=True): """ Update associated logs for escrow of partially delegated or otherwise @@ -3169,8 +3180,8 @@ def escrowPDEvent(self, serder, sigers=None, wigers=None, Parameters: serder is SerderKERI instance of event - wigers is list of Siger instance of indexed witness sigs sigers is optional list of Siger instances of indexed controller sigs + wigers is list of Siger instance of indexed witness sigs seqner is Seqner instance of sn of seal source event of delegator/issuer saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic @@ -3209,7 +3220,7 @@ def escrowPDEvent(self, serder, sigers=None, wigers=None, f"{serder.ked}\n.") return self.db.pdes.add(keys =(serder.preb, serder.sn), val=serder.saidb) - + # not used anymore deprecate? def escrowPACouple(self, serder, seqner, saider, local=True): """ Update associated logs for escrow of partially authenticated issued event. From 37380e1f81057844979e885ebfaf950ba48aa889 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 12:59:06 -0600 Subject: [PATCH 328/418] enabled ProcessPDEscrow --- src/keri/core/eventing.py | 233 ++++++++++++++++++++++---------------- 1 file changed, 133 insertions(+), 100 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index a44d9e8a2..7a92731df 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1561,16 +1561,17 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits, - local=local, - delseqner=delseqner, - delsaider=delsaider) + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel( + serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + local=local, + delseqner=delseqner, + delsaider=delsaider) self.delpre = delpre # may be None self.delegated = True if self.delpre else False @@ -1921,7 +1922,8 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # returned sigers and wigers are verified signatures # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel( + serder=serder, sigers=sigers, verfers=serder.verfers, tholder=tholder, @@ -1988,7 +1990,8 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, + sigers, wigers, delpre, _, _ = self.valSigsWigsDel( + serder=serder, sigers=sigers, verfers=self.verfers, tholder=self.tholder, @@ -2169,8 +2172,9 @@ def deriveBacks(self, serder): def valSigsWigsDel(self, serder, sigers, verfers, tholder, - wigers, toader, wits, local=True, - delseqner=None, delsaider=None): + wigers, toader, wits, *, + delseqner=None, delsaider=None, eager=False, + local=True): """ Returns triple (sigers, wigers, delegator) where: sigers is unique validated signature verified members of inputed sigers @@ -2195,14 +2199,19 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, toader (Number): instance of backer witness threshold wits (list): of qb64 non-transferable prefixes of witnesses used to derive werfers for wigers - local (bool): event source for validation logic - True means event source is local (protected). - False means event source is remote (unprotected). - Event validation logic is a function of local or remote delseqner (Seqner | None): instance of delegating event sequence number. If this event is not delegated then seqner is ignored delsaider (Saider | None): instance of of delegating event said. If this event is not delegated then saider is ignored + eager (bool): True means try harder to find delegation seals by + walking KEL of delegator. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing source seal couples + if any, either attached or in database. + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ if len(verfers) < tholder.size: @@ -2255,8 +2264,6 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, if not tholder.satisfy(indices): # at least one but not enough self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - #if delseqner and delsaider: - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" f" on sigs for {[siger.qb64 for siger in sigers]}" f" for evt = {serder.ked}.") @@ -2269,8 +2276,6 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, if not self.ntholder.satisfy(indices=ondices): self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider,local=local) - #if delseqner and delsaider: # save in case not attached later - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingSignatureError(f"Failure satisfying prior nsith=" f"{self.ntholder.sith} with exposed " f"sigs= {[siger.qb64 for siger in sigers]}" @@ -2297,41 +2302,50 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f" delegated by {delpre} of" f"event = {serder.ked}.") - # short circuit witness validation when either locallyOwned or locallyWitnessed - # otherwise must validate fully witnessed - if not (self.locallyOwned() or self.locallyMembered() or self.locallyWitnessed(wits=wits)): - if wits: # is witnessed - if toader.num < 1 or toader.num > len(wits): # out of bounds toad - raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - else: # not witnessed - if toader.num != 0: # invalid toad - raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - - if len(windices) < toader.num: # not fully witnessed yet - if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, - seqner=delseqner, saider=delsaider, - local=local): - # cue to query for witness receipts - self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) - raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " - f"on witness sigs=" - f"{[siger.qb64 for siger in wigers]} " - f"for event={serder.ked}.") - - if delpre: - #if not (delseqner and delsaider): - #seal = dict(i=serder.ked["i"], s=serder.snh, d=serder.said) - #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - #if srdr is not None: - #delseqner = coring.Seqner(sn=srdr.sn) - #delsaider = coring.Saider(qb64=srdr.said) - - self.validateDelegation(serder, sigers=sigers, - wigers=wigers, wits=wits, - local=local, delpre=delpre, - ) # delseqner=delseqner, delsaider=delsaider - - return (sigers, wigers, delpre) # delseqner, delsaider + # this point the sigers have been verified and the wigers have been verified + # even if locallyOwned or locallyMembered or locallyWitnessed. + # But the toad has not yet been verified. + + if not wits: + if toader.num != 0: # bad toad non-zero and not witnessed bad event + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + else: # wits so may need to validate toad + # The toad is only verified against the wigers as fully witnessed if + # not (locallyOwned or locallyMembered or locallyWitnessed) + if (not (self.locallyOwned() or self.locallyMembered() or + self.locallyWitnessed(wits=wits))): + if wits: # is witnessed + if toader.num < 1 or toader.num > len(wits): # out of bounds toad + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + else: # not witnessed + if toader.num != 0: # invalid toad + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + if len(windices) < toader.num: # not fully witnessed yet + if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, + seqner=delseqner, saider=delsaider, + local=local): + # cue to query for witness receipts + self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) + raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " + f"on witness sigs=" + f"{[siger.qb64 for siger in wigers]} " + f"for event={serder.ked}.") + + # returns (None, None) when delegation validation does not apply. + # Raises ValidationError if validation applies but does not validate. + delseqner, delsaider = self.validateDelegation(serder, + sigers=sigers, + wigers=wigers, + wits=wits, + delpre=delpre, + delseqner=delseqner, + delsaider=delsaider, + eager=eager, + local=local) + + return (sigers, wigers, delpre, delseqner, delsaider) def exposeds(self, sigers): @@ -2381,8 +2395,8 @@ def exposeds(self, sigers): return odxs - def validateDelegation(self, serder, sigers, wigers, wits, local=True, - delpre=None, delseqner=None, delsaider=None): + def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, + delseqner=None, delsaider=None, eager=False, local=True): """ Returns delegator's qb64 identifier prefix if validation successful. Assumes that local vs remote source checks have been applied before @@ -2410,15 +2424,24 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, delegated event. Assumes wigers is list of unique verified sigs wits (list[str]): of qb64 non-transferable prefixes of witnesses used to derive werfers for wigers - local (bool): event source for validation logic - True means event source is local (protected). - False means event source is remote (unprotected). - Event validation logic is a function of local or remote - delpre (str | None): qb64 prefix of delegator if any + delpre (str): qb64 prefix of delegator delseqner (Seqner | None): instance of delegating event sequence number. If this event is not delegated then ignored delsaider (Saider | None): instance of of delegating event digest. If this event is not delegated ignored + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + eager (bool): True means try harder to find delegation seals by + walking KEL of delegator. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing source seal couples + if any, either attached or in database. + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote Returns: None @@ -2612,7 +2635,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, """ if not delpre: # not delegable so no delegation validation needed - return + return (None, None) # non-delegated so delseqner delsaider must be None # if we are the delegatee, accept the event without requiring the # delegator validation via an anchored delegation seal or by requiring @@ -2621,8 +2644,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # seal to be anchored in delegator's KEL. # Witness accepts without waiting for delegation seal to be anchored in # delegator's KEL. Witness cue in Kevery will then generate receipt - if self.locallyOwned() or self.locallyWitnessed(wits=wits): - return + if (self.locallyOwned() or self.locallyMembered() or + self.locallyWitnessed(wits=wits)): + return (None, None) # not validated so delseqner delsaider must be None if self.kevers is None or delpre not in self.kevers: # missing delegator # ToDo XXXX cue a trigger to get the KEL of the delegator. This may @@ -2736,6 +2760,10 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, f"from {delpre} in {dserder.seals} for " f"evt = {serder.ked}.") + # valid anchoring seal of delegator delpre + delseqner = Seqner(snh=dserder.snh) + delsaider = Saider(qb64=dserder.said) + # Found anchor so can confirm delegation successful unless its one of # the superseding conditions. Valid if not superseding drt of drt. # Assumes database is reverified each bootup or otherwise when @@ -2749,7 +2777,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, (serder.sner.num == self.sner.num and # superseding event self.ilk == Ilks.ixn and # superseded is ixn and serder.ilk == Ilks.drt)): # recovery rotation superseding ixn - return # indicates delegation valid + return (None, None) # not validated so delseqner delsaider must be None # indicates delegation valid # get to here means drt rotation superseding another drt rotation # Kever.logEvent saves authorizer (delegator) seal source couple in @@ -2760,7 +2788,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, serfn = serder # new potentially superseding delegated event i.e. serf new bossn = dserder # new delegating event of superseding delegated event i.e. boss new serfo = self.serder # original delegated event i.e. serf original - bosso = self.fetchDelegatingEvent(delpre, serfo) + bosso = self.fetchDelegatingEvent(delpre, serfo, eager=eager) while (True): # superseding delegated rotation of rotation recovery rules # Only get to here if same sn for drt existing and drt superseding @@ -2768,7 +2796,8 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, if (bossn.sn > bosso.sn or # later supersedes (bossn.Ilk == Ilks.drt and bosso.Ilk == Ilks.ixn) ): # drt supersedes ixn - return # delegator # valid superseding delegation + # valid superseding delegation up chain so tail link valid + return (delseqner, delsaider) # tail event's delegation source if bossn.said == bosso.said: # same delegating event nseals = [SealEvent(**seal) for seal in bossn.seals @@ -2780,7 +2809,8 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, if nindex > oindex: # later seal supersedes # assumes index can't be None - return # delegator # valid superseding delegation + # valid superseding delegation up chain so tail link valid + return (delseqner, delsaider) # tail event's delegation source else: # ToDo: XXXX may want to cue up business logic for delegator @@ -2790,10 +2820,12 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # tie condition same sn and drt so need to climb delegation chain serfn = bossn - bossn = self.fetchDelegatingEvent(delpre, serfn) + bossn = self.fetchDelegatingEvent(delpre, serfn, eager=eager) serfo = bosso - bosso = self.fetchDelegatingEvent(delpre, serfo) + bosso = self.fetchDelegatingEvent(delpre, serfo, eager=eager) # repeat + # should never get to here + def fetchDelegatingEvent(self, delpre, serder, *, @@ -2899,7 +2931,7 @@ def fetchDelegatingEvent(self, delpre, serder, *, # database broken this should never happen so do not supersede raise ValidationError(f"Missing delegation source seal for {serder.ked}") - return dserder # get actually found delseqner, delsaider from dserder + return dserder # extract delseqner, delsaider from actually found dserder def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, @@ -3221,31 +3253,31 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, return self.db.pdes.add(keys =(serder.preb, serder.sn), val=serder.saidb) # not used anymore deprecate? - def escrowPACouple(self, serder, seqner, saider, local=True): - """ - Update associated logs for escrow of partially authenticated issued event. - Assuming signatures are provided elsewhere. Partial authentication results - from either a partially signed event or a fully signed delegated event - but whose delegation is not yet verified. - - Escrow allows escrow processor to retrieve serder from key and source - couple from val in order to to re-verify authentication status. Sigs - are escrowed elsewhere. - - Parameters: - serder is SerderKERI instance of delegated or issued event - seqner is Seqner instance of sn of seal source event of delegator/issuer - saider is Saider instance of said of delegator/issuer - local (bool): event source for validation logic - True means event source is local (protected). - False means event source is remote (unprotected). - Event validation logic is a function of local or remote - """ - local = True if local else False # ignored since not escrowing serder here - dgkey = dgKey(serder.preb, serder.saidb) - self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent - logger.debug("Kever state: Escrowed source couple for partially signed " - "or delegated event = %s\n", serder.ked) + #def escrowPACouple(self, serder, seqner, saider, local=True): + #""" + #Update associated logs for escrow of partially authenticated issued event. + #Assuming signatures are provided elsewhere. Partial authentication results + #from either a partially signed event or a fully signed delegated event + #but whose delegation is not yet verified. + + #Escrow allows escrow processor to retrieve serder from key and source + #couple from val in order to to re-verify authentication status. Sigs + #are escrowed elsewhere. + + #Parameters: + #serder is SerderKERI instance of delegated or issued event + #seqner is Seqner instance of sn of seal source event of delegator/issuer + #saider is Saider instance of said of delegator/issuer + #local (bool): event source for validation logic + #True means event source is local (protected). + #False means event source is remote (unprotected). + #Event validation logic is a function of local or remote + #""" + #local = True if local else False # ignored since not escrowing serder here + #dgkey = dgKey(serder.preb, serder.saidb) + #self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + #logger.debug("Kever state: Escrowed source couple for partially signed " + #"or delegated event = %s\n", serder.ked) @@ -5224,6 +5256,7 @@ def processEscrows(self): self.processEscrowUnverWitness() self.processEscrowUnverNonTrans() self.processEscrowUnverTrans() + self.processEscrowPartialDels() self.processEscrowPartialWigs() self.processEscrowPartialSigs() self.processEscrowDuplicitous() @@ -5761,7 +5794,7 @@ def processEscrowPartialDels(self): If successful then remove from escrow table """ - for ekey, edig in self.db.pdes.getItemIter(key=b''): + for ekey, edig in self.db.pdes.getItemIter(keys=b''): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) From a4c8df781c7dd08ab6508cc5b9df5eaa9ad8ec02 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 13:24:52 -0600 Subject: [PATCH 329/418] more careful refactoring --- src/keri/core/eventing.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 7a92731df..ed6f39e72 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2651,23 +2651,19 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, if self.kevers is None or delpre not in self.kevers: # missing delegator # ToDo XXXX cue a trigger to get the KEL of the delegator. This may # require OOBIing with the delegator. - # The processPSEvent should also cue a trigger to get KEL + # The processPDEvent should also cue a trigger to get KEL # of delegator if still missing when processing escrow later. - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - #if delseqner and delsaider: # save in case not attached later - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError(f"Missing KEL of delegator " f"{delpre} of evt = {serder.ked}.") - - dkever = self.kevers[delpre] + dkever = self.kevers[delpre] # get delegator's KEL if dkever.doNotDelegate: # drop event if delegation not allowed raise ValidationError(f"Delegator = {delpre} for evt = {serder.ked}," f" does not allow delegation.") - # Delegator accepts here without waiting for delegation seal to be anchored in # delegator's KEL. But has already waited for fully receipted above. # Once fully receipted, cue in Kevery will then trigger cue to approve @@ -2685,18 +2681,15 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, f" delegation by {delpre} of" f"event = {serder.ked}.") - else: # not local delegator so escrow PSEvent - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + else: # otherwise escrow PDEvent since no source seal + # If eager should attempt to walk KEL to find it in delegator's KEL + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - # since delseqner or delsaider is None there is no PACouple to escrow here - #if delseqner and delsaider: # save in case not attached later - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError(f"No delegation seal for delegator " f"{delpre} of evt = {serder.ked}.") - ssn = Number(num=delseqner.sn).validate(inceptive=False).sn # ToDo XXXX need to replace Seqners with Numbers - + ssn = Number(num=delseqner.sn).validate(inceptive=False).sn # get the dig of the delegating event. Using getKeLast ensures delegating # event has not already been superceded key = snKey(pre=delpre, sn=ssn) # database key @@ -2712,12 +2705,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # escrow event here inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False - #sn = validateSN(sn=serder.snh, inceptive=inceptive) sn = Number(num=serder.sn).validate(inceptive=inceptive).sn - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - #if delseqner and delsaider: # save in case not attached later - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError("No delegating event from {} at {} for " "evt = {}.".format(delpre, delsaider.qb64, @@ -2739,7 +2729,6 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." "".format(delpre, ddig, serder.ked)) - found = False # find event seal of delegated event in delegating data # purported delegating event from source seal couple # XXXX ToDo need to change logic here to support native CESR seals not just dicts @@ -5862,6 +5851,8 @@ def processEscrowPartialDels(self): wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] # seal source (delegator issuer if any) + # If delegator KEL not available should also cue a trigger to + # get it if still missing when processing escrow. delseqner = delsaider = None if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): delseqner, delsaider = couple # provided From b31f74ed43b3fb4da47d64dd38fb135bf76bebbe Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 14:34:32 -0600 Subject: [PATCH 330/418] refactored misfit in valDelSigsWigs --- src/keri/core/eventing.py | 291 ++++++++++++++++++++++++++++++++------ 1 file changed, 245 insertions(+), 46 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index ed6f39e72..24cdef8ea 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1663,11 +1663,13 @@ def locallyOwned(self, pre: str | None = None): Returns: (bool): True if pre is local hab but not group hab + When pre="" empty then returns False Parameters: pre (str|None): qb64 identifier prefix if any. Default None None means use self.prefixer.qb64 + """ pre = pre if pre is not None else self.prefixer.qb64 return pre in self.prefixes and pre not in self.groups @@ -2244,16 +2246,15 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, raise ValidationError("No verified signatures for evt = {}." "".format(serder.ked)) + # at least one valid controller signature # Misfit check events that must be locally sourced (protected) get # escrowed in order to repair the protection when appropriate - if (not local and - (self.locallyOwned() or - self.locallyWitnessed(wits=wits))): + if (not local and self.locallyOwned()): self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally owned" - f" or locally witnessed event" - f" = {serder.ked}, {wits}, {self.prefixes}") + raise MisfitEventSourceError(f"Nonlocal source for locally owned " + f"event = {serder.ked}, {wits}, " + f"{self.prefixes}") werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers # get unique verified wigers and windices lists from wigers list @@ -2282,25 +2283,12 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f" for new est evt={serder.ked}.") - # get delegator if any - if serder.ilk == Ilks.dip: - delpre = serder.delpre # delegator from dip event - if not delpre: # empty or None - raise ValidationError(f"Empty or missing delegator for delegated" - f" inception event = {serder.ked}.") - elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation - delpre = self.delpre - else: # not delegable event icp, rot, ixn - delpre = None - - # delpre maybe None so ensure not None to pass into .locallyOwned which - # defaults to self.prefixer.qb64 when None - if not local and self.locallyOwned(delpre if delpre is not None else ''): + if (not local and self.locallyWitnessed(wits=wits)): self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally" - f" delegated by {delpre} of" - f"event = {serder.ked}.") + seqner=delseqner, saider=delsaider, local=local) + raise MisfitEventSourceError(f"Nonlocal source for locally " + f"witnessed eventf = {serder.ked}," + f"{wits}, {self.prefixes}") # this point the sigers have been verified and the wigers have been verified # even if locallyOwned or locallyMembered or locallyWitnessed. @@ -2333,6 +2321,27 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f"{[siger.qb64 for siger in wigers]} " f"for event={serder.ked}.") + # get delegator if any + if serder.ilk == Ilks.dip: + delpre = serder.delpre # delegator from dip event + if not delpre: # empty or None + raise ValidationError(f"Empty or missing delegator for delegated" + f" inception event = {serder.ked}.") + elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation + delpre = self.delpre + else: # not delegable event icp, rot, ixn + delpre = None + + # delpre maybe None so ensure not None to pass into .locallyOwned which + # defaults to self.prefixer.qb64 when None. When delpre empty "" then + # returns locallyOwned(delpre) returns False. + if not local and self.locallyOwned(delpre if delpre is not None else ''): + self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MisfitEventSourceError(f"Nonlocal source for locally" + f" delegated by {delpre} of" + f"event = {serder.ked}.") + # returns (None, None) when delegation validation does not apply. # Raises ValidationError if validation applies but does not validate. delseqner, delsaider = self.validateDelegation(serder, @@ -2348,6 +2357,184 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, return (sigers, wigers, delpre, delseqner, delsaider) + #def valSigsWigsDel(self, serder, sigers, verfers, tholder, + #wigers, toader, wits, *, + #delseqner=None, delsaider=None, eager=False, + #local=True): + #""" + #Returns triple (sigers, wigers, delegator) where: + #sigers is unique validated signature verified members of inputed sigers + #wigers is unique validated signature verified members of inputed wigers + #delegator is qb64 delegator prefix if delegated else None + + #Validates sigers signatures by validating indexes, verifying signatures, and + #validating threshold sith. + #Validate witness receipts by validating indexes, verifying + #witness signatures and validating toad. + #Witness validation is a function of wits .prefixes and .local + + #Parameters: + #serder (SerderKERI): instance of event + #sigers (list): of Siger instances of indexed controllers signatures. + #Index is offset into verfers list from which public key may be derived. + #verfers (list): of Verfer instances of keys from latest est event + #tholder (Tholder): instance of sith threshold + #wigers (list): of Siger instances of indexed witness signatures. + #Index is offset into wits list of associated witness nontrans pre + #from which public key may be derived. + #toader (Number): instance of backer witness threshold + #wits (list): of qb64 non-transferable prefixes of witnesses used to + #derive werfers for wigers + #delseqner (Seqner | None): instance of delegating event sequence number. + #If this event is not delegated then seqner is ignored + #delsaider (Saider | None): instance of of delegating event said. + #If this event is not delegated then saider is ignored + #eager (bool): True means try harder to find delegation seals by + #walking KEL of delegator. Enables only being eager + #in escrow processing not initial parsing. + #False means only use pre-existing source seal couples + #if any, either attached or in database. + #local (bool): event source for validation logic + #True means event source is local (protected). + #False means event source is remote (unprotected). + #Event validation logic is a function of local or remote + + #""" + #if len(verfers) < tholder.size: + #raise ValidationError("Invalid sith = {} for keys = {} for evt = {}." + #"".format(tholder.sith, + #[verfer.qb64 for verfer in verfers], + #serder.ked)) + + ## Filters sigers to remove any signatures from locally membered groups + ## when not local (remote) event source. So that attacker can't source + ## compromised signature remotely to satisfy threshold. + + #if not local and self.locallyMembered(): # is this Kever's pre a local group + #if indices := self.locallyContributedIndices(verfers): + #for siger in list(sigers): # copy so clean del on original elements + #if siger.index in indices: + #sigers.remove(siger) + #if self.cues: + #self.cues.push(dict(kin="remoteMemberedSig", + #serder=serder, + #index=siger.index)) + + + ## get unique verified sigers and indices lists from sigers list + #sigers, indices = verifySigs(raw=serder.raw, sigers=sigers, verfers=verfers) + ## sigers now have .verfer assigned + + ## check if minimally signed in order to continue processing + #if not indices: # must have a least one verified sig + #raise ValidationError("No verified signatures for evt = {}." + #"".format(serder.ked)) + + ## at least one valid controller signature + ## Misfit check events that must be locally sourced (protected) get + ## escrowed in order to repair the protection when appropriate + #if (not local and + #(self.locallyOwned() or + #self.locallyWitnessed(wits=wits))): + #self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, + #seqner=delseqner, saider=delsaider, local=local) + #raise MisfitEventSourceError(f"Nonlocal source for locally owned" + #f" or locally witnessed event" + #f" = {serder.ked}, {wits}, {self.prefixes}") + + #werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers + ## get unique verified wigers and windices lists from wigers list + #wigers, windices = verifySigs(raw=serder.raw, sigers=wigers, verfers=werfers) + ## each wiger now has added to it a werfer of its wit in its .verfer property + + ## escrow if not fully signed vs signing threshold + #if not tholder.satisfy(indices): # at least one but not enough + #self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + #seqner=delseqner, saider=delsaider, local=local) + #raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" + #f" on sigs for {[siger.qb64 for siger in sigers]}" + #f" for evt = {serder.ked}.") + + + ## escrow if not fully signed vs prior next rotation threshold + #if serder.ilk in (Ilks.rot, Ilks.drt): # rotation so check prior next threshold + ## prior next threshold in .ntholder and digers in .ndigers + #ondices = self.exposeds(sigers) + #if not self.ntholder.satisfy(indices=ondices): + #self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + #seqner=delseqner, saider=delsaider,local=local) + #raise MissingSignatureError(f"Failure satisfying prior nsith=" + #f"{self.ntholder.sith} with exposed " + #f"sigs= {[siger.qb64 for siger in sigers]}" + #f" for new est evt={serder.ked}.") + + + ## get delegator if any + #if serder.ilk == Ilks.dip: + #delpre = serder.delpre # delegator from dip event + #if not delpre: # empty or None + #raise ValidationError(f"Empty or missing delegator for delegated" + #f" inception event = {serder.ked}.") + #elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation + #delpre = self.delpre + #else: # not delegable event icp, rot, ixn + #delpre = None + + ## delpre maybe None so ensure not None to pass into .locallyOwned which + ## defaults to self.prefixer.qb64 when None + #if not local and self.locallyOwned(delpre if delpre is not None else ''): + #self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, + #seqner=delseqner, saider=delsaider, local=local) + #raise MisfitEventSourceError(f"Nonlocal source for locally" + #f" delegated by {delpre} of" + #f"event = {serder.ked}.") + + ## this point the sigers have been verified and the wigers have been verified + ## even if locallyOwned or locallyMembered or locallyWitnessed. + ## But the toad has not yet been verified. + + #if not wits: + #if toader.num != 0: # bad toad non-zero and not witnessed bad event + #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + #else: # wits so may need to validate toad + ## The toad is only verified against the wigers as fully witnessed if + ## not (locallyOwned or locallyMembered or locallyWitnessed) + #if (not (self.locallyOwned() or self.locallyMembered() or + #self.locallyWitnessed(wits=wits))): + #if wits: # is witnessed + #if toader.num < 1 or toader.num > len(wits): # out of bounds toad + #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + #else: # not witnessed + #if toader.num != 0: # invalid toad + #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + #if len(windices) < toader.num: # not fully witnessed yet + #if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, + #seqner=delseqner, saider=delsaider, + #local=local): + ## cue to query for witness receipts + #self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) + #raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " + #f"on witness sigs=" + #f"{[siger.qb64 for siger in wigers]} " + #f"for event={serder.ked}.") + + ## returns (None, None) when delegation validation does not apply. + ## Raises ValidationError if validation applies but does not validate. + #delseqner, delsaider = self.validateDelegation(serder, + #sigers=sigers, + #wigers=wigers, + #wits=wits, + #delpre=delpre, + #delseqner=delseqner, + #delsaider=delsaider, + #eager=eager, + #local=local) + + #return (sigers, wigers, delpre, delseqner, delsaider) + + def exposeds(self, sigers): """Returns list of ondices (indices) suitable for Tholder.satisfy from self.ndigers (prior next key digests ) as exposed by event sigers. @@ -2667,28 +2854,34 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # Delegator accepts here without waiting for delegation seal to be anchored in # delegator's KEL. But has already waited for fully receipted above. # Once fully receipted, cue in Kevery will then trigger cue to approve - # delegation - - if delseqner is None or delsaider is None: # missing delegation seal ref - if self.locallyOwned(delpre): # local delegator so escrow delegable. - # Won't get to here if not local and locallyOwned(delpre) because - # valSigsWigsDel will send nonlocal sourced delegable event to - # misfit escrow first. Mistfit escrow must first - # promote to local and reprocess event before we get to here + # delegation. Delegator approves delegation by attaching valid source + # seal and reprocessing event which shows up here as delseqner, delsaider. + # Won't get to here if not local and locallyOwned(delpre) because + # valSigsWigsDel will send nonlocal sourced delegable event to + # misfit escrow first. Mistfit escrow must first promote to local and + # reprocess event before we get to here. Assumes that attached source + # seal in this case can't be malicious since sourced locally. + + if self.locallyOwned(delpre): # local delegator + if delseqner is None or delsaider is None: # missing delegation seal + # so escrow delegable. So local delegator can approve OOB. self.escrowDelegableEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) raise MissingDelegableApprovalError(f"Missing approval for " f" delegation by {delpre} of" f"event = {serder.ked}.") - else: # otherwise escrow PDEvent since no source seal - # If eager should attempt to walk KEL to find it in delegator's KEL - self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegation seal for delegator " - f"{delpre} of evt = {serder.ked}.") + if delseqner is None or delsaider is None: # missing delegation seal ref + # If eager should walk KEL here to attempt to find in delegator's KEL + # if not found then escrow to be found later delegating event may + # not yet have been created + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegation seal for delegator " + f"{delpre} of evt = {serder.ked}.") # ToDo XXXX need to replace Seqners with Numbers + # Get delegating event from delseqner and delpre ssn = Number(num=delseqner.sn).validate(inceptive=False).sn # get the dig of the delegating event. Using getKeLast ensures delegating # event has not already been superceded @@ -2702,10 +2895,12 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, self.cues.push(dict(kin="query", q=dict(pre=delpre, sn=delseqner.snh, dig=delsaider.qb64))) - # escrow event here inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False sn = Number(num=serder.sn).validate(inceptive=inceptive).sn + # if we were to allow for malicous local delegator source seal then we + # must check for locallyOwned(delpre) first and escrowDelegable. + # otherwise escrowPDEvent self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError("No delegating event from {} at {} for " @@ -2723,7 +2918,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event - # compare digests to make sure they match here. + # compare saids to ensure match of delegating event and source seal # should never fail unless database broken if not dserder.compare(said=delsaider.qb64): # drop event raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." @@ -2742,19 +2937,22 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, found = True break - if not found: # drop event but may want to try harder here by walking KEL - # source seal couple is bad could be malicious so need to nullify as - # also walk KEL to find if any. + if not found: # drop event but may want to try harder here. + # if try harder assume source seal was malicious so nullify it and + # attempt to repair by searching for valid one in KEL + # may not find it in KEL because not yet delegated. + # Assumes bad and unrepairable so raise ValidationError raise ValidationError(f"Missing delegation seal in designated event" f"from {delpre} in {dserder.seals} for " f"evt = {serder.ked}.") - # valid anchoring seal of delegator delpre + # Found valid anchoring seal of delegator delpre delseqner = Seqner(snh=dserder.snh) delsaider = Saider(qb64=dserder.said) - # Found anchor so can confirm delegation successful unless its one of - # the superseding conditions. Valid if not superseding drt of drt. + # Since found valid anchoring seal so can confirm delegation successful + # unless its one of the superseding conditions. + # Valid if not superseding drt of drt. # Assumes database is reverified each bootup or otherwise when # chain-of-custody of disc has been broken. # Rule for non-supeding delegated rotation of rotation. @@ -3098,6 +3296,7 @@ def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): logger.debug("Kever state: escrowed delegable event=\n%s\n", json.dumps(serder.ked, indent=1)) + def escrowPSEvent(self, serder, *, sigers=None, wigers=None, seqner=None, saider=None, local=True): """ From 6dae8ef2c93a3dde6a10bc64e0fbb05c81959031 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 15:24:30 -0600 Subject: [PATCH 331/418] fixed misfit escrow checks in valSigsWigsDel to occur upfront before event can go into any other escrow. --- src/keri/core/eventing.py | 344 ++++++++------------------------------ 1 file changed, 71 insertions(+), 273 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 24cdef8ea..c5b54d4f0 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1675,6 +1675,27 @@ def locallyOwned(self, pre: str | None = None): return pre in self.prefixes and pre not in self.groups + def locallyDelegated(self, pre: str): + """Returns True if pre is in .prefixes and not in .groups + False otherwise. + Indicates that provided identifier prefix is controlled by a local + controller from .prefixes but is not a group with local member. + i.e pre is a locally owned (controlled) AID (identifier prefix) + Because delpre may be None, changes the default to "" instead of + self.prefixer.pre because self.prefixer.pre is delegate not delegator + of self. Unaccepted dip events do not have self.delpre set yet. + + Returns: + (bool): True if pre is local hab but not group hab + When pre="" empty or None then returns False + + Parameters: + pre (str): qb64 identifier prefix if any. + """ + pre = pre if pre is not None else "" + return self.locallyOwned(pre=pre) + + def locallyWitnessed(self, *, wits: list[str]=None, serder: (str)=None): """Returns True if a local controller is a witness of this Kever's KEL of wits in serder of if None then current wits for this Kever. @@ -2224,8 +2245,7 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # Filters sigers to remove any signatures from locally membered groups # when not local (remote) event source. So that attacker can't source - # compromised signature remotely to satisfy threshold. - + # remotely compromised but locally membered signatures to satisfy threshold. if not local and self.locallyMembered(): # is this Kever's pre a local group if indices := self.locallyContributedIndices(verfers): for siger in list(sigers): # copy so clean del on original elements @@ -2246,15 +2266,34 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, raise ValidationError("No verified signatures for evt = {}." "".format(serder.ked)) - # at least one valid controller signature - # Misfit check events that must be locally sourced (protected) get - # escrowed in order to repair the protection when appropriate - if (not local and self.locallyOwned()): + # at least one valid controller signature so we can now escrow event + + # Misfit check events that must be locally sourced (protected). + # Misfit escrow process may repair (upgrade) the protection when appropriate + # Put all misfit checks up front so never goes into any escrow but + # misfit if fails any misfit checks. + # get delegator's delpre if any for misfit check + if serder.ilk == Ilks.dip: # dip so delpre in event "di" field + delpre = serder.delpre # delegator from dip event + if not delpre: # empty or None + raise ValidationError(f"Empty or missing delegator for delegated" + f" inception event = {serder.ked}.") + elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation + delpre = self.delpre # delpre in kever state + else: # not delegable event icp, rot, ixn + delpre = None + + # Misfit escrow checks + if (not local and (self.locallyOwned() or + self.locallyWitnessed(wits=wits) or + self.locallyDelegated(pre=delpre))): self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally owned " - f"event = {serder.ked}, {wits}, " - f"{self.prefixes}") + raise MisfitEventSourceError(f"Nonlocal source for locally owned or" + f"locally witnessed or locally delegated" + f"event={serder.ked}, local aids=" + f"{self.prefixes}, {wits=}, " + f"delgator={delpre}.") werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers # get unique verified wigers and windices lists from wigers list @@ -2282,14 +2321,6 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f"sigs= {[siger.qb64 for siger in sigers]}" f" for new est evt={serder.ked}.") - - if (not local and self.locallyWitnessed(wits=wits)): - self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally " - f"witnessed eventf = {serder.ked}," - f"{wits}, {self.prefixes}") - # this point the sigers have been verified and the wigers have been verified # even if locallyOwned or locallyMembered or locallyWitnessed. # But the toad has not yet been verified. @@ -2321,29 +2352,31 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f"{[siger.qb64 for siger in wigers]} " f"for event={serder.ked}.") - # get delegator if any - if serder.ilk == Ilks.dip: - delpre = serder.delpre # delegator from dip event - if not delpre: # empty or None - raise ValidationError(f"Empty or missing delegator for delegated" - f" inception event = {serder.ked}.") - elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation - delpre = self.delpre - else: # not delegable event icp, rot, ixn - delpre = None - # delpre maybe None so ensure not None to pass into .locallyOwned which - # defaults to self.prefixer.qb64 when None. When delpre empty "" then - # returns locallyOwned(delpre) returns False. - if not local and self.locallyOwned(delpre if delpre is not None else ''): - self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally" - f" delegated by {delpre} of" - f"event = {serder.ked}.") + # Delegator approves delegation by attaching valid source + # seal and reprocessing event which shows up here as delseqner, delsaider. + # Won't get to here if not local and locallyDelegated(delpre) misfit + # checks above will send nonlocal sourced delegable event to + # misfit escrow first. Mistfit escrow must first promote to local and + # reprocess event before we get to here. Assumes that attached source + # seal in this case can't be malicious since sourced locally. + # Doesn't get to here until fully signed and witnessed. + + if self.locallyDelegated(delpre): # local delegator + if delseqner is None or delsaider is None: # missing delegation seal + # so escrow delegable. So local delegator can approve OOB. + # and create delegator event with valid event seal of this + # delegated event and then reprocess event with attached source + # seal to delegating event, i.e. delseqner, delsaider. + self.escrowDelegableEvent(serder=serder, sigers=sigers, + wigers=wigers, local=local) + raise MissingDelegableApprovalError(f"Missing approval for " + f" delegation by {delpre} of" + f"event = {serder.ked}.") - # returns (None, None) when delegation validation does not apply. - # Raises ValidationError if validation applies but does not validate. + # validateDelegation returns (None, None) when delegation validation + # does not apply. Raises ValidationError if validation applies but + # does not validate. delseqner, delsaider = self.validateDelegation(serder, sigers=sigers, wigers=wigers, @@ -2357,182 +2390,6 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, return (sigers, wigers, delpre, delseqner, delsaider) - #def valSigsWigsDel(self, serder, sigers, verfers, tholder, - #wigers, toader, wits, *, - #delseqner=None, delsaider=None, eager=False, - #local=True): - #""" - #Returns triple (sigers, wigers, delegator) where: - #sigers is unique validated signature verified members of inputed sigers - #wigers is unique validated signature verified members of inputed wigers - #delegator is qb64 delegator prefix if delegated else None - - #Validates sigers signatures by validating indexes, verifying signatures, and - #validating threshold sith. - #Validate witness receipts by validating indexes, verifying - #witness signatures and validating toad. - #Witness validation is a function of wits .prefixes and .local - - #Parameters: - #serder (SerderKERI): instance of event - #sigers (list): of Siger instances of indexed controllers signatures. - #Index is offset into verfers list from which public key may be derived. - #verfers (list): of Verfer instances of keys from latest est event - #tholder (Tholder): instance of sith threshold - #wigers (list): of Siger instances of indexed witness signatures. - #Index is offset into wits list of associated witness nontrans pre - #from which public key may be derived. - #toader (Number): instance of backer witness threshold - #wits (list): of qb64 non-transferable prefixes of witnesses used to - #derive werfers for wigers - #delseqner (Seqner | None): instance of delegating event sequence number. - #If this event is not delegated then seqner is ignored - #delsaider (Saider | None): instance of of delegating event said. - #If this event is not delegated then saider is ignored - #eager (bool): True means try harder to find delegation seals by - #walking KEL of delegator. Enables only being eager - #in escrow processing not initial parsing. - #False means only use pre-existing source seal couples - #if any, either attached or in database. - #local (bool): event source for validation logic - #True means event source is local (protected). - #False means event source is remote (unprotected). - #Event validation logic is a function of local or remote - - #""" - #if len(verfers) < tholder.size: - #raise ValidationError("Invalid sith = {} for keys = {} for evt = {}." - #"".format(tholder.sith, - #[verfer.qb64 for verfer in verfers], - #serder.ked)) - - ## Filters sigers to remove any signatures from locally membered groups - ## when not local (remote) event source. So that attacker can't source - ## compromised signature remotely to satisfy threshold. - - #if not local and self.locallyMembered(): # is this Kever's pre a local group - #if indices := self.locallyContributedIndices(verfers): - #for siger in list(sigers): # copy so clean del on original elements - #if siger.index in indices: - #sigers.remove(siger) - #if self.cues: - #self.cues.push(dict(kin="remoteMemberedSig", - #serder=serder, - #index=siger.index)) - - - ## get unique verified sigers and indices lists from sigers list - #sigers, indices = verifySigs(raw=serder.raw, sigers=sigers, verfers=verfers) - ## sigers now have .verfer assigned - - ## check if minimally signed in order to continue processing - #if not indices: # must have a least one verified sig - #raise ValidationError("No verified signatures for evt = {}." - #"".format(serder.ked)) - - ## at least one valid controller signature - ## Misfit check events that must be locally sourced (protected) get - ## escrowed in order to repair the protection when appropriate - #if (not local and - #(self.locallyOwned() or - #self.locallyWitnessed(wits=wits))): - #self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - #seqner=delseqner, saider=delsaider, local=local) - #raise MisfitEventSourceError(f"Nonlocal source for locally owned" - #f" or locally witnessed event" - #f" = {serder.ked}, {wits}, {self.prefixes}") - - #werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers - ## get unique verified wigers and windices lists from wigers list - #wigers, windices = verifySigs(raw=serder.raw, sigers=wigers, verfers=werfers) - ## each wiger now has added to it a werfer of its wit in its .verfer property - - ## escrow if not fully signed vs signing threshold - #if not tholder.satisfy(indices): # at least one but not enough - #self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, - #seqner=delseqner, saider=delsaider, local=local) - #raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" - #f" on sigs for {[siger.qb64 for siger in sigers]}" - #f" for evt = {serder.ked}.") - - - ## escrow if not fully signed vs prior next rotation threshold - #if serder.ilk in (Ilks.rot, Ilks.drt): # rotation so check prior next threshold - ## prior next threshold in .ntholder and digers in .ndigers - #ondices = self.exposeds(sigers) - #if not self.ntholder.satisfy(indices=ondices): - #self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, - #seqner=delseqner, saider=delsaider,local=local) - #raise MissingSignatureError(f"Failure satisfying prior nsith=" - #f"{self.ntholder.sith} with exposed " - #f"sigs= {[siger.qb64 for siger in sigers]}" - #f" for new est evt={serder.ked}.") - - - ## get delegator if any - #if serder.ilk == Ilks.dip: - #delpre = serder.delpre # delegator from dip event - #if not delpre: # empty or None - #raise ValidationError(f"Empty or missing delegator for delegated" - #f" inception event = {serder.ked}.") - #elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation - #delpre = self.delpre - #else: # not delegable event icp, rot, ixn - #delpre = None - - ## delpre maybe None so ensure not None to pass into .locallyOwned which - ## defaults to self.prefixer.qb64 when None - #if not local and self.locallyOwned(delpre if delpre is not None else ''): - #self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - #seqner=delseqner, saider=delsaider, local=local) - #raise MisfitEventSourceError(f"Nonlocal source for locally" - #f" delegated by {delpre} of" - #f"event = {serder.ked}.") - - ## this point the sigers have been verified and the wigers have been verified - ## even if locallyOwned or locallyMembered or locallyWitnessed. - ## But the toad has not yet been verified. - - #if not wits: - #if toader.num != 0: # bad toad non-zero and not witnessed bad event - #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - - #else: # wits so may need to validate toad - ## The toad is only verified against the wigers as fully witnessed if - ## not (locallyOwned or locallyMembered or locallyWitnessed) - #if (not (self.locallyOwned() or self.locallyMembered() or - #self.locallyWitnessed(wits=wits))): - #if wits: # is witnessed - #if toader.num < 1 or toader.num > len(wits): # out of bounds toad - #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - #else: # not witnessed - #if toader.num != 0: # invalid toad - #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - - #if len(windices) < toader.num: # not fully witnessed yet - #if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, - #seqner=delseqner, saider=delsaider, - #local=local): - ## cue to query for witness receipts - #self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) - #raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " - #f"on witness sigs=" - #f"{[siger.qb64 for siger in wigers]} " - #f"for event={serder.ked}.") - - ## returns (None, None) when delegation validation does not apply. - ## Raises ValidationError if validation applies but does not validate. - #delseqner, delsaider = self.validateDelegation(serder, - #sigers=sigers, - #wigers=wigers, - #wits=wits, - #delpre=delpre, - #delseqner=delseqner, - #delsaider=delsaider, - #eager=eager, - #local=local) - - #return (sigers, wigers, delpre, delseqner, delsaider) def exposeds(self, sigers): @@ -2851,25 +2708,6 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, raise ValidationError(f"Delegator = {delpre} for evt = {serder.ked}," f" does not allow delegation.") - # Delegator accepts here without waiting for delegation seal to be anchored in - # delegator's KEL. But has already waited for fully receipted above. - # Once fully receipted, cue in Kevery will then trigger cue to approve - # delegation. Delegator approves delegation by attaching valid source - # seal and reprocessing event which shows up here as delseqner, delsaider. - # Won't get to here if not local and locallyOwned(delpre) because - # valSigsWigsDel will send nonlocal sourced delegable event to - # misfit escrow first. Mistfit escrow must first promote to local and - # reprocess event before we get to here. Assumes that attached source - # seal in this case can't be malicious since sourced locally. - - if self.locallyOwned(delpre): # local delegator - if delseqner is None or delsaider is None: # missing delegation seal - # so escrow delegable. So local delegator can approve OOB. - self.escrowDelegableEvent(serder=serder, sigers=sigers, - wigers=wigers, local=local) - raise MissingDelegableApprovalError(f"Missing approval for " - f" delegation by {delpre} of" - f"event = {serder.ked}.") if delseqner is None or delsaider is None: # missing delegation seal ref # If eager should walk KEL here to attempt to find in delegator's KEL @@ -3865,28 +3703,6 @@ def processEvent(self, serder, sigers, *, wigers=None, # one receipt is generated not two self.cues.push(dict(kin="witness", serder=serder)) - if self.local and kever.locallyOwned(): - # ToDo XXXX process this cue of query to send event to delegator - # to trigger generation of anchor in delegating event - # note for remote validators there is query cue in - # validateDelegation to query for anchoring event seal - pass - - # Moved logic to kever.validateDelegation trigger for delegation escrow - #if (self.local and - #kever.locallyOwned(kever.delpre if kever.delpre is not None else '')): # delpre may be None - ## ToDo XXXX need to cue task here to approve delegation by generating - ## an anchoring SealEvent of serder in delegators KEL - ## may include MFA and or business logic for the delegator i.e. is local - ## event that designates this controller as delegator triggers - ## this cue to approave delegation - #self.cues.push(dict(kin="approveDelegation", - #delegator=kever.delpre, - #serder=serder)) - - - - else: # not inception so can't verify sigs etc, add to out-of-order escrow self.escrowOOEvent(serder=serder, sigers=sigers, @@ -3972,24 +3788,6 @@ def processEvent(self, serder, sigers, *, wigers=None, # one receipt is generated not two self.cues.push(dict(kin="witness", serder=serder)) - if self.local and kever.locallyOwned(): - # ToDo XXXX process this cue of query to send event to delegator - # to trigger generation of anchor in delegating event - # note for remote validators there is query cue in - # validateDelegation to query for anchoring event seal - pass - - # Moved logic to kever.validateDelegation trigger for delegation escrow - #if (self.local and - #kever.locallyOwned(kever.delpre if kever.delpre is not None else '')): # delpre may be None - ## ToDo XXXX need to cue task here to approve delegation by generating - ## an anchoring SealEvent of serder in delegators KEL - ## may include MFA and or business logic for the delegator i.e. is local - ## event that designates this controller as delegator triggers - ## this cue to approave delegation - #self.cues.push(dict(kin="approveDelegation", - #delegator=kever.delpre, - #serder=serder)) else: # maybe duplicitous # check if duplicate of existing valid accepted event From 2445710fbb095d886f5a636562b89bae5dc372ee Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 15:42:48 -0600 Subject: [PATCH 332/418] added eager parameter to processEvent and connected it through to validateDelegation --- src/keri/core/eventing.py | 60 ++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index c5b54d4f0..2e7bb0241 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1494,7 +1494,7 @@ class Kever: def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, db=None, estOnly=None, delseqner=None, delsaider=None, firner=None, - dater=None, cues=None, local=True, check=False): + dater=None, cues=None, eager=False, local=True, check=False): """ Create incepting kever and state from inception serder Verify incepting serder against sigers raises ValidationError if not @@ -1522,6 +1522,11 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, When dater provided then use dater for first seen datetime cues (Deck | None): reference to Kevery.cues Deck when provided i.e. notices of events or requests to respond to + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -1569,9 +1574,10 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, wigers=wigers, toader=self.toader, wits=self.wits, - local=local, delseqner=delseqner, - delsaider=delsaider) + delsaider=delsaider, + eager=eager, + local=local) self.delpre = delpre # may be None self.delegated = True if self.delpre else False @@ -1884,7 +1890,7 @@ def config(self, serder, estOnly=None, doNotDelegate=None): def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, - firner=None, dater=None, local=True, check=False): + firner=None, dater=None, eager=False, local=True, check=False): """ Not an inception event. Verify event serder and indexed signatures in sigers and update state @@ -1908,6 +1914,11 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, dater (Dater | None): Dater instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -1953,9 +1964,10 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, wigers=wigers, toader=toader, wits=wits, - local=local, delseqner=delseqner, - delsaider=delsaider) + delsaider=delsaider, + eager=eager, + local=local) @@ -2021,6 +2033,7 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, wigers=wigers, toader=self.toader, wits=self.wits, + eager=eager, local=local) # .validateSigsDelWigs above ensures thresholds met otherwise raises exception @@ -2226,11 +2239,11 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, If this event is not delegated then seqner is ignored delsaider (Saider | None): instance of of delegating event said. If this event is not delegated then saider is ignored - eager (bool): True means try harder to find delegation seals by - walking KEL of delegator. Enables only being eager + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager in escrow processing not initial parsing. - False means only use pre-existing source seal couples - if any, either attached or in database. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -2477,11 +2490,11 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, True means event source is local (protected). False means event source is remote (unprotected). Event validation logic is a function of local or remote - eager (bool): True means try harder to find delegation seals by - walking KEL of delegator. Enables only being eager + eager (bool): True means try harder to validate event by + walking KELs. Enables only being eager in escrow processing not initial parsing. - False means only use pre-existing source seal couples - if any, either attached or in database. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -3619,7 +3632,7 @@ def fetchWitnessState(self, pre, sn): def processEvent(self, serder, sigers, *, wigers=None, delseqner=None, delsaider=None, - firner=None, dater=None, local=None): + firner=None, dater=None, eager=False, local=None): """ Process one event serder with attached indexd signatures sigers @@ -3639,6 +3652,11 @@ def processEvent(self, serder, sigers, *, wigers=None, dater (Dater|None): instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool|None): True means local (protected) event source. False means remote (unprotected). None means use default .local . @@ -3677,6 +3695,7 @@ def processEvent(self, serder, sigers, *, wigers=None, firner=firner if self.cloned else None, dater=dater if self.cloned else None, cues=self.cues, + eager=eager, local=local, check=self.check) self.kevers[pre] = kever # not exception so add to kevers @@ -3763,7 +3782,7 @@ def processEvent(self, serder, sigers, *, wigers=None, delseqner=delseqner, delsaider=delsaider, firner=firner if self.cloned else None, dater=dater if self.cloned else None, - local=local, check=self.check) + eager=eager, local=local, check=self.check) # At this point the non-inceptive event (rot, drt, or ixn) # given by serder together with its attachments has been @@ -5520,7 +5539,8 @@ def processEscrowPartialSigs(self): sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) # If process does NOT validate sigs or delegation seal (when delegated), # but there is still one valid signature then process will @@ -5706,7 +5726,8 @@ def processEscrowPartialWigs(self): self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) # If process does NOT validate wigs then process will attempt # to re-escrow and then raise MissingWitnessSignatureError @@ -5866,7 +5887,8 @@ def processEscrowPartialDels(self): self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) # If process does NOT validate delegation then process will attempt # to re-escrow and then raise MissingDelegationError From 1709aa1372f0de7352b0d5ab30da2a146342e606 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 16:59:51 -0600 Subject: [PATCH 333/418] refactored FetchDelegatingEvent to respect eager parameter and to also repair .aess when proper to do so. --- src/keri/core/eventing.py | 137 ++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 63 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 2e7bb0241..3c03478ce 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2754,26 +2754,25 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # otherwise escrowPDEvent self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError("No delegating event from {} at {} for " - "evt = {}.".format(delpre, - delsaider.qb64, - serder.ked)) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") # get the delegating event from dig ddig = bytes(raw) key = dgKey(pre=delpre, dig=ddig) # database key raw = self.db.getEvt(key) # get actual last event if raw is None: # drop event should never happen unless database is broken - raise ValidationError("Missing delegation from {} at event dig = {} for evt = {}." - "".format(delpre, ddig, serder.ked)) + raise ValidationError(f"Missing delegation from {delpre} at event " + f"dig = {ddig} for evt = {serder.ked}.") dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event # compare saids to ensure match of delegating event and source seal # should never fail unless database broken if not dserder.compare(said=delsaider.qb64): # drop event - raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." - "".format(delpre, ddig, serder.ked)) + raise ValidationError(f"Invalid delegation from {delpre} at event" + f" dig={ddig} for evt={serder.ked}.") found = False # find event seal of delegated event in delegating data # purported delegating event from source seal couple @@ -2825,8 +2824,13 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # set up recursive search for superseding delegations serfn = serder # new potentially superseding delegated event i.e. serf new bossn = dserder # new delegating event of superseding delegated event i.e. boss new - serfo = self.serder # original delegated event i.e. serf original - bosso = self.fetchDelegatingEvent(delpre, serfo, eager=eager) + serfo = self.serder # original accepted delegated event i.e. serf original + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") while (True): # superseding delegated rotation of rotation recovery rules # Only get to here if same sn for drt existing and drt superseding @@ -2858,52 +2862,62 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # tie condition same sn and drt so need to climb delegation chain serfn = bossn - bossn = self.fetchDelegatingEvent(delpre, serfn, eager=eager) + if not (bossn := self.fetchDelegatingEvent(delpre, serfn, eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") serfo = bosso - bosso = self.fetchDelegatingEvent(delpre, serfo, eager=eager) + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") # repeat # should never get to here - def fetchDelegatingEvent(self, delpre, serder, *, - delseqner=None, delsaider=None, eager=False): + def fetchDelegatingEvent(self, delpre, serder, *, eager=False): """Returns delegating event of delegator given by its aid delpre of delegated event given by serder otherwise raises ValidationError. - Assumes serder is already delegated event + Assumes delegation event must have been accepted at some point even if + it has subsequently been superseded. Therefore only looks in .aess for + delegating event source seal references. So do not use for fetching + delegating eventsthat have yet to be accepted. + Checks that found delegation events have both been accepted. + + When delegated event in serder has been accepted then will repair its + .aess entry if needed. Returns: - dserder (SerderKERI): key event with delegating seal + dserder (SerderKERI): delegating key event with delegating seal of + delegated event serder. If can't fetch then + returns None when eager==False or raises + ValidationError if can't fetch but eager==True Parameters: delpre (str): qb64 of identifier prefix of delegator serder (SerderKERI): delegated serder - delseqner (Seqner | None): instance of delegating event sequence number. - delsaider (Saider | None): instance of of delegating event digest. eager (bool): True means do more expensive KEL walk instead of escrow - False means do not do expensive KEL walk now + False means do not do expensive KEL walk now. Assumes db.aes source seal couple of delegating event cannot be written unless that event has been accepted (first seen) into delegator's KEL + even if it has later been superseded. + Confirms this by checking the .fons database of the delegator. - ToDo XXXX - Use the db.fons database to lookup delegating events to ensure that - lookup only uses delegating events that were indeed accepted (first seen) - at some point even though they may now be superseded. - - Looking up in the db.evts from dig in db.aes could be malicious escrows - of delegating events?? But a malicious escrow of delegating event would - only write source seal couple to db.udes not db.aes - - When escrowing .escrowPACouple, the delegating seal source couple goes - in db.udes indexed by delegating event pre,dig + A malicious escrow of delegating event could only write source seal + couple to escrow in db.udes not in accepted db.aes Delegating event may have been superceded but delegated event validator does not know it yet because db.aes keeps original delegating event source seal from before is was superseded. - Therefore lookup of delegating event needs to find delegating event via + Therefore lookup of delegating event needs to check delegating event via db.fons and not last key event in .kels which would only return the superseding event. @@ -2912,49 +2926,31 @@ def fetchDelegatingEvent(self, delpre, serder, *, KEL. This method does not distinguish between superseding and superseded so can't assume that the delegating event is the last event at its sn, i.e. the superseding event. - So this method must use db.fons to lookup to ensure that delegating + So this method must use db.fons to ensure that delegating event was accepted (first seen) even if it has subsequently been superseded. - This assumes that the delegating source seal in the AES db could - not have been written into the delegate's db.aes unless the delegating - event had been validated and accepted (first seen) into the - delegator's kel even though the delegating event may have later - been superseded at the time of this fetch. - - - Fetch delegating event has two cases: - 1. Delegated event has yet to be accepted (not seen) so can't repair - aes db because aes db must not be written until delegated event has - been accepted (first seen) bu .logEvent. - So must indicate the newly found dserder generates new source couple. - Returned dserder has the info to generate source couple but - does not indicate that a new one is needed. - 2. Delegated event has been accepted so can repair aes source couple entry - when necessary. + When the .aess entry is missing and eager is True and a valid delegation + is found and if delegate has been accepted then repairs missing .aess + entry. Otherwise does not repair but simply returns found delegation. - Need to indicate both if delegated event has been accepted and when - aes has been repaired. - seen=True (delegated event has been accepted first seen) - repair=True found new couple so repair externally if seen == False - or else repair just happened. - - When eager == False then may return None which means to escrow event and - try later with eager = True such as in escrow processor. + Found delegation may not be superseding so do not repair .aess unless + delegate was already accepted. """ + dserder = None # when not found and not eager so caller should reescrow dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key of delegate + # extra careful double check that delegate has been accepted by checkin + # fner = first seen Number instance index if (couple := self.db.getAes(dgkey)): # delegation source couple at delegate seqner, saider = deSourceCouple(couple) deldig = saider.qb64 # dig of delegating event # extra careful double check that .aes is valid by getting # fner = first seen Number instance index if not self.db.fons.get(keys=(delpre, deldig)): # None - raise ValidationError(f"Invalid delagation authorizing source " + raise ValidationError(f"Invalid delegation authorizing source " f"seal couple for {serder.ked}") - - - dgkey = dgKey(pre=delpre, dig=deldig) # event at its said - if not (raw := self.db.getEvt(dgkey)): + ddgkey = dgKey(pre=delpre, dig=deldig) # database key of delegation + if not (raw := self.db.getEvt(ddgkey)): # database broken this should never happen so do not supersede # ToDo XXXX should repair by deleting the erroneous aes entry and # returning found one @@ -2963,12 +2959,27 @@ def fetchDelegatingEvent(self, delpre, serder, *, # original delegating event i.e. boss original dserder = serdering.SerderKERI(raw=bytes(raw)) - else: #try to find seal the hard way by walking the delegator's KEL + elif eager: #missing but try to find seal by walking delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict if not (dserder := self.db.findAnchoringSealEvent(pre=delpre, seal=seal)): - # database broken this should never happen so do not supersede + # database broken this should never happen so do not validate raise ValidationError(f"Missing delegation source seal for {serder.ked}") + # extra careful double check that .aes is valid by getting + # fner = first seen Number instance index of delegation + if not self.db.fons.get(keys=(dserder.pre, dserder.dig)): # None + raise ValidationError(f"Invalid delegation authorizing source " + f"seal couple for {serder.ked}") + # Only repair when delegated has been accepted + if self.db.fons.get(keys=(preb, serder.saidb)): + # Repair .aess of delegated event by writing found source + # seal couple of delegation. This is safe becaause we confirmed + # delegation event was accepted in delegator's kel. + couple = dserder.sner.huge.encode() + dserder.saidb + self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + + #else not found so return None to escrow and get fixed later + return dserder # extract delseqner, delsaider from actually found dserder From 04619260276554cbf791c175dd4b61ed25c6265c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 17:33:31 -0600 Subject: [PATCH 334/418] added eager lookup to validateDelegation when missing source couple --- src/keri/core/eventing.py | 15 ++++++++++++--- src/keri/db/basing.py | 14 +++++++------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 3c03478ce..8e66a78ff 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2721,16 +2721,25 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, raise ValidationError(f"Delegator = {delpre} for evt = {serder.ked}," f" does not allow delegation.") - + dserder = None # no delegation event yet if delseqner is None or delsaider is None: # missing delegation seal ref - # If eager should walk KEL here to attempt to find in delegator's KEL + # If eager could walk KEL here to attempt to find in delegator's KEL # if not found then escrow to be found later delegating event may - # not yet have been created + # not yet have been created. + if eager: + seal = dict(i=serder.pre, s=serder.snh, d=serder.said) + dserder = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if dserder is not None: # found seal in dserder + delseqner = coring.Seqner(sn=dserder.sn) + delsaider = coring.Saider(qb64=dserder.said) + + if delseqner is None or delsaider is None: # missing delegation seal ref self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegation seal for delegator " f"{delpre} of evt = {serder.ked}.") + # if delseqner and delsaider and not dserder # ToDo XXXX need to replace Seqners with Numbers # Get delegating event from delseqner and delpre ssn = Number(num=delseqner.sn).validate(inceptive=False).sn diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 219805589..5c1235784 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1694,13 +1694,13 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): return None - def findAnchoringSeal(self, pre, seal, sn=0): - """ - Search through a KEL for the event that contains an anchored - Seal with same Seal type as provided seal but in dict form. - Searchs from sn forward (default = 0). Only searches last event at any - sn therefore does not search any disputed or superseded events. - Returns the Serder of the first event with the anchored Seal seal, + def findAnchoringSealLast(self, pre, seal, sn=0): + """Only searches last event at any sn therefore does not search + any disputed or superseded events. + Search through last event at each sn in KEL for the event that contains + an anchored Seal with same Seal type as provided seal but in dict form. + Searchs from sn forward (default = 0). + Returns the Serder of the first found event with the anchored Seal seal, None if not found Parameters: From f04b1d9ff6c72c2e786f2e7f115ebec2f23c3b50 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 18:03:24 -0600 Subject: [PATCH 335/418] refactor logic when can't find delegation when given source seal to nullify and walk kel --- src/keri/core/eventing.py | 159 +++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 79 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 8e66a78ff..9556f526b 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2723,91 +2723,92 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, dserder = None # no delegation event yet if delseqner is None or delsaider is None: # missing delegation seal ref - # If eager could walk KEL here to attempt to find in delegator's KEL - # if not found then escrow to be found later delegating event may - # not yet have been created. - if eager: + if eager: # walk kel here to find seal = dict(i=serder.pre, s=serder.snh, d=serder.said) dserder = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) if dserder is not None: # found seal in dserder delseqner = coring.Seqner(sn=dserder.sn) delsaider = coring.Saider(qb64=dserder.said) - if delseqner is None or delsaider is None: # missing delegation seal ref - self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegation seal for delegator " - f"{delpre} of evt = {serder.ked}.") - - # if delseqner and delsaider and not dserder - # ToDo XXXX need to replace Seqners with Numbers - # Get delegating event from delseqner and delpre - ssn = Number(num=delseqner.sn).validate(inceptive=False).sn - # get the dig of the delegating event. Using getKeLast ensures delegating - # event has not already been superceded - key = snKey(pre=delpre, sn=ssn) # database key - raw = self.db.getKeLast(key) # get dig of delegating event as index last - - if raw is None: # no index to delegating event at key pre, sn - # Have to wait until delegating event at sn shows up in kel - # ToDo XXXX process this cue of query to fetch delegating event from - # delegator - self.cues.push(dict(kin="query", q=dict(pre=delpre, - sn=delseqner.snh, - dig=delsaider.qb64))) - # escrow event here - inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False - sn = Number(num=serder.sn).validate(inceptive=inceptive).sn - # if we were to allow for malicous local delegator source seal then we - # must check for locallyOwned(delpre) first and escrowDelegable. - # otherwise escrowPDEvent - self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegating event from {delpre}" - f" at {delsaider.qb64} for " - f"evt = {serder.ked}.") - - # get the delegating event from dig - ddig = bytes(raw) - key = dgKey(pre=delpre, dig=ddig) # database key - raw = self.db.getEvt(key) # get actual last event - if raw is None: # drop event should never happen unless database is broken - raise ValidationError(f"Missing delegation from {delpre} at event " - f"dig = {ddig} for evt = {serder.ked}.") - - dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event - - # compare saids to ensure match of delegating event and source seal - # should never fail unless database broken - if not dserder.compare(said=delsaider.qb64): # drop event - raise ValidationError(f"Invalid delegation from {delpre} at event" - f" dig={ddig} for evt={serder.ked}.") - - found = False # find event seal of delegated event in delegating data - # purported delegating event from source seal couple - # XXXX ToDo need to change logic here to support native CESR seals not just dicts - # for JSON, CBOR, MGPK - for dseal in dserder.seals: # find delegating seal anchor - if tuple(dseal) == SealEvent._fields: - seal = SealEvent(**dseal) - if (seal.i == serder.pre and - seal.s == serder.sner.numh and - serder.compare(said=seal.d)): - found = True - break - - if not found: # drop event but may want to try harder here. - # if try harder assume source seal was malicious so nullify it and - # attempt to repair by searching for valid one in KEL - # may not find it in KEL because not yet delegated. - # Assumes bad and unrepairable so raise ValidationError - raise ValidationError(f"Missing delegation seal in designated event" - f"from {delpre} in {dserder.seals} for " - f"evt = {serder.ked}.") - - # Found valid anchoring seal of delegator delpre - delseqner = Seqner(snh=dserder.snh) - delsaider = Saider(qb64=dserder.said) + if not dserder: # just escrow and try later + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegation seal for delegator " + f"{delpre} of evt = {serder.ked}.") + + if delseqner and delsaider and not dserder: # given couple not found + # ToDo XXXX need to replace Seqners with Numbers + # Get delegating event from delseqner and delpre + ssn = Number(num=delseqner.sn).validate(inceptive=False).sn + # get the dig of the delegating event. Using getKeLast ensures delegating + # event has not already been superceded + key = snKey(pre=delpre, sn=ssn) # database key + raw = self.db.getKeLast(key) # get dig of delegating event as index last + + if raw is None: # no index to delegating event at key pre, sn + # Have to wait until delegating event at sn shows up in kel + # ToDo XXXX process this cue of query to fetch delegating event from + # delegator + self.cues.push(dict(kin="query", q=dict(pre=delpre, + sn=delseqner.snh, + dig=delsaider.qb64))) + # escrow event here + inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False + sn = Number(num=serder.sn).validate(inceptive=inceptive).sn + # if we were to allow for malicous local delegator source seal then we + # must check for locallyOwned(delpre) first and escrowDelegable. + # otherwise escrowPDEvent + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") + + # get the delegating event from dig + ddig = bytes(raw) + key = dgKey(pre=delpre, dig=ddig) # database key + raw = self.db.getEvt(key) # get actual last event + if raw is None: # drop event should never happen unless database is broken + raise ValidationError(f"Missing delegation from {delpre} at event " + f"dig = {ddig} for evt = {serder.ked}.") + + dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event + + # compare saids to ensure match of delegating event and source seal + # should never fail unless database broken + if not dserder.compare(said=delsaider.qb64): # drop event + raise ValidationError(f"Invalid delegation from {delpre} at event" + f" dig={ddig} for evt={serder.ked}.") + + found = False # find event seal of delegated event in delegating data + # purported delegating event from source seal couple + # XXXX ToDo need to change logic here to support native CESR seals not just dicts + # for JSON, CBOR, MGPK + for dseal in dserder.seals: # find delegating seal anchor + if tuple(dseal) == SealEvent._fields: + seal = SealEvent(**dseal) + if (seal.i == serder.pre and + seal.s == serder.sner.numh and + serder.compare(said=seal.d)): + found = True + break + + if not found: # nullify and escrow to try harder later + # worst case assume source seal was malicious so nullify it and + # attempt to repair by escrowing and eager search later + delseqner = delsaider = None # nullify + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegation seal for delegator " + f"{delpre} of evt = {serder.ked}.") + # Assumes bad and unrepairable so raise ValidationError + #raise ValidationError(f"Missing delegation seal in designated event" + #f"from {delpre} in {dserder.seals} for " + #f"evt = {serder.ked}.") + + # Found valid anchoring seal of delegator delpre + delseqner = Seqner(snh=dserder.snh) + delsaider = Saider(qb64=dserder.said) # Since found valid anchoring seal so can confirm delegation successful # unless its one of the superseding conditions. From 05912a1f14f0929f639f2034fc89b134e066b19f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 18:20:37 -0600 Subject: [PATCH 336/418] removed estra walks from partial wig and sig escrows now only partial del escrow --- src/keri/core/eventing.py | 76 +++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 9556f526b..3db805314 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5538,23 +5538,19 @@ def processEscrowPartialSigs(self): delseqner = delsaider = None if (couple := self.db.udes.get(keys=dgkey)): delseqner, delsaider = couple - #couple = self.db.getUde(dgkey) - #if couple is not None: - #delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - #couple = delseqner.qb64b + delsaider.qb64b - #self.db.putUde(dgkey, couple) - # idempotent - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + ## idempotent + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) # process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] @@ -5578,17 +5574,16 @@ def processEscrowPartialSigs(self): # Non re-escrow ValidationError means some other issue so unescrow. # No error at all means processed successfully so also unescrow. - except (MissingSignatureError, MissingDelegationError) as ex: + except MissingSignatureError as ex: # MissingDelegationError) # still waiting on missing sigs or missing seal to validate # processEvent idempotently reescrowed if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow + # error other than waiting on sigs so remove from escrow self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any + #self.db.udes.rem(keys=dgkey) # leave here since could PartialDelegationEscrow if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): self.cues.push(dict(kin="psUnescrow", serder=eserder)) @@ -5603,7 +5598,6 @@ def processEscrowPartialSigs(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any self.db.udes.rem(keys=dgkey) # remove escrow if any if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): @@ -5728,23 +5722,19 @@ def processEscrowPartialWigs(self): delseqner = delsaider = None if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): delseqner, delsaider = couple - #couple = self.db.getUde(dgKey(pre, bytes(edig))) - #if couple is not None: - #delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - #couple = delseqner.qb64b + delsaider.qb64b - #self.db.putUde(dgkey, couple) - # idempotent - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + ## idempotent + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=delseqner, delsaider=delsaider, @@ -5767,17 +5757,16 @@ def processEscrowPartialWigs(self): # validation will be successful as event would not be in # partially witnessed escrow unless they had already validated - except (MissingWitnessSignatureError, MissingDelegationError) as ex: + except MissingWitnessSignatureError as ex: # MissingDelegationError # still waiting on missing witness sigs or delegation # processEvent idempotently reescrowed if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow + # error other than waiting on wigs so remove from escrow self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any + #self.db.udes.rem(keys=dgkey) # leave here since could PartialDelegationEscrow if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) else: @@ -5788,7 +5777,6 @@ def processEscrowPartialWigs(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any self.db.udes.rem(keys=dgkey) # remove escrow if any logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) From c9def3ffa54c860fe7c492fc6d0f2af4cf28623c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 18:30:35 -0600 Subject: [PATCH 337/418] finished refactor of delegation escrow. --- src/keri/core/eventing.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 3db805314..f60bb589a 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5883,17 +5883,18 @@ def processEscrowPartialDels(self): delseqner = delsaider = None if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): delseqner, delsaider = couple # provided - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): # walk kel to find - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: # found seal in srdr - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): # walk kel to find + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: # found seal in srdr + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=delseqner, delsaider=delsaider, From 697068be21c76d028b94e71806aeed75096d65de Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 7 Sep 2024 10:28:39 -0600 Subject: [PATCH 338/418] fixup delegation source seal escrow to be non-idempotent so can nullify erroneous seals and/or repair missing seals --- src/keri/core/eventing.py | 106 ++++++++++++++++---------------------- src/keri/db/basing.py | 43 ++++++++++++++-- 2 files changed, 83 insertions(+), 66 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index f60bb589a..df2c50d3d 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2705,7 +2705,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, self.locallyWitnessed(wits=wits)): return (None, None) # not validated so delseqner delsaider must be None - if self.kevers is None or delpre not in self.kevers: # missing delegator + if self.kevers is None or delpre not in self.kevers: # missing delegator KEL # ToDo XXXX cue a trigger to get the KEL of the delegator. This may # require OOBIing with the delegator. # The processPDEvent should also cue a trigger to get KEL @@ -2725,10 +2725,11 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, if delseqner is None or delsaider is None: # missing delegation seal ref if eager: # walk kel here to find seal = dict(i=serder.pre, s=serder.snh, d=serder.said) - dserder = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + dserder = self.db.FetchSealingEventLastByEventSeal(pre=delpre, + seal=seal) if dserder is not None: # found seal in dserder - delseqner = coring.Seqner(sn=dserder.sn) - delsaider = coring.Saider(qb64=dserder.said) + delseqner = coring.Seqner(sn=dserder.sn) # replace with found + delsaider = coring.Saider(qb64=dserder.said) # replace with found if not dserder: # just escrow and try later self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, @@ -2743,9 +2744,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # get the dig of the delegating event. Using getKeLast ensures delegating # event has not already been superceded key = snKey(pre=delpre, sn=ssn) # database key - raw = self.db.getKeLast(key) # get dig of delegating event as index last - - if raw is None: # no index to delegating event at key pre, sn + # get dig of last delegating event purported at sn + raw = self.db.getKeLast(key) # last means not disputed or superseded + if raw is None: # no delegatint event yet. no index at key pre, sn # Have to wait until delegating event at sn shows up in kel # ToDo XXXX process this cue of query to fetch delegating event from # delegator @@ -2764,7 +2765,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, f" at {delsaider.qb64} for " f"evt = {serder.ked}.") - # get the delegating event from dig + # get the latest delegating event candidate from dig given by pre,sn index ddig = bytes(raw) key = dgKey(pre=delpre, dig=ddig) # database key raw = self.db.getEvt(key) # get actual last event @@ -2774,22 +2775,17 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event - # compare saids to ensure match of delegating event and source seal - # should never fail unless database broken - if not dserder.compare(said=delsaider.qb64): # drop event - raise ValidationError(f"Invalid delegation from {delpre} at event" - f" dig={ddig} for evt={serder.ked}.") - found = False # find event seal of delegated event in delegating data - # purported delegating event from source seal couple - # XXXX ToDo need to change logic here to support native CESR seals not just dicts - # for JSON, CBOR, MGPK + # search purported delegating event from source seal couple + for dseal in dserder.seals: # find delegating seal anchor + # XXXX ToDo need to change logic here to support native CESR seals not just dicts + # for JSON, CBOR, MGPK if tuple(dseal) == SealEvent._fields: - seal = SealEvent(**dseal) - if (seal.i == serder.pre and - seal.s == serder.sner.numh and - serder.compare(said=seal.d)): + dseal = SealEvent(**dseal) + if (dseal.i == serder.pre and + dseal.s == serder.sner.numh and + serder.compare(said=dseal.d)): found = True break @@ -2801,14 +2797,18 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegation seal for delegator " f"{delpre} of evt = {serder.ked}.") - # Assumes bad and unrepairable so raise ValidationError - #raise ValidationError(f"Missing delegation seal in designated event" - #f"from {delpre} in {dserder.seals} for " - #f"evt = {serder.ked}.") - # Found valid anchoring seal of delegator delpre - delseqner = Seqner(snh=dserder.snh) - delsaider = Saider(qb64=dserder.said) + ## Found valid anchoring seal of delegator delpre + ## compare saids to ensure match of delegating event and source seal + ## repair source seal to match last event at sn in source seal. + ## Original delegating event in seal may have been disputed or + ## superseded, but if latest matches then we want to repair + #if not dserder.compare(said=delsaider.qb64): # drop event + #raise ValidationError(f"Invalid delegation from {delpre} at event" + #f" dig={ddig} for evt={serder.ked}.") + + delseqner = Seqner(snh=dserder.snh) # replace with found + delsaider = Saider(qb64=dserder.said) # replace with found # Since found valid anchoring seal so can confirm delegation successful # unless its one of the superseding conditions. @@ -3262,11 +3262,15 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, """ Update associated logs for escrow of partially delegated or otherwise authorized issued event. - Assumes controller signatures, sigs, and witness signatures, wigs are + Assumes sigs (controller signatures) and wigs (witness signatures) are provided elsewhere. Partial authentication occurs once an event is fully signed and witnessed but the authorizing (delegating) source seal in the authorizer's (delegator's) key has not yet been verified. + The escrow of the authorizing event ref via source seal is not idempotent + so that malicious or erroneous source seals may be nullified and/or + replaced by found source seals. + Escrow allows escrow processor to retrieve serder from key and source couple from val in order to to re-verify authentication status. @@ -3290,11 +3294,16 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) if wigers: # idempotent self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) - if seqner and saider: # idempotent - self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent - logger.debug(f"Kever state: Escrowed source couple sn={seqner.sn}, " - f"said={saider.said} for partially delegated/authorized" - f" event said={serder.said}.") + if seqner and saider: # non-idempotent pin to repair replace + self.db.udes.pin(keys=dgkey, val=(seqner, saider)) # non-idempotent + logger.debug(f"Kever state: Replaced escrow source couple sn=" + f"{seqner.sn}, said={saider.said} for partially " + f"delegated/authorized event said={serder.said}.") + else: + self.db.udes.rem(keys=dgkey) # nullify non-idempotent + logger.debug(f"Kever state: Nullified escrow source couple for " + f"partially delegated/authorized event said=" + f"{serder.said}.") self.db.putEvt(dgkey, serder.raw) # idempotent @@ -3312,35 +3321,6 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, f"{serder.ked}\n.") return self.db.pdes.add(keys =(serder.preb, serder.sn), val=serder.saidb) - # not used anymore deprecate? - #def escrowPACouple(self, serder, seqner, saider, local=True): - #""" - #Update associated logs for escrow of partially authenticated issued event. - #Assuming signatures are provided elsewhere. Partial authentication results - #from either a partially signed event or a fully signed delegated event - #but whose delegation is not yet verified. - - #Escrow allows escrow processor to retrieve serder from key and source - #couple from val in order to to re-verify authentication status. Sigs - #are escrowed elsewhere. - - #Parameters: - #serder is SerderKERI instance of delegated or issued event - #seqner is Seqner instance of sn of seal source event of delegator/issuer - #saider is Saider instance of said of delegator/issuer - #local (bool): event source for validation logic - #True means event source is local (protected). - #False means event source is remote (unprotected). - #Event validation logic is a function of local or remote - #""" - #local = True if local else False # ignored since not escrowing serder here - #dgkey = dgKey(serder.preb, serder.saidb) - #self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent - #logger.debug("Kever state: Escrowed source couple for partially signed " - #"or delegated event = %s\n", serder.ked) - - - def state(self): """ diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 5c1235784..d5b6994d3 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1686,7 +1686,7 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): for evt in self.getEvtPreIter(pre=pre, sn=sn): # includes disputed & superseded srdr = serdering.SerderKERI(raw=evt.tobytes()) - for eseal in srdr.seals or []: + for eseal in srdr.seals or []: # or [] for seals 'a' field missing if tuple(eseal) == eventing.SealEvent._fields: eseal = eventing.SealEvent(**eseal) # convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): @@ -1694,7 +1694,44 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): return None - def findAnchoringSealLast(self, pre, seal, sn=0): + def FetchSealingEventLastByEventSeal(self, pre, seal, sn=0): + """ + Search through a KEL for the last event at any sn but that contains a + specific anchored event seal of namedtuple SealEvent type that matches + the provided seal in dict form and is also fully witnessed. + Searchs from provided sn forward (default = 0). + Searches only last events in KEL of pre so does not include disputed + and/or superseded events. + + Returns: + srdr (Serder): instance of the first event with the matching + anchoring SealEvent seal, + None if not found + + Parameters: + pre (bytes|str): identifier of the KEL to search + seal (dict): dict form of Seal of any type SealEvent to find in anchored + seals list of each event + sn (int): beginning sn to search + + """ + if tuple(seal) != eventing.SealEvent._fields: # wrong type of seal + return None + + seal = eventing.SealEvent(**seal) #convert to namedtuple + + for evt in self.getEvtLastPreIter(pre=pre, sn=sn): # no disputed or superseded + srdr = serdering.SerderKERI(raw=evt.tobytes()) + for eseal in srdr.seals or []: # or [] for seals 'a' field missing + if tuple(eseal) == eventing.SealEvent._fields: + eseal = eventing.SealEvent(**eseal) # convert to namedtuple + if seal == eseal and self.fullyWitnessed(srdr): + return srdr + return None + + + + def FetchSealingEventLastBySeal(self, pre, seal, sn=0): """Only searches last event at any sn therefore does not search any disputed or superseded events. Search through last event at each sn in KEL for the event that contains @@ -1715,7 +1752,7 @@ def findAnchoringSealLast(self, pre, seal, sn=0): for evt in self.getEvtLastPreIter(pre=pre, sn=sn): # only last evt at sn srdr = serdering.SerderKERI(raw=evt.tobytes()) - for eseal in srdr.seals or []: + for eseal in srdr.seals or []: # or [] for seals 'a' field missing if tuple(eseal) == Seal._fields: # same type of seal eseal = Seal(**eseal) #convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): From ca21e191f40c2a74552f2a608de4dbf93d5fa2fc Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 7 Sep 2024 13:01:10 -0600 Subject: [PATCH 339/418] more refinement of logic for delegation --- src/keri/core/eventing.py | 163 +++++++++++++++++++++++++------------- src/keri/db/basing.py | 9 ++- 2 files changed, 114 insertions(+), 58 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index df2c50d3d..3cbff9e38 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2638,17 +2638,27 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, at the same sn under either of the following conditions: B1. The superseding rotation's delegating event is later than - the superseded rotation's delegating event in the delegator's KEL, i.e. the - sn of the superseding event's delegation is higher than the superseded event's - delegation. - - B2. The sn of the superseding rotation's delegating event is the same as - the sn of the superseded rotation's delegating event in the delegator's KEL - and the superseding rotation's delegating event is a rotation and the - superseded rotation's delegating event is an interaction, - i.e. the superseding rotation's delegating event is itself a superseding - rotation of the superseded rotations delegating interaction event in the - delgator's KEL + the superseded rotation's delegating event in the delegator's KEL, + i.e. the sn of the superseding event's delegation is higher than + the superseded event's delegation. + + B2. The sn of the superseding rotation's delegating event is the + same as the sn of the superseded rotation's delegating event in the + delegator's KEL both have the same delegating event and the index + of the delegating seal of the superceding delegation seal is later + (higher) than the index of the delegating seal of the supderseded + rotation. In other words the delegating event is the same event but + the superseding delegation seal appears later in the seal list of the + delegating event than the superseded delegation seal. + + B3. The sn of the superseding rotation's delegating event is the + same as the sn of the superseded rotation's delegating event in the + delegator's KEL and the superseding rotation's delegating event is + a rotation and the superseded rotation's delegating event is an + interaction, i.e. the delegating events are not the same event and + the superseding rotation's delegating event is itself a superseding + rotation of the superseded rotations delegating interaction event + in the delgator's KEL. C. IF Neither A nor B is satisfied, then recursively apply rules A. and B. to the delegating events of those delegating events and so on until @@ -2663,22 +2673,41 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, A. or B. must be satisfied, or else the superseding rotation must be discarded. - Note: The latest seen deleagated rotation constraint means that any earlier - delegated rotations can NOT be superseded. This greatly simplifies the + Note: The latest seen delegated rotation constraint means that any earlier + delegated rotations CAN NOT be superseded. This greatly simplifies the validation logic and avoids a potential infinite regress of forks in the - delegated identifier's KEL. + delegated identifier's KEL while allowing the delegate to + detect that a compromised delegation has occurred and give an + opportunity for the delegator to refuse to approve a subsequent + delegated rotation without additional verification with the delegate + that the subsequent delegated rotation was not compromised. In order to capture control of a delegated identifier the attacker must issue a delegated rotation that rotates to keys under the control of the attacker that must be approved by the delegator. A recovery rotation must - therefore superseded the compromised rotation. If the attacker is able - to issue and get approved by the delegator a second rotation + therefore supersede the compromised rotation. If the attacker is able + to issue and get approved by the delegator a subsequent rotation that follows but does not supersede the compromising rotation then recovery is no longer possible because the delegatee would no longer control the privete keys needed to verifiably sign a recovery rotation. + One way that detectability may be assured is when the delegator imposes + a minimum time between approvals of a delegated rotation that is + sufficient for the delgate to detect a compromised rotation recovery. + Attempts to rotate sooner than the minimum time since the immediately + prior rotation are refused until further verification has occurred. - Repair of approval soruce seal couple in 'aes' database on recursive + A delegated rotation that occurs after the minimum time since the + immediately prior delegated rotation might be automatically approved + to minimize latency. While a subsequent delegated rotation that occurs + within the minimum time would not be approed to maximize safety. + The minimum time window is designed to give the delegate enough time + to detect a comprimised or duplicitious superseding rotation and + prevent the additional verification from proceding. + + ToDo: + + Repair the approval source seal couple in the 'aess' database on recursive climb the kel tree. Once an event has been accepted into its kel. Later adding a source seal couple to 'aes' should then be OK from a security perspective since its only making discovery less expensive. @@ -2725,7 +2754,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, if delseqner is None or delsaider is None: # missing delegation seal ref if eager: # walk kel here to find seal = dict(i=serder.pre, s=serder.snh, d=serder.said) - dserder = self.db.FetchSealingEventLastByEventSeal(pre=delpre, + dserder = self.db.fetchLastSealingEventByEventSeal(pre=delpre, seal=seal) if dserder is not None: # found seal in dserder delseqner = coring.Seqner(sn=dserder.sn) # replace with found @@ -2819,12 +2848,12 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # Returning delegator indicates success and eventually results in acceptance # via Kever.logEvent which also writes the delgating event source couple to # db.aess so we can find it later - if ((serder.ilk == Ilks.dip) or # delegated inception - (serder.sner.num == self.sner.num + 1) or # inorder event - (serder.sner.num == self.sner.num and # superseding event - self.ilk == Ilks.ixn and # superseded is ixn and - serder.ilk == Ilks.drt)): # recovery rotation superseding ixn - return (None, None) # not validated so delseqner delsaider must be None # indicates delegation valid + if ((serder.ilk == Ilks.dip) or # superseding is delegated inception or + (serder.sner.num == self.sner.num + 1) or # superseding is inorder later or + (serder.sner.num == self.sner.num and # superseding event at same sn and + self.ilk == Ilks.ixn and # superseded is interaction and + serder.ilk == Ilks.drt)): # superseding is rotation + return (delseqner, delsaider) # indicates delegation valid # get to here means drt rotation superseding another drt rotation # Kever.logEvent saves authorizer (delegator) seal source couple in @@ -2832,39 +2861,46 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # recusively look up delegating events # set up recursive search for superseding delegations - serfn = serder # new potentially superseding delegated event i.e. serf new - bossn = dserder # new delegating event of superseding delegated event i.e. boss new + # get original potentially superseded delegation serfo = self.serder # original accepted delegated event i.e. serf original - if not (bosso := self.fetchDelegatingEvent(delpre, serfo, eager=eager)): + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, original=True, + eager=eager)): self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) + seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegating event from {delpre}" f" at {delsaider.qb64} for " f"evt = {serder.ked}.") + # already have new potential superseding delegation + serfn = serder # new potentially superseding delegated event i.e. serf new + bossn = dserder # new delegating event of superseding delegated event i.e. boss new while (True): # superseding delegated rotation of rotation recovery rules - # Only get to here if same sn for drt existing and drt superseding - - if (bossn.sn > bosso.sn or # later supersedes - (bossn.Ilk == Ilks.drt and - bosso.Ilk == Ilks.ixn) ): # drt supersedes ixn + # Only get to here if same sn for delegated event and both + # existing and superseding delegated events are rotations + if (bossn.sn > bosso.sn or # superseding delgation is later or + (bossn.Ilk == Ilks.drt and # superseding delegation is rotation and + bosso.Ilk == Ilks.ixn) ): # superseded delegation is interaction # valid superseding delegation up chain so tail link valid return (delseqner, delsaider) # tail event's delegation source if bossn.said == bosso.said: # same delegating event nseals = [SealEvent(**seal) for seal in bossn.seals if tuple(seal) == SealEvent._fields] - nindex = nseals.index(SealEvent(i=serfn.pre, s=serfn.snh, d=serfn.said)) + nindex = nseals.index(SealEvent(i=serfn.pre, + s=serfn.snh, + d=serfn.said)) oseals = [SealEvent(**seal) for seal in bosso.seals if tuple(seal) == SealEvent._fields] - oindex = oseals.index(SealEvent(i=serfo.pre, s=serfo.snh, d=serfo.said)) + oindex = oseals.index(SealEvent(i=serfo.pre, + s=serfo.snh, + d=serfo.said)) - if nindex > oindex: # later seal supersedes + if nindex > oindex: # superseding delegation seal is later # assumes index can't be None # valid superseding delegation up chain so tail link valid return (delseqner, delsaider) # tail event's delegation source - else: + else: # not superseded # ToDo: XXXX may want to cue up business logic for delegator # if self.mine(delegator): # failed attempt at recovery raise ValidationError(f"Invalid delegation recovery rotation" @@ -2872,14 +2908,18 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # tie condition same sn and drt so need to climb delegation chain serfn = bossn - if not (bossn := self.fetchDelegatingEvent(delpre, serfn, eager=eager)): + if not (bossn := self.fetchDelegatingEvent(delpre, serfn, + original=False, + eager=eager)): self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegating event from {delpre}" f" at {delsaider.qb64} for " f"evt = {serder.ked}.") serfo = bosso - if not (bosso := self.fetchDelegatingEvent(delpre, serfo, eager=eager)): + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, + original=True, + eager=eager)): self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegating event from {delpre}" @@ -2890,7 +2930,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, - def fetchDelegatingEvent(self, delpre, serder, *, eager=False): + def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): """Returns delegating event of delegator given by its aid delpre of delegated event given by serder otherwise raises ValidationError. @@ -2912,6 +2952,12 @@ def fetchDelegatingEvent(self, delpre, serder, *, eager=False): Parameters: delpre (str): qb64 of identifier prefix of delegator serder (SerderKERI): delegated serder + original (bool): True means delegated event is the original candidate + to be superseded. This means kel walk search should + include superseded or disputed events. + False means the delegated event is new candidate to + supersede. This means kel walk search should not + include superseded or disputed events. eager (bool): True means do more expensive KEL walk instead of escrow False means do not do expensive KEL walk now. @@ -2949,39 +2995,46 @@ def fetchDelegatingEvent(self, delpre, serder, *, eager=False): """ dserder = None # when not found and not eager so caller should reescrow dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key of delegate - # extra careful double check that delegate has been accepted by checkin - # fner = first seen Number instance index + if (couple := self.db.getAes(dgkey)): # delegation source couple at delegate seqner, saider = deSourceCouple(couple) deldig = saider.qb64 # dig of delegating event # extra careful double check that .aes is valid by getting # fner = first seen Number instance index - if not self.db.fons.get(keys=(delpre, deldig)): # None + if not self.db.fons.get(keys=(delpre, deldig)): # Not first seen raise ValidationError(f"Invalid delegation authorizing source " f"seal couple for {serder.ked}") ddgkey = dgKey(pre=delpre, dig=deldig) # database key of delegation if not (raw := self.db.getEvt(ddgkey)): # database broken this should never happen so do not supersede - # ToDo XXXX should repair by deleting the erroneous aes entry and - # returning found one + # ToDo XXXX repair by deleting the erroneous aes and + # returning None so gets escrowed and subsequent check will + # search below and repair raise ValidationError(f"Missing delegation event for {serder.ked}") - # original delegating event i.e. boss original dserder = serdering.SerderKERI(raw=bytes(raw)) - elif eager: #missing but try to find seal by walking delegator's KEL + elif eager: #missing aes but try to find seal by walking delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict - if not (dserder := self.db.findAnchoringSealEvent(pre=delpre, seal=seal)): - # database broken this should never happen so do not validate - raise ValidationError(f"Missing delegation source seal for {serder.ked}") - - # extra careful double check that .aes is valid by getting - # fner = first seen Number instance index of delegation + if original: # search all events in delegator's kel not just last + if not (dserder:=self.db.fetchAllSealingEventByEventSeal(pre=delpre, + seal=seal)): + # database broken this should never happen so do not validate + raise ValidationError(f"Missing delegation source seal for {serder.ked}") + else: # only search last events in delegator's kel + if not (dserder:=self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal)): + # database broken this should never happen so do not validate + raise ValidationError(f"Missing delegation source seal for {serder.ked}") + + # extra careful double check that found event is/was accepted event + # fner = first seen Number instance index of delegation from .fons if not self.db.fons.get(keys=(dserder.pre, dserder.dig)): # None raise ValidationError(f"Invalid delegation authorizing source " f"seal couple for {serder.ked}") - # Only repair when delegated has been accepted - if self.db.fons.get(keys=(preb, serder.saidb)): + + # Only repair when delegated has been accepted i.e has .fons entry + if self.db.fons.get(keys=(serder.pre, serder.said)): # Repair .aess of delegated event by writing found source # seal couple of delegation. This is safe becaause we confirmed # delegation event was accepted in delegator's kel. diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index d5b6994d3..e754b670a 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1662,7 +1662,7 @@ def cloneDelegation(self, kever): for dmsg in self.clonePreIter(pre=kever.delpre, fn=0): yield dmsg - def findAnchoringSealEvent(self, pre, seal, sn=0): + def fetchAllSealingEventByEventSeal(self, pre, seal, sn=0): """ Search through a KEL for the event that contains a specific anchored SealEvent type of provided seal but in dict form and is also fully @@ -1693,8 +1693,11 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): return srdr return None + # use alias here until can change everywhere for backwards compatibility + findAnchoringSealEvent = fetchAllSealingEventByEventSeal # alias - def FetchSealingEventLastByEventSeal(self, pre, seal, sn=0): + + def fetchLastSealingEventByEventSeal(self, pre, seal, sn=0): """ Search through a KEL for the last event at any sn but that contains a specific anchored event seal of namedtuple SealEvent type that matches @@ -1731,7 +1734,7 @@ def FetchSealingEventLastByEventSeal(self, pre, seal, sn=0): - def FetchSealingEventLastBySeal(self, pre, seal, sn=0): + def fetchLastSealingEventBySeal(self, pre, seal, sn=0): """Only searches last event at any sn therefore does not search any disputed or superseded events. Search through last event at each sn in KEL for the event that contains From 9cfccdf00732fe9fa60c040ec79c3e7192e2c764 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 7 Sep 2024 13:46:58 -0600 Subject: [PATCH 340/418] finally finished delegation logic --- src/keri/core/eventing.py | 45 +++++++++++++++++++++------------------ src/keri/db/basing.py | 1 - 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 3cbff9e38..d733ee7e3 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2993,7 +2993,6 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): Found delegation may not be superseding so do not repair .aess unless delegate was already accepted. """ - dserder = None # when not found and not eager so caller should reescrow dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key of delegate if (couple := self.db.getAes(dgkey)): # delegation source couple at delegate @@ -3001,18 +3000,20 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): deldig = saider.qb64 # dig of delegating event # extra careful double check that .aes is valid by getting # fner = first seen Number instance index - if not self.db.fons.get(keys=(delpre, deldig)): # Not first seen - raise ValidationError(f"Invalid delegation authorizing source " - f"seal couple for {serder.ked}") + if not self.db.fons.get(keys=(delpre, deldig)): # Not first seen yet? + if original: # should not happen aes database broken + # repair by deleting aes and returning None so it escrows + # and then next time around find below with repair it + self.db.delAes(dgkey) # delete aes so next time repairs it + # superseding may not have happened yet so let it escrow + return None ddgkey = dgKey(pre=delpre, dig=deldig) # database key of delegation - if not (raw := self.db.getEvt(ddgkey)): - # database broken this should never happen so do not supersede - # ToDo XXXX repair by deleting the erroneous aes and - # returning None so gets escrowed and subsequent check will - # search below and repair + if not (raw := self.db.getEvt(ddgkey)): # in fons but no event + # database broken this should never happen raise ValidationError(f"Missing delegation event for {serder.ked}") # original delegating event i.e. boss original dserder = serdering.SerderKERI(raw=bytes(raw)) + return dserder elif eager: #missing aes but try to find seal by walking delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict @@ -3020,20 +3021,20 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): if not (dserder:=self.db.fetchAllSealingEventByEventSeal(pre=delpre, seal=seal)): # database broken this should never happen so do not validate + # since original must have been validated so it must have + # all its delegation chain. raise ValidationError(f"Missing delegation source seal for {serder.ked}") else: # only search last events in delegator's kel if not (dserder:=self.db.fetchLastSealingEventByEventSeal(pre=delpre, seal=seal)): - # database broken this should never happen so do not validate - raise ValidationError(f"Missing delegation source seal for {serder.ked}") - - # extra careful double check that found event is/was accepted event - # fner = first seen Number instance index of delegation from .fons - if not self.db.fons.get(keys=(dserder.pre, dserder.dig)): # None - raise ValidationError(f"Invalid delegation authorizing source " - f"seal couple for {serder.ked}") - - # Only repair when delegated has been accepted i.e has .fons entry + # superseding delegation may not have happened yet so escrow + # ToDo XXXX need to cue up to get latest events in + # delegator's kel. + #raise ValidationError(f"Missing delegation source seal for {serder.ked}") + return None + + # Only repair .aess when found delegation is for delegated event that + # has been accepted delegated event i.e has .fons entry if self.db.fons.get(keys=(serder.pre, serder.said)): # Repair .aess of delegated event by writing found source # seal couple of delegation. This is safe becaause we confirmed @@ -3041,9 +3042,11 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): couple = dserder.sner.huge.encode() + dserder.saidb self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal - #else not found so return None to escrow and get fixed later + return dserder + + else: # not found so return None to escrow and get fixed later + return None - return dserder # extract delseqner, delsaider from actually found dserder def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index e754b670a..0f7e32790 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1908,7 +1908,6 @@ def getEvtPreIter(self, pre, sn=0): for dig in self.getKelIter(pre, sn=sn): try: - dgkey = dbing.dgKey(pre, dig) # get message if not (raw := self.getEvt(key=dgkey)): raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) From e62f8a332b8c4a28bd1705c64f4e93222f9c4032 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 7 Sep 2024 13:55:59 -0600 Subject: [PATCH 341/418] refactored find to fetch more descriptive so use the right one --- src/keri/app/cli/commands/ipex/grant.py | 4 ++-- src/keri/app/delegating.py | 2 +- src/keri/app/grouping.py | 3 ++- src/keri/app/querying.py | 2 +- src/keri/core/eventing.py | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/keri/app/cli/commands/ipex/grant.py b/src/keri/app/cli/commands/ipex/grant.py index d6e309d72..cc766664a 100644 --- a/src/keri/app/cli/commands/ipex/grant.py +++ b/src/keri/app/cli/commands/ipex/grant.py @@ -111,8 +111,8 @@ def grantDo(self, tymth, tock=0.0): iserder = serdering.SerderKERI(raw=bytes(iss)) seqner = coring.Seqner(sn=iserder.sn) - serder = self.hby.db.findAnchoringSealEvent(creder.sad['i'], - seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) + serder = self.hby.db.fetchAllSealingEventByEventSeal(creder.sad['i'], + seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) anc = self.hby.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said) exn, atc = protocoling.ipexGrantExn(hab=self.hab, recp=recp, message=self.message, acdc=acdc, iss=iss, anc=anc, diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 6b90c53ef..98b468087 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -138,7 +138,7 @@ def processUnanchoredEscrow(self): dkever = self.hby.kevers[kever.delpre] seal = dict(i=serder.pre, s=serder.snh, d=serder.said) - if dserder := self.hby.db.findAnchoringSealEvent(dkever.prefixer.qb64, seal=seal): + if dserder := self.hby.db.fetchAllSealingEventByEventSeal(dkever.prefixer.qb64, seal=seal): seqner = coring.Seqner(sn=dserder.sn) couple = seqner.qb64b + dserder.saidb dgkey = dbing.dgKey(kever.prefixer.qb64b, kever.serder.saidb) diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 41f8249c4..096798f73 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -171,7 +171,8 @@ def processDelegateEscrow(self): self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) else: # Not witnesser, we need to look for the anchor and then wait for receipts - if serder := self.hby.db.findAnchoringSealEvent(kever.delpre, seal=anchor): + if serder := self.hby.db.fetchAllSealingEventByEventSeal(kever.delpre, + seal=anchor): aseq = coring.Seqner(sn=serder.sn) couple = aseq.qb64b + serder.saidb dgkey = dbing.dgKey(pre, saider.qb64b) diff --git a/src/keri/app/querying.py b/src/keri/app/querying.py index 11541640f..d5bfdbcae 100644 --- a/src/keri/app/querying.py +++ b/src/keri/app/querying.py @@ -137,7 +137,7 @@ def recur(self, tyme, deeds=None): return False kever = self.hab.kevers[self.pre] - if self.hby.db.findAnchoringSealEvent(self.pre, seal=self.anchor): + if self.hby.db.fetchAllSealingEventByEventSeal(self.pre, seal=self.anchor): self.remove([self.witq]) return True diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index d733ee7e3..9a9215811 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -4888,7 +4888,7 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): kever = self.kevers[pre] if anchor: - if not self.db.findAnchoringSealEvent(pre=pre, seal=anchor): + if not self.db.fetchAllSealingEventByEventSeal(pre=pre, seal=anchor): self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) raise QueryNotFoundError("Query not found error={}.".format(ked)) From 85e82c4a874608f41a4eea8043ca1ab090b0ed94 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 8 Sep 2024 13:22:56 -0600 Subject: [PATCH 342/418] attempting to fix unit tests failing with new delegation logic --- tests/core/test_escrow.py | 125 ++++++++++++++++++++---------------- tests/core/test_eventing.py | 36 ++++++++++- 2 files changed, 102 insertions(+), 59 deletions(-) diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index 3b96e7275..9ad0f014e 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -466,67 +466,77 @@ def test_missing_delegator_escrow(): couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) assert couple == seqner.qb64b + bobSrdr.saidb + # Did not set up Habery so need to setup Del's prefixes so Del is locally + # owned + delDB.prefixes.add(delPre) + assert delPre in delDB.prefixes + # apply Del's inception msg to Del's Kevery # Dels event will fail but will add to its escrow psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) # delKvy.process(ims=bytearray(msg)) # process remote copy of msg - assert delPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - #escrow entry for event - escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - - # verify Kevery process partials escrow is idempotent to previously escrowed events - # assuming not stale but nothing else has changed - delKvy.processEscrowPartialSigs() - assert delPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - #escrow entry for event - escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - # apply Bob's inception to Dels' Kvy - psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(bobIcpMsg)) # process remote copy of msg - assert bobPre in delKvy.kevers # message accepted - delKvy.processEscrowPartialSigs() # process escrow - assert delPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - #escrow entry for event - escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - # apply Bob's delegating interaction to Dels' Kvy - psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(bobIxnMsg)) # process remote copy of msg - delKvy.processEscrowPartialSigs() # process escrows - assert delPre in delKvy.kevers # event removed from escrow + assert delPre in delKvy.kevers delK = delKvy.kevers[delPre] - assert delK.delegated - assert delK.serder.said == delSrdr.said - couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - assert couple == seqner.qb64b + bobSrdr.saidb - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 0 - #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - #assert escrow is None # delegated inception delegation couple - #escrow entry for event delegated inception delegation couple - escrow = self.db.udes.get(keys=(delPre, delSrdr.said)) - assert escrow is None + + #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + #escrows = delKvy.db.getPwes(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + #escrows = delKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + #assert len(escrows) == 1 + #assert escrows[0] == delSrdr.saidb # escrow entry for event + ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + ##escrow entry for event + #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb + + + ## verify Kevery process partials escrow is idempotent to previously escrowed events + ## assuming not stale but nothing else has changed + #delKvy.processEscrowPartialSigs() + #assert delPre not in delKvy.kevers + #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + #assert len(escrows) == 1 + #assert escrows[0] == delSrdr.saidb # escrow entry for event + ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + ##escrow entry for event + #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb + + ## apply Bob's inception to Dels' Kvy + #psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) + ## delKvy.process(ims=bytearray(bobIcpMsg)) # process remote copy of msg + #assert bobPre in delKvy.kevers # message accepted + #delKvy.processEscrowPartialSigs() # process escrow + #assert delPre not in delKvy.kevers + #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + #assert len(escrows) == 1 + #assert escrows[0] == delSrdr.saidb # escrow entry for event + ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + ##escrow entry for event + #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb + + ## apply Bob's delegating interaction to Dels' Kvy + #psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy, local=True) + ## delKvy.process(ims=bytearray(bobIxnMsg)) # process remote copy of msg + #delKvy.processEscrowPartialSigs() # process escrows + #assert delPre in delKvy.kevers # event removed from escrow + #delK = delKvy.kevers[delPre] + #assert delK.delegated + #assert delK.serder.said == delSrdr.said + #couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + #assert couple == seqner.qb64b + bobSrdr.saidb + + #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + #assert len(escrows) == 0 + ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + ##assert escrow is None # delegated inception delegation couple + ##escrow entry for event delegated inception delegation couple + #escrow = self.db.udes.get(keys=(delPre, delSrdr.said)) + #assert escrow is None # Setup Del rotation event verfers, digers = delMgr.rotate(pre=delPre, temp=True) @@ -586,8 +596,9 @@ def test_missing_delegator_escrow(): # delKvy.process(ims=bytearray(msg)) # process remote copy of msg assert delK.delegated assert delK.serder.said == delSrdr.said - couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - assert couple == seqner.qb64b + bobSrdr.saidb + assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + #couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + #assert couple == seqner.qb64b + bobSrdr.saidb # apply Del's delegated Rotation event message to bob's Kevery psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index a4474d54a..2b799a984 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -4907,6 +4907,40 @@ def test_load_event(mockHelpingNowUTC): # Endorse Tee's inception event with Wan's Hab just so we have non-trans receipts evt = eventing.loadEvent(wanHab.db, teeHab.pre, teeHab.pre) + #assert evt == {'ked': {'a': [], + #'b': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP'], + #'bt': '1', + #'c': [], + #'d': 'EDnrWpxagMvr5BBCwCOh3q5M9lvurboZ66vxR-GnIgQo', + #'di': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'i': 'EDnrWpxagMvr5BBCwCOh3q5M9lvurboZ66vxR-GnIgQo', + #'k': ['DLDlVl1H2Q138A5tftVRpyy834ejsY33BB71kXLRNP2h'], + #'kt': '1', + #'n': ['EBTtZqMkJOO4nf3cCt6SdezwkoCKtx2fGUKHeFApj_Yx'], + #'nt': '1', + #'s': '0', + #'t': 'dip', + #'v': 'KERI10JSON00018d_'}, + #'receipts': {'nontransferable': [{'prefix': 'BEXrSXVksXpnfno_Di6RBX2Lsr9VWRAihjLhowfjNOQQ', + #'signature': '0BCQOeNT3mwAHxh6mYU9K_B2VmbtjJh7_8115k4JrBPR3c4' + #'3jUSO197H2J73vWMi61qzOovNkSWQbnRx3NFnrk8I'}], + #'transferable': [{'prefix': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'said': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'sequence': '0AAAAAAAAAAAAAAAAAAAAAAA', + #'signature': 'AADGbcmUNw_SX7OVNX-PQYl41UZx_pgJXHOoMWrcfmCDGgkc1-' + #'MqXJjMD9S9moJ-lpPL9-AiXgITemMZL_QYGzIA'}]}, + #'signatures': [{'index': 0, + #'signature': 'AAC1-NTntZ0xkgHwooNcKxe9G4XC-rgkSryVz0B_QrZR2kkv4IKi7DMkfMBd4Eck-' + #'2NAi0DMuZeXnlvch6ZP0coO'}], + #'source_seal': {'said': 'EF7pHYN6XABC9znRdzprt5frW-MMry9rfrCI-_t5Y8VD', + #'sequence': 1}, + #'stored': True, + #'timestamp': '2021-01-01T00:00:00.000000+00:00', + #'witness_signatures': [{'index': 0, + #'signature': 'AABPMW3J1iZyMC-elPOkdIhddhZB_BJYHTdYv5SxcrOfJL_5igDVB6zKD' + #'AQiTj_cNa7oP-l6xSRRxwlHDwqgSwcB'}], + #'witnesses': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP']} + # no source seal in load assert evt == {'ked': {'a': [], 'b': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP'], 'bt': '1', @@ -4932,8 +4966,6 @@ def test_load_event(mockHelpingNowUTC): 'signatures': [{'index': 0, 'signature': 'AAC1-NTntZ0xkgHwooNcKxe9G4XC-rgkSryVz0B_QrZR2kkv4IKi7DMkfMBd4Eck-' '2NAi0DMuZeXnlvch6ZP0coO'}], - 'source_seal': {'said': 'EF7pHYN6XABC9znRdzprt5frW-MMry9rfrCI-_t5Y8VD', - 'sequence': 1}, 'stored': True, 'timestamp': '2021-01-01T00:00:00.000000+00:00', 'witness_signatures': [{'index': 0, From 1447885d69c3ef73e3cc8e837248a601d655c381 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 8 Sep 2024 13:27:48 -0600 Subject: [PATCH 343/418] added clarifying comment. --- src/keri/app/habbing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index ba2959222..d98699707 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -2284,6 +2284,9 @@ def make(self, *, secrecies=None, iridx=0, code=coring.MtrDex.Blake3_256, dcode= # may want db method that updates .habs. and .prefixes together habord = basing.HabitatRecord(hid=self.pre, name=self.name, domain=self.ns) + # must add self.pre to self.prefixes before calling processEvent so that + # Kever.locallyOwned or Kever.locallyDelegated or Kever.locallyWitnessed + # evaluates correctly when processing own inception event. if not hidden: self.save(habord) self.prefixes.add(self.pre) From 4d7bde29c6b38a549ce1d730ddbe4540492baaf2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 9 Sep 2024 13:02:44 -0600 Subject: [PATCH 344/418] fixed bugs in testing partial delegated escrow --- src/keri/core/eventing.py | 49 +++++--- src/keri/db/basing.py | 2 +- tests/core/test_escrow.py | 235 +++++++++++++++++++++----------------- tests/db/test_basing.py | 2 +- 4 files changed, 165 insertions(+), 123 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 9a9215811..a02fc66e8 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1683,7 +1683,10 @@ def locallyOwned(self, pre: str | None = None): def locallyDelegated(self, pre: str): """Returns True if pre is in .prefixes and not in .groups - False otherwise. + False otherwise. Use when pre is a delegator for some event and + want to confirm that pre is also locallyOwned thereby making the + associated event locallyDelegated. + Indicates that provided identifier prefix is controlled by a local controller from .prefixes but is not a group with local member. i.e pre is a locally owned (controlled) AID (identifier prefix) @@ -2156,6 +2159,7 @@ def rotate(self, serder): return tholder, toader, wits, cuts, adds + def deriveBacks(self, serder): """Derives and return tuple of (wits, cuts, adds) for backers given current set and any changes provided by serder. @@ -3353,7 +3357,7 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, if seqner and saider: # non-idempotent pin to repair replace self.db.udes.pin(keys=dgkey, val=(seqner, saider)) # non-idempotent logger.debug(f"Kever state: Replaced escrow source couple sn=" - f"{seqner.sn}, said={saider.said} for partially " + f"{seqner.sn}, said={saider.qb64} for partially " f"delegated/authorized event said={serder.said}.") else: self.db.udes.rem(keys=dgkey) # nullify non-idempotent @@ -3375,7 +3379,7 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, logger.debug(f"Kever state: Escrowed partially delegated event=\n" f"{serder.ked}\n.") - return self.db.pdes.add(keys =(serder.preb, serder.sn), val=serder.saidb) + return self.db.pdes.add(keys=snKey(serder.preb, serder.sn), val=serder.saidb) def state(self): @@ -5472,6 +5476,7 @@ def processEscrowOutOfOrders(self): break key = ekey # setup next while iteration, with key after ekey + def processEscrowPartialSigs(self): """ Process events escrowed by Kever that were only partially fulfilled, @@ -5846,10 +5851,10 @@ def processEscrowPartialDels(self): If successful then remove from escrow table """ - for ekey, edig in self.db.pdes.getItemIter(keys=b''): + for (epre,), esn, edig in self.db.pdes.getOnItemIter(keys=b''): try: - pre, sn = splitSnKey(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) + #pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(epre, edig) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below raise ValidationError("Missing escrowed event source " @@ -5877,7 +5882,7 @@ def processEscrowPartialDels(self): "at dig = {}.".format(bytes(edig))) # get the escrowed event using edig - eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + eraw = self.db.getEvt(dgkey) if eraw is None: # no event so so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event at." @@ -5889,7 +5894,7 @@ def processEscrowPartialDels(self): eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event # get sigs - sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs + sigs = self.db.getSigs(dgkey) # list of sigs if not sigs: # empty list # no sigs so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event sigs at." @@ -5900,14 +5905,20 @@ def processEscrowPartialDels(self): # get witness signatures (wigs not wits) assumes wont be in this # escrow if wigs not needed because no wits - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - if not wigs: # empty list - # no wigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event wigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt wigs at " - "dig = {}.".format(bytes(edig))) + wigs = self.db.getWigs(dgkey) # list of wigs if any + # may want to checks wits and wigs here. We are assuming that + # never get to this escrow if wits and not wigs + #if wits and not wigs: # non empty wits but empty wigs + ## wigs maybe empty if not wits or if wits while waiting + ## for first witness signature + ## which may not arrive until some time after event is fully signed + ## so just log for debugging but do not unescrow by raising + ## ValidationError + #logger.info("Kevery unescrow error: Missing event wigs at." + #"dig = %s", bytes(edig)) + + #raise ValidationError("Missing escrowed evt wigs at " + #"dig = {}.".format(bytes(edig))) # setup parameters to process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] @@ -5917,7 +5928,7 @@ def processEscrowPartialDels(self): # If delegator KEL not available should also cue a trigger to # get it if still missing when processing escrow. delseqner = delsaider = None - if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + if (couple := self.db.udes.get(keys=(epre, edig))): delseqner, delsaider = couple # provided #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): # walk kel to find @@ -5962,7 +5973,7 @@ def processEscrowPartialDels(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow # removes one event escrow at key val - self.db.pdes.rem(keys=(pre, sn), val=edig) # event idx escrow + self.db.pdes.remOn(keys=snKey(epre, esn), val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) @@ -5974,7 +5985,7 @@ def processEscrowPartialDels(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. # removes one event escrow at key val - self.db.pdes.rem(keys=(pre, sn), val=edig) # event idx escrow + self.db.pdes.rem(keys=snKey(epre, esn), val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 0f7e32790..f5450dfe8 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1016,7 +1016,7 @@ def reopen(self, **kwa): self.vres = self.env.open_db(key=b'vres.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) - self.pdes = subing.IoDupSuber(db=self, subkey='pdes.') + self.pdes = subing.OnIoDupSuber(db=self, subkey='pdes.') self.udes = subing.CatCesrSuber(db=self, subkey='udes.', klas=(coring.Seqner, coring.Saider)) self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index 9ad0f014e..7808bdc13 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -352,39 +352,77 @@ def test_partial_signed_escrow(): def test_missing_delegator_escrow(): """ Test missing delegator escrow + + bod is the delegator + del is the delegate + wat is the watcher """ - # bob is the delegator del is bob's delegate bobSalt = core.Salter(raw=b'0123456789abcdef').qb64 delSalt = core.Salter(raw=b'abcdef0123456789').qb64 + watSalt = core.Salter(raw=b'wxyzabcdefghijkl').qb64 psr = parsing.Parser() - with basing.openDB(name="bob") as bobDB, \ - keeping.openKS(name="bob") as bobKS, \ - basing.openDB(name="del") as delDB, \ - keeping.openKS(name="del") as delKS: + with (basing.openDB(name="bob") as bobDB, + keeping.openKS(name="bob") as bobKS, + basing.openDB(name="del") as delDB, + keeping.openKS(name="del") as delKS, + basing.openDB(name="wat") as watDB, \ + keeping.openKS(name="wat") as watKS ): # Init key pair managers bobMgr = keeping.Manager(ks=bobKS, salt=bobSalt) delMgr = keeping.Manager(ks=delKS, salt=delSalt) + watMgr = keeping.Manager(ks=watKS, salt=watSalt) # Init Keverys bobKvy = eventing.Kevery(db=bobDB) delKvy = eventing.Kevery(db=delDB) + watKvy = eventing.Kevery(db=watDB) + + # Setup Wat with own inception event + verfers, digers = watMgr.incept(stem='wat', temp=True) # algo default salty and rooted + + watSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + watPre = watSrdr.pre + watMgr.move(old=verfers[0].qb64, new=watPre) # move key pair label to prefix + # Setup wat's prefixes so wat's KEL will be Kever.locallyOwned() + watDB.prefixes.add(watPre) + assert watPre in watDB.prefixes + # setup wat's on kel + sigers = watMgr.sign(ser=watSrdr.raw, verfers=verfers) + msg = bytearray(watSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + watIcpMsg = msg # save for later + + # apply msg to wats's Kevery + psr.parse(ims=bytearray(watIcpMsg), kvy=watKvy, local=True) + watK = watKvy.kevers[watPre] + assert watK.prefixer.qb64 == watPre + assert watK.serder.said == watSrdr.said - # Setup Bob by creating inception event + # Setup Bob with own inception event verfers, digers = bobMgr.incept(stem='bob', temp=True) # algo default salty and rooted bobSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], ndigs=[diger.qb64 for diger in digers], code=coring.MtrDex.Blake3_256) - bobPre = bobSrdr.ked["i"] - + bobPre = bobSrdr.pre bobMgr.move(old=verfers[0].qb64, new=bobPre) # move key pair label to prefix + # Setup Bob's prefixes so bob's KEL will be Kever.locallyOwned() and + # Del's KEL will be Kever.locallyDelegated() + bobDB.prefixes.add(bobPre) + assert bobPre in bobDB.prefixes sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) - msg = bytearray(bobSrdr.raw) counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) @@ -395,28 +433,32 @@ def test_missing_delegator_escrow(): bobIcpMsg = msg # save for later # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(bobIcpMsg), kvy=bobKvy, local=True) bobK = bobKvy.kevers[bobPre] assert bobK.prefixer.qb64 == bobPre assert bobK.serder.said == bobSrdr.said + assert bobK.sn == 0 # apply msg to del's Kevery so he knows about the AID - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) + psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) assert bobK.prefixer.qb64 in delKvy.kevers + delBobK = bobKvy.kevers[bobPre] # bobs kever in dels kevery + assert delBobK.sn == 0 - # Setup Del's inception event assuming that Bob's next event will be an ixn delegating event + # Setup Del's inception event assuming that Bob's next event will be + # an ixn delegating event verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted - delSrdr = eventing.delcept(keys=[verfer.qb64 for verfer in verfers], delpre=bobPre, ndigs=[diger.qb64 for diger in digers]) - delPre = delSrdr.ked["i"] - + delPre = delSrdr.pre delMgr.move(old=verfers[0].qb64, new=delPre) # move key pair label to prefix + # Setup Del's prefixes so Del's KEL will be Kever.locallyOwned() + delDB.prefixes.add(delPre) + assert delPre in delDB.prefixes - # Now create delegating event + # Now create delegating event for Bob seal = eventing.SealEvent(i=delPre, s=delSrdr.ked["s"], d=delSrdr.said) @@ -433,14 +475,14 @@ def test_missing_delegator_escrow(): msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - bobIxnMsg = msg + bobIxnMsg1 = msg # delegating event with attachments # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(bobIxnMsg1), kvy=bobKvy, local=True) assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + assert bobK.sn == 1 - # now create msg with Del's delegated inception event + # now create Del's delegated inception event msg sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) @@ -455,9 +497,12 @@ def test_missing_delegator_escrow(): seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) msg.extend(bobSrdr.saidb) + delIcpMsg = msg # apply Del's delegated inception event message to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) + # because the attachment includes valid source seal then the Delegables + # escrow is bypassed and is validated and shows up in AES + psr.parse(ims=bytearray(delIcpMsg), kvy=bobKvy, local=True) # bobKvy.process(ims=bytearray(msg)) # process local copy of msg assert delPre in bobKvy.kevers # successfully validated bobDelK = bobKvy.kevers[delPre] # delK in bobs kevery @@ -466,77 +511,51 @@ def test_missing_delegator_escrow(): couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) assert couple == seqner.qb64b + bobSrdr.saidb - # Did not set up Habery so need to setup Del's prefixes so Del is locally - # owned - delDB.prefixes.add(delPre) - assert delPre in delDB.prefixes - # apply Del's inception msg to Del's Kevery - # Dels event will fail but will add to its escrow - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + # Because locallyOwned by delegate event does not validate delegation + # and ignores the attached source seal + psr.parse(ims=bytearray(delIcpMsg), kvy=delKvy, local=True) assert delPre in delKvy.kevers delK = delKvy.kevers[delPre] + # no AES entry for del's own delegated event when locallyOwned + assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + + # apply Del's delegated inception event message to wats's Kevery as remote + # because the attachment includes valid source seal but wat does not + # yet have Bob's delegating event entry. The event goes into partial + # delegated event escrow + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + # Now apply Bob's incept to wat's kvy and process escrow + psr.parse(ims=bytearray(bobIcpMsg), kvy=watKvy, local=False) + assert bobPre in watKvy.kevers + watBobK = watKvy.kevers[bobPre] + assert watBobK.sn == 0 + watKvy.processEscrows() + assert not delPre in watKvy.kevers + escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + # Now apply Bob's ixn to wat's kvy and process escrow + psr.parse(ims=bytearray(bobIxnMsg1), kvy=watKvy, local=False) + watKvy.processEscrows() + escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + assert len(escrows) == 0 + assert watBobK.sn == 1 + + assert delPre in watKvy.kevers # successfully validated + watDelK = watKvy.kevers[delPre] # delK in wats kevery + assert watDelK.delegated + assert watDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = watKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb - #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - #escrows = delKvy.db.getPwes(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - #escrows = delKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) - #assert len(escrows) == 1 - #assert escrows[0] == delSrdr.saidb # escrow entry for event - ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - ##escrow entry for event - #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - - ## verify Kevery process partials escrow is idempotent to previously escrowed events - ## assuming not stale but nothing else has changed - #delKvy.processEscrowPartialSigs() - #assert delPre not in delKvy.kevers - #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - #assert len(escrows) == 1 - #assert escrows[0] == delSrdr.saidb # escrow entry for event - ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - ##escrow entry for event - #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - ## apply Bob's inception to Dels' Kvy - #psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) - ## delKvy.process(ims=bytearray(bobIcpMsg)) # process remote copy of msg - #assert bobPre in delKvy.kevers # message accepted - #delKvy.processEscrowPartialSigs() # process escrow - #assert delPre not in delKvy.kevers - #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - #assert len(escrows) == 1 - #assert escrows[0] == delSrdr.saidb # escrow entry for event - ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - ##escrow entry for event - #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - ## apply Bob's delegating interaction to Dels' Kvy - #psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy, local=True) - ## delKvy.process(ims=bytearray(bobIxnMsg)) # process remote copy of msg - #delKvy.processEscrowPartialSigs() # process escrows - #assert delPre in delKvy.kevers # event removed from escrow - #delK = delKvy.kevers[delPre] - #assert delK.delegated - #assert delK.serder.said == delSrdr.said - #couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - #assert couple == seqner.qb64b + bobSrdr.saidb - - #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - #assert len(escrows) == 0 - ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - ##assert escrow is None # delegated inception delegation couple - ##escrow entry for event delegated inception delegation couple - #escrow = self.db.udes.get(keys=(delPre, delSrdr.said)) - #assert escrow is None # Setup Del rotation event verfers, digers = delMgr.rotate(pre=delPre, temp=True) @@ -565,19 +584,25 @@ def test_missing_delegator_escrow(): for siger in sigers: msg.extend(siger.qb64b) - # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + bobIxnMsg2 = msg + + # apply bobs IXN msg to bob's Kevery + psr.parse(ims=bytearray(bobIxnMsg2), kvy=bobKvy, local=True) assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + assert bobK.sn == 2 # apply msg to del's Kevery - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg - assert delKvy.kevers[bobPre].serder.said == bobSrdr.said + psr.parse(ims=bytearray(bobIxnMsg2), kvy=delKvy, local=True) + assert delBobK.serder.said == bobSrdr.said + assert delBobK.sn == 2 + + # apply msg to wat's Kevery + psr.parse(ims=bytearray(bobIxnMsg2), kvy=watKvy, local=True) + assert watBobK.serder.said == bobSrdr.said + assert watBobK.sn == 2 # now create msg from Del's delegated rotation event sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) - msg = bytearray(delSrdr.raw) counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) @@ -591,23 +616,28 @@ def test_missing_delegator_escrow(): msg.extend(seqner.qb64b) msg.extend(bobSrdr.saidb) + delRotMsg = msg + # apply Del's delegated Rotation event message to del's Kevery - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + psr.parse(ims=bytearray(delRotMsg), kvy=delKvy, local=True) assert delK.delegated assert delK.serder.said == delSrdr.said assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - #couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - #assert couple == seqner.qb64b + bobSrdr.saidb # apply Del's delegated Rotation event message to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(delRotMsg), kvy=bobKvy, local=True) assert bobDelK.delegated assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) assert couple == seqner.qb64b + bobSrdr.saidb + # apply Del's delegated Rotation event message to wats's Kevery + psr.parse(ims=bytearray(delRotMsg), kvy=watKvy, local=True) + assert watDelK.delegated + assert watDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = watKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + assert not os.path.exists(delKS.path) @@ -1449,5 +1479,6 @@ def test_unverified_trans_receipt_escrow(): if __name__ == "__main__": - test_unverified_receipt_escrow() + #test_unverified_receipt_escrow() + test_missing_delegator_escrow() diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index b4de48322..659403c21 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -784,7 +784,7 @@ def test_baser(): assert key == f'{preb.decode("utf-8")}.{digb.decode("utf-8")}'.encode("utf-8") # test .pdes SerderIoSetSuber methods - assert isinstance(db.pdes, subing.IoDupSuber) + assert isinstance(db.pdes, subing.OnIoDupSuber) # test .udes CatCesrSuber sub db methods From 1a03114d9b08971297e1194206eb79ee2b684e94 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 9 Sep 2024 14:01:04 -0600 Subject: [PATCH 345/418] fixed bug typo --- src/keri/core/eventing.py | 2 +- src/keri/db/dbing.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index a02fc66e8..368680ff7 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5973,7 +5973,7 @@ def processEscrowPartialDels(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow # removes one event escrow at key val - self.db.pdes.remOn(keys=snKey(epre, esn), val=edig) # event idx escrow + self.db.pdes.rem(keys=snKey(epre, esn), val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 57a3f491c..5787c5ac4 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1869,12 +1869,6 @@ def getOnIoDupLastItemIter(self, db, key=b'', on=0, *, sep=b'.'): return # no values end of db raises StopIteration - - - # getOnIoDupItemBackIter symmetric with getOnIoDupItemIter - # getOnIoDupValBackIter symmetric with getOnIoDupValIter - - def getOnIoDupValBackIter(self, db, key=b'', on=0, *, sep=b'.'): """Returns iterator going backwards of values, of insertion ordered item at each key over all ordinal numbered keys From f810077e404ada22f2e34a00ff565dba6a78bfa3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 9 Sep 2024 15:16:40 -0600 Subject: [PATCH 346/418] added local check --- src/keri/core/eventing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 368680ff7..b02d5ca23 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2380,7 +2380,7 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # Doesn't get to here until fully signed and witnessed. if self.locallyDelegated(delpre): # local delegator - if delseqner is None or delsaider is None: # missing delegation seal + if delseqner is None or delsaider is None or not local: # missing delegation seal # so escrow delegable. So local delegator can approve OOB. # and create delegator event with valid event seal of this # delegated event and then reprocess event with attached source From fe7ce5d6638cff8b9e2302080df3c1ca0cd4472a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 9 Sep 2024 15:20:43 -0600 Subject: [PATCH 347/418] removed local test since redundant already caught in misfit check --- src/keri/core/eventing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index b02d5ca23..d15cb65b9 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2380,7 +2380,8 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # Doesn't get to here until fully signed and witnessed. if self.locallyDelegated(delpre): # local delegator - if delseqner is None or delsaider is None or not local: # missing delegation seal + # must be local if locallyDelegated or caught above as misfit + if delseqner is None or delsaider is None: # missing delegation seal # so escrow delegable. So local delegator can approve OOB. # and create delegator event with valid event seal of this # delegated event and then reprocess event with attached source From cadfa44c8bf98acbe4e511d876659dcc7f59e6b3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 9 Sep 2024 19:31:41 -0600 Subject: [PATCH 348/418] finish fleshing out OnIoDupSuber methods to better support keri escrows for KELs --- src/keri/db/dbing.py | 172 ++++++++++++++++++++++++++++++++++++++++ src/keri/db/subing.py | 124 ++++++++++++++++++++++++++++- tests/db/test_dbing.py | 29 ++++++- tests/db/test_subing.py | 33 ++++++++ 4 files changed, 354 insertions(+), 4 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 5787c5ac4..4bd1e67c0 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -662,6 +662,65 @@ def delTopVal(self, db, top=b''): # For subdbs the use keys with trailing part the is monotonically # ordinal number serialized as 32 hex bytes + + # used in OnSuberBase + def putOnVal(self, db, key, on=0, val=b'', *, sep=b'.'): + """Write serialized bytes val to location at onkey consisting of + key + sep + serialized on in db. + Does not overwrite. + + Returns: + result (bool): True if successful write i.e onkey not already in db + False otherwise + + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which write + val (bytes): to be written at onkey + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=True, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return (txn.put(onkey, val, overwrite=False)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{onkey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + # used in OnSuberBase + def setOnVal(self, db, key, on=0, val=b'', *, sep=b'.'): + """ + Write serialized bytes val to location at onkey consisting of + key + sep + serialized on in db. + Overwrites pre-existing value at onkey if any. + + Returns: + result (bool): True if successful write i.e onkey not already in db + False otherwise + + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which write + val (bytes): to be written at onkey + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=True, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return (txn.put(onkey, val)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{onkey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + # used in OnSuberBase def appendOnVal(self, db, key, val, *, sep=b'.'): """ @@ -721,6 +780,35 @@ def appendOnVal(self, db, key, val, *, sep=b'.'): raise ValueError(f"Failed appending {val=} at {key=}.") return on + + # used in OnSuberBase + def getOnVal(self, db, key, on=0, *, sep=b'.'): + """Gets value at onkey consisting of key + sep + serialized on in db. + + Returns: + val (bytes | memoryview): entry at onkey consisting of key + sep + + serialized on in db. + None if no entry at key + + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which to retrieve + sep (bytes): separator character for split + + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return(txn.get(onkey)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{onkey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + # used in OnSuberBase def delOnVal(self, db, key, on=0, *, sep=b'.'): """ @@ -1713,6 +1801,34 @@ def getTopIoDupItemIter(self, db, top=b''): # this is so we do the proem add and strip here not in some higher level class # like suber + def addOnIoDupVal(self, db, key, on=0, val=b'', sep=b'.'): + """ + Add val bytes as dup at onkey consisting of key + sep + serialized on in db. + Adds to existing values at key if any + Returns True if written else False if dup val already exists + + Duplicates are inserted in lexocographic order not insertion order. + Lmdb does not insert a duplicate unless it is a unique value for that + key. + + Does inclusion test to dectect of duplicate already exists + Uses a python set for the duplicate inclusion test. Set inclusion scales + with O(1) whereas list inclusion scales with O(n). + + Returns: + result (bool): True if duplicate val added at onkey idempotent + False if duplicate val preexists at onkey + + Parameters: + db is opened named sub db with dupsort=True + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to add at onkey as dup + sep (bytes): separator character for split + """ + onkey = onKey(key, on, sep=sep) + return (self.addIoDupVal(db, key=onkey, val=val)) + + # used in OnIoDupSuber def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): """ @@ -1742,6 +1858,62 @@ def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) + def delOnIoDupVals(self, db, key, on=0, sep=b'.'): + """Deletes all dup iovals at onkey consisting of key + sep + serialized + on in db. + + Assumes DB opened with dupsort=True + + Duplicates are inserted in lexocographic order not insertion order. + Lmdb does not insert a duplicate unless it is a unique value for that + key. + + Does inclusion test to dectect of duplicate already exists + Uses a python set for the duplicate inclusion test. Set inclusion scales + with O(1) whereas list inclusion scales with O(n). + + Returns: + result (bool): True if onkey present so all dups at onkey deleted + False if onkey not present + + Parameters: + db is opened named sub db with dupsort=True + key (bytes): key within sub db's keyspace plus trailing part on + sep (bytes): separator character for split + """ + onkey = onKey(key, on, sep=sep) + return (self.delIoDupVals(db, key=onkey)) + + + def delOnIoDupVal(self, db, key, on=0, val=b'', sep=b'.'): + """Deletes dup ioval at key onkey consisting of key + sep + serialized + on in db. + Returns True if deleted else False if dup val not present + Assumes DB opened with dupsort=True + + Duplicates are inserted in lexocographic order not insertion order. + Lmdb does not insert a duplicate unless it is a unique value for that + key. + + Does inclusion test to dectect of duplicate already exists + Uses a python set for the duplicate inclusion test. Set inclusion scales + with O(1) whereas list inclusion scales with O(n). + + Returns: + result (bool): True if duplicate val found and deleted + False if duplicate val does not exist at onkey + + Parameters: + db is opened named sub db with dupsort=True + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized dup value to del at onkey + sep (bytes): separator character for split + """ + onkey = onKey(key, on, sep=sep) + return (self.delIoDupVal(db, key=onkey, val=val)) + + + # used in OnIoDupSuber def getOnIoDupValIter(self, db, key=b'', on=0, *, sep=b'.'): """ diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 214cf5ed1..4e6abf311 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -436,6 +436,47 @@ def __init__(self, *pa, **kwa): super(OnSuberBase, self).__init__(*pa, **kwa) + def putOn(self, keys: str | bytes | memoryview, on: int=0, + val: str | bytes | memoryview=''): + """ + Returns + result (bool): True if onkey made from key+sep+serialized on is + not found in database so value is written + idempotently. + False otherwise + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + val (str | bytes | memoryview): serialization + """ + return (self.db.putOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode())) + + def pinOn(self, keys: str | bytes | memoryview, on: int=0, + val: str | bytes | memoryview=''): + """ + Returns + result (bool): True if value is written or overwritten at onkey + False otherwise + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + val (str | bytes | memoryview): serialization + """ + return (self.db.setOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode())) + + def appendOn(self, keys: str | bytes | memoryview, val: str | bytes | memoryview): """ @@ -446,7 +487,6 @@ def appendOn(self, keys: str | bytes | memoryview, keys (str | bytes | memoryview | Iterable): top keys as prefix to be combined with serialized on suffix and sep to form key val (str | bytes | memoryview): serialization - on (int): ordinal number used with onKey(key,on) to form key. """ return (self.db.appendOnVal(db=self.sdb, key=self._tokey(keys), @@ -454,6 +494,25 @@ def appendOn(self, keys: str | bytes | memoryview, sep=self.sep.encode())) + def getOn(self, keys: str | bytes | memoryview, on: int=0): + """ + Returns + val (str): serialization at onkey if any + None if no entry at onkey + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + """ + val = self.db.getOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode()) + return (self._des(val) if val is not None else None) + + + def remOn(self, keys: str | bytes | memoryview, on: int=0): """ Returns @@ -2018,7 +2077,7 @@ def getLast(self, keys: str | bytes | memoryview | Iterable): def rem(self, keys: str | bytes | memoryview | Iterable, - val: str | bytes | memoryview = b''): + val: str | bytes | memoryview = ''): """ Removes entry at key made from keys and dup val that matches val if any, notwithstanding hidden ordinal proem. Otherwise deletes all dup values @@ -2141,6 +2200,32 @@ def __init__(self, *pa, **kwa): super(OnIoDupSuber, self).__init__(*pa, **kwa) + def addOn(self, keys: str | bytes | memoryview | Iterable, on: int=0, + val: str | bytes | memoryview = ''): + """ + Add val idempotently at key made from keys in insertion order using hidden + ordinal proem. Idempotently means do not add val that is already in + dup vals at key. Does not overwrite. + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(pre,on) to form onkey. + val (str | bytes | memoryview): serialization + + Returns: + result (bool): True means unique value added among duplications, + False means duplicate of same value already exists. + + """ + return (self.db.addOnIoDupVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode())) + + + def appendOn(self, keys: str | bytes | memoryview, val: str | bytes | memoryview): """ @@ -2151,13 +2236,46 @@ def appendOn(self, keys: str | bytes | memoryview, keys (str | bytes | memoryview | Iterable): top keys as prefix to be combined with serialized on suffix and sep to form key val (str | bytes | memoryview): serialization - on (int): ordinal number used with onKey(pre,on) to form key. """ return (self.db.appendOnIoDupVal(db=self.sdb, key=self._tokey(keys), val=self._ser(val), sep=self.sep.encode())) + def remOn(self, keys: str | bytes | memoryview | Iterable, on: int=0, + val: str | bytes | memoryview = ''): + """ + Removes entry at key made from keys and dup val that matches val if any, + notwithstanding hidden ordinal proem. Otherwise deletes all dup values + at key if any. + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + + on (int): ordinal number used with onKey(pre,on) to form key. + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key + + Returns: + result (bool): True if onkey with dup val exists so rem successful. + False otherwise + + """ + if val: + return self.db.delOnIoDupVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode()) + else: + return self.db.delOnIoDupVals(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode()) + + def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 9c2348c56..309a366df 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -311,7 +311,7 @@ def test_lmdber(): items = [ (key, bytes(val)) for key, val in dber.getTopItemIter(db=db )] assert items == [(b'b.1', b'woo')] - # test OrdVal OrdItem ordinal numbered event sub db + # test Ordinal Numbered ON keyed value methods db = dber.env.open_db(key=b'seen.') preA = b'BBKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc' @@ -346,8 +346,18 @@ def test_lmdber(): assert dber.putVal(db, keyA0, val=digA) == False assert dber.setVal(db, keyA0, val=digA) == True assert dber.getVal(db, keyA0) == digA + assert dber.getOnVal(db, preA, 0) == digA assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None + assert dber.getOnVal(db, preA, 0) == None + + assert dber.putOnVal(db, preA, 0, val=digA) == True + assert dber.getOnVal(db, preA, 0) == digA + assert dber.putOnVal(db, preA, 0, val=digA) == False + assert dber.setOnVal(db, preA, 0, val=digA) == True + assert dber.getOnVal(db, preA, 0) == digA + assert dber.delOnVal(db, preA, 0) == True + assert dber.getOnVal(db, preA, 0) == None # test appendOnValPre # empty database @@ -981,7 +991,24 @@ def test_lmdber(): (b'Z', 1, b'l'), (b'Z', 0, b'k')] + key = b'Y' + assert dber.addOnIoDupVal(ldb, key, on=0, val=b'r') + assert dber.addOnIoDupVal(ldb, key, on=0, val=b's') + assert dber.addOnIoDupVal(ldb, key, on=1, val=b't') + assert dber.addOnIoDupVal(ldb, key, on=1, val=b'u') + + assert dber.cntOnVals(ldb, key) == 4 + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key)] + assert items == [(b'Y', 0, b'r'), + (b'Y', 0, b's'), + (b'Y', 1, b't'), + (b'Y', 1, b'u')] + assert dber.delOnIoDupVal(ldb, key, on=0, val=b's') + assert dber.delOnIoDupVals(ldb, key, on=1) + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key)] + assert items == [(b'Y', 0, b'r')] # test IoSetVals insertion order set of vals methods. key0 = b'ABC.ZYX' diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 6b97d3c55..8ac6d5e52 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -351,6 +351,14 @@ def test_on_suber(): assert items == [(('a',), 0, 'Blue dog'), (('a',), 2, 'Red apple')] + assert onsuber.putOn(keys='d', on=0, val='moon') + assert onsuber.getOn(keys='d', on=0) == 'moon' + assert not onsuber.putOn(keys='d', on=0, val='moon') + assert onsuber.pinOn(keys='d', on=0, val='sun') + assert onsuber.getOn(keys='d', on=0) == 'sun' + assert onsuber.remOn(keys='d', on=0) + + assert not os.path.exists(db.path) assert not db.opened @@ -763,6 +771,29 @@ def test_on_iodup_suber(): y = "Red apple" z = "White snow" + # test addOn remOn + assert onsuber.addOn(keys="z", on=0, val=w) + assert onsuber.addOn(keys="z", on=0, val=x) + assert onsuber.addOn(keys="z", on=1, val=y) + assert onsuber.addOn(keys="z", on=1, val=z) + + assert onsuber.cntOn(keys=("z",)) == 4 + + items = [item for item in onsuber.getOnItemIter(keys='z')] + assert items == [(('z',), 0, 'Blue dog'), + (('z',), 0, 'Green tree'), + (('z',), 1, 'Red apple'), + (('z',), 1, 'White snow')] + + assert onsuber.remOn(keys='z', on=0, val=w) + assert onsuber.remOn(keys='z', on=1) + items = [item for item in onsuber.getOnItemIter(keys='z')] + assert items == [(('z',), 0, 'Green tree')] + assert onsuber.remOn(keys='z', on=0, val=x) + + assert onsuber.cntOn(keys=("z",)) == 0 + + # test append assert 0 == onsuber.appendOn(keys=("a",), val=w) assert 1 == onsuber.appendOn(keys=("a",), val=x) @@ -1004,6 +1035,8 @@ def test_on_iodup_suber(): ] + + assert not os.path.exists(db.path) assert not db.opened From 9e06c8f2e19928062f3b0c37812e37980c9e5102 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 9 Sep 2024 20:06:34 -0600 Subject: [PATCH 349/418] refactore db.pdes to use new suber methods to make more convenient added unit tests --- src/keri/core/eventing.py | 6 +++--- src/keri/db/dbing.py | 2 +- src/keri/db/subing.py | 22 ++++++++++++++++++++++ tests/core/test_escrow.py | 6 +++--- tests/db/test_subing.py | 4 ++++ 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index a7bd66193..b244af681 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -3380,7 +3380,7 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, logger.debug(f"Kever state: Escrowed partially delegated event=\n" f"{serder.ked}\n.") - return self.db.pdes.add(keys=snKey(serder.preb, serder.sn), val=serder.saidb) + return self.db.pdes.addOn(keys=serder.pre, on=serder.sn, val=serder.said) def state(self): @@ -5976,7 +5976,7 @@ def processEscrowPartialDels(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow # removes one event escrow at key val - self.db.pdes.rem(keys=snKey(epre, esn), val=edig) # event idx escrow + self.db.pdes.remOn(keys=epre, on=esn, val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) @@ -5988,7 +5988,7 @@ def processEscrowPartialDels(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. # removes one event escrow at key val - self.db.pdes.rem(keys=snKey(epre, esn), val=edig) # event idx escrow + self.db.pdes.remOn(keys=epre, on=esn, val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 4bd1e67c0..667edb2b2 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1917,7 +1917,7 @@ def delOnIoDupVal(self, db, key, on=0, val=b'', sep=b'.'): # used in OnIoDupSuber def getOnIoDupValIter(self, db, key=b'', on=0, *, sep=b'.'): """ - Returns iterator of triples (key, on, val), at each key over all ordinal + Returns iterator of val at each key over all ordinal numbered keys with same key + sep + on in db. Values are sorted by onKey(key, on) where on is ordinal number int and key is prefix sans on. Values duplicates are sorted internally by hidden prefixed insertion order diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 4e6abf311..521c17886 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -2242,6 +2242,28 @@ def appendOn(self, keys: str | bytes | memoryview, val=self._ser(val), sep=self.sep.encode())) + + def getOn(self, keys: str | bytes | memoryview | Iterable, on: int = 0): + """ + Gets dup vals list at key made from keys + + Parameters: + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + on (int): ordinal number used with onKey(pre,on) to form key. + + Returns: + vals (list): each item in list is str + empty list if no entry at keys + + """ + return [self._des(val) for val in + self.db.getOnIoDupValIter(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode())] + + def remOn(self, keys: str | bytes | memoryview | Iterable, on: int=0, val: str | bytes | memoryview = ''): """ diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index 7808bdc13..b6d9b4dfb 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -527,7 +527,7 @@ def test_missing_delegator_escrow(): psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) assert not bobPre in watKvy.kevers assert not delPre in watKvy.kevers - escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) assert len(escrows) == 1 assert escrows[0] == delSrdr.said # escrow entry for event @@ -538,14 +538,14 @@ def test_missing_delegator_escrow(): assert watBobK.sn == 0 watKvy.processEscrows() assert not delPre in watKvy.kevers - escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) assert len(escrows) == 1 assert escrows[0] == delSrdr.said # escrow entry for event # Now apply Bob's ixn to wat's kvy and process escrow psr.parse(ims=bytearray(bobIxnMsg1), kvy=watKvy, local=False) watKvy.processEscrows() - escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) assert len(escrows) == 0 assert watBobK.sn == 1 diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 8ac6d5e52..58c72ccdc 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -773,9 +773,13 @@ def test_on_iodup_suber(): # test addOn remOn assert onsuber.addOn(keys="z", on=0, val=w) + assert onsuber.getOn(keys="z", on=0) == [w] assert onsuber.addOn(keys="z", on=0, val=x) + assert onsuber.getOn(keys="z", on=0) == [w, x] assert onsuber.addOn(keys="z", on=1, val=y) + assert onsuber.getOn(keys="z", on=1) == [y] assert onsuber.addOn(keys="z", on=1, val=z) + assert onsuber.getOn(keys="z", on=1) == [y, z] assert onsuber.cntOn(keys=("z",)) == 4 From c9f4efb16a7a91d1b0dc248494b347168bd7186c Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Wed, 11 Sep 2024 10:34:00 -0400 Subject: [PATCH 350/418] 1.2.0 release candidate Signed-off-by: Kevin Griffin --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4bf4ec2fe..926f97aba 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.0-dev13 +VERSION=1.2.0-rc1 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index a704266a4..4340dfa25 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-dev13', # also change in src/keri/__init__.py + version='1.2.0-rc1', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 09ed18bf2..a05d64ce8 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-dev13' # also change in setup.py +__version__ = '1.2.0-rc1' # also change in setup.py From b92bbeb977a7db8c937cbf576bc9b59f477409d4 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Fri, 13 Sep 2024 17:47:51 -0700 Subject: [PATCH 351/418] Minor update to parsing ESSR attachments. (#861) Signed-off-by: pfeairheller --- src/keri/core/parsing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index db0a86fa5..d186c69f1 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -9,7 +9,7 @@ from ..kering import Vrsn_1_0, Vrsn_2_0 from .coring import (Ilks, Seqner, Cigar, - Dater, Verfer, Prefixer, Saider, Pather, Texter) + Dater, Verfer, Prefixer, Saider, Pather, Matter) from .counting import Counter, Codens, CtrDex_1_0 from .indexing import (Siger, ) from . import serdering @@ -960,7 +960,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, elif ctr.code == CtrDex_1_0.ESSRPayloadGroup: for i in range(ctr.count): texter = yield from self._extractor(ims, - klas=Texter, + klas=Matter, cold=cold, abort=pipelined) essrs.append(texter) From c43002cfea7c360aa4ea2e6c53a8acc17c909ff1 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 13 Sep 2024 18:12:48 -0700 Subject: [PATCH 352/418] Version Bump --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 926f97aba..907de599d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.0-rc1 +VERSION=1.2.0-rc2 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index 4340dfa25..52bb722b4 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-rc1', # also change in src/keri/__init__.py + version='1.2.0-rc2', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index a05d64ce8..0d3f9095b 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-rc1' # also change in setup.py +__version__ = '1.2.0-rc2' # also change in setup.py From 5911c13caecf0bb1323fec5989b5a4d7f6739da2 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Fri, 4 Oct 2024 12:49:53 -0600 Subject: [PATCH 353/418] fix: update smids and rmids for group lead on new event (#858) The prior code would fix smids and rmids for all new rotation members on each rotation yet would not fix the smids and rmids for the group lead submitting the multisig event because of the early return when the serder arg is not present. This fixes that Signed-off-by: Kent Bull --- src/keri/app/cli/commands/multisig/rotate.py | 2 +- src/keri/app/habbing.py | 29 ++--- tests/app/test_grouping.py | 110 +++++++++++++++++-- 3 files changed, 118 insertions(+), 23 deletions(-) diff --git a/src/keri/app/cli/commands/multisig/rotate.py b/src/keri/app/cli/commands/multisig/rotate.py index 248882903..56f84ec97 100644 --- a/src/keri/app/cli/commands/multisig/rotate.py +++ b/src/keri/app/cli/commands/multisig/rotate.py @@ -205,7 +205,7 @@ def rotateDo(self, tymth, tock=0.0, **opts): seqner = coring.Seqner(sn=ghab.kever.sn+1) rot = ghab.rotate(isith=self.isith, nsith=self.nsith, toad=self.toad, cuts=list(self.cuts), adds=list(self.adds), data=self.data, - verfers=merfers, digers=migers) + verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) # Create a notification EXN message to send to the other agents diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index d98699707..ec2aa781f 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -2750,25 +2750,26 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, def rotate(self, smids=None, rmids=None, serder=None, **kwargs): - if serder is None: - return super(GroupHab, self).rotate(**kwargs) - if (habord := self.db.habs.get(keys=(self.pre,))) is None: raise kering.ValidationError(f"Missing HabitatRecord for pre={self.pre}") - # sign handles group hab with .mhab case - sigers = self.sign(ser=serder.raw, verfers=serder.verfers, rotated=True) + if serder is None: + msg = super(GroupHab, self).rotate(**kwargs) + else: - # update own key event verifier state - msg = eventing.messagize(serder, sigers=sigers) + # sign handles group hab with .mhab case + sigers = self.sign(ser=serder.raw, verfers=serder.verfers, rotated=True) - try: - self.kvy.processEvent(serder=serder, sigers=sigers) - except MissingSignatureError: - pass - except Exception as ex: - raise kering.ValidationError("Improper Habitat rotation for " - "pre={self.pre}.") from ex + # update own key event verifier state + msg = eventing.messagize(serder, sigers=sigers) + + try: + self.kvy.processEvent(serder=serder, sigers=sigers) + except MissingSignatureError: + pass + except Exception as ex: + raise kering.ValidationError("Improper Habitat rotation for " + "pre={self.pre}.") from ex self.smids = smids self.rmids = rmids diff --git a/tests/app/test_grouping.py b/tests/app/test_grouping.py index c50d9402d..39f8a1c4e 100644 --- a/tests/app/test_grouping.py +++ b/tests/app/test_grouping.py @@ -37,7 +37,7 @@ def test_counselor(): parsing.Parser().parse(ims=bytearray(icp3), kvy=kev2, local=True) smids = [hab1.pre, hab2.pre, hab3.pre] - rmids = None # need to fixe this + rmids = [hab1.pre, hab2.pre, hab3.pre] inits = dict(isith='["1/2", "1/2", "1/2"]', nsith='["1/2", "1/2", "1/2"]', toad=0, wits=[]) # Create group hab with init params @@ -84,7 +84,7 @@ def test_counselor(): migers = [hab1.kever.ndigers[0], hab2.kever.ndigers[0]] prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) - rot = ghab.rotate(isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rot = ghab.rotate(isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor.start(ghab=ghab, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -139,7 +139,7 @@ def test_counselor(): migers = [hab1.kever.ndigers[0], hab2.kever.ndigers[0], hab3.kever.ndigers[0]] prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) - rot = ghab.rotate(isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rot = ghab.rotate(isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor.start(ghab=ghab, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -195,7 +195,7 @@ def test_counselor(): migers = [hab1.kever.ndigers[0], hab3.kever.ndigers[0]] prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) - rot = ghab.rotate(isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rot = ghab.rotate(isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor.start(ghab=ghab, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -274,7 +274,7 @@ def test_the_seven(): parsing.Parser().parse(ims=bytearray(icp), kvy=kev, local=True) smids = [hab1.pre, hab2.pre, hab3.pre, hab4.pre, hab5.pre, hab6.pre, hab7.pre] - rmids = None # need to fixe this + rmids = [hab1.pre, hab2.pre, hab3.pre, hab4.pre, hab5.pre, hab6.pre, hab7.pre] inits = dict(isith='["1/3", "1/3", "1/3", "1/3", "1/3", "1/3", "1/3"]', nsith='["1/3", "1/3", "1/3", "1/3", "1/3", "1/3", "1/3"]', toad=0, wits=[]) @@ -377,7 +377,7 @@ def test_the_seven(): prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) rot = ghab.rotate(isith='["1/3", "1/3", "1/3"]', nsith='["1/3", "1/3", "1/3", "1/3", "1/3", "1/3", "1/3"]', - toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor.start(ghab=ghab, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -441,7 +441,7 @@ def test_the_seven(): prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) rot = ghab.rotate(isith='["1/3", "1/3", "1/3"]', nsith='["1/3", "1/3", "1/3", "1/3", "1/3", "1/3", "1/3"]', - toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor.start(ghab=ghab, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -520,7 +520,7 @@ def test_the_seven(): prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) rot = ghab4.rotate(isith='["1/3", "1/3", "1/3"]', nsith='["1/3", "1/3", "1/3"]', - toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor4.start(ghab=ghab4, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -673,6 +673,100 @@ def test_multisig_rotate(mockHelpingNowUTC): assert data["gid"] == ghab1.pre assert "rot" in exn.ked["e"] +def test_multisig_rotate_new_group_member_updates_smids(mockHelpingNowUTC): + # Create a multisig with three members, test_1, test_2, and test_3 + with openMultiSig(prefix="smidstest") as ((hby1, ghab1), (hby2, ghab2), (hby3, ghab3)): + # Create a new member, test_4 + with habbing.openHab(name="smidstest_4", salt=b'0123456789abcdef', transferable=True, temp=True) as (hby4, hab4): + icp4 = hab4.makeOwnEvent(sn=0) # Get test_4's inception event to introduce to group members + + hab1 = hby1.habByName("smidstest_1") + hab2 = hby2.habByName("smidstest_2") + hab3 = hby3.habByName("smidstest_3") + # Create member Kevery instances to parse each other's events and update their keystate + kev1 = eventing.Kevery(db=hab1.db, lax=True, local=False) + kev2 = eventing.Kevery(db=hab2.db, lax=True, local=False) + kev3 = eventing.Kevery(db=hab3.db, lax=True, local=False) + kev4 = eventing.Kevery(db=hab4.db, lax=True, local=False) + + # Introduce test_4 member to group by parsing test_4's inception event (latest key state) + parsing.Parser().parse(ims=bytearray(icp4), kvy=kev1) + parsing.Parser().parse(ims=bytearray(icp4), kvy=kev2) + parsing.Parser().parse(ims=bytearray(icp4), kvy=kev3) + # introduce each member to 4 + parsing.Parser().parse(ims=bytearray(hab1.makeOwnEvent(sn=0)), kvy=kev4) + parsing.Parser().parse(ims=bytearray(hab2.makeOwnEvent(sn=0)), kvy=kev4) + parsing.Parser().parse(ims=bytearray(hab3.makeOwnEvent(sn=0)), kvy=kev4) + + # rotate each individual hab to satisfy the rotation threshold with new keys + hab1.rotate() + hab2.rotate() + hab3.rotate() + + # Update keystate in each hab for each other member + rot1 = hab1.makeOwnEvent(sn=1) # get latest event for hab1 and update keystate for other members + parsing.Parser().parse(ims=bytearray(rot1), kvy=kev2) + parsing.Parser().parse(ims=bytearray(rot1), kvy=kev3) + parsing.Parser().parse(ims=bytearray(rot1), kvy=kev4) + + rot2 = hab2.makeOwnEvent(sn=1) # get latest event for hab2 and update keystate for other members + parsing.Parser().parse(ims=bytearray(rot2), kvy=kev1) + parsing.Parser().parse(ims=bytearray(rot2), kvy=kev3) + parsing.Parser().parse(ims=bytearray(rot2), kvy=kev4) + + rot3 = hab3.makeOwnEvent(sn=1) # get latest event for hab3 and update keystate for other members + parsing.Parser().parse(ims=bytearray(rot3), kvy=kev1) + parsing.Parser().parse(ims=bytearray(rot3), kvy=kev2) + parsing.Parser().parse(ims=bytearray(rot3), kvy=kev4) + + # create signing and rotation member AID lists for upcoming rotation + smids = [hab1.pre, hab2.pre, hab3.pre, hab4.pre] + rmids = [hab1.pre, hab2.pre, hab3.pre, hab4.pre] + + # make group hab for test_4 + ghab4 = hby4.joinGroupHab(hab4.pre, group="smidstest_group4", mhab=hab4, smids=smids, rmids=rmids) + + isith = '["1/4", "1/4", "1/4", "1/4"]' + nsith = '["1/4", "1/4", "1/4", "1/4"]' + merfers = [hab1.kever.verfers[0], hab2.kever.verfers[0], hab3.kever.verfers[0], hab4.kever.verfers[0]] + migers = [hab1.kever.ndigers[0], hab2.kever.ndigers[0], hab3.kever.ndigers[0], hab4.kever.ndigers[0]] + rot = ghab1.rotate(smids=smids, rmids=rmids, isith=isith, nsith=nsith, toad=None, cuts=None, adds=None, data=None, + verfers=merfers, digers=migers) + rserder = serdering.SerderKERI(raw=rot) + + # start counselor for group hab 1 + prefixer = coring.Prefixer(qb64=ghab1.pre) + seqner = coring.Seqner(sn=ghab1.kever.sn + 1) + saider = coring.Saider(qb64=rserder.said) + counselor = grouping.Counselor(hby=hby1) + counselor.start(ghab=ghab1, prefixer=prefixer, seqner=seqner, saider=saider) + + # hab2 signs rotation event and parse into hab1 Kevery + sigers2 = hab2.mgr.sign(rserder.raw, verfers=hab2.kever.verfers, indexed=True, indices=[1]) + msg2 = eventing.messagize(serder=rserder, sigers=sigers2) + parsing.Parser().parse(ims=bytearray(msg2), kvy=kev1) + + # hab3 signs rotation event and parse into hab1 Kevery + sigers3 = hab3.mgr.sign(rserder.raw, verfers=hab3.kever.verfers, indexed=True, indices=[2]) + msg3 = eventing.messagize(serder=rserder, sigers=sigers3) + parsing.Parser().parse(ims=bytearray(msg3), kvy=kev1) + + # hab4 signs rotation event and parse into hab1 Kevery. This should commit the event + sigers4 = hab4.mgr.sign(rserder.raw, verfers=hab4.kever.verfers, indexed=True, indices=[3]) + msg4 = eventing.messagize(serder=rserder, sigers=sigers4) + parsing.Parser().parse(ims=bytearray(msg4), kvy=kev1) + + kev1.processEscrows() # Runs escrows for Kevery1 so he processes all sigs together + + counselor.processEscrows() # Get the rest of the way through counselor. + assert counselor.complete(prefixer=prefixer, seqner=seqner, saider=saider) + assert ghab4.smids == smids + assert ghab4.rmids == rmids + hby1.loadHabs() + ghab1 = hby1.habByName("smidstest_group1") # reload hab to get updated smids and rmids values + assert ghab1.smids == smids + assert ghab1.rmids == rmids + def test_multisig_interact(mockHelpingNowUTC): with openMultiSig(prefix="test") as ((hby1, ghab1), (_, _), (_, _)): From 470a9abebe0dae7ffa5701f9ed4c2bb9bb614920 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Fri, 4 Oct 2024 12:14:00 -0700 Subject: [PATCH 354/418] Two command line changes: (#871) - OOBI resolve now waits until the AID is in Kevers before completing - Witness authenticate includes the CESR http header. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/oobi/resolve.py | 5 +++++ src/keri/app/cli/commands/witness/authenticate.py | 4 +++- src/keri/end/ending.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/keri/app/cli/commands/oobi/resolve.py b/src/keri/app/cli/commands/oobi/resolve.py index dc6e62dc4..747c67c50 100644 --- a/src/keri/app/cli/commands/oobi/resolve.py +++ b/src/keri/app/cli/commands/oobi/resolve.py @@ -107,6 +107,11 @@ def waitDo(self, tymth, tock=0.0): yield 0.25 obr = self.obi.hby.db.roobi.get(keys=(self.oobi,)) + if self.force: + while obr.cid not in self.hby.kevers: + self.hby.kvy.processEscrows() + yield 0.25 + print(self.oobi, obr.state) self.remove([self.hbyDoer, *self.obi.doers, *self.authn.doers]) diff --git a/src/keri/app/cli/commands/witness/authenticate.py b/src/keri/app/cli/commands/witness/authenticate.py index f0a2b4192..532415361 100644 --- a/src/keri/app/cli/commands/witness/authenticate.py +++ b/src/keri/app/cli/commands/witness/authenticate.py @@ -16,6 +16,7 @@ from keri.app import httping, connecting from keri.app.agenting import httpClient from keri.app.cli.common import existing +from keri.app.httping import CESR_DESTINATION_HEADER from keri.core import coring logger = help.ogler.getLogger() @@ -107,7 +108,8 @@ def authDo(self, tymth, tock=0.0): fargs['delkel'] = delkel.decode("utf-8") headers = (Hict([ - ("Content-Type", "multipart/form-data") + ("Content-Type", "multipart/form-data"), + (CESR_DESTINATION_HEADER, self.witness) ])) client, clientDoer = httpClient(self.hab, self.witness) diff --git a/src/keri/end/ending.py b/src/keri/end/ending.py index 6ff62ef65..72729316f 100644 --- a/src/keri/end/ending.py +++ b/src/keri/end/ending.py @@ -275,7 +275,7 @@ def siginput(name, method, path, headers, fields, hab=None, signers=None, expire expires (str): iso8601 formated date string indicating exiration of header signature signers (list): Optional signer objects used to sign the values hab (Hab): Optional Hab used to sign the values. One of signers or Hab is required - fields (str): Fields in request to sign. Includes special fields as well as Header fields + fields (list): Fields in request to sign. Includes special fields as well as Header fields headers (dict): HTTP request headers path (str): HTTP request path method (str): HTTP request method (POST, GET, PUT, etc) From 480d608428ba6c64053039345deda97872157722 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Fri, 4 Oct 2024 12:33:45 -0700 Subject: [PATCH 355/418] Version Bump --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 907de599d..1e343f073 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.0-rc2 +VERSION=1.2.0-rc3 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index 52bb722b4..e81cdf0a0 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-rc2', # also change in src/keri/__init__.py + version='1.2.0-rc3', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 0d3f9095b..a9d2b5e65 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-rc2' # also change in setup.py +__version__ = '1.2.0-rc3' # also change in setup.py From d98f43fea9c11578199f1dd842f4f7b5a07da16a Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Thu, 10 Oct 2024 09:13:55 -0700 Subject: [PATCH 356/418] Fix calls to out of order escrow (#873) * Fix calls to out of order escrow to include `local=local` to propagate the locality of the event across out of order escrowing. Signed-off-by: pfeairheller * Fix OOBI generation to favor HTTPS over HTTP and to honor the full URL of the service endpoint. Use the "Last" variant of the method for finding event seals in all locations. Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller --- scripts/demo/basic/challenge.sh | 3 +++ src/keri/app/cli/commands/did/generate.py | 16 ++++++++-------- src/keri/app/cli/commands/ipex/grant.py | 4 ++-- src/keri/app/cli/commands/oobi/generate.py | 22 ++++++++++------------ src/keri/app/delegating.py | 2 +- src/keri/app/grouping.py | 4 ++-- src/keri/app/oobiing.py | 11 +++++------ src/keri/app/querying.py | 3 +-- src/keri/core/eventing.py | 18 +++++++++--------- src/keri/db/basing.py | 1 - 10 files changed, 41 insertions(+), 43 deletions(-) diff --git a/scripts/demo/basic/challenge.sh b/scripts/demo/basic/challenge.sh index c3fcc6ccb..28d412855 100755 --- a/scripts/demo/basic/challenge.sh +++ b/scripts/demo/basic/challenge.sh @@ -11,6 +11,9 @@ kli ends add --name cha2 --alias cha2 --eid BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZE cha1_oobi="$(kli oobi generate --name cha1 --alias cha1 --role witness | sed -n '2 p')" cha2_oobi="$(kli oobi generate --name cha2 --alias cha2 --role witness | sed -n '2 p')" +echo "${cha1_oobi}" +echo "${cha2_oobi}" + kli oobi resolve --name cha1 --oobi-alias cha2 --oobi "${cha2_oobi}" kli oobi resolve --name cha2 --oobi-alias cha1 --oobi "${cha1_oobi}" diff --git a/src/keri/app/cli/commands/did/generate.py b/src/keri/app/cli/commands/did/generate.py index 0200d40ba..0aca604ce 100644 --- a/src/keri/app/cli/commands/did/generate.py +++ b/src/keri/app/cli/commands/did/generate.py @@ -73,20 +73,20 @@ def generate(tymth, tock=0.0, **opts): sys.exit(-1) wit = random.choice(hab.kever.wits) - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: raise kering.ConfigurationError(f"unable to query witness {wit}, no http endpoint") - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - enc = urllib.parse.quote_plus(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + enc = urllib.parse.quote_plus(f"{url.rstrip("/")}/oobi/{hab.pre}/witness") print(f"did:keri:{hab.pre}?oobi={enc}") elif role in (kering.Roles.controller,): - urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) if not urls: print(f"{alias} identifier {hab.pre} does not have any controller endpoints") return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - enc = urllib.parse.quote_plus(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + enc = urllib.parse.quote_plus(f"{url.rstrip("/")}/oobi/{hab.pre}/controller") print(f"did:keri:{hab.pre}?oobi={enc}") diff --git a/src/keri/app/cli/commands/ipex/grant.py b/src/keri/app/cli/commands/ipex/grant.py index cc766664a..5787089f6 100644 --- a/src/keri/app/cli/commands/ipex/grant.py +++ b/src/keri/app/cli/commands/ipex/grant.py @@ -111,8 +111,8 @@ def grantDo(self, tymth, tock=0.0): iserder = serdering.SerderKERI(raw=bytes(iss)) seqner = coring.Seqner(sn=iserder.sn) - serder = self.hby.db.fetchAllSealingEventByEventSeal(creder.sad['i'], - seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) + serder = self.hby.db.fetchLastSealingEventByEventSeal(creder.sad['i'], + seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) anc = self.hby.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said) exn, atc = protocoling.ipexGrantExn(hab=self.hab, recp=recp, message=self.message, acdc=acdc, iss=iss, anc=anc, diff --git a/src/keri/app/cli/commands/oobi/generate.py b/src/keri/app/cli/commands/oobi/generate.py index e35a53354..406d5f08f 100644 --- a/src/keri/app/cli/commands/oobi/generate.py +++ b/src/keri/app/cli/commands/oobi/generate.py @@ -4,7 +4,6 @@ """ import argparse -from urllib.parse import urlparse import sys from hio import help @@ -66,21 +65,21 @@ def generate(tymth, tock=0.0, **opts): sys.exit(-1) for wit in hab.kever.wits: - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: raise kering.ConfigurationError(f"unable to query witness {wit}, no http endpoint") - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - print(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + print(f"{url.rstrip("/")}/oobi/{hab.pre}/witness") elif role in (kering.Roles.controller,): - urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) if not urls: print(f"{alias} identifier {hab.pre} does not have any controller endpoints") return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - print(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + print(f"{url.rstrip("/")}/oobi/{hab.pre}/controller") elif role in (kering.Roles.mailbox,): for (_, _, eid), end in hab.db.ends.getItemIter(keys=(hab.pre, kering.Roles.mailbox, )): if not (end.allowed and end.enabled is not False): @@ -91,6 +90,5 @@ def generate(tymth, tock=0.0, **opts): if not urls: print(f"{alias} identifier {hab.pre} does not have any mailbox endpoints") return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - print(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/mailbox/{eid}") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + print(f"{url.rstrip("/")}/oobi/{hab.pre}/mailbox/{eid}") diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index bddb7dd4e..1e09e6596 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -138,7 +138,7 @@ def processUnanchoredEscrow(self): dkever = self.hby.kevers[kever.delpre] seal = dict(i=serder.pre, s=serder.snh, d=serder.said) - if dserder := self.hby.db.fetchAllSealingEventByEventSeal(dkever.prefixer.qb64, seal=seal): + if dserder := self.hby.db.fetchLastSealingEventByEventSeal(dkever.prefixer.qb64, seal=seal): seqner = coring.Seqner(sn=dserder.sn) couple = seqner.qb64b + dserder.saidb dgkey = dbing.dgKey(kever.prefixer.qb64b, kever.serder.saidb) diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 096798f73..f87ef2490 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -171,8 +171,8 @@ def processDelegateEscrow(self): self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) else: # Not witnesser, we need to look for the anchor and then wait for receipts - if serder := self.hby.db.fetchAllSealingEventByEventSeal(kever.delpre, - seal=anchor): + if serder := self.hby.db.fetchLastSealingEventByEventSeal(kever.delpre, + seal=anchor): aseq = coring.Seqner(sn=serder.sn) couple = aseq.qb64b + serder.saidb dgkey = dbing.dgKey(pre, saider.qb64b) diff --git a/src/keri/app/oobiing.py b/src/keri/app/oobiing.py index cb8b41d3a..7228b9187 100644 --- a/src/keri/app/oobiing.py +++ b/src/keri/app/oobiing.py @@ -115,15 +115,15 @@ def on_get_alias(self, req, rep, alias=None): if role in (kering.Roles.witness,): # Fetch URL OOBIs for all witnesses oobis = [] for wit in hab.kever.wits: - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: rep.status = falcon.HTTP_404 rep.text = f"unable to query witness {wit}, no http endpoint" return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - oobis.append(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness/{wit}") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + oobis.append(f"{url.rstrip("/")}/oobi/{hab.pre}/witness/{wit}") res["oobis"] = oobis elif role in (kering.Roles.controller,): # Fetch any controller URL OOBIs oobis = [] @@ -134,8 +134,7 @@ def on_get_alias(self, req, rep, alias=None): rep.text = f"unable to query controller {hab.pre}, no http endpoint" return url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - oobis.append(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + oobis.append(f"{url.rstrip("/")}/oobi/{hab.pre}/controller") res["oobis"] = oobis else: rep.status = falcon.HTTP_404 diff --git a/src/keri/app/querying.py b/src/keri/app/querying.py index b6a401ada..21ff01297 100644 --- a/src/keri/app/querying.py +++ b/src/keri/app/querying.py @@ -140,8 +140,7 @@ def recur(self, tyme, deeds=None): if self.pre not in self.hab.kevers: return False - kever = self.hab.kevers[self.pre] - if self.hby.db.fetchAllSealingEventByEventSeal(self.pre, seal=self.anchor): + if self.hby.db.fetchLastSealingEventByEventSeal(self.pre, seal=self.anchor): self.remove([self.witq]) return True diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index b244af681..3e8ba393f 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -3020,22 +3020,22 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): dserder = serdering.SerderKERI(raw=bytes(raw)) return dserder - elif eager: #missing aes but try to find seal by walking delegator's KEL + elif eager: # missing aes but try to find seal by walking delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict if original: # search all events in delegator's kel not just last - if not (dserder:=self.db.fetchAllSealingEventByEventSeal(pre=delpre, - seal=seal)): + if not (dserder := self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal)): # database broken this should never happen so do not validate # since original must have been validated so it must have # all its delegation chain. raise ValidationError(f"Missing delegation source seal for {serder.ked}") - else: # only search last events in delegator's kel - if not (dserder:=self.db.fetchLastSealingEventByEventSeal(pre=delpre, - seal=seal)): + else: # only search last events in delegator's kel + if not (dserder := self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal)): # superseding delegation may not have happened yet so escrow # ToDo XXXX need to cue up to get latest events in # delegator's kel. - #raise ValidationError(f"Missing delegation source seal for {serder.ked}") + # raise ValidationError(f"Missing delegation source seal for {serder.ked}") return None # Only repair .aess when found delegation is for delegated event that @@ -3788,7 +3788,7 @@ def processEvent(self, serder, sigers, *, wigers=None, else: # not inception so can't verify sigs etc, add to out-of-order escrow self.escrowOOEvent(serder=serder, sigers=sigers, - seqner=delseqner, saider=delsaider, wigers=wigers) + seqner=delseqner, saider=delsaider, wigers=wigers, local=local) raise OutOfOrderError("Out-of-order event={}.".format(ked)) else: # already accepted inception event for pre so already first seen @@ -3829,7 +3829,7 @@ def processEvent(self, serder, sigers, *, wigers=None, if sn > sno: # sn later than sno so out of order escrow # escrow out-of-order event self.escrowOOEvent(serder=serder, sigers=sigers, - seqner=delseqner, saider=delsaider, wigers=wigers) + seqner=delseqner, saider=delsaider, wigers=wigers, local=local) raise OutOfOrderError("Out-of-order event={}.".format(ked)) elif ((sn == sno) or # inorder event (ixn, rot, drt) or diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 14e9bfb53..98f109312 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1700,7 +1700,6 @@ def fetchAllSealingEventByEventSeal(self, pre, seal, sn=0): # use alias here until can change everywhere for backwards compatibility findAnchoringSealEvent = fetchAllSealingEventByEventSeal # alias - def fetchLastSealingEventByEventSeal(self, pre, seal, sn=0): """ Search through a KEL for the last event at any sn but that contains a From e3d0ff5b5949f81cd261fe5d1a2288d2be91f58f Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Thu, 10 Oct 2024 10:27:56 -0600 Subject: [PATCH 357/418] feat: migrations for 0.6.7 to 1.2.0 (#875) * feat: migrations for 0.6.7 to 1.2.0 * refactor: use db object, not cli args for Reger --- src/keri/app/cli/commands/migrate.py | 162 ------------------ src/keri/app/cli/commands/migrate/list.py | 4 +- src/keri/app/cli/commands/migrate/run.py | 12 +- src/keri/db/basing.py | 42 +++-- .../add_key_and_reg_state_schemas.py | 147 ++++++++++++++++ src/keri/db/migrations/hab_data_rename.py | 109 ++++++++++++ src/keri/db/migrations/rekey_habs.py | 24 ++- 7 files changed, 313 insertions(+), 187 deletions(-) delete mode 100644 src/keri/app/cli/commands/migrate.py create mode 100644 src/keri/db/migrations/add_key_and_reg_state_schemas.py create mode 100644 src/keri/db/migrations/hab_data_rename.py diff --git a/src/keri/app/cli/commands/migrate.py b/src/keri/app/cli/commands/migrate.py deleted file mode 100644 index 878587790..000000000 --- a/src/keri/app/cli/commands/migrate.py +++ /dev/null @@ -1,162 +0,0 @@ -# -*- encoding: utf-8 -*- -""" -KERI -keri.kli.commands module - -""" -import argparse - -from hio import help -from hio.base import doing - -from keri import kering -from keri.app.cli.common import existing -from keri.core import coring, serdering -from keri.db import koming, subing, dbing -from keri.db.basing import KeyStateRecord, StateEERecord -from keri.kering import ConfigurationError, Version -from keri.vdr import viring - -logger = help.ogler.getLogger() - -parser = argparse.ArgumentParser(description='View status of a local AID') -parser.set_defaults(handler=lambda args: handler(args), - transferable=True) -parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) -parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', - required=False, default="") -parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', - dest="bran", default=None) # passcode => bran -parser.add_argument('--force', action="store_true", required=False, - help='True means perform migration without prompting the user') - - -def handler(args): - if not args.force: - print() - print("This command will migrate your datastore to the next version of KERIpy and is not reversible.") - print("After this command, you will not be able to access your data store with this version.") - print() - yn = input("Are you sure you want to continue? [y|N]: ") - - if yn not in ("y", "Y"): - print("...exiting") - return [] - - kwa = dict(args=args) - return [doing.doify(migrate, **kwa)] - - -def migrate(tymth, tock=0.0, **opts): - """ Command line status handler - - """ - _ = (yield tock) - args = opts["args"] - name = args.name - base = args.base - bran = args.bran - - try: - with dbing.openLMDB(name=name, base=base, bran=bran, temp=False) as db: - print(db.path) - states = koming.Komer(db=db, - schema=dict, - subkey='stts.') - nstates = koming.Komer(db=db, - schema=KeyStateRecord, - subkey='stts.') - - for keys, sad in states.getItemIter(): - ksr = KeyStateRecord( - vn=Version, # version number as list [major, minor] - i=sad['i'], # qb64 prefix - s=sad['s'], # lowercase hex string no leading zeros - p=sad['p'], - d=sad['d'], - f=sad['f'], # lowercase hex string no leading zeros - dt=sad['dt'], - et=sad['et'], - kt=sad['kt'], - k=sad['k'], - nt=sad['nt'], - n=sad['n'], - bt=sad['bt'], - b=sad['b'], - c=sad['c'], - ee=StateEERecord._fromdict(sad['ee']), # latest est event dict - di=sad['di'] if sad['di'] else None - ) - - nstates.pin(keys=keys, val=ksr) - - with existing.existingHby(name=name, base=base, bran=bran) as hby: - rgy = viring.Reger(name=name, base=base, db=hby.db, temp=False, - reopen=True) - - rstates = koming.Komer(db=rgy, - schema=dict, - subkey='stts.') - - for _, sad in rstates.getItemIter(): - rsr = viring.RegStateRecord( - vn=list(Version), # version number as list [major, minor] - i=sad['i'], # qb64 registry SAID - s=sad['s'], # lowercase hex string no leading zeros - d=sad['d'], - ii=sad['ii'], - dt=sad['dt'], - et=sad['et'], - bt=sad['bt'], # hex string no leading zeros lowercase - b=sad['b'], # list of qb64 may be empty - c=sad['c'], - ) - # ksr = stateFromKever(kever) - rgy.states.pin(sad['i'], val=rsr) - - for (said,), _ in rgy.creds.getItemIter(): - snkey = dbing.snKey(said, 0) - dig = rgy.getTel(key=snkey) - - prefixer = coring.Prefixer(qb64=said) - seqner = coring.Seqner(sn=0) - saider = coring.Saider(qb64b=bytes(dig)) - rgy.cancs.pin(keys=said, val=[prefixer, seqner, saider]) - - migrateKeys(hby.db) - - # clear escrows - print("clearing escrows") - hby.db.gpwe.trim() - hby.db.gdee.trim() - hby.db.dpwe.trim() - hby.db.gpse.trim() - hby.db.epse.trim() - hby.db.dune.trim() - - except ConfigurationError: - print(f"identifier prefix for {name} does not exist, incept must be run first", ) - return -1 - - -def migrateKeys(db): - # public keys mapped to the AID and event seq no they appeared in - pubs = subing.CatCesrIoSetSuber(db=db, subkey="pubs.", - klas=(coring.Prefixer, coring.Seqner)) - - # next key digests mapped to the AID and event seq no they appeared in - digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", - klas=(coring.Prefixer, coring.Seqner)) - - for pre, fn, dig in db.getFelItemAllPreIter(): - dgkey = dbing.dgKey(pre, dig) # get message - if not (raw := db.getEvt(key=dgkey)): - raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) - serder = serdering.SerderKERI(raw=bytes(raw)) - val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) - verfers = serder.verfers or [] - for verfer in verfers: - pubs.add(keys=(verfer.qb64,), val=val) - ndigers = serder.ndigers or [] - for diger in ndigers: - digs.add(keys=(diger.qb64,), val=val) diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py index 104fa8baf..152850436 100644 --- a/src/keri/app/cli/commands/migrate/list.py +++ b/src/keri/app/cli/commands/migrate/list.py @@ -16,7 +16,7 @@ def handler(args): """ - Launch KERI database initialization + List local LMDB database migrations and their completion status Args: args(Namespace): arguments object from command line @@ -25,7 +25,7 @@ def handler(args): return [lister] -parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser = argparse.ArgumentParser(description='Lists the local LMDB migrations and their completion status') parser.set_defaults(handler=handler, transferable=True) diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py index 9a9e4c818..03d92ebc5 100644 --- a/src/keri/app/cli/commands/migrate/run.py +++ b/src/keri/app/cli/commands/migrate/run.py @@ -18,16 +18,16 @@ def handler(args): """ - Launch KERI database initialization + Launch KERI database migrator Args: args(Namespace): arguments object from command line """ - clean = MigrateDoer(args) - return [clean] + migrator = MigrateDoer(args) + return [migrator] -parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore up to the latest source code version') parser.set_defaults(handler=handler, transferable=True) @@ -60,8 +60,8 @@ def recur(self, tyme): except kering.DatabaseError: pass - print("Migrating...") + print(f"Migrating {self.args.name}...") db.migrate() - print("Finished") + print(f"Finished migrating {self.args.name}") return True diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 98f109312..14c184bd5 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -49,7 +49,9 @@ MIGRATIONS = [ - ("1.1.0", ["rekey_habs"]) + ("0.6.8", ["hab_data_rename"]), + ("1.0.0", ["add_key_and_reg_state_schemas"]), + ("1.2.0", ["rekey_habs"]) ] @@ -1312,7 +1314,7 @@ def reload(self): """ # Check migrations to see if this database is up to date. Error otherwise if not self.current: - raise kering.DatabaseError("Database migrations must be run.") + raise kering.DatabaseError(f"Database migrations must be run. DB version {self.version}; current {keri.__version__}") removes = [] for keys, data in self.habs.getItemIter(): @@ -1346,10 +1348,18 @@ def migrate(self): """ for (version, migrations) in MIGRATIONS: - # Check to see if this is for an older version + # Only run migration if current source code version is at or below the migration version + ver = semver.VersionInfo.parse(keri.__version__) + ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) + if self.version is not None and semver.compare(version, ver_no_prerelease) > 0: + print( + f"Skipping migration {version} as higher than the current KERI version {keri.__version__}") + continue + # Skip migrations already run - where version less than (-1) or equal to (0) database version if self.version is not None and semver.compare(version, self.version) != 1: continue + print(f"Migrating database v{self.version} --> v{version}") for migration in migrations: modName = f"keri.db.migrations.{migration}" if self.migs.get(keys=(migration,)) is not None: @@ -1357,13 +1367,13 @@ def migrate(self): mod = importlib.import_module(modName) try: - print(f"running migration {modName}") mod.migrate(self) except Exception as e: - print(f"\nAbandoning migration {migration} with error: {e}") + print(f"\nAbandoning migration {migration} at version {version} with error: {e}") return self.migs.pin(keys=(migration,), val=coring.Dater()) + self.version = version # update database version after successful migration self.version = keri.__version__ @@ -1382,13 +1392,16 @@ def current(self): return True # If database version is ahead of library version, throw exception - if self.version is not None and semver.compare(self.version, keri.__version__) == 1: + ver = semver.VersionInfo.parse(keri.__version__) + ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) + if self.version is not None and semver.compare(self.version, ver_no_prerelease) == 1: raise kering.ConfigurationError( f"Database version={self.version} is ahead of library version={keri.__version__}") last = MIGRATIONS[-1] - # If we aren't at latest version, but there are no outstanding migrations, reset version to latest - if self.migs.get(keys=(last[1][0],)) is not None: + # If we aren't at latest version, but there are no outstanding migrations, + # reset version to latest (rightmost (-1) migration is latest) + if self.migs.get(keys=(last[1][-1],)) is not None: return True # We have migrations to run @@ -1407,12 +1420,15 @@ def complete(self, name=None): migrations = [] if not name: for version, migs in MIGRATIONS: - for mig in migs: - dater = self.migs.get(keys=(mig,)) - migrations.append((mig, dater)) + # Print entries only for migrations that have been run + if self.version is not None and semver.compare(version, self.version) <= 0: + for mig in migs: + dater = self.migs.get(keys=(mig,)) + migrations.append((mig, dater)) else: - if name not in MIGRATIONS or not self.migs.get(keys=(name,)): - raise ValueError(f"No migration named {name}") + for version, migs in MIGRATIONS: # check all migrations for each version + if name not in migs or not self.migs.get(keys=(name,)): + raise ValueError(f"No migration named {name}") migrations.append((name, self.migs.get(keys=(name,)))) return migrations diff --git a/src/keri/db/migrations/add_key_and_reg_state_schemas.py b/src/keri/db/migrations/add_key_and_reg_state_schemas.py new file mode 100644 index 000000000..3a632f466 --- /dev/null +++ b/src/keri/db/migrations/add_key_and_reg_state_schemas.py @@ -0,0 +1,147 @@ +from keri import help +from keri.core import coring, serdering +from keri.db import koming, subing, dbing +from keri.db.basing import StateEERecord, KeyStateRecord +from keri.db.dbing import dgKey, splitKey +from keri.kering import ConfigurationError, Version +from keri.vdr import viring + +logger = help.ogler.getLogger() + +def _check_if_needed(db): + states = koming.Komer(db=db, + schema=dict, + subkey='stts.') + first = next(states.getItemIter(), None) + if first is None: + return False + keys, sad = first + if 'vn' in sad: + return False + return True + +def migrate(db): + """Adds schema for KeyStateRecord, RegStateRecord, and migrates the rgy.cancs., hby.db.pubs., + and hby.db.digs. to be up to date as of 2022-??-?? + This migration performs the following: + - hby.db -> "stts." schema from dict -> KeyStateRecord + - rgy -> "stts." schema from dict -> RegStateRecord + - rgy -> "cancs." reset to (ACDC SAID, SN 0, TEL evt 0 digest) + - hby.db -> "pubs." and + hby.db -> "digs." + that don't exist are populated with verification keys and event digests for the first seen events and + Keys: + "pubs." Verfer of each Verfer for each FEL event + "digs." Diger of next Diger (ndiger) of each FEL event + Value: (prefix, sn) of each event + Parameters: + db(Baser): Baser database object on which to run the migration + """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, database already in correct state") + return + + try: + logger.debug(f"Migrating keystate and regstate dict to schema for {db.path}") + states = koming.Komer(db=db, + schema=dict, + subkey='stts.') + nstates = koming.Komer(db=db, + schema=KeyStateRecord, + subkey='stts.') + + for keys, sad in states.getItemIter(): + ksr = KeyStateRecord( + vn=Version, # version number as list [major, minor] + i=sad['i'], # qb64 prefix + s=sad['s'], # lowercase hex string no leading zeros + p=sad['p'], + d=sad['d'], + f=sad['f'], # lowercase hex string no leading zeros + dt=sad['dt'], + et=sad['et'], + kt=sad['kt'], + k=sad['k'], + nt=sad['nt'], + n=sad['n'], + bt=sad['bt'], + b=sad['b'], + c=sad['c'], + ee=StateEERecord._fromdict(sad['ee']), # latest est event dict + di=sad['di'] if sad['di'] else None + ) + + nstates.pin(keys=keys, val=ksr) + + rgy = viring.Reger(name=db.name, base=db.base, db=db, temp=db.temp, reopen=True) + + rstates = koming.Komer(db=rgy, + schema=dict, + subkey='stts.') + + for _, sad in rstates.getItemIter(): + rsr = viring.RegStateRecord( + vn=list(Version), # version number as list [major, minor] + i=sad['i'], # qb64 registry SAID + s=sad['s'], # lowercase hex string no leading zeros + d=sad['d'], + ii=sad['ii'], + dt=sad['dt'], + et=sad['et'], + bt=sad['bt'], # hex string no leading zeros lowercase + b=sad['b'], # list of qb64 may be empty + c=sad['c'], + ) + # ksr = stateFromKever(kever) + rgy.states.pin(sad['i'], val=rsr) + + for (said,), _ in rgy.saved.getItemIter(): + snkey = dbing.snKey(said, 0) + dig = rgy.getTel(key=snkey) + + prefixer = coring.Prefixer(qb64=said) + seqner = coring.Seqner(sn=0) + saider = coring.Saider(qb64b=bytes(dig)) + rgy.cancs.pin(keys=said, val=[prefixer, seqner, saider]) + + migrateKeys(db) + + # clear escrows + logger.info("clearing escrows") + db.gpwe.trim() + db.gdee.trim() + db.dpwe.trim() + db.gpse.trim() + db.epse.trim() + db.dune.trim() + db.qnfs.trim() + + except ConfigurationError: + logger.error(f"identifier prefix for {db.name} does not exist, incept must be run first", ) + return -1 + + +def migrateKeys(db): + # public keys mapped to the AID and event seq no they appeared in + pubs = subing.CatCesrIoSetSuber(db=db, subkey="pubs.", + klas=(coring.Prefixer, coring.Seqner)) + + # next key digests mapped to the AID and event seq no they appeared in + digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", + klas=(coring.Prefixer, coring.Seqner)) + + for pre, fn, dig in db.getFelItemAllPreIter(): + dgkey = dbing.dgKey(pre, dig) # get message + if not (raw := db.getEvt(key=dgkey)): + logger.info(f"Migrate keys: missing event for dig={dig}, skipped.") + continue + serder = serdering.SerderKERI(raw=bytes(raw)) + val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) + verfers = serder.verfers or [] + for verfer in verfers: + pubs.add(keys=(verfer.qb64,), val=val) + ndigers = serder.ndigers or [] + for diger in ndigers: + digs.add(keys=(diger.qb64,), val=val) \ No newline at end of file diff --git a/src/keri/db/migrations/hab_data_rename.py b/src/keri/db/migrations/hab_data_rename.py new file mode 100644 index 000000000..09e377966 --- /dev/null +++ b/src/keri/db/migrations/hab_data_rename.py @@ -0,0 +1,109 @@ +from dataclasses import dataclass, field, asdict +from typing import Optional + +from keri.db import koming, basing +from keri.db.basing import HabitatRecord, Baser +from keri.vdr.viring import Reger + + +@dataclass +class HabitatRecordV0_6_7: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + prefix (str): identifier prefix of hab qb64 + pid (str | None): group member identifier qb64 when hid is group + aids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + """ + prefix: str # aid qb64 + pid: Optional[str] # participant aid of group aid + aids: Optional[list] # all identifiers participating in the group identity + + watchers: list[str] = field(default_factory=list) # aids qb64 of watchers + +@dataclass +class HabitatRecordV0_6_8: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + hid (str): identifier prefix of hab qb64 + mid (str | None): group member identifier qb64 when hid is group + smids (list | None): group signing member identifiers qb64 when hid is group + rmids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + + + """ + hid: str # hab own identifier prefix qb64 + mid: str | None = None # group member identifier qb64 when hid is group + smids: list | None = None # group signing member ids when hid is group + rmids: list | None = None # group rotating member ids when hid is group + sid: str | None = None # Signify identifier qb64 when hid is Signify + watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers + +def _check_if_needed(db): + """ + Check if the migration is needed + Parameters: + db(Baser): Baser database object on which to run the migration + Returns: + bool: True if the migration is needed, False otherwise + """ + habs = koming.Komer(db=db, subkey='habs.', schema=dict, ) + first = next(habs.getItemIter(), None) + if first is None: + return False + name, habord = first + if 'prefix' in habord: + return True + return False + +def migrate(db): + """Rename data in HabitatRecord from the old labels to the new labels as of 2022-10-17 + + This migration performs the following: + 1. rename prefix -> hid + 2. rename pid -> mid + 3. rename aids -> smids, rmids + + Parameters: + db(Baser): Baser database object on which to run the migration + """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, database already in correct state") + return + + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_7, ) + + habords = dict() + # Update Hab records from .habs with name + for name, habord in habs.getItemIter(): + existing = asdict(habord) + habord_0_6_7 = HabitatRecordV0_6_7(**existing) + habord_0_6_8 = HabitatRecordV0_6_8( + hid=habord_0_6_7.prefix, + mid=habord_0_6_7.pid, + smids=habord_0_6_7.aids, + rmids=habord_0_6_7.aids, + sid=None, + watchers=habord_0_6_7.watchers + ) + habords[habord_0_6_8.hid] = habord_0_6_8 + + habs.trim() # remove existing records + + # Add in the renamed records + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_8, ) + + for pre, habord in habords.items(): + habs.pin(keys=(pre,), val=habord) + diff --git a/src/keri/db/migrations/rekey_habs.py b/src/keri/db/migrations/rekey_habs.py index e21dfdb9f..4d84b99b3 100644 --- a/src/keri/db/migrations/rekey_habs.py +++ b/src/keri/db/migrations/rekey_habs.py @@ -4,7 +4,7 @@ @dataclass -class OldHabitatRecord: # baser.habs +class HabitatRecordV0_6_8: # baser.habs """ Habitat application state information keyed by habitat name (baser.habs) @@ -24,6 +24,17 @@ class OldHabitatRecord: # baser.habs sid: str | None = None # Signify identifier qb64 when hid is Signify watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers +def _check_if_needed(db): + habs = koming.Komer(db=db, + subkey='habs.', + schema=dict, ) + first = next(habs.getItemIter(), None) + if first is None: + return False + name, habord = first + if 'domain' in habord: + return False + return True def migrate(db): """ Re-key habs migration for changing the key for .habs and introducing the .names database @@ -36,16 +47,21 @@ def migrate(db): Parameters: db(Baser): Baser database object on which to run the migration - """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, database already in correct state") + return + habs = koming.Komer(db=db, subkey='habs.', - schema=OldHabitatRecord, ) + schema=HabitatRecordV0_6_8, ) # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix nmsp = koming.Komer(db=db, subkey='nmsp.', - schema=OldHabitatRecord, ) + schema=HabitatRecordV0_6_8, ) habords = dict() # Update Hab records from .habs with name From 68d862c64f4c23be09e0c25123f04a8b5253e534 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Thu, 10 Oct 2024 09:52:16 -0700 Subject: [PATCH 358/418] Version Bump --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1e343f073..de6389b3e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.0-rc3 +VERSION=1.2.0-rc4 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index e81cdf0a0..97d395049 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-rc3', # also change in src/keri/__init__.py + version='1.2.0-rc4', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index a9d2b5e65..01d63d664 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-rc3' # also change in setup.py +__version__ = '1.2.0-rc4' # also change in setup.py From 9b5323eca7dbceea66c63f6ec792bdb15d98ba90 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Mon, 18 Nov 2024 07:29:42 -0800 Subject: [PATCH 359/418] Last bit of clean up for V1.2.0 rc4 (#890) * Fix delegation "catchup" logic for witnesses during delegated rotation. Fix witness authenticate to handle service endpoint urls with a trailing slash. Signed-off-by: pfeairheller * Minor fix to parsing for local delegators and 3 typo fixes Signed-off-by: pfeairheller --------- Signed-off-by: pfeairheller --- src/keri/app/agenting.py | 4 ++-- src/keri/app/cli/commands/interact.py | 2 +- src/keri/app/cli/commands/rotate.py | 2 +- src/keri/app/cli/commands/witness/authenticate.py | 4 ++-- src/keri/core/eventing.py | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 334e3e17d..890fe8172 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -70,7 +70,7 @@ def receipt(self, pre, sn=None, auths=None): ser = serdering.SerderKERI(raw=msg) # If we are a rotation event, may need to catch new witnesses up to current key state - if ser.ked['t'] in (coring.Ilks.rot,): + if ser.ked['t'] in (coring.Ilks.rot, coring.Ilks.drt,): adds = ser.ked["ba"] for wit in adds: yield from self.catchup(ser.pre, wit) @@ -351,7 +351,7 @@ def receiptDo(self, tymth=None, tock=0.0): witer.msgs.append(bytearray(dmsg)) if ser.ked['t'] in (coring.Ilks.icp, coring.Ilks.dip) or \ - "ba" in ser.ked and wit in ser.ked["ba"]: # Newly added witness, must send full KEL to catch up + "ba" in ser.ked and wit in ser.ked["ba"]: # Newly added witness, must catch up for fmsg in hab.db.clonePreIter(pre=pre): witer.msgs.append(bytearray(fmsg)) diff --git a/src/keri/app/cli/commands/interact.py b/src/keri/app/cli/commands/interact.py index 07a98ea54..138143511 100644 --- a/src/keri/app/cli/commands/interact.py +++ b/src/keri/app/cli/commands/interact.py @@ -119,7 +119,7 @@ def interactDo(self, tymth, tock=0.0, **opts): for wit in hab.kever.wits: if wit in auths: continue - code = input(f"Entire code for {wit}: ") + code = input(f"Enter code for {wit}: ") auths[wit] = f"{code}#{helping.nowIso8601()}" if self.endpoint: diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index 6edb0955a..84d32f20b 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -210,7 +210,7 @@ def rotateDo(self, tymth, tock=0.0): for wit in hab.kever.wits: if wit in auths: continue - code = input(f"Entire code for {wit}: ") + code = input(f"Enter code for {wit}: ") auths[wit] = f"{code}#{helping.nowIso8601()}" if hab.kever.delpre: diff --git a/src/keri/app/cli/commands/witness/authenticate.py b/src/keri/app/cli/commands/witness/authenticate.py index 532415361..81a4d341e 100644 --- a/src/keri/app/cli/commands/witness/authenticate.py +++ b/src/keri/app/cli/commands/witness/authenticate.py @@ -117,7 +117,7 @@ def authDo(self, tymth, tock=0.0): client.request( method="POST", - path=f"{client.requester.path}/aids", + path=f"{client.requester.path.rstrip('/')}/aids", headers=headers, fargs=fargs ) @@ -131,7 +131,7 @@ def authDo(self, tymth, tock=0.0): totp = data["totp"] m = coring.Matter(qb64=totp) # refactor this to use cipher d = coring.Matter(qb64=self.hab.decrypt(ser=m.raw)) - otpurl = f"otpauth://totp/KERIpy:{self.witness}?secret={d.raw.decode('utf-8')}&issuer=KERIpy" + otpurl = f"otpauth://totp/KERI:{self.witness}?secret={d.raw.decode('utf-8')}&issuer=KERI" if not self.urlOnly: qr = qrcode.QRCode() diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 3e8ba393f..d1bcaf745 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2379,9 +2379,9 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # seal in this case can't be malicious since sourced locally. # Doesn't get to here until fully signed and witnessed. - if self.locallyDelegated(delpre): # local delegator + if self.locallyDelegated(delpre) and not self.locallyOwned(): # local delegator # must be local if locallyDelegated or caught above as misfit - if delseqner is None or delsaider is None: # missing delegation seal + if delseqner is None or delsaider is None: # missing delegation seal # so escrow delegable. So local delegator can approve OOB. # and create delegator event with valid event seal of this # delegated event and then reprocess event with attached source From 31d985d482d2cb7c340dfe52ef8554de06f7ed02 Mon Sep 17 00:00:00 2001 From: pfeairheller Date: Mon, 18 Nov 2024 07:53:40 -0800 Subject: [PATCH 360/418] Version Bump to release 1.2.0 Signed-off-by: pfeairheller --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index de6389b3e..7e6a53bcb 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.0-rc4 +VERSION=1.2.0 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index 97d395049..a466e63cf 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0-rc4', # also change in src/keri/__init__.py + version='1.2.0', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 01d63d664..e8823ef7d 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0-rc4' # also change in setup.py +__version__ = '1.2.0' # also change in setup.py From 89f1bd34cd91ba5cb719804892c4233f3b4991da Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Fri, 13 Dec 2024 09:06:55 -0500 Subject: [PATCH 361/418] pass through salts for vc create to facilitate command line multisig Signed-off-by: Kevin Griffin --- src/keri/app/cli/commands/vc/create.py | 21 +++++++++++++++++---- src/keri/vc/proving.py | 21 ++++++++++----------- src/keri/vdr/credentialing.py | 15 +++++++++++---- tests/vc/test_proving.py | 6 +++--- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/keri/app/cli/commands/vc/create.py b/src/keri/app/cli/commands/vc/create.py index b16e9a178..294e0257b 100644 --- a/src/keri/app/cli/commands/vc/create.py +++ b/src/keri/app/cli/commands/vc/create.py @@ -1,5 +1,6 @@ import argparse import json +from typing import Optional from hio import help from hio.base import doing @@ -38,6 +39,10 @@ parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) parser.add_argument("--private", help="flag to indicate if this credential needs privacy preserving features", action="store_true") +parser.add_argument("--private-credential-nonce", help="nonce for vc", + action="store_true") +parser.add_argument("--private-subject-nonce", help="nonce for subject", + action="store_true") parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--time", help="timestamp for the credential creation", required=False, default=None) @@ -101,7 +106,10 @@ def issueCredential(args): rules=rules, credential=credential, timestamp=args.time, - private=args.private) + private=args.private, + private_credential_nonce=args.private_credential_nonce, + private_subject_nonce=args.private_subject_nonce, + ) doers = [issueDoer] return doers @@ -114,7 +122,8 @@ class CredentialIssuer(doing.DoDoer): """ def __init__(self, name, alias, base, bran, registryName=None, schema=None, edges=None, recipient=None, data=None, - rules=None, credential=None, timestamp=None, private=False): + rules=None, credential=None, timestamp=None, private:bool=False, private_credential_nonce:Optional[str]=None, + private_subject_nonce:Optional[str]=None,): """ Create DoDoer for issuing a credential and managing the processes needed to complete issuance Parameters: @@ -126,7 +135,9 @@ def __init__(self, name, alias, base, bran, registryName=None, schema=None, edge data: (dict) credential data dict credential: (dict) full credential to issue when joining a multisig issuance out (str): Filename for credential output - private: (bool) privacy preserving + private (bool): apply nonce used for privacy preserving ACDC + private_credential_nonce (Optional[str]): nonce used for privacy vc + private_subject_nonce (Optional[str]): nonce used for subject """ self.name = name @@ -175,7 +186,9 @@ def __init__(self, name, alias, base, bran, registryName=None, schema=None, edge source=edges, rules=rules, data=data, - private=private) + private=private, + private_credential_nonce=private_credential_nonce, + private_subject_nonce=private_subject_nonce) else: self.creder = serdering.SerderACDC(sad=credential) # proving.Creder(ked=credential) self.credentialer.validate(creder=self.creder) diff --git a/src/keri/vc/proving.py b/src/keri/vc/proving.py index bd0a3e015..bf6efb055 100644 --- a/src/keri/vc/proving.py +++ b/src/keri/vc/proving.py @@ -4,17 +4,14 @@ """ -from collections.abc import Iterable -from typing import Union - -from .. import help +from typing import Optional from .. import core +from .. import help from ..core import coring, serdering from ..core.coring import (Kinds, versify) -from ..db import subing -from ..kering import Version from ..help import helping +from ..kering import Version KERI_REGISTRY_TYPE = "KERICredentialRegistry" @@ -25,8 +22,9 @@ def credential(schema, issuer, data, recipient=None, - private=False, - salt=None, + private:bool=False, + private_credential_nonce:Optional[str]=None, + private_subject_nonce:Optional[str]=None, status=None, source=None, rules=None, @@ -42,7 +40,8 @@ def credential(schema, recipient (Option[str|None]): qb64 identifier prefix of the recipient data (dict): of the values being assigned to the subject of this credential private (bool): apply nonce used for privacy preserving ACDC - salt (string): salt for nonce + private_credential_nonce (Optional[str]): nonce used for privacy vc + private_subject_nonce (Optional[str]): nonce used for subject source (dict | list): of source credentials to which this credential is chained rules (dict | list): ACDC rules section for credential version (Version): version instance @@ -64,8 +63,8 @@ def credential(schema, ) if private: - vc["u"] = salt if salt is not None else core.Salter().qb64 - subject["u"] = salt if salt is not None else core.Salter().qb64 + vc["u"] = private_credential_nonce if private_credential_nonce is not None else core.Salter().qb64 + subject["u"] = private_subject_nonce if private_subject_nonce is not None else core.Salter().qb64 if recipient is not None: subject['i'] = recipient diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index 416bb0af0..93c47e6a7 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -5,14 +5,16 @@ VC issuer support """ +from typing import Optional + from hio.base import doing from hio.help import decking from keri.vdr import viring -from .. import kering, help +from .. import help +from .. import kering, core from ..app import agenting from ..app.habbing import GroupHab -from .. import kering, core from ..core import parsing, coring, scheming, serdering from ..core.coring import Seqner, MtrDex from ..core.eventing import TraitDex @@ -772,7 +774,8 @@ def __init__(self, hby, rgy, registrar, verifier): super(Credentialer, self).__init__(doers=doers) - def create(self, regname, recp: str, schema, source, rules, data, private=False): + def create(self, regname, recp: str, schema, source, rules, data, private: bool = False, + private_credential_nonce: Optional[str] = None, private_subject_nonce: Optional[str] = None): """ Create and validate a credential returning the fully populated Creder Parameters: @@ -782,7 +785,9 @@ def create(self, regname, recp: str, schema, source, rules, data, private=False) source: rules: data: - private: add nonce for privacy preserving + private (bool): apply nonce used for privacy preserving ACDC + private_credential_nonce (Optional[str]): nonce used for privacy vc + private_subject_nonce (Optional[str]): nonce used for subject Returns: Creder: Creder class for the issued credential @@ -803,6 +808,8 @@ def create(self, regname, recp: str, schema, source, rules, data, private=False) data=data, source=source, private=private, + private_credential_nonce=private_credential_nonce, + private_subject_nonce=private_subject_nonce, rules=rules, status=registry.regk) self.validate(creder) diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index 798478be3..649bc2351 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -253,17 +253,17 @@ def test_privacy_preserving_credential(mockHelpingNowIso8601): engagementContextRole="Project Manager", ) - salt = core.Salter(raw=b'0123456789abcdef').qb64 cred = credential(schema="EZllThM1rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q", recipient="EM_S2MdMaKgP6P2Yyno6-flV6GqrwPencTIw8tCMR7iB", private=True, - salt=salt, + private_credential_nonce=core.Salter(raw=b'0123456789abcdef').qb64, + private_subject_nonce=core.Salter(raw=b'abcdef0123456789').qb64, issuer="EMZeK1yLZd1JV6Ktdq_YUt-YbyoTWB9UMcFzuiDly2Y6", data=d, status="ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M") assert cred.size == len(cred.raw) assert "u" in cred.sad - print(cred.raw) + assert cred.raw == (b'{"v":"ACDC10JSON00021c_","d":"ELFOCm58xUlId994cS6m6bsfYOkNHEKoe15Cav-Sj8__",' b'"u":"0AAwMTIzNDU2Nzg5YWJjZGVm","i":"EMZeK1yLZd1JV6Ktdq_YUt-YbyoTWB9UMcFzuiDl' b'y2Y6","ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EZllThM1rLBSM' From a7adf393cdb57cccddd96253746f88c41b1952f5 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Fri, 13 Dec 2024 14:38:23 -0500 Subject: [PATCH 362/418] fixes nonce pass through to support multisig vc create with --private. Version bump. Signed-off-by: Kevin Griffin --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- tests/vc/test_proving.py | 16 ++++++++-------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 7e6a53bcb..22ee28c91 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.0 +VERSION=1.2.1 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index a466e63cf..5584535eb 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.0', # also change in src/keri/__init__.py + version='1.2.1', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index e8823ef7d..3167ff791 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.0' # also change in setup.py +__version__ = '1.2.1' # also change in setup.py diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index 649bc2351..383e38e5b 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -264,14 +264,14 @@ def test_privacy_preserving_credential(mockHelpingNowIso8601): assert cred.size == len(cred.raw) assert "u" in cred.sad - assert cred.raw == (b'{"v":"ACDC10JSON00021c_","d":"ELFOCm58xUlId994cS6m6bsfYOkNHEKoe15Cav-Sj8__",' - b'"u":"0AAwMTIzNDU2Nzg5YWJjZGVm","i":"EMZeK1yLZd1JV6Ktdq_YUt-YbyoTWB9UMcFzuiDl' - b'y2Y6","ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EZllThM1rLBSM' - b'Z_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EFwWs1d_fe_VeLZ0vQQKO-gkRvGrpfWAR' - b'bI4e9tzcqlV","u":"0AAwMTIzNDU2Nzg5YWJjZGVm","i":"EM_S2MdMaKgP6P2Yyno6-flV6Gq' - b'rwPencTIw8tCMR7iB","dt":"2021-06-27T21:26:21.233257+00:00","LEI":"254900OPPU' - b'84GM83MG36","personLegalName":"John Doe","engagementContextRole":"Project Ma' - b'nager"}}') + assert cred.raw == (b'{"v":"ACDC10JSON00021c_","d":"EMMDzhHHlpQP0XNMRThDeIFkYD1WkDHF7Tp-8kt8X5pn",' + b'"u":"0AAwMTIzNDU2Nzg5YWJjZGVm","i":"EMZeK1yLZd1JV6Ktdq_YUt-YbyoTWB9UMcFzuiDly2Y6",' + b'"ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M",' + b'"s":"EZllThM1rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q",' + b'"a":{"d":"EK3MRnlg-bMUnHtYKyZ8HD_IbBeI0v4N8YB4UnNVBqrv","u":"0ABhYmNkZWYwMTIzNDU2Nzg5",' + b'"i":"EM_S2MdMaKgP6P2Yyno6-flV6GqrwPencTIw8tCMR7iB","dt":"2021-06-27T21:26:21.233257+00:00",' + b'"LEI":"254900OPPU84GM83MG36","personLegalName":"John Doe","engagementContextRole":"Project ' + b'Manager"}}') """End Test""" From 69fe0fce1daac4a875392ac64701623d484a6a52 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Fri, 13 Dec 2024 16:26:33 -0500 Subject: [PATCH 363/418] updates type hints for all Signed-off-by: Kevin Griffin --- src/keri/vc/proving.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/keri/vc/proving.py b/src/keri/vc/proving.py index bf6efb055..5f92717fa 100644 --- a/src/keri/vc/proving.py +++ b/src/keri/vc/proving.py @@ -4,7 +4,7 @@ """ -from typing import Optional +from typing import Optional, Union from .. import core from .. import help @@ -18,37 +18,37 @@ logger = help.ogler.getLogger() -def credential(schema, - issuer, - data, - recipient=None, +def credential(schema:str, + issuer:str, + data:dict, + recipient:Optional[str]=None, private:bool=False, private_credential_nonce:Optional[str]=None, private_subject_nonce:Optional[str]=None, - status=None, - source=None, - rules=None, - version=Version, - kind=Kinds.json): + status:str=None, + source:Union[dict, list]=None, + rules:Union[dict, list]=None, + version:Version=Version, + kind:Kinds=Kinds.json): """Utility function to create an ACDC. Creates dict SAD for credential from parameters and Saidifyies it before creation. Parameters: - schema (SAID): of schema for this credential + schema (str): SAID of schema for this credential issuer (str): qb64 identifier prefix of the issuer - status (str): qb64 said of the credential registry - recipient (Option[str|None]): qb64 identifier prefix of the recipient data (dict): of the values being assigned to the subject of this credential + recipient (Optional[str]): qb64 identifier prefix of the recipient private (bool): apply nonce used for privacy preserving ACDC private_credential_nonce (Optional[str]): nonce used for privacy vc private_subject_nonce (Optional[str]): nonce used for subject - source (dict | list): of source credentials to which this credential is chained - rules (dict | list): ACDC rules section for credential + status (str): qb64 said of the credential registry + source (Union[dict, list]): of source credentials to which this credential is chained + rules (Union[dict, list]): ACDC rules section for credential version (Version): version instance - kind (Serials): serialization kind + kind (Kinds): serialization kind Returns: - SerderACDC: credential instance + serdering.SerderACDC: credential instance """ vs = versify(protocol=coring.Protocols.acdc, version=version, kind=kind, size=0) From e787333b0ff381048f5116585c57802eac0b3655 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Thu, 2 Jan 2025 09:09:04 -0500 Subject: [PATCH 364/418] fixes command line args (#918) Signed-off-by: Kevin Griffin --- src/keri/app/cli/commands/vc/create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keri/app/cli/commands/vc/create.py b/src/keri/app/cli/commands/vc/create.py index 294e0257b..bf0584332 100644 --- a/src/keri/app/cli/commands/vc/create.py +++ b/src/keri/app/cli/commands/vc/create.py @@ -40,9 +40,9 @@ parser.add_argument("--private", help="flag to indicate if this credential needs privacy preserving features", action="store_true") parser.add_argument("--private-credential-nonce", help="nonce for vc", - action="store_true") + action="store") parser.add_argument("--private-subject-nonce", help="nonce for subject", - action="store_true") + action="store") parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--time", help="timestamp for the credential creation", required=False, default=None) From 60ef7489cd12e95b785064853ac691973c73ad29 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Thu, 2 Jan 2025 09:12:30 -0500 Subject: [PATCH 365/418] version bump Signed-off-by: Kevin Griffin --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 22ee28c91..08fed7d1b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.1 +VERSION=1.2.2 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index 5584535eb..155e66f81 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.1', # also change in src/keri/__init__.py + version='1.2.2', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 3167ff791..8e44a52b4 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.1' # also change in setup.py +__version__ = '1.2.2' # also change in setup.py From 22e24a6b39f14a850c3257254e9ac08e98be9612 Mon Sep 17 00:00:00 2001 From: Arshdeep Singh Date: Wed, 8 Jan 2025 19:49:48 +0530 Subject: [PATCH 366/418] feat: add endpoint in witness to query tel event of an aid (#887) * feat: add endpoint in witness to query tel event of an aid - added /query endpoint in witness agent to query for TEL or KEL events using HTTP GET request Signed-off-by: arshdeep singh * cleanup imports Signed-off-by: arshdeep singh --------- Signed-off-by: arshdeep singh --- src/keri/app/indirecting.py | 103 ++++++++++++++++++++++++++++ tests/app/test_indirecting.py | 124 ++++++++++++++++++++++++++++++++-- 2 files changed, 222 insertions(+), 5 deletions(-) diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 482d79124..e531a4402 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -88,6 +88,8 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo app.add_route("/", httpEnd) receiptEnd = ReceiptEnd(hab=hab, inbound=cues, aids=aids) app.add_route("/receipts", receiptEnd) + queryEnd = QueryEnd(hab=hab) + app.add_route("/query", queryEnd) server = createHttpServer(host, httpPort, app, keypath, certpath, cafilepath) if not server.reopen(): @@ -1183,3 +1185,104 @@ def interceptDo(self, tymth=None, tock=0.0): yield self.tock yield self.tock + + +class QueryEnd: + """ Endpoint class for quering witness for KELs and TELs using HTTP GET + + """ + + def __init__(self, hab): + self.hab = hab + self.reger = viring.Reger(name=hab.name, db=hab.db, temp=False) + + def on_get(self, req, rep): + """ Handles GET requests to query KEL or TEL events of a pre from a witness. + + Parameters: + req (Request) Falcon HTTP request + rep (Response) Falcon HTTP response + + Query Parameters: + typ (string): The type of event data to query for. Accepted values are: + - 'kel': Retrieve KEL events for a specified 'pre'. + - 'tel': Retrieve TEL events based on 'reg' or 'vcid'. + pre (string, optional): For 'kel' queries, the specific 'pre' to query. + sn (int, optional): For "kel" queries. If provided, returns events with seq-num + greater than or equal to `sn`. + reg (string, optional): For 'tel' queries, registry pre. required if `vcid` is not provided. + vcid (string, optional): For 'tel' queries, credential said. required if `reg` is not provided. + + Response: + - 200 OK: Returns event data in "application/json+cesr" format. + - 400 Bad Request: Returned if required query parameters are missing or if an invalid `typ` is specified. + + Example: + - /query?typ=kel&pre=ELZ1KBCFOmdj1RPu6kMUnzgMBTl4YsHfpw7wIGvLgW5W + - /query?typ=kel&pre=ELZ1KBCFOmdj1RPu6kMUnzgMBTl4YsHfpw7wIGvLgW5W&sn=5 + - /query?typ=tel®=EHrbPfpRLU9wpFXTzGY-LIo2FjMiljjEnt238eWHb7yZ&vcid=EO5y0jMXS5XKTYBKjCUPmNKPr1FWcWhtKwB2Go2ozvr0 + + """ + + typ = req.get_param("typ") + + if not typ: + raise falcon.HTTPBadRequest(description="'typ' query param is required") + + if typ == "kel": + pre = req.get_param("pre") + + if not pre: + raise falcon.HTTPBadRequest(description="'pre' query param is required") + + evnts = bytearray() + + sn = req.get_param_as_int("sn") + if sn is not None: ## query for event with seq-num >= sn + preb = pre.encode("utf-8") + dig = self.hab.db.getKeLast(key=dbing.snKey(pre=preb, + sn=sn)) + if dig is None: + raise falcon.HTTPBadRequest(description=f"non-existant event at seq-num {sn}") + + for dig in self.hab.db.getKelIter(pre, sn=sn): + try: + msg = self.hab.db.cloneEvtMsg(pre=pre, fn=0, dig=dig) + except Exception: + continue # skip this event + evnts.extend(msg) + else: + for msg in self.hab.db.clonePreIter(pre=pre): + evnts.extend(msg) + + + rep.set_header('Content-Type', "application/json+cesr") + rep.status = falcon.HTTP_200 + rep.data = bytes(evnts) + + elif typ == "tel": + regk = req.get_param("reg") + vcid = req.get_param("vcid") + + if not regk and not vcid: + raise falcon.HTTPBadRequest(description="Either 'reg' or 'vcid' query param is required for TEL query") + + evnts = bytearray() + if regk is not None: + cloner = self.reger.clonePreIter(pre=regk) + for msg in cloner: + evnts.extend(msg) + + if vcid is not None: + cloner = self.reger.clonePreIter(pre=vcid) + for msg in cloner: + evnts.extend(msg) + + rep.set_header('Content-Type', "application/json+cesr") + rep.status = falcon.HTTP_200 + rep.data = bytes(evnts) + + else: + rep.set_header('Content-Type', "application/json") + rep.text = "unkown query type." + rep.status = falcon.HTTP_400 \ No newline at end of file diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index 07377d761..43815925c 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -4,15 +4,20 @@ """ import json +import time import falcon +from falcon import testing import hio import pytest -from hio.core import tcp, http + +from hio.core import http +from hio.base import doing, tyming from hio.help import decking -from keri.app import indirecting, storing, habbing -from keri.core import coring, serdering +from keri import kering +from keri import core +from keri.app import indirecting, storing, habbing, agenting def test_mailbox_iter(): @@ -104,9 +109,9 @@ def test_qrymailbox_iter(): with habbing.openHab(name="test", transferable=True, temp=True, salt=b'0123456789abcdef') as (hby, hab): assert hab.pre == 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3' icp = hab.makeOwnInception() - icpSrdr = serdering.SerderKERI(raw=icp) + icpSrdr = core.serdering.SerderKERI(raw=icp) qry = hab.query(pre=hab.pre, src=hab.pre, route="/mbx") - srdr = serdering.SerderKERI(raw=qry) + srdr = core.serdering.SerderKERI(raw=qry) cues = decking.Deck() mbx = storing.Mailboxer(temp=True) @@ -152,6 +157,114 @@ def test_qrymailbox_iter(): next(mbi) +def test_wit_query_ends(seeder): + with habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby: + + wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, tcpPort=5634, httpPort=5644) + witDoer = agenting.Receiptor(hby=palHby) + + wesHab = wesHby.habByName(name="wes") + seeder.seedWitEnds(palHby.db, witHabs=[wesHab], protocols=[kering.Schemes.http]) + + app = falcon.App() + query_endpoint = indirecting.QueryEnd(wesHab) + app.add_route("/query", query_endpoint) + + wesClient = testing.TestClient(app) + + opts = dict( + wesHab=wesHab, + palHby=palHby, + witDoer=witDoer, + wesClient=wesClient + ) + + doers = wesDoers + [witDoer, doing.doify(wit_querier_test_do, **opts)] + + limit = 1.0 + tock = 0.03125 + doist = doing.Doist(tock=tock, limit=limit, doers=doers) + doist.enter() + + tymer = tyming.Tymer(tymth=doist.tymen(), duration=doist.limit) + + while not tymer.expired: + doist.recur() + time.sleep(doist.tock) + # doist.do(doers=doers) + + assert doist.limit == limit + + doist.exit() + + +def wit_querier_test_do(tymth=None, tock=0.0, **opts): + yield tock # enter context + + wesHab = opts["wesHab"] + palHby = opts["palHby"] + witDoer = opts["witDoer"] + wesClient = opts["wesClient"] + + palHab = palHby.makeHab(name="pal", wits=[wesHab.pre], transferable=True) + + assert palHab.pre == "EEWz3RVIvbGWw4VJC7JEZnGCLPYx4-QgWOwAzGnw-g8y" + + witDoer.msgs.append(dict(pre=palHab.pre)) + while not witDoer.cues: + yield tock + + witDoer.cues.popleft() + msg = next(wesHab.db.clonePreIter(pre=palHab.pre)) + + # Test valid KEL query with 'pre' + res = wesClient.simulate_get("/query", params={"typ": "kel", "pre": palHab.pre}) + assert res.status_code == 200 + assert res.headers['Content-Type'] == "application/json+cesr" + assert bytearray(res.content) == bytearray(msg) + + # Test KEL query without 'pre' + res = wesClient.simulate_get("/query", params={"typ": "kel"}) + assert res.status_code == 400 + assert res.headers['Content-Type'] == "application/json" + assert "'pre' query param is required" in res.text + + # Test KEL query with 'sn' parameter + res = wesClient.simulate_get("/query", params={"typ": "kel", "pre": palHab.pre, "sn": 0}) + assert res.status_code == 200 + assert res.headers['Content-Type'] == "application/json+cesr" + + # Test KEL query with non-existant 'sn' parameter + res = wesClient.simulate_get("/query", params={"typ": "kel", "pre": palHab.pre, "sn": 5}) + assert res.status_code == 400 + assert res.headers['Content-Type'] == "application/json" + assert "non-existant event at seq-num 5" in res.text + + # Test valid TEL query with 'reg' + res = wesClient.simulate_get("/query", params={"typ": "tel", "reg": "mock_reg"}) + assert res.status_code == 200 + assert res.headers['Content-Type'] == "application/json+cesr" + + # Test valid TEL query with 'vcid' + res = wesClient.simulate_get("/query", params={"typ": "tel", "vcid": "mock_vcid"}) + assert res.status_code == 200 + assert res.headers['Content-Type'] == "application/json+cesr" + + # Test TEL query missing both 'reg' and 'vcid' + res = wesClient.simulate_get("/query", params={"typ": "tel"}) + assert res.status_code == 400 + assert res.headers['Content-Type'] == "application/json" + assert "Either 'reg' or 'vcid' query param is required for TEL query" in res.text + + # Test invalid 'typ' parameter + res = wesClient.simulate_get("/query", params={"typ": "invalid"}) + assert res.status_code == 400 + assert res.headers['Content-Type'] == "application/json" + assert "unkown query type" in res.text + + + class MockServerTls: def __init__(self, certify, keypath, certpath, cafilepath, port): pass @@ -183,3 +296,4 @@ def test_createHttpServer(monkeypatch): if __name__ == "__main__": test_mailbox_iter() test_qrymailbox_iter() + test_wit_query_ends() From 3a7ba5e1eaa651004cb5db27ce0fd3eaea5d444b Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 13 Jan 2025 12:02:09 -0500 Subject: [PATCH 367/418] version bump Signed-off-by: Kevin Griffin --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 08fed7d1b..8b2250e5f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.2 +VERSION=1.2.3 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index 155e66f81..7b1c02a82 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.2', # also change in src/keri/__init__.py + version='1.2.3', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 8e44a52b4..b88b62144 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.2' # also change in setup.py +__version__ = '1.2.3' # also change in setup.py From d9bf2e83121258f9249182fa1e04e8d866a6e23c Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Fri, 17 Jan 2025 11:39:29 -0500 Subject: [PATCH 368/418] adds escrow clear (#926) Signed-off-by: Kevin Griffin --- src/keri/app/cli/commands/escrow/__init__.py | 3 + src/keri/app/cli/commands/escrow/clear.py | 52 ++++++ src/keri/app/cli/commands/escrow/list.py | 159 +++++++++++++++++++ src/keri/db/basing.py | 40 ++++- tests/app/cli/test_kli_commands.py | 2 +- tests/db/test_basing.py | 110 +++++++++++-- 6 files changed, 347 insertions(+), 19 deletions(-) create mode 100644 src/keri/app/cli/commands/escrow/__init__.py create mode 100644 src/keri/app/cli/commands/escrow/clear.py create mode 100644 src/keri/app/cli/commands/escrow/list.py diff --git a/src/keri/app/cli/commands/escrow/__init__.py b/src/keri/app/cli/commands/escrow/__init__.py new file mode 100644 index 000000000..defc725e9 --- /dev/null +++ b/src/keri/app/cli/commands/escrow/__init__.py @@ -0,0 +1,3 @@ +import argparse + +parser = argparse.ArgumentParser(description="A collection of escrow operations") \ No newline at end of file diff --git a/src/keri/app/cli/commands/escrow/clear.py b/src/keri/app/cli/commands/escrow/clear.py new file mode 100644 index 000000000..5d030b91f --- /dev/null +++ b/src/keri/app/cli/commands/escrow/clear.py @@ -0,0 +1,52 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.escrow module + +""" +import argparse + +from hio.base import doing +from keri import help +from keri.app.cli.common import existing + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Clear escrows') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--force', action="store_true", required=False, + help='True means perform clear without prompting the user') + + +def handler(args): + if not args.force: + print() + print("This command will clear all escrows and is not reversible.") + print() + yn = input("Are you sure you want to continue? [y|N]: ") + + if yn not in ("y", "Y"): + print("...exiting") + return [] + + kwa = dict(args=args) + return [doing.doify(clear, **kwa)] + + +def clear(tymth, tock=0.0, **opts): + """ Command line clear handler + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + + with existing.existingHby(name=name, base=base, bran=bran) as hby: + hby.db.clearEscrows() diff --git a/src/keri/app/cli/commands/escrow/list.py b/src/keri/app/cli/commands/escrow/list.py new file mode 100644 index 000000000..418a82e62 --- /dev/null +++ b/src/keri/app/cli/commands/escrow/list.py @@ -0,0 +1,159 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.escrow module + +""" +import argparse +import json + +from hio.base import doing + +from keri import help +from keri.core import eventing +from keri.app.cli.common import existing +from keri.db import dbing +from keri.kering import ConfigurationError +from keri.vdr import viring + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Views events in escrow state.') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran + +parser.add_argument("--escrow", "-e", help="show values for one specific escrow", default=None) + + +def handler(args): + """ Command line escrow handler + + """ + kwa = dict(args=args) + return [doing.doify(escrows, **kwa)] + + +def escrows(tymth, tock=0.0, **opts): + _ = (yield tock) + + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + escrow = args.escrow + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + reger = viring.Reger(name=hby.name, db=hby.db, temp=False) + + escrows = dict() + if (not escrow) or escrow == "out-of-order-events": + oots = list() + key = ekey = b'' # both start same. when not same means escrows found + while True: + for ekey, edig in hby.db.getOoeItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item + + try: + oots.append(eventing.loadEvent(hby.db, pre, edig)) + except ValueError as e: + raise e + + if ekey == key: # still same so no escrows found on last while iteration + break + key = ekey # setup next while iteration, with key after ekey + + escrows["out-of-order-events"] = oots + + if (not escrow) or escrow == "partially-witnessed-events": + pwes = list() + key = ekey = b'' # both start same. when not same means escrows found + while True: # break when done + for ekey, edig in hby.db.getPweItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item + + try: + pwes.append(eventing.loadEvent(hby.db, pre, edig)) + except ValueError as e: + raise e + + if ekey == key: # still same so no escrows found on last while iteration + break + key = ekey # setup next while iteration, with key after ekey + + escrows["partially-witnessed-events"] = pwes + + if (not escrow) or escrow == "partially-signed-events": + pses = list() + key = ekey = b'' # both start same. when not same means escrows found + while True: # break when done + for ekey, edig in hby.db.getPseItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item + + try: + pses.append(eventing.loadEvent(hby.db, pre, edig)) + except ValueError as e: + raise e + + if ekey == key: # still same so no escrows found on last while iteration + break + key = ekey # setup next while iteration, with key after ekey + + escrows["partially-signed-events"] = pses + + if (not escrow) or escrow == "likely-duplicitous-events": + ldes = list() + key = ekey = b'' # both start same. when not same means escrows found + while True: # break when done + for ekey, edig in hby.db.getLdeItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item + + try: + ldes.append(eventing.loadEvent(hby.db, pre, edig)) + except ValueError as e: + raise e + + if ekey == key: # still same so no escrows found on last while iteration + break + key = ekey # setup next while iteration, with key after ekey + + escrows["likely-duplicitous-events"] = ldes + + if (not escrow) or escrow == "missing-registry-escrow": + creds = list() + for (said,), dater in reger.mre.getItemIter(): + creder, *_ = reger.cloneCred(said) + creds.append(creder.sad) + + escrows["missing-registry-escrow"] = creds + + if (not escrow) or escrow == "broken-chain-escrow": + creds = list() + for (said,), dater in reger.mce.getItemIter(): + creder, *_ = reger.cloneCred(said) + creds.append(creder.sad) + + escrows["broken-chain-escrow"] = creds + + if (not escrow) or escrow == "missing-schema-escrow": + creds = list() + for (said,), dater in reger.mse.getItemIter(): + creder, *_ = reger.cloneCred(said) + creds.append(creder.sad) + + escrows["missing-schema-escrow"] = creds + + print(json.dumps(escrows, indent=2)) + + if not(escrow) or escrow == 'tel-partial-witness-escrow': + for (regk, snq), (prefixer, seqner, saider) in reger.tpwe.getItemIter(): + pass + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 14c184bd5..a452c676e 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1351,7 +1351,7 @@ def migrate(self): # Only run migration if current source code version is at or below the migration version ver = semver.VersionInfo.parse(keri.__version__) ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) - if self.version is not None and semver.compare(version, ver_no_prerelease) > 0: + if self.version is not None and semver.compare(version, str(ver_no_prerelease)) > 0: print( f"Skipping migration {version} as higher than the current KERI version {keri.__version__}") continue @@ -1367,16 +1367,45 @@ def migrate(self): mod = importlib.import_module(modName) try: + print(f"running migration {modName}") mod.migrate(self) except Exception as e: print(f"\nAbandoning migration {migration} at version {version} with error: {e}") return self.migs.pin(keys=(migration,), val=coring.Dater()) - self.version = version # update database version after successful migration + + # update database version after successful migration + self.version = version self.version = keri.__version__ + def clearEscrows(self): + """ + Clear all escrows + """ + for (k, _) in self.getUreItemIter(): + self.delUres(key=k) + for (k, _) in self.getVreItemIter(): + self.delVres(key=k) + for (k, _) in self.getPseItemIter(): + self.delPses(key=k) + for (k, _) in self.getPweItemIter(): + self.delPwes(key=k) + for (k, _) in self.getUweItemIter(): + self.delUwes(key=k) + for (k, _) in self.getOoeItemIter(): + self.delOoes(key=k) + for (k, _) in self.getLdeItemIter(): + self.delLdes(key=k) + for (pre, said), edig in self.qnfs.getItemIter(): + self.qnfs.rem(keys=(pre, said)) + + + for escrow in [self.qnfs, self.misfits, self.delegables, self.pdes, self.udes, self.rpes, self.epsd, self.eoobi, + self.dpub, self.gpwe, self.gdee, self.dpwe, self.gpse, self.epse, self.dune]: + escrow.trim() + @property def current(self): """ Current property determines if we are at the current database migration state. @@ -1391,10 +1420,9 @@ def current(self): if self.version == keri.__version__: return True - # If database version is ahead of library version, throw exception ver = semver.VersionInfo.parse(keri.__version__) ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) - if self.version is not None and semver.compare(self.version, ver_no_prerelease) == 1: + if self.version is not None and semver.compare(self.version, str(ver_no_prerelease)) == 1: raise kering.ConfigurationError( f"Database version={self.version} is ahead of library version={keri.__version__}") @@ -1426,7 +1454,7 @@ def complete(self, name=None): dater = self.migs.get(keys=(mig,)) migrations.append((mig, dater)) else: - for version, migs in MIGRATIONS: # check all migrations for each version + for version, migs in MIGRATIONS: # check all migrations for each version if name not in migs or not self.migs.get(keys=(name,)): raise ValueError(f"No migration named {name}") migrations.append((name, self.migs.get(keys=(name,)))) @@ -2349,7 +2377,7 @@ def getUreLast(self, key): def getUreItemIter(self, key=b''): """ - Use sgKey() + Use snKey() Return iterator of partial signed escrowed event triple items at next key after key. Items is (key, val) where proem has already been stripped from val diff --git a/tests/app/cli/test_kli_commands.py b/tests/app/cli/test_kli_commands.py index 94d960424..ec30511df 100644 --- a/tests/app/cli/test_kli_commands.py +++ b/tests/app/cli/test_kli_commands.py @@ -239,7 +239,7 @@ def test_standalone_kli_commands(helpers, capsys): '\t3. DEMwUl3u8mJ-cWxSnReA0rQesIgZ8SFoHp0U2WyiZjRt\n' '\n') - args = parser.parse_args(["escrow", "--name", "test"]) + args = parser.parse_args(["escrow", "list", "--name", "test"]) assert args.handler is not None doers = args.handler(args) directing.runController(doers=doers) diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 659403c21..97cc31958 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -7,28 +7,23 @@ import os from dataclasses import dataclass, asdict -import lmdb import pytest -from hio.base import doing - -from keri.help.helping import datify, dictify +import lmdb +from hio.base import doing from keri import core +from keri.app import habbing from keri.core import coring, eventing, serdering - -from keri.core.coring import Kinds, versify - +from keri.core.coring import Kinds, versify, Seqner from keri.core.eventing import incept, rotate, interact, Kever - -from keri.app import habbing - +from keri.core.serdering import Serder from keri.db import basing from keri.db import dbing from keri.db import subing -from keri.db.basing import openDB, Baser, KeyStateRecord +from keri.db.basing import openDB, Baser, KeyStateRecord, OobiRecord from keri.db.dbing import (dgKey, onKey, snKey) from keri.db.dbing import openLMDB - +from keri.help.helping import datify, dictify # this breaks when running as __main__ better to do a custom import call to # walk the directory tree and import explicity rather than depend on it # being a known package. Works with pytest because pytest contructs a path @@ -1824,6 +1819,97 @@ def test_KERI_BASER_MAP_SIZE_handles_bad_values(caplog): os.environ.pop("KERI_BASER_MAP_SIZE") +def test_clear_escrows(): + with openDB() as db: + key = b'A' + vals = [b"z", b"m", b"x", b"a"] + + db.putUres(key, vals) + db.putVres(key, vals) + db.putPses(key, vals) + db.putPwes(key, vals) + db.putUwes(key, vals) + db.putOoes(key, vals) + db.putLdes(key, vals) + + pre = b'k' + snh = b'snh' + saidb = b'saidb' + db.qnfs.add(keys=(pre, saidb), val=b"z") + assert db.qnfs.cnt(keys=(pre, saidb)) == 1 + + db.misfits.add(keys=(pre, snh), val=saidb) + assert db.misfits.cnt(keys=(pre, snh)) == 1 + + db.delegables.add(snKey(pre, 0), saidb) + assert db.delegables.cnt(keys=snKey(pre, 0)) == 1 + + db.pdes.addOn(keys=pre, on=0, val=saidb) + assert db.pdes.cnt(keys=snKey(pre, 0)) == 1 + + udesKey = dgKey('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc'.encode("utf-8"), + 'EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4'.encode("utf-8")) + db.udes.put(keys=udesKey, val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), + coring.Saider(qb64b=b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E'))) + assert db.udes.get(keys=udesKey) is not None + + saider = coring.Saider(qb64b='EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') + db.rpes.put(keys=('route',), vals=[saider]) + assert db.rpes.cnt(keys=('route',)) == 1 + + db.epsd.put(keys=('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc',), val=coring.Dater()) + assert db.epsd.get(keys=('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc',)) is not None + + db.eoobi.pin(keys=('url',), val=OobiRecord()) + assert db.eoobi.cntAll() == 1 + + serder = Serder(raw=b'{"v":"KERI10JSON0000cb_","t":"ixn","d":"EG8WAmM29ZBdoXbnb87yiPxQw4Y7gcQjqZS74vBAKsRm","i":"DApYGFaqnrALTyejaJaGAVhNpSCtqyerPqWVK9ZBNZk0","s":"4","p":"EAskHI462CuIMS_gNkcl_QewzrRSKH2p9zHQIO132Z30","a":[]}') + db.dpub.put(keys=(pre, 'said'), val=serder) + assert db.dpub.get(keys=(pre, 'said')) is not None + + db.gpwe.add(keys=(pre,), val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), saider)) + assert db.gpwe.cnt(keys=(pre,)) == 1 + + db.gdee.add(keys=(pre,), val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), saider)) + assert db.gdee.cnt(keys=(pre,)) == 1 + + db.dpwe.pin(keys=(pre, 'said'), val=serder) + assert db.dpwe.get(keys=(pre, 'said')) is not None + + db.gpse.add(keys=('qb64',), val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), saider)) + assert db.gpse.cnt(keys=('qb64',)) == 1 + + db.epse.put(keys=('dig',), val=serder) + assert db.epse.get(keys=('dig',)) is not None + + db.dune.pin(keys=(pre, 'said'), val=serder) + assert db.dune.get(keys=(pre, 'said')) is not None + + db.clearEscrows() + + assert db.getUres(key) == [] + assert db.getVres(key) == [] + assert db.getPses(key) == [] + assert db.getPwes(key) == [] + assert db.getUwes(key) == [] + assert db.getOoes(key) == [] + assert db.getLdes(key) == [] + assert db.qnfs.cnt(keys=(pre, saidb)) == 0 + assert db.misfits.cnt(keys=(pre, snh)) == 0 + assert db.delegables.cnt(keys=snKey(pre, 0)) == 0 + assert db.pdes.cnt(keys=snKey(pre, 0)) == 0 + assert db.udes.get(keys=udesKey) is None + assert db.rpes.cnt(keys=('route',)) == 0 + assert db.epsd.get(keys=('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc',)) is None + assert db.eoobi.cntAll() == 0 + assert db.dpub.get(keys=(pre, 'said')) is None + assert db.gpwe.cnt(keys=(pre,)) == 0 + assert db.gdee.cnt(keys=(pre,)) == 0 + assert db.dpwe.get(keys=(pre, 'said')) is None + assert db.gpse.cnt(keys=('qb64',)) == 0 + assert db.epse.get(keys=('dig',)) is None + assert db.dune.get(keys=(pre, 'said')) is None + if __name__ == "__main__": test_baser() test_clean_baser() From e5dbdc05367482655ff38fb733cb479de7375765 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Sun, 19 Jan 2025 19:06:35 -0500 Subject: [PATCH 369/418] version bump Signed-off-by: Kevin Griffin --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8b2250e5f..c1dc35f0e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.3 +VERSION=1.2.4 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index 7b1c02a82..5616bb0e7 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.3', # also change in src/keri/__init__.py + version='1.2.4', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index b88b62144..6c3f1ca01 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.3' # also change in setup.py +__version__ = '1.2.4' # also change in setup.py From f937eaf6ddc76110a7022f31e42efd519b95f710 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Mon, 3 Mar 2025 12:34:15 -0700 Subject: [PATCH 370/418] to 1.2.4: Add trace logging and consistent logging (#932) * fix: hio.help import -> keri.help * feat: add consistent logging for message bodies * feat: add TRACE log level and turn down some escrow logs * feat: add rich logging format string * refactor: remove class name prefixes and log format string --- src/keri/app/cli/commands/aid.py | 2 +- .../app/cli/commands/challenge/generate.py | 2 +- .../app/cli/commands/challenge/respond.py | 2 + src/keri/app/cli/commands/challenge/verify.py | 2 +- src/keri/app/cli/commands/clean.py | 2 +- src/keri/app/cli/commands/contacts/list.py | 2 +- src/keri/app/cli/commands/did/generate.py | 2 +- src/keri/app/cli/commands/ends/add.py | 2 +- src/keri/app/cli/commands/ends/export.py | 2 +- src/keri/app/cli/commands/ends/list.py | 2 +- src/keri/app/cli/commands/escrow.py | 2 +- src/keri/app/cli/commands/event.py | 2 +- src/keri/app/cli/commands/export.py | 2 +- src/keri/app/cli/commands/incept.py | 2 +- src/keri/app/cli/commands/init.py | 2 +- src/keri/app/cli/commands/ipex/list.py | 2 +- src/keri/app/cli/commands/kevers.py | 2 +- src/keri/app/cli/commands/list.py | 2 +- src/keri/app/cli/commands/local/watch.py | 2 +- src/keri/app/cli/commands/location/add.py | 2 +- src/keri/app/cli/commands/mailbox/add.py | 2 +- src/keri/app/cli/commands/mailbox/debug.py | 2 +- src/keri/app/cli/commands/mailbox/list.py | 2 +- src/keri/app/cli/commands/mailbox/update.py | 2 +- src/keri/app/cli/commands/migrate/list.py | 2 +- src/keri/app/cli/commands/migrate/run.py | 2 +- src/keri/app/cli/commands/migrate/show.py | 2 +- .../app/cli/commands/multisig/continue.py | 2 +- .../app/cli/commands/multisig/interact.py | 2 +- src/keri/app/cli/commands/multisig/notice.py | 2 +- src/keri/app/cli/commands/multisig/rotate.py | 4 +- src/keri/app/cli/commands/multisig/update.py | 2 +- src/keri/app/cli/commands/oobi/clean.py | 2 +- src/keri/app/cli/commands/oobi/generate.py | 2 +- src/keri/app/cli/commands/oobi/resolve.py | 2 +- src/keri/app/cli/commands/passcode/remove.py | 2 +- src/keri/app/cli/commands/passcode/set.py | 2 +- src/keri/app/cli/commands/query.py | 2 +- src/keri/app/cli/commands/rename.py | 2 +- src/keri/app/cli/commands/rollback.py | 2 +- src/keri/app/cli/commands/saidify.py | 2 +- src/keri/app/cli/commands/ssh/export.py | 2 +- src/keri/app/cli/commands/status.py | 2 +- src/keri/app/cli/commands/vc/create.py | 2 +- src/keri/app/cli/commands/vc/export.py | 2 +- src/keri/app/cli/commands/vc/list.py | 2 +- .../app/cli/commands/vc/registry/incept.py | 2 +- src/keri/app/cli/commands/vc/registry/list.py | 2 +- .../app/cli/commands/vc/registry/status.py | 2 +- src/keri/app/cli/commands/watcher/add.py | 2 +- .../app/cli/commands/watcher/adjudicate.py | 2 +- src/keri/app/cli/commands/watcher/list.py | 2 +- .../app/cli/commands/witness/authenticate.py | 2 +- src/keri/app/cli/commands/witness/demo.py | 12 +- src/keri/app/cli/commands/witness/list.py | 2 +- src/keri/app/cli/commands/witness/submit.py | 2 +- src/keri/app/delegating.py | 10 +- src/keri/app/grouping.py | 26 +- src/keri/app/indirecting.py | 12 +- src/keri/app/storing.py | 2 + src/keri/core/eventing.py | 865 +++++++++--------- src/keri/core/parsing.py | 48 +- src/keri/core/routing.py | 62 +- src/keri/db/escrowing.py | 31 +- src/keri/demo/demo_kev.py | 2 +- src/keri/help/__init__.py | 14 +- src/keri/peer/exchanging.py | 96 +- src/keri/vdr/credentialing.py | 4 +- src/keri/vdr/eventing.py | 93 +- src/keri/vdr/verifying.py | 9 +- 70 files changed, 734 insertions(+), 664 deletions(-) diff --git a/src/keri/app/cli/commands/aid.py b/src/keri/app/cli/commands/aid.py index 856f65dda..953e463e8 100644 --- a/src/keri/app/cli/commands/aid.py +++ b/src/keri/app/cli/commands/aid.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/challenge/generate.py b/src/keri/app/cli/commands/challenge/generate.py index e26e6f10e..169ea93c7 100644 --- a/src/keri/app/cli/commands/challenge/generate.py +++ b/src/keri/app/cli/commands/challenge/generate.py @@ -6,7 +6,7 @@ import argparse import json -from hio import help +from keri import help from hio.base import doing from mnemonic import mnemonic diff --git a/src/keri/app/cli/commands/challenge/respond.py b/src/keri/app/cli/commands/challenge/respond.py index 1bca6554a..d942312da 100644 --- a/src/keri/app/cli/commands/challenge/respond.py +++ b/src/keri/app/cli/commands/challenge/respond.py @@ -7,6 +7,7 @@ from hio.base import doing +from keri import help from keri.app import habbing, forwarding, connecting from keri.app.cli.common import existing from keri.app.habbing import GroupHab @@ -26,6 +27,7 @@ parser.add_argument('--recipient', '-r', help='Contact alias of the AID to send the signed words to', action="store", required=True) +logger = help.ogler.getLogger() def respond(args): """ diff --git a/src/keri/app/cli/commands/challenge/verify.py b/src/keri/app/cli/commands/challenge/verify.py index 4811e6098..09f0dd5d9 100644 --- a/src/keri/app/cli/commands/challenge/verify.py +++ b/src/keri/app/cli/commands/challenge/verify.py @@ -8,7 +8,7 @@ import datetime import sys -from hio import help +from keri import help from hio.base import doing from keri.app import indirecting, challenging, connecting, signaling diff --git a/src/keri/app/cli/commands/clean.py b/src/keri/app/cli/commands/clean.py index 63c3b4907..95bdd6e81 100644 --- a/src/keri/app/cli/commands/clean.py +++ b/src/keri/app/cli/commands/clean.py @@ -5,7 +5,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/contacts/list.py b/src/keri/app/cli/commands/contacts/list.py index d4c626d37..5daf4d411 100644 --- a/src/keri/app/cli/commands/contacts/list.py +++ b/src/keri/app/cli/commands/contacts/list.py @@ -7,7 +7,7 @@ import argparse import json -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/did/generate.py b/src/keri/app/cli/commands/did/generate.py index 0aca604ce..d373e6a61 100644 --- a/src/keri/app/cli/commands/did/generate.py +++ b/src/keri/app/cli/commands/did/generate.py @@ -9,7 +9,7 @@ from urllib.parse import urlparse import sys -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/ends/add.py b/src/keri/app/cli/commands/ends/add.py index 44f4fd639..08a0e7fd4 100644 --- a/src/keri/app/cli/commands/ends/add.py +++ b/src/keri/app/cli/commands/ends/add.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/ends/export.py b/src/keri/app/cli/commands/ends/export.py index 6eed26a16..1ac939a90 100644 --- a/src/keri/app/cli/commands/ends/export.py +++ b/src/keri/app/cli/commands/ends/export.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/ends/list.py b/src/keri/app/cli/commands/ends/list.py index 28e9cc3d9..f03f357ec 100644 --- a/src/keri/app/cli/commands/ends/list.py +++ b/src/keri/app/cli/commands/ends/list.py @@ -7,7 +7,7 @@ import argparse import json -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/escrow.py b/src/keri/app/cli/commands/escrow.py index 277ad4b7b..43d3dda62 100644 --- a/src/keri/app/cli/commands/escrow.py +++ b/src/keri/app/cli/commands/escrow.py @@ -7,7 +7,7 @@ import argparse import json -from hio import help +from keri import help from hio.base import doing from keri.core import eventing diff --git a/src/keri/app/cli/commands/event.py b/src/keri/app/cli/commands/event.py index b750d99f6..647df2a38 100644 --- a/src/keri/app/cli/commands/event.py +++ b/src/keri/app/cli/commands/event.py @@ -7,7 +7,7 @@ import argparse import json -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/export.py b/src/keri/app/cli/commands/export.py index 5f6b2e418..dc44b5376 100644 --- a/src/keri/app/cli/commands/export.py +++ b/src/keri/app/cli/commands/export.py @@ -7,7 +7,7 @@ import argparse import sys -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/incept.py b/src/keri/app/cli/commands/incept.py index e521d064e..cf30bae92 100644 --- a/src/keri/app/cli/commands/incept.py +++ b/src/keri/app/cli/commands/incept.py @@ -6,7 +6,7 @@ import argparse from dataclasses import dataclass -from hio import help +from keri import help from hio.base import doing from keri.app import habbing, agenting, indirecting, configing, delegating, forwarding diff --git a/src/keri/app/cli/commands/init.py b/src/keri/app/cli/commands/init.py index 52fc51398..0d5070ca3 100644 --- a/src/keri/app/cli/commands/init.py +++ b/src/keri/app/cli/commands/init.py @@ -8,7 +8,7 @@ import os import sys -from hio import help +from keri import help from hio.base import doing import keri.app.oobiing diff --git a/src/keri/app/cli/commands/ipex/list.py b/src/keri/app/cli/commands/ipex/list.py index 4dd3a50ca..6d01fb2ba 100644 --- a/src/keri/app/cli/commands/ipex/list.py +++ b/src/keri/app/cli/commands/ipex/list.py @@ -11,7 +11,7 @@ import json import sys -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/kevers.py b/src/keri/app/cli/commands/kevers.py index a1badf246..4aa0edb13 100644 --- a/src/keri/app/cli/commands/kevers.py +++ b/src/keri/app/cli/commands/kevers.py @@ -8,7 +8,7 @@ import datetime import sys -from hio import help +from keri import help from hio.base import doing from keri.app import indirecting diff --git a/src/keri/app/cli/commands/list.py b/src/keri/app/cli/commands/list.py index 8e15b3b1b..4cb8d5567 100644 --- a/src/keri/app/cli/commands/list.py +++ b/src/keri/app/cli/commands/list.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/local/watch.py b/src/keri/app/cli/commands/local/watch.py index b93aeba3c..484ecdf6e 100644 --- a/src/keri/app/cli/commands/local/watch.py +++ b/src/keri/app/cli/commands/local/watch.py @@ -8,7 +8,7 @@ import sys import time -from hio import help +from keri import help from hio.base import doing from keri.app import agenting, indirecting, habbing, forwarding from keri.app.cli.common import existing, terming diff --git a/src/keri/app/cli/commands/location/add.py b/src/keri/app/cli/commands/location/add.py index d549c0ded..344ef3c2d 100644 --- a/src/keri/app/cli/commands/location/add.py +++ b/src/keri/app/cli/commands/location/add.py @@ -7,7 +7,7 @@ import argparse from urllib.parse import urlparse -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/mailbox/add.py b/src/keri/app/cli/commands/mailbox/add.py index 67abf1990..46f7df4e3 100644 --- a/src/keri/app/cli/commands/mailbox/add.py +++ b/src/keri/app/cli/commands/mailbox/add.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from hio.help import Hict diff --git a/src/keri/app/cli/commands/mailbox/debug.py b/src/keri/app/cli/commands/mailbox/debug.py index 873c9ad05..f11a53035 100644 --- a/src/keri/app/cli/commands/mailbox/debug.py +++ b/src/keri/app/cli/commands/mailbox/debug.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/mailbox/list.py b/src/keri/app/cli/commands/mailbox/list.py index 369e749df..d03d6e443 100644 --- a/src/keri/app/cli/commands/mailbox/list.py +++ b/src/keri/app/cli/commands/mailbox/list.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app import connecting diff --git a/src/keri/app/cli/commands/mailbox/update.py b/src/keri/app/cli/commands/mailbox/update.py index d648702b6..05d6c341d 100644 --- a/src/keri/app/cli/commands/mailbox/update.py +++ b/src/keri/app/cli/commands/mailbox/update.py @@ -5,7 +5,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py index 152850436..0191b02b3 100644 --- a/src/keri/app/cli/commands/migrate/list.py +++ b/src/keri/app/cli/commands/migrate/list.py @@ -5,7 +5,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from prettytable import PrettyTable diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py index 03d92ebc5..0e9e1d52c 100644 --- a/src/keri/app/cli/commands/migrate/run.py +++ b/src/keri/app/cli/commands/migrate/run.py @@ -6,7 +6,7 @@ import argparse import keri -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/migrate/show.py b/src/keri/app/cli/commands/migrate/show.py index 29c140889..538d7f511 100644 --- a/src/keri/app/cli/commands/migrate/show.py +++ b/src/keri/app/cli/commands/migrate/show.py @@ -5,7 +5,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/multisig/continue.py b/src/keri/app/cli/commands/multisig/continue.py index a93caa190..9ca75f360 100644 --- a/src/keri/app/cli/commands/multisig/continue.py +++ b/src/keri/app/cli/commands/multisig/continue.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app import indirecting, grouping, agenting diff --git a/src/keri/app/cli/commands/multisig/interact.py b/src/keri/app/cli/commands/multisig/interact.py index 5b45a11ac..0f5a9345e 100644 --- a/src/keri/app/cli/commands/multisig/interact.py +++ b/src/keri/app/cli/commands/multisig/interact.py @@ -7,7 +7,7 @@ import argparse from ordered_set import OrderedSet as oset -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/multisig/notice.py b/src/keri/app/cli/commands/multisig/notice.py index e866984e4..88fb8b67e 100644 --- a/src/keri/app/cli/commands/multisig/notice.py +++ b/src/keri/app/cli/commands/multisig/notice.py @@ -6,7 +6,7 @@ import argparse from ordered_set import OrderedSet as oset -from hio import help +from keri import help from hio.base import doing from keri.app import habbing, forwarding, grouping diff --git a/src/keri/app/cli/commands/multisig/rotate.py b/src/keri/app/cli/commands/multisig/rotate.py index 56f84ec97..3a6dc385b 100644 --- a/src/keri/app/cli/commands/multisig/rotate.py +++ b/src/keri/app/cli/commands/multisig/rotate.py @@ -7,7 +7,7 @@ import argparse from ordered_set import OrderedSet as oset -from hio import help +from keri import help from hio.base import doing from keri import kering @@ -44,7 +44,7 @@ def rotateGroupIdentifier(args): Performs a rotation on the group identifier specified as an argument. The identifier prefix of the environment represented by the name parameter must be a member of the group identifier. This command will perform a rotation of the local identifier if the sequence number of the local identifier is the same as the group identifier sequence - number. It will wait for all other members of the group to acheive the same sequence number (group + 1) and then + number. It will wait for all other members of the group to achieve the same sequence number (group + 1) and then publish the signed rotation event for the group identifier to all witnesses and wait for receipts. Parameters: diff --git a/src/keri/app/cli/commands/multisig/update.py b/src/keri/app/cli/commands/multisig/update.py index 37f4d6d84..3527c8dd4 100644 --- a/src/keri/app/cli/commands/multisig/update.py +++ b/src/keri/app/cli/commands/multisig/update.py @@ -6,7 +6,7 @@ import argparse import time -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/oobi/clean.py b/src/keri/app/cli/commands/oobi/clean.py index 0af65e04f..c09545d50 100644 --- a/src/keri/app/cli/commands/oobi/clean.py +++ b/src/keri/app/cli/commands/oobi/clean.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/oobi/generate.py b/src/keri/app/cli/commands/oobi/generate.py index 406d5f08f..38c378ce7 100644 --- a/src/keri/app/cli/commands/oobi/generate.py +++ b/src/keri/app/cli/commands/oobi/generate.py @@ -6,7 +6,7 @@ import argparse import sys -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/oobi/resolve.py b/src/keri/app/cli/commands/oobi/resolve.py index 747c67c50..221d0ed97 100644 --- a/src/keri/app/cli/commands/oobi/resolve.py +++ b/src/keri/app/cli/commands/oobi/resolve.py @@ -5,7 +5,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing import keri.app.oobiing diff --git a/src/keri/app/cli/commands/passcode/remove.py b/src/keri/app/cli/commands/passcode/remove.py index 61b3dcf79..8feba43d2 100644 --- a/src/keri/app/cli/commands/passcode/remove.py +++ b/src/keri/app/cli/commands/passcode/remove.py @@ -5,7 +5,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/passcode/set.py b/src/keri/app/cli/commands/passcode/set.py index e9b30b55c..99e7b9b51 100644 --- a/src/keri/app/cli/commands/passcode/set.py +++ b/src/keri/app/cli/commands/passcode/set.py @@ -6,7 +6,7 @@ import argparse import getpass -from hio import help +from keri import help from hio.base import doing diff --git a/src/keri/app/cli/commands/query.py b/src/keri/app/cli/commands/query.py index f3b9414f5..3505e9411 100644 --- a/src/keri/app/cli/commands/query.py +++ b/src/keri/app/cli/commands/query.py @@ -7,7 +7,7 @@ import datetime import json -from hio import help +from keri import help from hio.base import doing from hio.help import decking diff --git a/src/keri/app/cli/commands/rename.py b/src/keri/app/cli/commands/rename.py index b14b2f6a5..83251b88f 100644 --- a/src/keri/app/cli/commands/rename.py +++ b/src/keri/app/cli/commands/rename.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/rollback.py b/src/keri/app/cli/commands/rollback.py index 386f631f1..249590bad 100644 --- a/src/keri/app/cli/commands/rollback.py +++ b/src/keri/app/cli/commands/rollback.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/saidify.py b/src/keri/app/cli/commands/saidify.py index 280035fb8..c89fb664b 100644 --- a/src/keri/app/cli/commands/saidify.py +++ b/src/keri/app/cli/commands/saidify.py @@ -6,7 +6,7 @@ import argparse import json -from hio import help +from keri import help from hio.base import doing from keri.core import coring diff --git a/src/keri/app/cli/commands/ssh/export.py b/src/keri/app/cli/commands/ssh/export.py index 542aead0e..21d1e15cb 100644 --- a/src/keri/app/cli/commands/ssh/export.py +++ b/src/keri/app/cli/commands/ssh/export.py @@ -11,7 +11,7 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import ed25519 -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/status.py b/src/keri/app/cli/commands/status.py index 96f531828..741c8acfb 100644 --- a/src/keri/app/cli/commands/status.py +++ b/src/keri/app/cli/commands/status.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import displaying, existing diff --git a/src/keri/app/cli/commands/vc/create.py b/src/keri/app/cli/commands/vc/create.py index bf0584332..0dba4c979 100644 --- a/src/keri/app/cli/commands/vc/create.py +++ b/src/keri/app/cli/commands/vc/create.py @@ -2,7 +2,7 @@ import json from typing import Optional -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/vc/export.py b/src/keri/app/cli/commands/vc/export.py index 71ce933dc..d8bbb0663 100644 --- a/src/keri/app/cli/commands/vc/export.py +++ b/src/keri/app/cli/commands/vc/export.py @@ -7,7 +7,7 @@ import argparse import sys -from hio import help +from keri import help from hio.base import doing from keri.app import signing diff --git a/src/keri/app/cli/commands/vc/list.py b/src/keri/app/cli/commands/vc/list.py index a7d5d8706..a97e9a4f1 100644 --- a/src/keri/app/cli/commands/vc/list.py +++ b/src/keri/app/cli/commands/vc/list.py @@ -9,7 +9,7 @@ import json import sys -from hio import help +from keri import help from hio.base import doing from keri import kering diff --git a/src/keri/app/cli/commands/vc/registry/incept.py b/src/keri/app/cli/commands/vc/registry/incept.py index 5a2411359..4de3d4a81 100644 --- a/src/keri/app/cli/commands/vc/registry/incept.py +++ b/src/keri/app/cli/commands/vc/registry/incept.py @@ -1,6 +1,6 @@ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app import indirecting, habbing, grouping, forwarding diff --git a/src/keri/app/cli/commands/vc/registry/list.py b/src/keri/app/cli/commands/vc/registry/list.py index 0087d6530..c7d99380e 100644 --- a/src/keri/app/cli/commands/vc/registry/list.py +++ b/src/keri/app/cli/commands/vc/registry/list.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import existing diff --git a/src/keri/app/cli/commands/vc/registry/status.py b/src/keri/app/cli/commands/vc/registry/status.py index ad7efdbaf..39bb2eadd 100644 --- a/src/keri/app/cli/commands/vc/registry/status.py +++ b/src/keri/app/cli/commands/vc/registry/status.py @@ -1,6 +1,6 @@ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app import indirecting, habbing, grouping diff --git a/src/keri/app/cli/commands/watcher/add.py b/src/keri/app/cli/commands/watcher/add.py index 546eec96c..e2ff67a77 100644 --- a/src/keri/app/cli/commands/watcher/add.py +++ b/src/keri/app/cli/commands/watcher/add.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app import connecting, habbing, forwarding diff --git a/src/keri/app/cli/commands/watcher/adjudicate.py b/src/keri/app/cli/commands/watcher/adjudicate.py index 41c9efc9a..0f441e4b1 100644 --- a/src/keri/app/cli/commands/watcher/adjudicate.py +++ b/src/keri/app/cli/commands/watcher/adjudicate.py @@ -9,7 +9,7 @@ import random import sys -from hio import help +from keri import help from hio.base import doing from keri.app import connecting, indirecting, querying, watching diff --git a/src/keri/app/cli/commands/watcher/list.py b/src/keri/app/cli/commands/watcher/list.py index ff54dc0ff..47644e599 100644 --- a/src/keri/app/cli/commands/watcher/list.py +++ b/src/keri/app/cli/commands/watcher/list.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app import connecting diff --git a/src/keri/app/cli/commands/witness/authenticate.py b/src/keri/app/cli/commands/witness/authenticate.py index 81a4d341e..8e01d8216 100644 --- a/src/keri/app/cli/commands/witness/authenticate.py +++ b/src/keri/app/cli/commands/witness/authenticate.py @@ -9,7 +9,7 @@ import sys import qrcode -from hio import help +from keri import help from hio.base import doing from hio.help import Hict diff --git a/src/keri/app/cli/commands/witness/demo.py b/src/keri/app/cli/commands/witness/demo.py index 912fa0417..27cfcdea6 100644 --- a/src/keri/app/cli/commands/witness/demo.py +++ b/src/keri/app/cli/commands/witness/demo.py @@ -8,6 +8,7 @@ import argparse import logging +import os from hio.base import doing @@ -19,18 +20,25 @@ parser = argparse.ArgumentParser(description="Run a demo collection of witnesses") +parser.add_argument("--loglevel", action="store", required=False, default=os.getenv("KERI_LOG_LEVEL", "CRITICAL"), + help="Set log level to DEBUG | INFO | WARNING | ERROR | CRITICAL. Default is CRITICAL") parser.set_defaults(handler=lambda args: demo(args)) -help.ogler.level = logging.INFO logger = help.ogler.getLogger() -def demo(_): +def demo(args): """ Run set of three witnesses for demo """ + base_formatter = logging.Formatter('%(asctime)s [keri] %(module)s.%(funcName)s-%(lineno)s %(levelname)-8s %(message)s') + base_formatter.default_msec_format = None + help.ogler.baseConsoleHandler.setFormatter(base_formatter) + help.ogler.level = logging.getLevelName(args.loglevel.upper()) + logger.setLevel(help.ogler.level) + help.ogler.reopen(name="keri", temp=True, clear=True) wancf = configing.Configer(name="wan", headDirPath="scripts", temp=False, reopen=True, clear=False) wilcf = configing.Configer(name="wil", headDirPath="scripts", temp=False, reopen=True, clear=False) diff --git a/src/keri/app/cli/commands/witness/list.py b/src/keri/app/cli/commands/witness/list.py index 9ab11e6e5..f86d8d9e5 100644 --- a/src/keri/app/cli/commands/witness/list.py +++ b/src/keri/app/cli/commands/witness/list.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app.cli.common import displaying, existing diff --git a/src/keri/app/cli/commands/witness/submit.py b/src/keri/app/cli/commands/witness/submit.py index 9f46a1a52..bc7402e1f 100644 --- a/src/keri/app/cli/commands/witness/submit.py +++ b/src/keri/app/cli/commands/witness/submit.py @@ -5,7 +5,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app import habbing, agenting, indirecting diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 1e09e6596..3178ac3d5 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -6,7 +6,7 @@ module for enveloping and forwarding KERI message """ -from hio import help +from keri import help from hio.base import doing from . import agenting, forwarding @@ -190,12 +190,18 @@ def processPartialWitnessEscrow(self): raise kering.ValidationError("no proxy to send messages for delegation") evt = hab.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said) + srdr = serdering.SerderKERI(raw=evt) exn, atc = delegateRequestExn(phab, delpre=delpre, evt=bytes(evt), aids=smids) + logger.info("Sending delegation request exn for %s from %s to delegator %s", + srdr.ilk, phab.pre, delpre) + logger.debug("Delegation request=\n%s\n", exn.pretty()) self.postman.send(hab=phab, dest=hab.kever.delpre, topic="delegate", serder=exn, attachment=atc) - srdr = serdering.SerderKERI(raw=evt) del evt[:srdr.size] + logger.info("Sending delegation event %s from %s to delegator %s", + srdr.ilk, phab.pre, delpre) + logger.debug("Delegated inception=\n%s\n", srdr.pretty()) self.postman.send(hab=phab, dest=delpre, topic="delegate", serder=srdr, attachment=evt) seal = dict(i=srdr.pre, s=srdr.snh, d=srdr.said) diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index f87ef2490..e5928074f 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -47,7 +47,9 @@ def start(self, ghab, prefixer, seqner, saider): saider (Saider): saider of event of group identifier """ - print(f"Waiting for other signatures for {prefixer.qb64}:{seqner.sn}...") + evt = ghab.makeOwnEvent(sn=seqner.sn, allowPartiallySigned=True) # used just for the log message + serder = serdering.SerderKERI(raw=evt) # used just for the log message + logger.info("Waiting for other signatures on %s for %s:%s...", serder.ilk, prefixer.qb64, seqner.sn) return self.hby.db.gpse.add(keys=(prefixer.qb64,), val=(seqner, saider)) def complete(self, prefixer, seqner, saider=None): @@ -129,7 +131,8 @@ def processPartialSignedEscrow(self): if kever.delegated and kever.ilk in (coring.Ilks.dip, coring.Ilks.drt): # We are a delegated identifier, must wait for delegator approval for dip and drt if witered: # We are elected to perform delegation and witnessing messaging - logger.info(f"We are the witnesser, sending {pre} to delegator") + logger.info("AID %s...%s: We are the witnesser, sending %s to delegator", + pre[:4], pre[-4:], pre) self.swain.delegation(pre=pre, sn=seqner.sn) else: anchor = dict(i=pre, s=seqner.snh, d=saider.qb64) @@ -138,15 +141,17 @@ def processPartialSignedEscrow(self): else: self.witq.query(src=ghab.mhab.pre, pre=kever.delpre, anchor=anchor) - logger.info("Waiting for delegation approval...") + logger.info("AID %s...%s: Waiting for delegation approval...", pre[:4], pre[-4:]) self.hby.db.gdee.add(keys=(pre,), val=(seqner, saider)) else: # Non-delegation, move on to witnessing if witered: # We are elected witnesser, send off event to witnesses - logger.info(f"We are the fully signed witnesser {seqner.sn}, sending to witnesses") + logger.info("AID %s...%s: We are the fully signed witnesser %s, sending to witnesses", + pre[:4], pre[-4:], seqner.sn) self.witDoer.msgs.append(dict(pre=pre, sn=seqner.sn)) # Move to escrow waiting for witness receipts - logger.info(f"Waiting for fully signed witness receipts for {seqner.sn}") + logger.info("AID %s...%s: Waiting for fully signed witness receipts for %s", + pre[:4], pre[-4:], seqner.sn) self.hby.db.gpwe.add(keys=(pre,), val=(seqner, saider)) def processDelegateEscrow(self): @@ -166,7 +171,8 @@ def processDelegateEscrow(self): if witer: # We are elected witnesser, We've already done out part in Boatswain, we are done. if self.swain.complete(prefixer=kever.prefixer, seqner=coring.Seqner(sn=kever.sn)): self.hby.db.gdee.rem(keys=(pre,)) - logger.info(f"Delegation approval for {pre} received.") + logger.info("AID %s...%s: Delegation approval for %s received.", + pre[:4], pre[-4:], pre) self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) @@ -178,10 +184,10 @@ def processDelegateEscrow(self): dgkey = dbing.dgKey(pre, saider.qb64b) self.hby.db.setAes(dgkey, couple) # authorizer event seal (delegator/issuer) self.hby.db.gdee.rem(keys=(pre,)) - logger.info(f"Delegation approval for {pre} received.") + logger.info("AID %s...%s: Delegation approval for %s received.", pre[:4], pre[-4:], pre) # Move to escrow waiting for witness receipts - logger.info(f"Waiting for witness receipts for {pre}") + logger.info("AID %s...%s: Waiting for witness receipts for %s", pre[:4], pre[-4:], pre) self.hby.db.gdee.rem(keys=(pre,)) self.hby.db.gpwe.add(keys=(pre,), val=(seqner, saider)) @@ -209,7 +215,7 @@ def processPartialWitnessEscrow(self): witnessed = True if not witnessed: continue - logger.info(f"Witness receipts complete, {pre} confirmed.") + logger.info("AID %s...%s: Witness receipts complete, %s confirmed.", pre[:4], pre[-4:], pre) self.hby.db.gpwe.rem(keys=(pre,)) self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) elif not witer: @@ -240,6 +246,8 @@ def handle(self, serder, attachments=None): attachments (list): list of tuples of pather, CESR SAD path attachments to the exn event """ + logger.info("Notification for %s event SAID=%s", self.resource, serder.said) + logger.debug("EXN Body=\n%s\n", serder.pretty()) self.mux.add(serder=serder) diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index e531a4402..3194af867 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -479,13 +479,11 @@ class MailboxDirector(doing.DoDoer): hby (Habitat: local controller's context Properties: - tyme (float): relative cycle time of associated Tymist, obtained - via injected .tymth function wrapper closure. - tymth (function): function wrapper closure returned by Tymist .tymeth() - method. When .tymth is called it returns associated Tymist .tyme. - .tymth provides injected dependency on Tymist tyme base. - tock (float): desired time in seconds between runs or until next run, - non negative, zero means run asap + hby (Habery): the Habery in which mailbox messages are routed + verifier (Verifier): TEL event acceptor and validator + exchanger (Exchanger): Exchange (exn) message delivery component + rep (Respondant): Respondant for reply messages + cues (Deck): Queue for new actions to schedule shared between the Revery, Kevery (and Kever), and Tevery (and Tever) Methods: diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index 80cf0f2fd..f817b67fc 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -224,6 +224,8 @@ def responseDo(self, tymth=None, tock=0.0): # sign the exn to get the signature eattach = senderHab.endorse(exn, last=False, pipelined=False) del eattach[:exn.size] + logger.info("Sending exn on %s from %s to %s", topic, sender, recipient) + logger.debug("exn body=\n%s\n", exn.pretty()) self.postman.send(recipient, topic=topic, serder=exn, hab=forwardHab, attachment=eattach) yield self.tock # throttle just do one cue at a time diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index d1bcaf745..93c88c432 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2318,12 +2318,16 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # each wiger now has added to it a werfer of its wit in its .verfer property # escrow if not fully signed vs signing threshold + pre = self.prefixer.qb64 if not tholder.satisfy(indices): # at least one but not enough self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" - f" on sigs for {[siger.qb64 for siger in sigers]}" - f" for evt = {serder.ked}.") + msg = (f"AID {pre[:4]}...{pre[-4:]}: Failure satisfying sith = {tholder.sith} " + f"on sigs {[siger.qb64 for siger in sigers]} " + f"for evt = {serder.said}") + logger.trace(msg) + logger.trace("Event Body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) # escrow if not fully signed vs prior next rotation threshold @@ -2333,10 +2337,13 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, if not self.ntholder.satisfy(indices=ondices): self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider,local=local) - raise MissingSignatureError(f"Failure satisfying prior nsith=" - f"{self.ntholder.sith} with exposed " - f"sigs= {[siger.qb64 for siger in sigers]}" - f" for new est evt={serder.ked}.") + msg = ( + f"AID {pre[:4]}...{pre[-4:]}: Failure satisfying prior nsith = {self.ntholder.sith} " + f"with exposed sigs {[siger.qb64 for siger in sigers]} " + f"for new est evt={serder.said}") + logger.trace(msg) + logger.trace("Event Body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) # this point the sigers have been verified and the wigers have been verified # even if locallyOwned or locallyMembered or locallyWitnessed. @@ -2364,10 +2371,12 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, local=local): # cue to query for witness receipts self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) - raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " - f"on witness sigs=" - f"{[siger.qb64 for siger in wigers]} " - f"for event={serder.ked}.") + msg = (f"AID {pre[:4]}...{pre[-4:]}: Failure satisfying toad={toader.num} " + f"on witness sigs {[siger.qb64 for siger in wigers]} " + f"for event={serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingWitnessSignatureError(msg) # Delegator approves delegation by attaching valid source @@ -2388,9 +2397,10 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # seal to delegating event, i.e. delseqner, delsaider. self.escrowDelegableEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - raise MissingDelegableApprovalError(f"Missing approval for " - f" delegation by {delpre} of" - f"event = {serder.ked}.") + msg = f"Missing approval for delegation by {delpre} of event = {serder.said}" + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegableApprovalError(msg) # validateDelegation returns (None, None) when delegation validation # does not apply. Raises ValidationError if validation applies but @@ -2746,14 +2756,19 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # of delegator if still missing when processing escrow later. self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"Missing KEL of delegator " - f"{delpre} of evt = {serder.ked}.") + msg = f"Missing KEL of delegator {delpre} of evt {serder.sn} {serder.ilk} {serder.said}" + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) dkever = self.kevers[delpre] # get delegator's KEL if dkever.doNotDelegate: # drop event if delegation not allowed - raise ValidationError(f"Delegator = {delpre} for evt = {serder.ked}," - f" does not allow delegation.") + msg = (f"Delegator {delpre} does not allow delegation on evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise ValidationError(msg) dserder = None # no delegation event yet if delseqner is None or delsaider is None: # missing delegation seal ref @@ -2768,8 +2783,11 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, if not dserder: # just escrow and try later self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegation seal for delegator " - f"{delpre} of evt = {serder.ked}.") + msg = (f"No delegation seal for delegator {delpre} on evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) if delseqner and delsaider and not dserder: # given couple not found # ToDo XXXX need to replace Seqners with Numbers @@ -2795,17 +2813,22 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # otherwise escrowPDEvent self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegating event from {delpre}" - f" at {delsaider.qb64} for " - f"evt = {serder.ked}.") + msg = (f"No delegating event from {delpre} at {delsaider.qb64} for evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) # get the latest delegating event candidate from dig given by pre,sn index ddig = bytes(raw) key = dgKey(pre=delpre, dig=ddig) # database key raw = self.db.getEvt(key) # get actual last event if raw is None: # drop event should never happen unless database is broken - raise ValidationError(f"Missing delegation from {delpre} at event " - f"dig = {ddig} for evt = {serder.ked}.") + msg = (f"Missing delegation from {delpre} at event dig = {ddig} for evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise ValidationError(msg) dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event @@ -2829,8 +2852,11 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, delseqner = delsaider = None # nullify self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegation seal for delegator " - f"{delpre} of evt = {serder.ked}.") + msg = (f"No delegation seal for delegator {delpre} of evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) ## Found valid anchoring seal of delegator delpre ## compare saids to ensure match of delegating event and source seal @@ -2872,9 +2898,11 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, eager=eager)): self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegating event from {delpre}" - f" at {delsaider.qb64} for " - f"evt = {serder.ked}.") + msg = (f"No delegating event from {delpre} at {delsaider.qb64} for evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) # already have new potential superseding delegation serfn = serder # new potentially superseding delegated event i.e. serf new bossn = dserder # new delegating event of superseding delegated event i.e. boss new @@ -2908,8 +2936,11 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, else: # not superseded # ToDo: XXXX may want to cue up business logic for delegator # if self.mine(delegator): # failed attempt at recovery - raise ValidationError(f"Invalid delegation recovery rotation" - f"of {serfo.ked} by {serfn.ked}") + msg = f"Invalid delegation recovery rotation of {serfo.pre} by {serfn.pre}" + logger.info(msg) + logger.debug("Delegate Event Body=\n%s\n", serfo.pretty()) + logger.debug("Delegator Event Body=\n%s\n", serfn.pretty()) + raise ValidationError(msg) # tie condition same sn and drt so need to climb delegation chain serfn = bossn @@ -2918,18 +2949,22 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, eager=eager)): self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegating event from {delpre}" - f" at {delsaider.qb64} for " - f"evt = {serder.ked}.") + msg = (f"No delegating event from {delpre} at {delsaider.qb64} for evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) serfo = bosso if not (bosso := self.fetchDelegatingEvent(delpre, serfo, original=True, eager=eager)): self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegating event from {delpre}" - f" at {delsaider.qb64} for " - f"evt = {serder.ked}.") + msg = (f"No delegating event from {delpre} at {delsaider.qb64} for evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) # repeat # should never get to here @@ -3015,7 +3050,10 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): ddgkey = dgKey(pre=delpre, dig=deldig) # database key of delegation if not (raw := self.db.getEvt(ddgkey)): # in fons but no event # database broken this should never happen - raise ValidationError(f"Missing delegation event for {serder.ked}") + msg = f"Missing delegation event for {serder.said}" + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise ValidationError(msg) # original delegating event i.e. boss original dserder = serdering.SerderKERI(raw=bytes(raw)) return dserder @@ -3028,7 +3066,10 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): # database broken this should never happen so do not validate # since original must have been validated so it must have # all its delegation chain. - raise ValidationError(f"Missing delegation source seal for {serder.ked}") + msg = f"Missing delegation source seal for {serder.said}" + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise ValidationError(msg) else: # only search last events in delegator's kel if not (dserder := self.db.fetchLastSealingEventByEventSeal(pre=delpre, seal=seal)): @@ -3125,27 +3166,29 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, esr = basing.EventSourceRecord(local=local) self.db.esrs.put(keys=dgkeys, val=esr) + pre = self.prefixer.qb64 if first: # append event dig to first seen database in order fn = self.db.appendFe(serder.preb, serder.saidb) if firner and fn != firner.sn: # cloned replay but replay fn not match if self.cues is not None: # cue to notice BadCloneFN self.cues.push(dict(kin="noticeBadCloneFN", serder=serder, fn=fn, firner=firner, dater=dater)) - logger.info("Kever Mismatch Cloned Replay FN: %s First seen " + logger.info("Kever: Mismatch Cloned Replay FN: %s First seen " "ordinal fn %s and clone fn %s, said=%s", serder.preb, fn, firner.sn, serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Event body=\n%s\n", serder.pretty()) if dater: # cloned replay use original's dts from dater dtsb = dater.dtsb self.db.setDts(dgkey, dtsb) # first seen so set dts to now self.db.fons.pin(keys=dgkey, val=Seqner(sn=fn)) - logger.info("Kever state: %s First seen ordinal %s at %s, said=%s", - serder.pre, fn, dtsb.decode("utf-8"), serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("AID %s...%s: First seen %s at sn=%s valid event SAID=%s for %s at %s", + pre[:4], pre[-4:], serder.ilk, fn, serder.said, + serder.pre, dtsb.decode("utf-8")) + logger.debug("Event Body=\n%s\n", serder.pretty()) self.db.addKe(snKey(serder.preb, serder.sn), serder.saidb) - logger.info("Kever state: %s Added to KEL valid said=%s", - serder.pre, serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("AID %s...%s: Added to KEL %s at sn=%s valid event SAID=%s", + pre[:4], pre[-4:], serder.ilk, serder.sn, serder.said) + logger.debug("Event Body=\n%s\n", serder.pretty()) return (fn, dtsb.decode("utf-8")) # (fn int, dts str) if first else (None, dts str) @@ -3188,8 +3231,7 @@ def escrowMFEvent(self, serder, sigers, wigers=None, res = self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed - logger.debug("Kever state: escrowed misfit event=\n%s\n", - json.dumps(serder.ked, indent=1)) + logger.debug("Kever state: escrowed misfit event=\n%s\n", serder.pretty()) def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): @@ -3226,8 +3268,7 @@ def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) self.db.delegables.add(snKey(serder.preb, serder.sn), serder.saidb) # log escrowed - logger.debug("Kever state: escrowed delegable event=\n%s\n", - json.dumps(serder.ked, indent=1)) + logger.debug("Kever state: escrowed delegable event=\n%s\n", serder.pretty()) def escrowPSEvent(self, serder, *, sigers=None, wigers=None, @@ -3313,8 +3354,8 @@ def escrowPWEvent(self, serder, *, sigers=None, wigers=None, esr = basing.EventSourceRecord(local=local) self.db.esrs.put(keys=dgkey, val=esr) - logger.debug("Kever state: Escrowed partially witnessed " - "event = %s\n", serder.ked) + logger.trace("Kever state: Escrowed partially witnessed event = %s", serder.said) + logger.trace("Event Body=\n%s\n", serder.pretty()) return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) @@ -3820,7 +3861,10 @@ def processEvent(self, serder, sigers, *, wigers=None, else: # escrow likely duplicitous event self.escrowLDEvent(serder=serder, sigers=sigers) - raise LikelyDuplicitousError("Likely Duplicitous event={}.".format(ked)) + msg = f"Likely Duplicitous Event sn={serder.sn} type={serder.ilk} SAID={serder.said}" + logger.debug(msg) + logger.debug("Duplicitous event body=\n%s\n", serder.pretty()) + raise LikelyDuplicitousError(msg) else: # rot, drt, or ixn, so sn matters kever = self.kevers[pre] # get existing kever for pre @@ -3830,7 +3874,10 @@ def processEvent(self, serder, sigers, *, wigers=None, # escrow out-of-order event self.escrowOOEvent(serder=serder, sigers=sigers, seqner=delseqner, saider=delsaider, wigers=wigers, local=local) - raise OutOfOrderError("Out-of-order event={}.".format(ked)) + msg = f"Out-of-order event sn={serder.sn} type={serder.ilk} SAID={serder.said}" + logger.debug(msg) + logger.debug("Out-of-order event body=\n%s\n", serder.pretty()) + raise OutOfOrderError(msg) elif ((sn == sno) or # inorder event (ixn, rot, drt) or (ilk == Ilks.rot and # superseding recovery rot or @@ -3899,7 +3946,10 @@ def processEvent(self, serder, sigers, *, wigers=None, else: # escrow likely duplicitous event self.escrowLDEvent(serder=serder, sigers=sigers) - raise LikelyDuplicitousError("Likely Duplicitous event={}.".format(ked)) + msg = f"Likely Duplicitous Event sn={serder.sn} type={serder.ilk} SAID={serder.said}" + logger.debug(msg) + logger.debug("Duplicitous event body=\n%s\n", serder.pretty()) + raise LikelyDuplicitousError(msg) def processReceiptWitness(self, serder, wigers, local=None): @@ -3944,8 +3994,10 @@ def processReceiptWitness(self, serder, wigers, local=None): lserder = serdering.SerderKERI(raw=raw) # deserialize event raw if not lserder.compare(said=ked["d"]): # stale receipt at sn discard - raise ValidationError("Stale receipt at sn = {} for rct = {}." - "".format(ked["s"], ked)) + msg = f"Stale receipt at sn = {ked['s']} for rct = {serder.said}." + logger.info(msg) + logger.debug("Stale receipt event body=\n%s\n", serder.pretty()) + raise ValidationError(msg) # process each couple verify sig and write to db wits = [wit.qb64 for wit in self.fetchWitnessState(pre, sn)] @@ -3960,15 +4012,15 @@ def processReceiptWitness(self, serder, wigers, local=None): if not self.lax and wiger.verfer.qb64 in self.prefixes: # own is witness if pre in self.prefixes: # skip own receiptor of own event # sign own events not receipt them - logger.info("Kevery process: skipped own receipt attachment" - " on own event receipt=%s", serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("Kevery: skipped own receipt attachment" + " on own event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) continue # skip own receipt attachment on own event if not local: # so skip own receipt on other event when non-local source - logger.info("Kevery process: skipped own receipt attachment" - " on nonlocal event receipt=%s", serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Kevery: skipped own receipt attachment" + " on nonlocal event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) continue # skip own receipt attachment on non-local event if wiger.verfer.verify(wiger.raw, lserder.raw): @@ -3978,8 +4030,10 @@ def processReceiptWitness(self, serder, wigers, local=None): else: # no events to be receipted yet at that sn so escrow # get digest from receipt message not receipted event self.escrowUWReceipt(serder=serder, wigers=wigers, said=ked["d"]) - raise UnverifiedWitnessReceiptError("Unverified witness receipt={}." - "".format(ked)) + msg = f"Unverified witness receipt={serder.said}" + logger.info(msg) + logger.debug("Event=\n%s\n", serder.pretty()) + raise UnverifiedWitnessReceiptError(msg) def processReceipt(self, serder, cigars, local=None): """ @@ -4035,15 +4089,15 @@ def processReceipt(self, serder, cigars, local=None): if not self.lax and cigar.verfer.qb64 in self.prefixes: # own is receiptor if pre in self.prefixes: # skip own receipter of own event # sign own events not receipt them - logger.info("Kevery process: skipped own receipt attachment" - " on own event receipt=%s", serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Kevery process: skipped own receipt attachment" + " on own event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) continue # skip own receipt attachment on own event if not local: # skip own receipt on other event when not local - logger.info("Kevery process: skipped own receipt attachment" - " on nonlocal event receipt=%s", serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Kevery process: skipped own receipt attachment" + " on nonlocal event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) continue # skip own receipt attachment on non-local event if cigar.verfer.verify(cigar.raw, lserder.raw): @@ -4060,7 +4114,10 @@ def processReceipt(self, serder, cigars, local=None): else: # no events to be receipted yet at that sn so escrow self.escrowUReceipt(serder, cigars, said=ked["d"]) # digest in receipt - raise UnverifiedReceiptError("Unverified receipt={}.".format(ked)) + msg = f"Unverified receipt = {serder.said}" + logger.info(msg) + logger.debug("event=\n%s\n", serder.pretty()) + raise UnverifiedReceiptError(msg) def processAttachedReceiptCouples(self, serder, cigars, firner=None, local=None): @@ -4119,15 +4176,15 @@ def processAttachedReceiptCouples(self, serder, cigars, firner=None, local=None) if not self.lax and cigar.verfer.qb64 in self.prefixes: # own is receiptor if pre in self.prefixes: # skip own receipter on own event # sign own events not receipt them - logger.info("Kevery process: skipped own receipt attachment" - " on own event receipt=%s", serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Kevery process: skipped own receipt attachment" + " on own event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) continue # skip own receipt attachment on own event if not local: # own receipt on other event when not local - logger.info("Kevery process: skipped own receipt attachment" - " on nonlocal event receipt=%s", serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Kevery process: skipped own receipt attachment" + " on nonlocal event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) continue # skip own receipt attachment on non-local event @@ -4333,12 +4390,9 @@ def processAttachedReceiptQuadruples(self, serder, trqs, firner=None, local=None siger.verfer = sverfers[siger.index] # assign verfer if not siger.verfer.verify(siger.raw, serder.raw): # verify sig - logger.info("Kevery unescrow error: Bad trans receipt sig." - "pre=%s sn=%x receipter=%s", pre, sn, sprefixer.qb64) - - raise ValidationError("Bad escrowed trans receipt sig at " - "pre={} sn={:x} receipter={}." - "".format(pre, sn, sprefixer.qb64)) + msg = f"Bad escrowed trans receipt sig pre={pre} sn={sn:x} receipter={sprefixer.qb64}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # good sig so write receipt quadruple to database @@ -4349,10 +4403,11 @@ def processAttachedReceiptQuadruples(self, serder, trqs, firner=None, local=None else: # escrow either receiptor or receipted event not yet in database self.escrowTRQuadruple(serder, sprefixer, sseqner, saider, siger) - raise UnverifiedTransferableReceiptError("Unverified receipt: " - "missing associated event for transferable " - "validator receipt quadruple for event={}." - "".format(ked)) + msg = (f"Unverified receipt: missing associated event for transferable validator" + f"receipt quadruple for event {serder.said}") + logger.info(msg) + logger.debug("Event=\n%s\n", serder.pretty()) + raise UnverifiedTransferableReceiptError(msg) def removeStaleReplyEndRole(self, saider): """ @@ -4467,8 +4522,10 @@ def processReplyEndRole(self, *, serder, saider, route, cigars=None, tsgs=None, aid=aid, osaider=osaider, cigars=cigars, tsgs=tsgs) if not accepted: - logger.debug(f"Unverified end role reply ked={serder.ked}") - raise UnverifiedReplyError(f"Unverified end role reply. {serder.said}") + msg = f"Unverified end role reply = {serder.said} role = {role}" + logger.debug(f"Kevery: %s", msg) + logger.debug(f"Event=\n%s\n", serder.pretty()) + raise UnverifiedReplyError(msg) self.updateEnd(keys=keys, saider=saider, allowed=allowed) # update .eans and .ends @@ -4565,7 +4622,10 @@ def processReplyLocScheme(self, *, serder, saider, route, aid=aid, osaider=osaider, cigars=cigars, tsgs=tsgs) if not accepted: - raise UnverifiedReplyError(f"Unverified loc scheme reply. {serder.ked}") + msg = f"Unverified loc scheme reply URL={url} SAID={serder.said}" + logger.debug(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise UnverifiedReplyError(msg) self.updateLoc(keys=keys, saider=saider, url=url) # update .lans and .locs @@ -4891,18 +4951,27 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): if pre not in self.kevers: self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - raise QueryNotFoundError("Query not found error={}.".format(ked)) + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) kever = self.kevers[pre] if anchor: if not self.db.fetchAllSealingEventByEventSeal(pre=pre, seal=anchor): self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - raise QueryNotFoundError("Query not found error={}.".format(ked)) + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) elif sn is not None: if kever.sner.num < sn or not self.db.fullyWitnessed(kever.serder): self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - raise QueryNotFoundError("Query not found error={}.".format(ked)) + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) msgs = list() # outgoing messages for msg in self.db.clonePreIter(pre=pre, fn=fn): @@ -4922,7 +4991,10 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): if pre not in self.kevers: self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - raise QueryNotFoundError("Query not found error={}.".format(ked)) + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) kever = self.kevers[pre] @@ -4932,7 +5004,10 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): if len(wigers) < kever.toader.num: self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - raise QueryNotFoundError("Query not found error={}.".format(ked)) + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) rserder = reply(route=f"/ksn/{src}", data=kever.state()._asdict()) self.cues.push(dict(kin="reply", src=src, route="/ksn", serder=rserder, @@ -4945,7 +5020,10 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): if pre not in self.kevers: self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - raise QueryNotFoundError("Query not found error={}.".format(ked)) + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) self.cues.push(dict(kin="stream", serder=serder, pre=pre, src=src, topics=topics)) # if pre in self.kevers: @@ -4954,7 +5032,10 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): # self.cues.push(dict(kin="stream", serder=serder, pre=pre, src=src, topics=topics)) else: self.cues.push(dict(kin="invalid", serder=serder)) - raise ValidationError("invalid query message {} for evt = {}".format(ilk, ked)) + msg = f"Invalid query message {ilk} for event route={route} SAID={serder.said}" + logger.info(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise ValidationError(msg) def fetchEstEvent(self, pre, sn): """ @@ -5027,8 +5108,7 @@ def escrowMFEvent(self, serder, sigers, wigers=None, self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed - logger.debug("Kevery process: escrowed misfit event=\n%s", - json.dumps(serder.ked, indent=1)) + logger.debug("Kevery process: escrowed misfit event=\n%s", serder.pretty()) def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None, local=True): @@ -5068,8 +5148,7 @@ def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None, l self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.addOoe(snKey(serder.preb, serder.sn), serder.saidb) # log escrowed - logger.debug("Kevery process: escrowed out of order event=\n%s", - json.dumps(serder.ked, indent=1)) + logger.debug("Kevery process: escrowed out of order event=\n%s", serder.pretty()) def escrowQueryNotFoundEvent(self, prefixer, serder, sigers, cigars=None): """ @@ -5092,8 +5171,8 @@ def escrowQueryNotFoundEvent(self, prefixer, serder, sigers, cigars=None): self.db.addRct(key=dgkey, val=cigar.verfer.qb64b + cigar.qb64b) # log escrowed - logger.debug("Kevery process: escrowed query not found event=\n%s", - json.dumps(serder.ked, indent=1)) + logger.trace("Kevery: escrowed query not found event = %s", serder.said) + logger.trace("Event Body=\n%s\n", serder.pretty()) def escrowLDEvent(self, serder, sigers, local=True): """ @@ -5333,9 +5412,8 @@ def processEscrows(self): except Exception as ex: # log diagnostics errors etc if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery escrow process error: %s", ex.args[0]) - else: - logger.error("Kevery escrow process error: %s", ex.args[0]) + logger.trace("Kevery: other escrow process error: %s\n", ex.args[0]) + logger.exception("Kevery other escrow process error: %s\n", ex.args[0]) raise ex def processEscrowOutOfOrders(self): @@ -5383,40 +5461,35 @@ def processEscrowOutOfOrders(self): pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) + # no local source so raise ValidationError which unescrows below + msg = f"OOO Missing escrowed event source at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # check date if expired then remove escrow. dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) + msg = f"OOO Missing escrowed event datetime at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutOOE): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + msg = f"OOO Stale event escrow at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get the escrowed event using edig eraw = self.db.getEvt(dgKey(pre, bytes(edig))) if eraw is None: # no event so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + msg = f"OOO Missing escrowed event at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event @@ -5424,11 +5497,9 @@ def processEscrowOutOfOrders(self): sigs = self.db.getSigs(dgKey(pre, bytes(edig))) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) + msg = f"OOO Missing escrowed event sigs at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] @@ -5455,25 +5526,25 @@ def processEscrowOutOfOrders(self): except OutOfOrderError as ex: # still waiting on missing prior event to validate - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Kevery OOO escrow unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery OOO escrow unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delOoe(snKey(pre, sn), edig) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + logger.debug("Kevery: OOO escrow other error on escrow: %s\n", ex.args[0]) + logger.exception("Kevery: OOO escrow other error on : %s\n", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delOoe(snKey(pre, sn), edig) # removes one escrow at key val - logger.info("Kevery unescrow succeeded in valid event: " + logger.info("Kevery out of order unescrow succeeded in valid event: " "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + logger.debug("Event=\n%s\n", eserder.pretty()) if ekey == key: # still same so no escrows found on last while iteration break @@ -5523,51 +5594,44 @@ def processEscrowPartialSigs(self): pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) + # no local source so raise ValidationError which unescrows below + msg = f"PSE Missing escrowed event source at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # check date if expired then remove escrow. dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) + msg = f"PSE Missing escrowed event datetime at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + msg = f"PSE Stale event escrow at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get the escrowed event using edig eraw = self.db.getEvt(dgkey) if eraw is None: # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + msg = f"PSE Missing escrowed evt at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event # get sigs and attach sigs = self.db.getSigs(dgkey) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) + msg = f"PSE Missing escrowed evt sigs at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs if not wigs: # empty list wigs witness sigs not wits # wigs maybe empty if not wits or if wits while waiting @@ -5621,8 +5685,9 @@ def processEscrowPartialSigs(self): except MissingSignatureError as ex: # MissingDelegationError) # still waiting on missing sigs or missing seal to validate # processEvent idempotently reescrowed - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Kevery: PSE unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: PSE unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs so remove from escrow @@ -5633,9 +5698,10 @@ def processEscrowPartialSigs(self): self.cues.push(dict(kin="psUnescrow", serder=eserder)) if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + logger.trace("Kevery: PSE other error on unescrow: %s\n", + ex.args[0]) + logger.exception("Kevery: PSE other error on unescrow: %s\n", + ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -5647,9 +5713,9 @@ def processEscrowPartialSigs(self): if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): self.cues.push(dict(kin="psUnescrow", serder=eserder)) - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + logger.trace("Kevery: partial sig escrow unescrow succeeded in valid event: " + "key = %s \tdigest = %s", bytes(ekey).decode(), bytes(edig).decode()) + logger.debug("event=\n%s\n", eserder.pretty()) #if ekey == key: # still same so no escrows found on last while iteration #break @@ -5696,40 +5762,35 @@ def processEscrowPartialWigs(self): pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) + # no local source so raise ValidationError which unescrows below + msg = f"PWE Missing escrowed event source at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # check date if expired then remove escrow. dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) + msg = f"PWE Missing escrowed event datetime at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + msg = f"PWE Stale event escrow at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get the escrowed event using edig eraw = self.db.getEvt(dgKey(pre, bytes(edig))) if eraw is None: # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + msg = f"PWE Missing escrowed evt at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event @@ -5737,11 +5798,9 @@ def processEscrowPartialWigs(self): sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs if not sigs: # empty list # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) + msg = f"PWE Missing escrowed evt sigs at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get witness signatures (wigs not wits) wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs @@ -5752,8 +5811,8 @@ def processEscrowPartialWigs(self): # which may not arrive until some time after event is fully signed # so just log for debugging but do not unescrow by raising # ValidationError - logger.debug("Kevery unescrow wigs: No event wigs yet at." - "dig = %s", bytes(edig)) + logger.debug("Kevery unescrow wigs: PW No event wigs yet at." + "dig = %s", bytes(edig).decode()) # raise ValidationError("Missing escrowed evt wigs at " # "dig = {}.".format(bytes(edig))) @@ -5804,17 +5863,19 @@ def processEscrowPartialWigs(self): except MissingWitnessSignatureError as ex: # MissingDelegationError # still waiting on missing witness sigs or delegation # processEvent idempotently reescrowed - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Kevery: PWE unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: PWE unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than waiting on wigs so remove from escrow self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val #self.db.udes.rem(keys=dgkey) # leave here since could PartialDelegationEscrow - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Kevery: PWE other error on unescrow: %s\n", + ex.args[0]) + logger.exception("Kevery: PWE other error unescrow: %s\n", + ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -5822,9 +5883,9 @@ def processEscrowPartialWigs(self): # valid event escrow. self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val self.db.udes.rem(keys=dgkey) # remove escrow if any - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + logger.info("Kevery: partial wig escrow unescrow succeeded in valid event: " + "key = %s \tdigest = %s", bytes(ekey).decode(), bytes(edig).decode()) + logger.debug("Event=\n%s\n", eserder.pretty()) def processEscrowPartialDels(self): @@ -5859,40 +5920,35 @@ def processEscrowPartialDels(self): #pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(epre, edig) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) + # no local source so raise ValidationError which unescrows below + msg = f"PDE Missing escrowed event source at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # check date if expired then remove escrow. dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) + msg = f"PDE Missing escrowed event datetime at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + msg = f"PDE Stale event escrow at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get the escrowed event using edig eraw = self.db.getEvt(dgkey) if eraw is None: # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + msg = f"PDE Missing escrowed evt at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event @@ -5900,11 +5956,9 @@ def processEscrowPartialDels(self): sigs = self.db.getSigs(dgkey) # list of sigs if not sigs: # empty list # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) + msg = f"PDE Missing escrowed evt sigs at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", bytes(edig)) + raise ValidationError(msg) # get witness signatures (wigs not wits) assumes wont be in this # escrow if wigs not needed because no wits @@ -5971,7 +6025,7 @@ def processEscrowPartialDels(self): # still waiting on missing delegation source seal # processEvent idempotently reescrowed if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + logger.exception("Kevery PDE unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow @@ -5979,9 +6033,9 @@ def processEscrowPartialDels(self): self.db.pdes.remOn(keys=epre, on=esn, val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) + logger.exception("Kevery PDE unescrowed: %s", ex.args[0]) else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + logger.error("Kevery PDE unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be @@ -5990,9 +6044,9 @@ def processEscrowPartialDels(self): # removes one event escrow at key val self.db.pdes.remOn(keys=epre, on=esn, val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any - logger.info("Kevery unescrow succeeded in valid event: " + logger.info("Kevery partial delegation escrow unescrow succeeded in valid event: " "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + logger.debug("Event=\n%s\n", eserder.pretty()) def processEscrowUnverWitness(self): @@ -6059,22 +6113,18 @@ def processEscrowUnverWitness(self): dtb = self.db.getDts(dgKey(pre, bytes(rdiger.qb64b))) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", rdiger.qb64b) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(rdiger.qb64b)) + msg = f"UWE Missing escrowed event datetime at dig = {rdiger.qb64b}" + logger.trace("Kevery unescrow error: %s", rdiger.qb64b) + raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutUWE): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", rdiger.qb64b) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(rdiger.qb64b)) + msg = f"UWE Stale event escrow at dig = {rdiger.qb64b}" + logger.trace("Kevery unescrow error: %s", rdiger.qb64b) + raise ValidationError(msg) # lookup database dig of the receipted event in pwes escrow # using pre and sn lastEvt @@ -6085,33 +6135,31 @@ def processEscrowUnverWitness(self): if not found: # no partial witness escrow of event found # so keep in escrow by raising UnverifiedWitnessReceiptError - logger.debug("Kevery unescrow error: Missing witness " - "receipted evt at pre=%s sn=%x", (pre, sn)) - - raise UnverifiedWitnessReceiptError("Missing witness " - "receipted evt at pre={} sn={:x}".format(pre, sn)) + msg = f"UWE Missing witness receipted evt at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) + raise UnverifiedWitnessReceiptError(msg) except UnverifiedWitnessReceiptError as ex: # still waiting on missing prior event to validate # only happens if we process above if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + logger.trace("Kevery: UWE unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: UWE unescrow failed: %s\n", + ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + logger.trace("Kevery: UWE other unescrow error: %s\n", ex.args[0]) + logger.exception("Kevery: UWE other unescrow error: %s\n", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val - logger.info("Kevery unescrow succeeded for event pre=%s " - "sn=%s", pre, sn) + logger.info("Kevery UWE unescrow succeeded for event pre=%s sn=%s", pre, sn) if ekey == key: # still same so no escrows found on last while iteration break @@ -6177,22 +6225,18 @@ def processEscrowUnverNonTrans(self): dtb = self.db.getDts(dgKey(pre, bytes(rsaider.qb64b))) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", rsaider.qb64b) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(rsaider.qb64b)) + msg = f"URE Missing escrowed event datetime at dig = {rsaider.qb64b}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutURE): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", rsaider.qb64b) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(rsaider.qb64b)) + msg = f"URE Stale event escrow at dig = {rsaider.qb64b}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # Is receipt for unverified witnessed event in .Pwes escrow # if found then try else clause will remove from escrow @@ -6209,41 +6253,31 @@ def processEscrowUnverNonTrans(self): dig = self.db.getKeLast(snKey(pre, sn)) if dig is None: # no receipted event so keep in escrow - logger.debug("Kevery unescrow error: Missing receipted " - "event at pre=%s sn=%x", pre, sn) - - raise UnverifiedReceiptError("Missing receipted evt " - "at pre={} sn={:x}".format(pre, sn)) + msg = f"URE Missing receipted evt at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) + raise UnverifiedReceiptError(msg) # get receipted event using pre and edig raw = self.db.getEvt(dgKey(pre, dig)) if raw is None: # receipted event superseded so remove from escrow - logger.info("Kevery unescrow error: Invalid receipted " - "event refereance at pre=%s sn=%x", pre, sn) - - raise ValidationError("Invalid receipted evt reference" - " at pre={} sn={:x}".format(pre, sn)) + msg = f"URE Invalid receipted event reference at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) serder = serdering.SerderKERI(raw=bytes(raw)) # receipted event # compare digs if rsaider.qb64b != serder.saidb: - logger.info("Kevery unescrow error: Bad receipt dig." - "pre=%s sn=%x receipter=%s", pre, sn, sprefixer.qb64) - - raise ValidationError("Bad escrowed receipt dig at " - "pre={} sn={:x} receipter={}." - "".format(pre, sn, sprefixer.qb64)) + msg = f"URE Bad escrowed receipt dig at pre={pre} sn={sn:x} receipter={sprefixer.qb64}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # verify sig verfer key is prefixer from triple if not cigar.verfer.verify(cigar.raw, serder.raw): # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Bad receipt sig." - "pre=%s sn=%x receipter=%s", pre, sn, sprefixer.qb64) - - raise ValidationError("Bad escrowed receipt sig at " - "pre={} sn={:x} receipter={}." - "".format(pre, sn, sprefixer.qb64)) + msg = f"URE Bad escrowed receipt sig at pre={pre} sn={sn:x} receipter={sprefixer.qb64}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get current wits from kever state assuming not stale # receipt. Need function here to compute wits for actual @@ -6265,23 +6299,24 @@ def processEscrowUnverNonTrans(self): except UnverifiedReceiptError as ex: # still waiting on missing prior event to validate # only happens if we process above - if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): # adds exception data + logger.trace("Kevery: URE escrow other error on unescrow: %s\n", ex.args[0]) + logger.exception("Kevery: URE escrow other error on unescrow: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delUre(snKey(pre, sn), etriplet) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrowed: %s", ex.args[0]) + logger.exception("Kevery URE unescrowed: %s", ex.args[0]) else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + logger.error("Kevery URE unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delUre(snKey(pre, sn), etriplet) # removes one escrow at key val - logger.info("Kevery unescrow succeeded for event pre=%s " + logger.info("Kevery UNT unescrow succeeded for event pre=%s " "sn=%s", pre, sn) if ekey == key: # still same so no escrows found on last while iteration @@ -6316,40 +6351,35 @@ def processEscrowDelegables(self): edig = dig.encode("utf-8") dgkey = dgKey(pre.encode("utf-8"), edig) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) + # no local source so raise ValidationError which unescrows below + msg = f"DEL Missing escrowed event source at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # check date if expired then remove escrow. dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) + msg = f"DEL Missing escrowed event datetime at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutOOE): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + msg = f"DEL Stale event escrow at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get the escrowed event using edig eraw = self.db.getEvt(dgKey(pre, bytes(edig))) if eraw is None: # no event so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + msg = f"DEL Missing escrowed evt at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event @@ -6357,11 +6387,9 @@ def processEscrowDelegables(self): sigs = self.db.getSigs(dgKey(pre, bytes(edig))) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) + msg = f"DEL Missing escrowed evt sigs at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] @@ -6400,9 +6428,9 @@ def processEscrowDelegables(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delegables.rem(keys=(pre, sn,), val=edig) # removes one escrow at key val - logger.info("Kevery unescrow succeeded in valid event: " + logger.info("Kevery delegables escrow unescrow succeeded in valid event: " "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + logger.debug("event=\n%s\n", eserder.pretty()) def processQueryNotFound(self): """ @@ -6440,32 +6468,26 @@ def processQueryNotFound(self): dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) + msg = f"QNF Missing escrowed event datetime at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutQNF): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale qry event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale qry event escrow " - "at dig = {}.".format(bytes(edig))) + msg = f"QNF Stale qry event escrow at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get the escrowed event using edig eraw = self.db.getEvt(dgkey) if eraw is None: # no event so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + msg = f"QNF Missing escrowed evt at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event @@ -6473,11 +6495,9 @@ def processQueryNotFound(self): sigs = self.db.getSigs(dgkey) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) + msg = f"QNF Missing escrowed evt sigs at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] @@ -6500,24 +6520,24 @@ def processQueryNotFound(self): except QueryNotFoundError as ex: # still waiting on missing prior event to validate - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Kevery: QNF escrow unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: QNF escrow unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.qnfs.rem(keys=(pre, said), val=edig) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + logger.debug("Kevery: QNF other unescrow error: %s\n", ex.args[0]) + logger.exception("Kevery: QNF other unescrow error: %s\n", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.qnfs.rem(keys=(pre, said), val=edig) # removes one escrow at key val - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + logger.info("Kevery: query not found escrow unescrow succeeded in valid event: " + "key = %s \tdigest = %s", ekey.decode(), edig) + logger.debug("Event=\n%s\n", eserder.pretty()) if ekey == key: # still same so no escrows found on last while iteration break @@ -6608,10 +6628,9 @@ def _processEscrowFindUnver(self, pre, sn, rsaider, wiger=None, cigar=None): elif wiger: # check index and assign verfier to wiger if wiger.index >= len(wits): # bad index # raise ValidationError which removes from escrow by caller - logger.info("Kevery unescrow error: Bad witness receipt" - " index=%i for pre=%s sn=%x", wiger.index, pre, sn) - raise ValidationError("Bad escrowed witness receipt index={}" - " at pre={} sn={:x}.".format(wiger.index, pre, sn)) + msg = f"FUV Bad escrowed witness receipt index={wiger.index} at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) wiger.verfer = Verfer(qb64=wits[wiger.index]) found = True @@ -6620,12 +6639,9 @@ def _processEscrowFindUnver(self, pre, sn, rsaider, wiger=None, cigar=None): if found: # verify signature and if verified write to .Wigs if not wiger.verfer.verify(wiger.raw, serder.raw): # not verify # raise ValidationError which unescrows .Uwes or .Ures in caller - logger.info("Kevery unescrow error: Bad witness receipt" - " wig. pre=%s sn=%x", pre, sn) - - raise ValidationError("Bad escrowed witness receipt wig" - " at pre={} sn={:x}." - "".format(pre, sn)) + msg = f"FUV Bad escrowed witness receipt wig at pre={pre} sn={sn:x}." + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) self.db.addWig(key=dgKey(pre, serder.said), val=wiger.qb64b) # processEscrowPartialWigs removes from this .Pwes escrow # when fully witnessed using self.db.delPwe(snkey, dig) @@ -6689,53 +6705,42 @@ def processEscrowUnverTrans(self): dtb = self.db.getDts(dgKey(pre, bytes(esaider.qb64b))) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", esaider.qb64b) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(esaider.qb64b)) + msg = f"VRE Missing escrowed event datetime at dig = {esaider.qb64b}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutVRE): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", esaider.qb64b) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(esaider.qb64b)) + msg = f"VRE Stale event escrow at dig = {esaider.qb64b}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get dig of the receipted event using pre and sn lastEvt raw = self.db.getKeLast(snKey(pre, sn)) if raw is None: # no event so keep in escrow - logger.debug("Kevery unescrow error: Missing receipted " - "event at pre=%s sn=%x", pre, sn) - - raise UnverifiedTransferableReceiptError("Missing receipted evt at pre={} " - " sn={:x}".format(pre, sn)) + msg = f"VRE Missing receipted evt at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) + raise UnverifiedTransferableReceiptError(msg) dig = bytes(raw) # get receipted event using pre and edig raw = self.db.getEvt(dgKey(pre, dig)) if raw is None: # receipted event superseded so remove from escrow - logger.info("Kevery unescrow error: Invalid receipted " - "event referenace at pre=%s sn=%x", pre, sn) - - raise ValidationError("Invalid receipted evt reference " - "at pre={} sn={:x}".format(pre, sn)) + msg = f"VRE Invalid receipted evt reference at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) serder = serdering.SerderKERI(raw=bytes(raw)) # receipted event # compare digs if esaider.qb64b != serder.saidb: - logger.info("Kevery unescrow error: Bad receipt dig." - "pre=%s sn=%x receipter=%s", (pre, sn, sprefixer.qb64)) - - raise ValidationError("Bad escrowed receipt dig at " - "pre={} sn={:x} receipter={}." - "".format(pre, sn, sprefixer.qb64)) + msg = f"VRE Bad escrowed receipt dig at pre={pre} sn={sn:x} receipter={sprefixer.qb64}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get receipter's last est event # retrieve dig of last event at sn of receipter. @@ -6743,11 +6748,9 @@ def processEscrowUnverTrans(self): sn=sseqner.sn)) if sdig is None: # no event so keep in escrow - logger.debug("Kevery unescrow error: Missing receipted " - "event at pre=%s sn=%x", pre, sn) - - raise UnverifiedTransferableReceiptError("Missing receipted evt at pre={} " - " sn={:x}".format(pre, sn)) + msg = f"VRE Missing receipted evt at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) + raise UnverifiedTransferableReceiptError(msg) # retrieve last event itself of receipter sraw = self.db.getEvt(key=dgKey(pre=sprefixer.qb64b, dig=bytes(sdig))) @@ -6755,31 +6758,31 @@ def processEscrowUnverTrans(self): sserder = serdering.SerderKERI(raw=bytes(sraw)) if not sserder.compare(said=ssaider.qb64): # seal dig not match event # this unescrows - raise ValidationError("Bad chit seal at sn = {} for rct = {}." - "".format(sseqner.sn, sserder.ked)) + msg = f"VRE Bad chit seal at sn = {sseqner.sn} for rct = {sserder.ked}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # verify sigs and if so write quadruple to database verfers = sserder.verfers if not verfers: - raise ValidationError("Invalid seal est. event dig = {} for " - "receipt from pre ={} no keys." - "".format(ssaider.qb64, sprefixer.qb64)) + msg = (f"VRE Invalid seal est. event dig = {ssaider.qb64} " + f"for receipt from pre = {sprefixer.qb64} no keys") + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # Set up quadruple sealet = sprefixer.qb64b + sseqner.qb64b + ssaider.qb64b if siger.index >= len(verfers): - raise ValidationError("Index = {} to large for keys." - "".format(siger.index)) + msg = f"VRE Index = {siger.index} too large for keys" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) siger.verfer = verfers[siger.index] # assign verfer if not siger.verfer.verify(siger.raw, serder.raw): # verify sig - logger.info("Kevery unescrow error: Bad trans receipt sig." - "pre=%s sn=%x receipter=%s", pre, sn, sprefixer.qb64) - - raise ValidationError("Bad escrowed trans receipt sig at " - "pre={} sn={:x} receipter={}." - "".format(pre, sn, sprefixer.qb64)) + msg = f"VRE Bad escrowed trans receipt sig at pre={pre} sn={sn:x} receipter={sprefixer.qb64}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # good sig so write receipt quadruple to database quadruple = sealet + siger.qb64b @@ -6789,24 +6792,24 @@ def processEscrowUnverTrans(self): except UnverifiedTransferableReceiptError as ex: # still waiting on missing prior event to validate # only happens if we process above - if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): # adds exception data + logger.trace("Kevery: VRE escrow unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: VRE escrow unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delVre(snKey(pre, sn), equinlet) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): # adds exception data - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + logger.debug("Kevery: VRE escrow other error on unescrow: %s\n", ex.args[0]) + logger.exception("Kevery: VRE escrow other error on unescrow: %s\n", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delVre(snKey(pre, sn), equinlet) # removes one escrow at key val - logger.info("Kevery unescrow succeeded for event = %s", serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("Kevery VRE unescrow succeeded for event = %s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) if ekey == key: # still same so no escrows found on last while iteration break @@ -6855,40 +6858,35 @@ def processEscrowDuplicitous(self): pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) + # no local source so raise ValidationError which unescrows below + msg = f"DUP Missing escrowed event source at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # check date if expired then remove escrow. dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) + msg = f"DUP Missing escrowed event datetime at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes dtnow = helping.nowUTC() dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutLDE): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + msg = f"DUP Stale event escrow at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # get the escrowed event using edig eraw = self.db.getEvt(dgKey(pre, bytes(edig))) if eraw is None: # no event so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + msg = f"DUP Missing escrowed evt at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event @@ -6896,11 +6894,9 @@ def processEscrowDuplicitous(self): sigs = self.db.getSigs(dgKey(pre, bytes(edig))) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) + msg = f"DUP Missing escrowed evt sigs at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] self.processEvent(serder=eserder, sigers=sigers, local=esr.local) @@ -6921,25 +6917,24 @@ def processEscrowDuplicitous(self): except LikelyDuplicitousError as ex: # still can't determine if duplicitous - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Kevery: DUP escrow unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: DUP escrow unescrow failed: %s\n",ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than likely duplicitous so remove from escrow self.db.delLde(snKey(pre, sn), edig) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + logger.trace("Kevery: DUP escrow other unescrow error: %s\n", ex.args[0]) + logger.exception("Kevery: DUP escrow other unescrow error: %s\n", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delLde(snKey(pre, sn), edig) # removes one escrow at key val - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + logger.info("Kevery duplicitous escrow unescrow succeeded in valid event: event=%s", eserder.said) + logger.debug("event=\n%s\n", eserder.pretty()) if ekey == key: # still same so no escrows found on last while iteration break diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index d186c69f1..98fe18e4b 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -443,9 +443,9 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream if logger.isEnabledFor(logging.DEBUG): - logger.exception("Parser msg extraction error: %s", ex.args[0]) + logger.exception("Parser sized group error: %s", ex.args[0]) else: - logger.error("Parser msg extraction error: %s", ex.args[0]) + logger.error("Parser sized group error: %s", ex.args[0]) except (kering.ColdStartError, kering.ExtractionError) as ex: # some extraction error if logger.isEnabledFor(logging.DEBUG): @@ -457,10 +457,10 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, except (kering.ValidationError, Exception) as ex: # non Extraction Error # Non extraction errors happen after successfully extracted from stream # so we don't flush rest of stream just resume + if logger.isEnabledFor(logging.TRACE): + logger.exception("Parser msg validation or non-extraction error: %s", ex) if logger.isEnabledFor(logging.DEBUG): - logger.exception("Parser msg non-extraction error: %s", ex) - else: - logger.error("Parser msg non-extraction error: %s", ex) + logger.error("Parser msg validation or non-extraction error: %s", ex) yield return True @@ -618,9 +618,9 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream if logger.isEnabledFor(logging.DEBUG): - logger.exception("Parser msg extraction error: %s", ex.args[0]) + logger.exception("Parser sized group error: %s", ex.args[0]) else: - logger.error("Parser msg extraction error: %s", ex.args[0]) + logger.error("Parser sized group error: %s", ex.args[0]) except (kering.ColdStartError, kering.ExtractionError) as ex: # some extraction error if logger.isEnabledFor(logging.DEBUG): @@ -632,9 +632,9 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, except (kering.ValidationError, Exception) as ex: # non Extraction Error # Non extraction errors happen after successfully extracted from stream # so we don't flush rest of stream just resume - if logger.isEnabledFor(logging.DEBUG): + if logger.isEnabledFor(logging.TRACE): logger.exception("Parser msg non-extraction error: %s", ex.args[0]) - else: + if logger.isEnabledFor(logging.DEBUG): logger.error("Parser msg non-extraction error: %s", ex.args[0]) yield @@ -1007,8 +1007,10 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, # when present assumes this is source seal of delegating event in delegator's KEL delseqner, delsaider = sscs[-1] if sscs else (None, None) # use last one if more than one if not sigers: - raise kering.ValidationError("Missing attached signature(s) for evt " - "= {}.".format(serder.ked)) + msg = f"Missing attached signature(s) for evt = {serder.ked['d']}" + logger.info(msg) + logger.debug("Event Body = \n%s\n", serder.pretty()) + raise kering.ValidationError(msg) try: kvy.processEvent(serder=serder, sigers=sigers, @@ -1027,13 +1029,17 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, firner=firner, local=local) except AttributeError as ex: - raise kering.ValidationError("No kevery to process so dropped msg" - "= {}.".format(serder.pretty())) from ex + msg = f"No kevery to process so dropped msg={serder.said}" + logger.info(msg) + logger.debug("Event Body = \n%s\n", serder.pretty()) + raise kering.ValidationError(msg) from ex elif ilk in [Ilks.rct]: # event receipt msg (nontransferable) if not (cigars or wigers or tsgs): - raise kering.ValidationError("Missing attached signatures on receipt" - "msg = {}.".format(serder.ked)) + msg = f"Missing attached signatures on receipt msg sn={serder.sn} SAID={serder.said}" + logger.info(msg) + logger.debug("Receipt body=\n%s\n", serder.pretty()) + raise kering.ValidationError(msg) try: if cigars: @@ -1051,6 +1057,12 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, except AttributeError: raise kering.ValidationError("No kevery to process so dropped msg" "= {}.".format(serder.pretty())) + except kering.UnverifiedReplyError as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Error processing reply = %s", e) + logger.debug("Reply Body=\n%s\n", serder.pretty()) + else: + logger.error("Error processing reply = %s", e) elif ilk in (Ilks.rpy,): # reply message if not (cigars or tsgs): @@ -1096,6 +1108,12 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, except AttributeError as e: raise kering.ValidationError("No tevery to process so dropped msg" "= {} from {}.".format(serder.pretty(), e)) + except kering.QueryNotFoundError as e: # catch escrow error and log it + if logger.isEnabledFor(logging.TRACE): + logger.exception("Error processing query = %s", e) + logger.trace("Query Body=\n%s\n", serder.pretty()) + else: + logger.error("Error processing query = %s", e) else: raise kering.ValidationError("Invalid resource type {} so dropped msg" diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 1418272de..7a8cfe22a 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -261,37 +261,37 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, for cigar in cigars: # process each couple to verify sig and write to db if cigar.verfer.transferable: # ignore invalid transferable verfers - logger.info("Kevery process: skipped invalid transferable verfers" - " on reply said=", serder.said) + logger.info("Revery: skipped invalid transferable verfers " + "on reply said = %s", serder.said) continue # skip invalid transferable if not self.lax and cigar.verfer.qb64 in self.prefixes: # own cig if not self.local: # own cig when not local so ignore - logger.info("Kevery process: skipped own attachment" - " on nonlocal reply said=", serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("Revery: skipped own attachment for AID %s" + " on non-local reply at route = %s", aid, serder.ked['r']) + logger.debug("Reply Body=\n%s\n", serder.pretty()) continue # skip own cig attachment on non-local reply msg if aid != cigar.verfer.qb64: # cig not by aid - logger.info("Kevery process: skipped cig not from aid=" - "%s on reply said=%s", aid, serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("Revery: skipped cig not from aid=" + "%s on reply at route %s", aid, serder.ked['r']) + logger.debug("Reply Body=\n%s\n", serder.pretty()) continue # skip invalid cig's verfer is not aid if odater: # get old compare datetimes to see if later if dater.datetime <= odater.datetime: - logger.info("Kevery process: skipped stale update from " - "%s of reply said=%s", aid, serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.trace("Revery: skipped stale update from " + "%s of reply at route= %s", aid, serder.ked['r']) + logger.trace("Reply Body=\n%s\n", serder.pretty()) continue # skip if not later # raise ValidationError(f"Stale update of {route} from {aid} " # f"via {Ilks.rpy}={serder.ked}.") if not cigar.verfer.verify(cigar.raw, serder.raw): # cig not verify - logger.info("Kevery process: skipped nonverifying cig from " - "%s on reply said=%s", cigar.verfer.qb64, serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("Revery: skipped non-verifying cig from " + "%s on reply at route = %s", cigar.verfer.qb64, serder.ked['r']) + logger.debug("Reply Body=\n%s\n", serder.pretty()) continue # skip if cig not verify # All constraints satisfied so update @@ -303,16 +303,16 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, for prefixer, seqner, ssaider, sigers in tsgs: # iterate over each tsg if not self.lax and prefixer.qb64 in self.prefixes: # own sig if not self.local: # own sig when not local so ignore - logger.info("Kevery process: skipped own attachment" - " on nonlocal reply said=%s", serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Revery: skipped own attachment " + "on nonlocal reply said=%s", serder.said) + logger.debug("event=\n%s\n", serder.pretty()) continue # skip own sig attachment on non-local reply msg spre = prefixer.qb64 if aid != spre: # sig not by aid - logger.info("Kevery process: skipped signature not from aid=" - "%s on reply said=%s", aid, serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Revery: skipped signature not from aid = " + "%s on reply said=%s", aid, serder.said) + logger.debug("event=\n%s\n", serder.pretty()) continue # skip invalid signature is not from aid if osaider: # check if later logic sn > or sn == and dt > @@ -320,19 +320,18 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, _, osqr, _, _ = otsgs[0] # zeroth should be authoritative if seqner.sn < osqr.sn: # sn earlier - logger.info("Kevery process: skipped stale key state sig" + logger.info("Revery: skipped stale key state sig " "from %s sn=%s<%s on reply said=%s", aid, seqner.sn, osqr.sn, serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("event=\n%s\n", serder.pretty()) continue # skip if sn earlier if seqner.sn == osqr.sn: # sn same so check datetime if odater: if dater.datetime <= odater.datetime: - logger.info("Kevery process: skipped stale key" - "state sig datetime from %s on reply said=%s", - aid, serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("Revery: skipped stale key state sig datetime " + "from %s on reply said=%s", aid, serder.said) + logger.debug("event=\n%s\n", serder.pretty()) continue # skip if not later # retrieve sdig of last event at sn of signer. @@ -340,7 +339,7 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, if sdig is None: # create cue here to request key state for sprefixer signer # signer's est event not yet in signer's KEL - logger.info("Kevery process: escrowing without key state for signer" + logger.info("Revery: escrowing without key state for signer" " on reply said=", serder.said) self.escrowReply(serder=serder, saider=saider, dater=dater, route=route, prefixer=prefixer, seqner=seqner, @@ -495,8 +494,8 @@ def processEscrowReply(self): except kering.UnverifiedReplyError as ex: # still waiting on missing prior event to validate - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Kevery unescrow attempt failed: %s\n", ex.args[0]) except Exception as ex: # other error so remove from reply escrow self.db.rpes.rem(keys=(route, ), val=saider) # remove escrow only @@ -508,9 +507,8 @@ def processEscrowReply(self): else: # unescrow succeded self.db.rpes.rem(keys=(route, ), val=saider) # remove escrow only - logger.info("Kevery unescrow succeeded for reply said=%s", - serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("Revery unescrow succeeded for reply said=%s", serder.said) + logger.debug("event=\n%s\n", serder.pretty()) except Exception as ex: # log diagnostics errors etc self.db.rpes.rem(keys=(route,), val=saider) # remove escrow only diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 36cdbc0e9..ea820edc5 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -90,8 +90,9 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): try: if not (dater and serder and (tsgs or vcigars)): - raise ValueError(f"Missing escrow artifacts at said={saider.qb64}" - f"for pre={pre}.") + msg = f"Missing escrow artifacts at said={saider.qb64} for pre={pre}." + logger.info("Broker %s: unescrow error: %s", typ, msg) + raise ValueError(msg) cigars = [] if vcigars: @@ -103,39 +104,39 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): if ((helping.nowUTC() - dater.datetime) > datetime.timedelta(seconds=self.timeout)): # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale txn state escrow " - " at pre = %s", pre) - - raise kering.ValidationError(f"Stale txn state escrow at pre = {pre}.") + msg = f"Escrow unescrow error: Stale txn state escrow at pre = {pre}" + logger.trace("Broker %s: %s", typ, msg) + raise kering.ValidationError(msg) processReply(serder=serder, saider=saider, route=serder.ked["r"], cigars=cigars, tsgs=tsgs, aid=aid) except extype as ex: # still waiting on missing prior event to validate - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Broker %s: unescrow attempt failed: %s\n", typ, ex.args[0]) + logger.exception("Broker %s: unescrow attempt failed: %s", typ, ex.args[0]) except Exception as ex: # other error so remove from reply escrow self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) + logger.exception("Broker %s: unescrowed due to error: %s", typ, ex.args[0]) else: - logger.error("Kevery unescrowed due to error: %s", ex.args[0]) + logger.error("Broker %s: unescrowed due to error: %s", typ, ex.args[0]) else: # unescrow succeded self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow - logger.info("Kevery unescrow succeeded for txn state=%s", - serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("Broker %s: unescrow succeeded for txn state=%s", + typ, serder.said) + logger.debug("TXN State Body=\n%s\n", serder.pretty()) except Exception as ex: # log diagnostics errors etc self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.removeState(saider) if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) + logger.exception("Broker %s: unescrowed due to error: %s", typ, ex.args[0]) else: - logger.error("Kevery unescrowed due to error: %s", ex.args[0]) + logger.error("Broker %s: unescrowed due to error: %s", typ, ex.args[0]) def escrowStateNotice(self, *, typ, pre, aid, serder, saider, dater, cigars=None, tsgs=None): """ diff --git a/src/keri/demo/demo_kev.py b/src/keri/demo/demo_kev.py index 00b9cca7c..96c70f282 100644 --- a/src/keri/demo/demo_kev.py +++ b/src/keri/demo/demo_kev.py @@ -8,7 +8,7 @@ import argparse import logging -from hio import help +from keri import help from hio.base import doing from ..app import habbing, keeping, apping diff --git a/src/keri/help/__init__.py b/src/keri/help/__init__.py index 2223582c8..a7bf9ad5a 100644 --- a/src/keri/help/__init__.py +++ b/src/keri/help/__init__.py @@ -6,13 +6,25 @@ utility functions """ +import logging + # Setup module global ogler as package logger factory. This must be done on # import to ensure global is defined so all modules in package have access to # logggers via ogling.ogler.getLoggers(). May always change level and reopen log file # if need be - from hio.help import ogling + +# Custom TRACE log level configuration +TRACE = 5 # TRACE (5) logging level value between DEBUG (10) and NOTSET (0) +logging.TRACE = TRACE # add TRACE logging level to logging module +logging.addLevelName(logging.TRACE, "TRACE") +def trace(self, message, *args, **kwargs): + """Trace logging function - logs message if TRACE (5) level enabled""" + if self.isEnabledFor(TRACE): + self._log(TRACE, message, args, **kwargs) +logging.Logger.trace = trace + # want help.ogler always defined by default ogler = ogling.initOgler(prefix='keri', syslogged=False) # inits once only on first import diff --git a/src/keri/peer/exchanging.py b/src/keri/peer/exchanging.py index daaf31d45..ad1e20159 100644 --- a/src/keri/peer/exchanging.py +++ b/src/keri/peer/exchanging.py @@ -25,7 +25,7 @@ class Exchanger: Peer to Peer KERI message Exchanger. """ - TimeoutPSE = 10 # seconds to timeout partially signed or delegated escrows + TimeoutPSE = 20 # seconds to timeout partially signed or delegated escrows def __init__(self, hby, handlers, cues=None, delta=ExchangeMessageTimeWindow): """ Initialize instance @@ -80,14 +80,19 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): if tsgs: for prefixer, seqner, ssaider, sigers in tsgs: # iterate over each tsg if sender != prefixer.qb64: # sig not by aid - raise MissingSignatureError(f"Exchange process: skipped signature not from aid=" - f"{sender}, from {prefixer.qb64} on exn msg=\n{serder.pretty()}\n") + msg = (f"Skipped signature not from aid = " + f"{sender}, from {prefixer.qb64} on exn msg = {serder.said}") + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) if prefixer.qb64 not in self.kevers or self.kevers[prefixer.qb64].sn < seqner.sn: if self.escrowPSEvent(serder=serder, tsgs=tsgs, pathed=pathed): self.cues.append(dict(kin="query", q=dict(r="logs", pre=prefixer.qb64, sn=seqner.snh))) - raise MissingSignatureError(f"Unable to find sender {prefixer.qb64} in kevers" - f" for evt = {serder.ked}.") + msg = f"Unable to find sender {prefixer.qb64} in kevers for evt = {serder.said}" + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) # Verify the signatures are valid and that the signature threshold as of the signing event is met tholder, verfers = self.hby.db.resolveVerifiers(pre=prefixer.qb64, sn=seqner.sn, dig=ssaider.qb64) @@ -96,23 +101,35 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): if not tholder.satisfy(indices): # We still don't have all the sigers, need to escrow if self.escrowPSEvent(serder=serder, tsgs=tsgs, pathed=pathed): self.cues.append(dict(kin="query", q=dict(r="logs", pre=prefixer.qb64, sn=seqner.snh))) - raise MissingSignatureError(f"Not enough signatures in {indices}" - f" for evt = {serder.ked}.") + msg = (f"Not enough signatures in idx={indices} route={route} " + f"for evt = {serder.said} recipient={serder.ked.get('rp', '')}") + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) elif cigars: for cigar in cigars: if sender != cigar.verfer.qb64: # cig not by aid - raise MissingSignatureError(" process: skipped cig not from aid=" - "%s on exn msg=\n%s\n", sender, serder.pretty()) + msg = (f"Skipped cig not from aid={sender} route={route} " + f"for exn evt = {serder.said} recipient={serder.ked.get('rp', '')}") + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) if not cigar.verfer.verify(cigar.raw, serder.raw): # cig not verify - raise MissingSignatureError("Failure satisfying exn on cigs for {}" - " for evt = {}.".format(cigar, - serder.ked)) + msg = (f"Failure satisfying exn on cigs for {cigar} route={route} " + f"for evt = {serder.said} recipient={serder.ked.get('rp', '')}") + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) else: self.escrowPSEvent(serder=serder, tsgs=[], pathed=pathed) - raise MissingSignatureError("Failure satisfying exn, no cigs or sigs" - " for evt = {}.".format(serder.ked)) + msg = ( + f"Failure satisfying exn, no cigs or sigs for evt = {serder.said} " + f"on route {route} recipient = {serder.ked.get('rp', '')}") + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) e = coring.Pather(path=["e"]) @@ -142,13 +159,13 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): # Perform behavior specific verification, think IPEX chaining requirements try: if not behavior.verify(serder=serder, **kwargs): - logger.info(f"exn event for route {route} failed behavior verfication. said={serder.said}") - logger.debug(f"event=\n{serder.pretty()}\n") + logger.error("exn event for route %s failed behavior verification. said=%s", route, serder.said) + logger.debug(f"Event=\n%s\n", serder.pretty()) return except AttributeError: - logger.info(f"Behavior for {route} missing or does not have verify for said={serder.said}") - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Behavior for %s missing or does not have verify for said %s", route, serder.said) + logger.debug("Exn Event Body=\n%s\n", serder.pretty()) # Always persist events self.logEvent(serder, pathed, tsgs, cigars, essrs) @@ -158,8 +175,8 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): try: behavior.handle(serder=serder, **kwargs) except AttributeError: - logger.info(f"Behavior for {route} missing or does not have handle for said={serder.said}") - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Behavior for %s missing or does not have handle for SAID=%s", route, serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) def processEscrow(self): """ Process all escrows for `exn` messages @@ -231,24 +248,23 @@ def processEscrowPartialSigned(self): self.processEvent(serder=serder, tsgs=tsgs, pathed=pathed, **kwargs) except MissingSignatureError as ex: - if logger.isEnabledFor(logging.DEBUG): - logger.info("Exchange partially signed unescrow failed: %s", ex.args[0]) - else: - logger.info("Exchange partially signed failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Exchange partially signed unescrow failed: %s\n", ex.args[0]) + logger.debug(f"Event body=\n%s\n", serder.pretty()) except Exception as ex: self.hby.db.epse.rem(dig) self.hby.db.epsd.rem(dig) self.hby.db.esigs.rem(dig) if logger.isEnabledFor(logging.DEBUG): - logger.exception("Exchange partially signed unescrowed: %s", ex.args[0]) + logger.exception("Exchanger: partially signed unescrowed: %s", ex.args[0]) else: - logger.error("Exchange partially signed unescrowed: %s", ex.args[0]) + logger.error("Exchanger: partially signed unescrowed: %s", ex.args[0]) else: self.hby.db.epse.rem(dig) self.hby.db.esigs.rem(dig) - logger.info("Exchanger unescrow succeeded in valid exchange: " + logger.info("Exchanger: unescrow succeeded in valid exchange: " "creder=%s", serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.debug("Event=\n%s\n", serder.pretty()) def logEvent(self, serder, pathed=None, tsgs=None, cigars=None, essrs=None): dig = serder.said @@ -538,27 +554,35 @@ def verify(hby, serder): accepted = False for prefixer, seqner, ssaider, sigers in tsgs: if prefixer.qb64 not in hby.kevers or hby.kevers[prefixer.qb64].sn < seqner.sn: - raise MissingSignatureError(f"Unable to find sender {prefixer.qb64} in kevers" - f" for evt = {serder.ked}.") + msg = f"Unable to find sender {prefixer.qb64} in kevers for evt = {serder.said}" + logger.info(msg) + logger.debug("Exn Body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) # Verify the signatures are valid and that the signature threshold as of the signing event is met tholder, verfers = hby.db.resolveVerifiers(pre=prefixer.qb64, sn=seqner.sn, dig=ssaider.qb64) _, indices = eventing.verifySigs(serder.raw, sigers, verfers) if not tholder.satisfy(indices): # We still don't have all the sigers, need to escrow - raise MissingSignatureError(f"Not enough signatures in {indices}" - f" for evt = {serder.ked}.") + msg = f"Not enough signatures in idx={indices} for evt = {serder.said}" + logger.info(msg) + logger.debug("Exn Body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) accepted = True cigars = hby.db.ecigs.get(keys=(serder.said,)) for cigar in cigars: if not cigar.verfer.verify(cigar.raw, serder.raw): # cig not verify - raise MissingSignatureError("Failure satisfying exn on cigs for {}" - " for evt = {}.".format(cigar, - serder.ked)) + msg = f"Failure satisfying exn on cigs for {cigar} for evt = {serder.said}" + logger.info(msg) + logger.debug("Exn Body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) accepted = True if not accepted: - raise MissingSignatureError(f"No valid signatures stored for evt = {serder.ked}") + msg = f"No valid signatures stored for evt = {serder.said}" + logger.info(msg) + logger.debug("Exn Body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) return tsgs, cigars diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index 93c47e6a7..87df5ddb8 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -235,8 +235,8 @@ def processEvent(self, serder): try: self.tvy.processEvent(serder=serder) except kering.MissingAnchorError: - logger.info("Credential registry missing anchor for inception = {}".format(serder.said)) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("Credential registry missing anchor for inception = %s", serder.said) + logger.debug("event=\n%s\n", serder.pretty()) def anchorMsg(self, pre, regd, seqner, saider): """ Create key event with seal to serder anchored as data. diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index e696babac..3f008bae2 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -1253,9 +1253,9 @@ def logEvent(self, pre, sn, serder, seqner, saider, bigers=None, baks=None): self.reger.tets.pin(keys=(pre.decode("utf-8"), dig.decode("utf-8")), val=coring.Dater()) self.reger.putTvt(key, serder.raw) self.reger.putTel(snKey(pre, sn), dig) - logger.info("Tever state: %s Added to TEL valid said=%s", - pre, serder.said) - logger.debug(f"event=\n{serder.pretty()}\n") + logger.info("Tever: Added to TEL valid %s event %s said=%s reg=%.8s iss=%.8s", + serder.ilk, pre.decode(), serder.said, self.regk, self.pre) + logger.debug("TEL Event Body=\n%s\n", serder.pretty()) def valAnchorBigs(self, serder, seqner, saider, bigers, toad, baks): """ Validate anchor and backer signatures (bigers) when provided. @@ -1306,12 +1306,11 @@ def valAnchorBigs(self, serder, seqner, saider, bigers, toad, baks): if len(bindices) < toad: # not fully witnessed yet self.escrowPWEvent(serder=serder, seqner=seqner, saider=saider, bigers=bigers) - - raise MissingWitnessSignatureError("Failure satisfying toad = {} " - "on witness sigs for {} for evt = {}.".format(toad, - [siger.qb64 for siger - in bigers], - serder.ked)) + msg = (f"Failure satisfying toad = {toad} on witness sigs " + f"for {[siger.qb64 for siger in bigers]} for evt = {serder.said}") + logger.info("Tever: %s", msg) + logger.debug(f"Event Body=\n%s\n", serder.pretty()) + raise MissingWitnessSignatureError(msg) return bigers def verifyAnchor(self, serder, seqner=None, saider=None): @@ -1570,13 +1569,19 @@ def processEvent(self, serder, seqner=None, saider=None, wigers=None): else: # out of order, need to escrow self.escrowOOEvent(serder=serder, seqner=seqner, saider=saider) - raise OutOfOrderError("escrowed out of order event {}".format(ked)) + msg = f"Escrowed out of order event of type = {ilk} pre = {pre} SAID = {serder.said}" + logger.debug("Tevery: %s", msg) + logger.debug("TEL Event Body=\n%s\n", serder.pretty()) + raise OutOfOrderError(msg) else: if ilk in (Ilks.vcp,): # we don't have multiple signatures to verify so this - # is already first seen and then lifely duplicitious - raise LikelyDuplicitousError("Likely Duplicitous event={}.".format(ked)) + # is already first seen and then likely duplicitious + msg = f"Likely Duplicitous Event of type={serder.ilk} sn={sn} SAID={serder.said}" + logger.debug("Tevery: %s", msg) + logger.debug("TEL Event Body=\n%s\n", serder.pretty()) + raise LikelyDuplicitousError(msg) tever = self.tevers[regk] tever.cues = self.cues @@ -1602,7 +1607,10 @@ def processEvent(self, serder, seqner=None, saider=None, wigers=None): # self.cues.append(dict(kin="receipt", serder=serder)) pass else: # duplicitious - raise LikelyDuplicitousError("Likely Duplicitous event={} with sn {}.".format(ked, sn)) + msg = f"Likely Duplicitous Event type={serder.ilk} sn={sn} SAID={serder.said}" + logger.debug("Tevery: %s", msg) + logger.debug("TEL Event Body=\n%s\n", serder.pretty()) + raise LikelyDuplicitousError(msg) def processQuery(self, serder, source=None, sigers=None, cigars=None): """ Process TEL query event message (qry) @@ -2039,11 +2047,9 @@ def processEscrowOutOfOrders(self): traw = self.reger.getTvt(dgkey) if traw is None: # no event so raise ValidationError which unescrows below - logger.info("Tevery unescrow error: Missing event at." - "dig = %s", bytes(digb)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(digb))) + msg = f"OOO Missing escrowed event at dig = {bytes(digb).decode()}" + logger.info("Tevery unescrow error: %s", msg) + raise ValidationError(msg) tserder = serdering.SerderKERI(raw=bytes(traw)) # escrowed event @@ -2053,11 +2059,9 @@ def processEscrowOutOfOrders(self): couple = self.reger.getAnc(dgkey) if couple is None: - logger.info("Tevery unescrow error: Missing anchor at." - "dig = %s", bytes(digb)) - - raise ValidationError("Missing escrowed anchor at dig = {}." - "".format(bytes(digb))) + msg = f"OOO Missing escrowed anchor at dig = {bytes(digb).decode()}" + logger.info("Tevery unescrow error: %s", msg) + raise ValidationError(msg) ancb = bytearray(couple) seqner = coring.Seqner(qb64b=ancb, strip=True) saider = coring.Saider(qb64b=ancb, strip=True) @@ -2066,27 +2070,26 @@ def processEscrowOutOfOrders(self): except OutOfOrderError as ex: # still waiting on missing prior event to validate - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery unescrow failed: %s", ex.args[0]) - else: - logger.error("Tevery unescrow failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Tevery: OOO unescrow failed: %s\n", ex.args[0]) + logger.exception("Tevery: OOO unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.reger.delOot(snKey(pre, sn)) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery unescrowed: %s", ex.args[0]) + logger.exception("Tevery: OOO unescrowed: %s", ex.args[0]) else: - logger.error("Tevery unescrowed: %s", ex.args[0]) + logger.error("Tevery: OOO unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.reger.delOot(snKey(pre, sn)) # removes from escrow - logger.info("Tevery unescrow succeeded in valid event: " + logger.info("Tevery: OOO unescrow succeeded in valid event: " "said=%s", tserder.said) - logger.debug(f"event=\n{tserder.pretty()}\n") + logger.debug("Event=\n%s\n", tserder.pretty()) def processEscrowAnchorless(self): """ Process escrow of TEL events received before the anchoring KEL event. @@ -2108,11 +2111,9 @@ def processEscrowAnchorless(self): traw = self.reger.getTvt(dgkey) if traw is None: # no event so raise ValidationError which unescrows below - logger.info("Tevery unescrow error: Missing event at." - "dig = %s", bytes(digb)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(digb))) + msg = f"ANC Missing escrowed event at dig = {bytes(digb).decode()}" + logger.trace("Tevery unescrow error: %s", msg) + raise ValidationError(msg) tserder = serdering.SerderKERI(raw=bytes(traw)) # escrowed event @@ -2122,11 +2123,9 @@ def processEscrowAnchorless(self): couple = self.reger.getAnc(dgkey) if couple is None: - logger.info("Tevery unescrow error: Missing anchor at." - "dig = %s", bytes(digb)) - - raise MissingAnchorError("Missing escrowed anchor at dig = {}." - "".format(bytes(digb))) + msg = f"ANC Missing escrowed anchor at dig = {bytes(digb).decode()}" + logger.trace("Tevery unescrow error: %s", msg) + raise MissingAnchorError(msg) ancb = bytearray(couple) seqner = coring.Seqner(qb64b=ancb, strip=True) saider = coring.Saider(qb64b=ancb, strip=True) @@ -2136,23 +2135,23 @@ def processEscrowAnchorless(self): except MissingAnchorError as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery unescrow failed: %s", ex.args[0]) + logger.exception("Tevery ANC unescrow failed: %s", ex.args[0]) else: - logger.error("Tevery unescrow failed: %s", ex.args[0]) + logger.error("Tevery ANC unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.reger.delTae(snKey(pre, sn)) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery unescrowed: %s", ex.args[0]) + logger.exception("Tevery ANC unescrowed: %s", ex.args[0]) else: - logger.error("Tevery unescrowed: %s", ex.args[0]) + logger.error("Tevery ANC unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.reger.delTae(snKey(pre, sn)) # removes from escrow - logger.info("Tevery unescrow succeeded in valid event: " + logger.info("Tevery ANC unescrow succeeded in valid event: " "said=%s", tserder.said) - logger.debug(f"event=\n{tserder.pretty()}\n") + logger.debug("event=\n%s\n", tserder.pretty()) diff --git a/src/keri/vdr/verifying.py b/src/keri/vdr/verifying.py index e9bdf5858..139d94564 100644 --- a/src/keri/vdr/verifying.py +++ b/src/keri/vdr/verifying.py @@ -263,10 +263,9 @@ def _processEscrow(self, db, timeout, etype: Type[Exception]): self.processCredential(creder, prefixer, seqner, saider) except etype as ex: - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Verifiery unescrow failed: %s", ex.args[0]) - else: - logger.error("Verifier unescrow failed: %s", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.trace("Verifier unescrow failed: %s\n", ex.args[0]) + logger.exception("Verifier unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than missing sigs so remove from PA escrow db.rem(said) @@ -278,7 +277,7 @@ def _processEscrow(self, db, timeout, etype: Type[Exception]): db.rem(said) logger.info("Verifier unescrow succeeded in valid group op: " "creder=%s", creder.said) - logger.debug(f"event=\n{creder.pretty()}\n") + logger.debug(f"Event=\n%s\n", creder.pretty()) def saveCredential(self, creder, prefixer, seqner, saider): """ Write the credential and associated indicies to the database From 5cb408ea7d6122671031f68b3c1435deaefcb495 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 3 Mar 2025 14:36:02 -0500 Subject: [PATCH 371/418] adds a naive count all to SuberBase (#940) Signed-off-by: Kevin Griffin --- src/keri/db/basing.py | 2 ++ src/keri/db/subing.py | 15 +++++++++++++++ tests/db/test_basing.py | 31 ++++++++++++++----------------- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index a452c676e..23cceaf30 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1404,7 +1404,9 @@ def clearEscrows(self): for escrow in [self.qnfs, self.misfits, self.delegables, self.pdes, self.udes, self.rpes, self.epsd, self.eoobi, self.dpub, self.gpwe, self.gdee, self.dpwe, self.gpse, self.epse, self.dune]: + count = escrow.cntAll() escrow.trim() + logger.info(f"KEL: Cleared {count} escrows from ({escrow}") @property def current(self): diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 521c17886..2daa09311 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -314,6 +314,21 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable="", top=self._tokey(keys, topive=topive)): yield (self._tokeys(key), self._des(val)) + def cntAll(self): + """ + Return iterator over the all the items in subdb + + Returns: + iterator: of tuples of keys tuple and val dataclass instance for + each entry in db. Raises StopIteration when done + + Example: + if key in database is "a.b" and val is serialization of dataclass + with attributes x and y then returns + (("a","b"), dataclass(x=1,y=2)) + """ + return self.db.cnt(db=self.sdb) + class Suber(SuberBase): """ diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 97cc31958..1db881361 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -180,9 +180,6 @@ def test_baser(): assert actual.local == record.local assert db.esrs.get(key) == record - - - # test first seen event log .fels sub db preA = b'BAKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc' preB = b'EH7Oq9oxCgYa-nnNLvwhp9sFZpALILlRYyB-6n4WDi7w' @@ -1894,21 +1891,21 @@ def test_clear_escrows(): assert db.getUwes(key) == [] assert db.getOoes(key) == [] assert db.getLdes(key) == [] - assert db.qnfs.cnt(keys=(pre, saidb)) == 0 - assert db.misfits.cnt(keys=(pre, snh)) == 0 - assert db.delegables.cnt(keys=snKey(pre, 0)) == 0 - assert db.pdes.cnt(keys=snKey(pre, 0)) == 0 - assert db.udes.get(keys=udesKey) is None - assert db.rpes.cnt(keys=('route',)) == 0 - assert db.epsd.get(keys=('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc',)) is None + assert db.qnfs.cntAll() == 0 + assert db.pdes.cntAll() == 0 + assert db.rpes.cntAll() == 0 assert db.eoobi.cntAll() == 0 - assert db.dpub.get(keys=(pre, 'said')) is None - assert db.gpwe.cnt(keys=(pre,)) == 0 - assert db.gdee.cnt(keys=(pre,)) == 0 - assert db.dpwe.get(keys=(pre, 'said')) is None - assert db.gpse.cnt(keys=('qb64',)) == 0 - assert db.epse.get(keys=('dig',)) is None - assert db.dune.get(keys=(pre, 'said')) is None + assert db.gpwe.cntAll() == 0 + assert db.gdee.cntAll() == 0 + assert db.dpwe.cntAll() == 0 + assert db.gpse.cntAll() == 0 + assert db.epse.cntAll() == 0 + assert db.dune.cntAll() == 0 + assert db.misfits.cntAll() == 0 + assert db.delegables.cntAll() == 0 + assert db.udes.cntAll() == 0 + assert db.epsd.cntAll() == 0 + assert db.dpub.cntAll() == 0 if __name__ == "__main__": test_baser() From 71f0dd28a14e2993665a0d1958b2d01d2a6c5ad7 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Mon, 3 Mar 2025 12:41:24 -0700 Subject: [PATCH 372/418] fix: merge instead of replace contact data on re-OOBI (#942) --- src/keri/app/oobiing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/app/oobiing.py b/src/keri/app/oobiing.py index 7228b9187..541999e90 100644 --- a/src/keri/app/oobiing.py +++ b/src/keri/app/oobiing.py @@ -503,7 +503,7 @@ def processClients(self): obr.cid = response["headers"][ending.OOBI_AID_HEADER] if obr.oobialias is not None and obr.cid: - self.org.replace(pre=obr.cid, data=dict(alias=obr.oobialias, oobi=url)) + self.org.update(pre=obr.cid, data=dict(alias=obr.oobialias, oobi=url)) self.hby.db.coobi.rem(keys=(url,)) obr.state = Result.resolved From a70a53c4548d75161bea9fc58351d169a870ebdf Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 3 Mar 2025 17:24:21 -0500 Subject: [PATCH 373/418] kli list notifications Signed-off-by: Kevin Griffin --- .../cli/commands/notifications/__init__.py | 0 .../app/cli/commands/notifications/list.py | 90 +++++++++++++++++ .../app/cli/commands/notifications/mark.py | 98 +++++++++++++++++++ .../app/cli/commands/notifications/rem.py | 98 +++++++++++++++++++ 4 files changed, 286 insertions(+) create mode 100644 src/keri/app/cli/commands/notifications/__init__.py create mode 100644 src/keri/app/cli/commands/notifications/list.py create mode 100644 src/keri/app/cli/commands/notifications/mark.py create mode 100644 src/keri/app/cli/commands/notifications/rem.py diff --git a/src/keri/app/cli/commands/notifications/__init__.py b/src/keri/app/cli/commands/notifications/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/keri/app/cli/commands/notifications/list.py b/src/keri/app/cli/commands/notifications/list.py new file mode 100644 index 000000000..dac919a7b --- /dev/null +++ b/src/keri/app/cli/commands/notifications/list.py @@ -0,0 +1,90 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse +import json + +from hio.base import doing +from keri import help +from keri.app import habbing, notifying +from keri.app.cli.common import existing + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Display notifications for an identifier') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") + + +def handler(args): + """ + List notifications for an identifier. + + Args: + args(Namespace): arguments object from command line + """ + + name = args.name + base = args.base + bran = args.bran + alias = args.alias + verbose = args.verbose + + notesDoer = NotesDoer(name=name, base=base, alias=alias, bran=bran, verbose=verbose) + + doers = [notesDoer] + return doers + + +class NotesDoer(doing.DoDoer): + def __init__(self, name, base, alias, bran, verbose): + + hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer + self.alias = alias + self.hby = hby + self.verbose = verbose + self.notifier = notifying.Notifier(hby=self.hby) + + doers = [self.hbyDoer, doing.doify(self.readDo)] + + super(NotesDoer, self).__init__(doers=doers) + + def readDo(self, tymth, tock=0.0): + """ + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + print("Waiting for notifications...") + + while self.notifier.noter.notes.cntAll() == 0: + yield self.tock + + for keys, notice in self.notifier.noter.notes.getItemIter(): + if self.verbose: + print(keys) + print(json.dumps(notice.pad, indent=4)) + else: + print(keys, notice.attrs.get('r', 'no route')) + + self.remove([self.hbyDoer,]) + return \ No newline at end of file diff --git a/src/keri/app/cli/commands/notifications/mark.py b/src/keri/app/cli/commands/notifications/mark.py new file mode 100644 index 000000000..78bcb786d --- /dev/null +++ b/src/keri/app/cli/commands/notifications/mark.py @@ -0,0 +1,98 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from hio.base import doing +from keri import help +from keri.app import habbing, notifying +from keri.app.cli.common import existing + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Display notifications for an identifier') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--rid", '-r', help='notification SAID to mark as read', default=None) +parser.add_argument("--all", help="mark all notifications as read", action="store_true") + + +def handler(args): + """ + List notifications for an identifier. + + Args: + args(Namespace): arguments object from command line + """ + + name = args.name + base = args.base + bran = args.bran + alias = args.alias + rid = args.rid + all = args.all + + markDoer = MarkDoer(name=name, base=base, alias=alias, bran=bran, rid=rid, all=all) + + doers = [markDoer] + return doers + + +class MarkDoer(doing.DoDoer): + def __init__(self, name, base, alias, bran, rid, all): + + hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer + self.alias = alias + self.hby = hby + self.notifier = notifying.Notifier(hby=self.hby) + self.rid = rid + self.all = all + + doers = [self.hbyDoer, doing.doify(self.markDo)] + + super(MarkDoer, self).__init__(doers=doers) + + def markDo(self, tymth, tock=0.0): + """ + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + if self.all: + print() + print("This command will mark all notifications as read") + print() + yn = input("Are you sure you want to continue? [y|N]: ") + + if yn not in ("y", "Y"): + print("...exiting") + else: + for n in self.notifier.getNotes(): + print(f"marking {n.rid} as read") + self.notifier.mar(rid=n.rid) + elif self.rid is not None: + print(f"marking {self.rid} as read") + self.notifier.mar(rid=self.rid) + else: + print("Must specify one of --rid or --all") + + self.remove([self.hbyDoer,]) + return \ No newline at end of file diff --git a/src/keri/app/cli/commands/notifications/rem.py b/src/keri/app/cli/commands/notifications/rem.py new file mode 100644 index 000000000..415d949c7 --- /dev/null +++ b/src/keri/app/cli/commands/notifications/rem.py @@ -0,0 +1,98 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from hio.base import doing +from keri import help +from keri.app import habbing, notifying +from keri.app.cli.common import existing + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Display notifications for an identifier') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--rid", '-r', help='notification SAID to mark as read', default=None) +parser.add_argument("--all", help="mark all notifications as read", action="store_true") + + +def handler(args): + """ + List notifications for an identifier. + + Args: + args(Namespace): arguments object from command line + """ + + name = args.name + base = args.base + bran = args.bran + alias = args.alias + rid = args.rid + all = args.all + + removeDoer = RemoveDoer(name=name, base=base, alias=alias, bran=bran, rid=rid, all=all) + + doers = [removeDoer] + return doers + + +class RemoveDoer(doing.DoDoer): + def __init__(self, name, base, alias, bran, rid, all): + + hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer + self.alias = alias + self.hby = hby + self.notifier = notifying.Notifier(hby=self.hby) + self.rid = rid + self.all = all + + doers = [self.hbyDoer, doing.doify(self.remDoer)] + + super(RemoveDoer, self).__init__(doers=doers) + + def remDoer(self, tymth, tock=0.0): + """ + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + if self.all: + print() + print("This command will remove all notifications") + print() + yn = input("Are you sure you want to continue? [y|N]: ") + + if yn not in ("y", "Y"): + print("...exiting") + else: + for n in self.notifier.getNotes(): + print(f"removing {n.rid}") + self.notifier.rem(rid=n.rid) + elif self.rid is not None: + print(f"removing {self.rid}") + self.notifier.rem(rid=self.rid) + else: + print("Must specify one of --rid or --all") + + self.remove([self.hbyDoer,]) + return \ No newline at end of file From ea7532146e98074f6a7faf7aa3a909c6b21d9671 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Sat, 12 Apr 2025 16:25:05 -0600 Subject: [PATCH 374/418] fix logging substitution error --- src/keri/core/routing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 7a8cfe22a..29c87a9e3 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -340,7 +340,7 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, # create cue here to request key state for sprefixer signer # signer's est event not yet in signer's KEL logger.info("Revery: escrowing without key state for signer" - " on reply said=", serder.said) + " on reply said=%s", serder.said) self.escrowReply(serder=serder, saider=saider, dater=dater, route=route, prefixer=prefixer, seqner=seqner, ssaider=ssaider, sigers=sigers) From b031efc4d86322f6d183e14ceeb75fe65776eb41 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Sat, 12 Apr 2025 21:15:04 -0400 Subject: [PATCH 375/418] adds escrow clear for reger (#951) * adds escrow clear for reger Signed-off-by: Kevin Griffin * adds escrow clear for reger Signed-off-by: Kevin Griffin --------- Signed-off-by: Kevin Griffin --- src/keri/app/cli/commands/escrow/clear.py | 3 ++ src/keri/vdr/viring.py | 49 +++++++++++++++++-- tests/vdr/test_viring.py | 59 +++++++++++++++++++++++ 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/src/keri/app/cli/commands/escrow/clear.py b/src/keri/app/cli/commands/escrow/clear.py index 5d030b91f..b3720a24a 100644 --- a/src/keri/app/cli/commands/escrow/clear.py +++ b/src/keri/app/cli/commands/escrow/clear.py @@ -9,6 +9,7 @@ from hio.base import doing from keri import help from keri.app.cli.common import existing +from keri.vdr import viring logger = help.ogler.getLogger() @@ -50,3 +51,5 @@ def clear(tymth, tock=0.0, **opts): with existing.existingHby(name=name, base=base, bran=bran) as hby: hby.db.clearEscrows() + reger = viring.Reger(name=hby.name, db=hby.db, temp=False) + reger.clearEscrows() diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index c2fccfdab..e0d0c45ae 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -9,19 +9,19 @@ """ from dataclasses import dataclass, field, asdict -from ordered_set import OrderedSet as oset -from ..db import koming, subing, escrowing +from ordered_set import OrderedSet as oset +from keri import help from .. import kering, core from ..app import signing from ..core import coring, serdering, indexing, counting from ..db import dbing, basing +from ..db import koming, subing, escrowing from ..db.dbing import snKey -from ..help import helping -from ..vc import proving from ..vdr import eventing +logger = help.ogler.getLogger() class rbdict(dict): """ Reger backed read through cache for registry state @@ -777,6 +777,13 @@ def delTwe(self, key): """ return self.delVal(self.twes, key) + def getTweItemIter(self): + """ + Return iterator of all items in .twes + + """ + return self.getTopItemIter(self.twes) + def putTae(self, key, val): """ Use snKey() @@ -972,6 +979,40 @@ def delBak(self, key, val): """ return self.delIoDupVal(self.baks, key, val) + def clearEscrows(self): + """Clear credential event escrows""" + # self.oots, self.twes, self.taes + count = 0 + for (k , _) in self.getOotItemIter(): + count += 1 + self.delOot(k) + logger.info(f"TEL: Cleared {count} out of order escrows.") + + count = 0 + for (k, _) in self.getTweItemIter(): + count += 1 + self.delTwe(k) + logger.info(f"TEL: Cleared {count} partially witnessed escrows.") + + count = 0 + for (k, _) in self.getTaeItemIter(): + count += 1 + self.delTae(k) + logger.info(f"TEL: Cleared {count} anchorless escrows.") + + for name, sub, desc in [ + ( 'mre', self.mre, 'missing registry escrows'), + ( 'mce', self.mce, 'broken chain escrows'), + ( 'mse', self.mse, 'missing schema escrows'), + ('cmse', self.cmse, 'missing signature escrows'), + ('tpwe', self.tpwe, 'partial witness escrows'), + ('tmse', self.tmse, 'multisig escrows'), + ('tede', self.tede, 'event dissemination escrows') + ]: + sub.trim() + logger.info(f"TEL: Cleared escrow ({name.ljust(5)}): {desc}") + logger.info("Cleared TEL escrows") + def buildProof(prefixer, seqner, diger, sigers): """ diff --git a/tests/vdr/test_viring.py b/tests/vdr/test_viring.py index 2e98dda6c..e69682110 100644 --- a/tests/vdr/test_viring.py +++ b/tests/vdr/test_viring.py @@ -9,7 +9,10 @@ import lmdb +from keri import kering +from keri.core import coring from keri.core.coring import Diger, versify, Kinds +from keri.core.serdering import SerderACDC from keri.db.dbing import openLMDB, dgKey, snKey from keri.vdr.viring import Reger @@ -341,6 +344,62 @@ def test_clone(): b'PjioY7Ycna6ouhSSH0QcKsEjce10HCXIW_XtmEYr9SrB5BA-GAB0AAAAAAAAAAAA' b'AAAAAAAAABCEzpq06UecHwzy-K9FpNoRxCJp2wIGM9u2Edk-PLMZ1H4') +def test_clearEscrows(): + with openLMDB(cls=Reger) as db: + regk = "EAWdT7a7fZwRz0jiZ0DJxZEM3vsNbLDPEUk-ODnif3O0".encode("utf-8") + sn = 0 + ooKey = snKey(regk, sn) + vs = versify(kind=coring.Kinds.json, size=20) + rarb = "BBjzaUuRMwh1ivT5BQrqNhbvx82lB-ofrHVHjL3WADbA".encode("utf-8") + vcp = dict(v=vs, i=regk.decode("utf-8"), + s="{:x}".format(sn), b=[rarb.decode("utf-8")], + t="vcp") + + vcpb = json.dumps(vcp, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + vdig = Diger(ser=vcpb) + + db.putOot(ooKey, val=vdig.qb64b) + db.putTwe(ooKey, val=vdig.qb64b) + db.putTae(ooKey, val=vdig.qb64b) + + pre = 'k' + db.mre.put(keys=(pre, ), val=coring.Dater()) + db.mce.put(keys=(pre, ), val=coring.Dater()) + db.mse.put(keys=(pre, ), val=coring.Dater()) + + creder = SerderACDC(makify=True, proto=kering.Protocols.acdc, verify=False) + db.cmse.put(("a", "b"), val=creder) + + prefixer = coring.Prefixer(qb64="EAD919wF4oiG7ck6mnBWTRD_Z-Io0wZKCxL0zjx5je9I") + seqner = coring.Seqner(sn=0) + saider = coring.Saider(qb64="EAD919wF4oiG7ck6mnBWTRD_Z-Io0wZKCxL0zjx5je9I") + db.tpwe.add(("a", "b"), (prefixer, seqner, saider)) + db.tmse.add(("a", "b"), (prefixer, seqner, saider)) + db.tede.add(("a", "b"), (prefixer, seqner, saider)) + + assert db.getOot(ooKey) == vdig.qb64b + assert db.getTwe(ooKey) == vdig.qb64b + assert db.getTae(ooKey) == vdig.qb64b + assert db.mre.get((pre, )) is not None + assert db.mce.get((pre, )) is not None + assert db.mse.get((pre, )) is not None + assert db.cmse.cntAll() == 1 + assert db.tpwe.cntAll() == 1 + assert db.tmse.cntAll() == 1 + assert db.tede.cntAll() == 1 + + db.clearEscrows() + + assert db.getOot(ooKey) is None + assert db.getTwe(ooKey) is None + assert db.getTae(ooKey) is None + assert db.mre.get((pre, )) is None + assert db.mce.get((pre, )) is None + assert db.mse.get((pre, )) is None + assert db.cmse.cntAll() == 0 + assert db.tpwe.cntAll() == 0 + assert db.tmse.cntAll() == 0 + assert db.tede.cntAll() == 0 if __name__ == "__main__": test_issuer() From b1b7aa20ca5a0cc4b911e5c07bdc9c3541a2a15e Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 14 Apr 2025 10:56:18 -0400 Subject: [PATCH 376/418] 1.2.5 Signed-off-by: Kevin Griffin --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c1dc35f0e..bfab24f4b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.4 +VERSION=1.2.5 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index 5616bb0e7..24b5c121c 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.4', # also change in src/keri/__init__.py + version='1.2.5', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 6c3f1ca01..13dbd8aaf 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.4' # also change in setup.py +__version__ = '1.2.5' # also change in setup.py From 74bb6b4bbe5b445e3f25a1ffcb9d15ccc37eddef Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 14 Apr 2025 11:35:03 -0400 Subject: [PATCH 377/418] pin hio Signed-off-by: Kevin Griffin --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 24b5c121c..8e3d6ed18 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ 'cbor2>=5.6.2', 'multidict>=6.0.5', 'ordered-set>=4.1.0', - 'hio>=0.6.14', + 'hio==0.6.14', 'multicommand>=1.0.0', 'jsonschema>=4.21.1', 'falcon>=3.1.3', From 42f674e7f1ddbd2b85f16fe026ea8476702b3149 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 14 Apr 2025 14:58:39 -0400 Subject: [PATCH 378/418] 1.2.6 Signed-off-by: Kevin Griffin --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index bfab24f4b..0c42d94d3 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.5 +VERSION=12.6 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index 8e3d6ed18..3a1b2e678 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.5', # also change in src/keri/__init__.py + version='12.6', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 13dbd8aaf..07cbd5c06 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.5' # also change in setup.py +__version__ = '12.6' # also change in setup.py From 1d02a1dbe6a35e139c129791779a05468522e4bd Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 14 Apr 2025 18:29:43 -0400 Subject: [PATCH 379/418] 1.2.6 Signed-off-by: Kevin Griffin --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0c42d94d3..13c3a83b0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=12.6 +VERSION=1.2.6 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index 3a1b2e678..f2a2d9c59 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='12.6', # also change in src/keri/__init__.py + version='1.2.6', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 07cbd5c06..cb99ceb42 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '12.6' # also change in setup.py +__version__ = '1.2.6' # also change in setup.py From 6785a9e5e460f4a5b3e86d6790a7fcefc51fc614 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Wed, 23 Apr 2025 19:24:51 -0400 Subject: [PATCH 380/418] fix export (#964) Signed-off-by: Kevin Griffin --- keri/cf/demo-witness-oobis.json | 0 src/keri/app/cli/commands/export.py | 15 +++++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) delete mode 100755 keri/cf/demo-witness-oobis.json diff --git a/keri/cf/demo-witness-oobis.json b/keri/cf/demo-witness-oobis.json deleted file mode 100755 index e69de29bb..000000000 diff --git a/src/keri/app/cli/commands/export.py b/src/keri/app/cli/commands/export.py index dc44b5376..e566b98ad 100644 --- a/src/keri/app/cli/commands/export.py +++ b/src/keri/app/cli/commands/export.py @@ -106,11 +106,10 @@ def outputEnds(self, pre): f = open(f"{pre}-ends.cesr", "w") msgs = self.hab.replyToOobi(aid=pre, role="controller") - for msg in msgs: - if f is not None: - f.write(msg.decode("utf-8")) - else: - serder = serdering.SerderKERI(raw=msg) - atc = msg[serder.size:] - sys.stdout.write(serder.raw.decode("utf-8")) - sys.stdout.write(atc.decode("utf-8")) + if f is not None: + f.write(msgs.decode("utf-8")) + else: + serder = serdering.SerderKERI(raw=msgs) + atc = msgs[serder.size:] + sys.stdout.write(serder.raw.decode("utf-8")) + sys.stdout.write(atc.decode("utf-8")) From 870b103044624ddd2c4d96e47f8d7e749ff6b2d1 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Tue, 20 May 2025 10:23:16 -0400 Subject: [PATCH 381/418] fixes imports (#1004) Signed-off-by: Kevin Griffin --- src/keri/app/cli/commands/local/watch.py | 3 ++- src/keri/app/cli/commands/multisig/update.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/keri/app/cli/commands/local/watch.py b/src/keri/app/cli/commands/local/watch.py index 484ecdf6e..eae8ccbb0 100644 --- a/src/keri/app/cli/commands/local/watch.py +++ b/src/keri/app/cli/commands/local/watch.py @@ -10,6 +10,7 @@ from keri import help from hio.base import doing +from hio.help import decking from keri.app import agenting, indirecting, habbing, forwarding from keri.app.cli.common import existing, terming from keri.app.habbing import GroupHab @@ -44,7 +45,7 @@ def __init__(self, name, base, bran, **kwa): doers = [] self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer - self.cues = help.decking.Deck() + self.cues = decking.Deck() self.mbd = indirecting.MailboxDirector(hby=self.hby, topics=["/replay", "/receipt", "/reply"]) self.postman = forwarding.Poster(hby=self.hby) diff --git a/src/keri/app/cli/commands/multisig/update.py b/src/keri/app/cli/commands/multisig/update.py index 3527c8dd4..5ef24266b 100644 --- a/src/keri/app/cli/commands/multisig/update.py +++ b/src/keri/app/cli/commands/multisig/update.py @@ -8,6 +8,7 @@ from keri import help from hio.base import doing +from hio.help import decking from keri import kering from keri.app import agenting, indirecting, habbing @@ -60,7 +61,7 @@ def __init__(self, name, alias, base, bran, wit, sn, said, **kwa): self.wit = wit self.sn = sn self.said = said - self.cues = help.decking.Deck() + self.cues = decking.Deck() self.mbd = indirecting.MailboxDirector(hby=self.hby, topics=["/replay", "/receipt", "/reply"]) self.witq = agenting.WitnessInquisitor(hby=self.hby) From 753f5d45e3f53adca320a531e64d4f804e029ff3 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Fri, 20 Jun 2025 13:39:04 -0600 Subject: [PATCH 382/418] Update Doer method signatures to support new Hio multiprocessing safety (#968) ported from Sam's commit to main in PR #945 --- setup.py | 2 +- src/keri/app/agenting.py | 14 ++++++------ src/keri/app/apping.py | 2 +- src/keri/app/cli/commands/incept.py | 2 +- src/keri/app/cli/commands/init.py | 2 +- src/keri/app/cli/commands/multisig/incept.py | 2 +- src/keri/app/cli/commands/rotate.py | 2 +- src/keri/app/configing.py | 2 +- src/keri/app/delegating.py | 2 +- src/keri/app/forwarding.py | 2 +- src/keri/app/grouping.py | 2 +- src/keri/app/habbing.py | 2 +- src/keri/app/httping.py | 2 +- src/keri/app/indirecting.py | 24 ++++++++++---------- src/keri/app/keeping.py | 4 ++-- src/keri/app/oobiing.py | 4 ++-- src/keri/app/signaling.py | 2 +- src/keri/app/storing.py | 4 ++-- src/keri/db/basing.py | 2 +- src/keri/vc/walleting.py | 2 +- src/keri/vdr/credentialing.py | 4 ++-- tests/app/test_agenting.py | 4 ++-- 22 files changed, 44 insertions(+), 44 deletions(-) diff --git a/setup.py b/setup.py index f2a2d9c59..c027f6c98 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ 'cbor2>=5.6.2', 'multidict>=6.0.5', 'ordered-set>=4.1.0', - 'hio==0.6.14', + 'hio>=0.6.14', 'multicommand>=1.0.0', 'jsonschema>=4.21.1', 'falcon>=3.1.3', diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 890fe8172..1f340b226 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -206,7 +206,7 @@ def catchup(self, pre, wit): self.remove([clientDoer]) - def witDo(self, tymth=None, tock=0.0): + def witDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery and .tevery escrows. @@ -235,7 +235,7 @@ def witDo(self, tymth=None, tock=0.0): yield self.tock - def gitDo(self, tymth=None, tock=0.0): + def gitDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery and .tevery escrows. @@ -295,7 +295,7 @@ def __init__(self, hby, msgs=None, cues=None, force=False, auths=None, **kwa): super(WitnessReceiptor, self).__init__(doers=[doing.doify(self.receiptDo)], **kwa) - def receiptDo(self, tymth=None, tock=0.0): + def receiptDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatible generator method (doer dog) @@ -680,7 +680,7 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): super(TCPMessenger, self).__init__(doers=doers) - def receiptDo(self, tymth=None, tock=0.0): + def receiptDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatible generator method (doer dog) @@ -774,7 +774,7 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): super(TCPStreamMessenger, self).__init__(doers=doers) - def receiptDo(self, tymth=None, tock=0.0): + def receiptDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatible generator method (doer dog) @@ -875,7 +875,7 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, auth=None, * super(HTTPMessenger, self).__init__(doers=doers, **kwa) - def msgDo(self, tymth=None, tock=0.0): + def msgDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatible generator method (doer dog) @@ -901,7 +901,7 @@ def msgDo(self, tymth=None, tock=0.0): yield self.tock - def responseDo(self, tymth=None, tock=0.0): + def responseDo(self, tymth=None, tock=0.0, **kwa): """ Processes responses from client and adds them to sent cue diff --git a/src/keri/app/apping.py b/src/keri/app/apping.py index 7478aa1d2..95d77249b 100644 --- a/src/keri/app/apping.py +++ b/src/keri/app/apping.py @@ -28,7 +28,7 @@ def __init__(self, db=None, console=None, **kwa): self.db = db if db is not None else basing.Baser() self.console = console if console is not None else serialing.Console() - def enter(self): + def enter(self, *, temp=None): """""" if not self.console.reopen(): raise IOError("Unable to open serial console.") diff --git a/src/keri/app/cli/commands/incept.py b/src/keri/app/cli/commands/incept.py index cf30bae92..47364d9ff 100644 --- a/src/keri/app/cli/commands/incept.py +++ b/src/keri/app/cli/commands/incept.py @@ -150,7 +150,7 @@ def __init__(self, name, base, alias, bran, endpoint, proxy=None, cnfg=None, **k self.alias = alias super(InceptDoer, self).__init__(doers=doers) - def inceptDo(self, tymth, tock=0.0): + def inceptDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/init.py b/src/keri/app/cli/commands/init.py index 0d5070ca3..60bcf0516 100644 --- a/src/keri/app/cli/commands/init.py +++ b/src/keri/app/cli/commands/init.py @@ -67,7 +67,7 @@ def __init__(self, args): self.args = args super(InitDoer, self).__init__(doers=[doing.doify(self.initialize)]) - def initialize(self, tymth, tock=0.0): + def initialize(self, tymth, tock=0.0, **kwa): # enter context self.wind(tymth) diff --git a/src/keri/app/cli/commands/multisig/incept.py b/src/keri/app/cli/commands/multisig/incept.py index 57478dfed..89378404d 100644 --- a/src/keri/app/cli/commands/multisig/incept.py +++ b/src/keri/app/cli/commands/multisig/incept.py @@ -105,7 +105,7 @@ def __init__(self, name, base, alias, bran, group, wait, **kwa): super(GroupMultisigIncept, self).__init__(doers=doers) - def inceptDo(self, tymth, tock=0.0): + def inceptDo(self, tymth, tock=0.0, **kwa): """ Create or participate in an inception event for a distributed multisig identifier Parameters: diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index 84d32f20b..bd1868683 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -167,7 +167,7 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No super(RotateDoer, self).__init__(doers=doers) - def rotateDo(self, tymth, tock=0.0): + def rotateDo(self, tymth, tock=0.0, **kwa): """ Returns: doifiable Doist compatible generator method Usage: diff --git a/src/keri/app/configing.py b/src/keri/app/configing.py index 72efc20f7..1ed28b896 100644 --- a/src/keri/app/configing.py +++ b/src/keri/app/configing.py @@ -225,7 +225,7 @@ def __init__(self, configer, **kwa): super(ConfigerDoer, self).__init__(**kwa) self.configer = configer - def enter(self): + def enter(self, *, temp=None): """""" if not self.configer.opened: self.configer.reopen() diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 3178ac3d5..252176911 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -95,7 +95,7 @@ def complete(self, prefixer, seqner, saider=None): return True - def escrowDo(self, tymth, tock=1.0): + def escrowDo(self, tymth, tock=1.0, **kwa): """ Process escrows of group multisig identifiers waiting to be compeleted. Steps involve: diff --git a/src/keri/app/forwarding.py b/src/keri/app/forwarding.py index 877a98330..fda62ebdc 100644 --- a/src/keri/app/forwarding.py +++ b/src/keri/app/forwarding.py @@ -39,7 +39,7 @@ def __init__(self, hby, mbx=None, evts=None, cues=None, **kwa): doers = [doing.doify(self.deliverDo)] super(Poster, self).__init__(doers=doers, **kwa) - def deliverDo(self, tymth=None, tock=0.0): + def deliverDo(self, tymth=None, tock=0.0, **kwa): """ Returns: doifiable Doist compatible generator method that processes a queue of messages and envelopes them in a `fwd` message diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index e5928074f..8b748a982 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -72,7 +72,7 @@ def complete(self, prefixer, seqner, saider=None): return True - def escrowDo(self, tymth, tock=1.0): + def escrowDo(self, tymth, tock=1.0, **kwa): """ Process escrows of group multisig identifiers waiting to be compeleted. Steps involve: diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index ec2aa781f..7d746434d 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -900,7 +900,7 @@ def __init__(self, habery, **kwa): super(HaberyDoer, self).__init__(**kwa) self.habery = habery - def enter(self): + def enter(self, *, temp=False): """ Enter context and set up Habery """ if not self.habery.inited: self.habery.setup(**self.habery._inits) diff --git a/src/keri/app/httping.py b/src/keri/app/httping.py index f568ae9fc..f2f484c2b 100644 --- a/src/keri/app/httping.py +++ b/src/keri/app/httping.py @@ -259,7 +259,7 @@ def remove(self, client): (_, doer, _) = tup super(Clienter, self).remove([doer]) - def clientDo(self, tymth, tock=0.0): + def clientDo(self, tymth, tock=0.0, **kwa): """ Periodically prune stale clients Process existing clients and prune any that have receieved a response longer than timeout diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 3194af867..a89c1e9cc 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -161,7 +161,7 @@ def __init__(self, hab, parser, kvy, tvy, rvy, exc, cues=None, replies=None, res doers = [doing.doify(self.start), doing.doify(self.msgDo), doing.doify(self.escrowDo), doing.doify(self.cueDo)] super().__init__(doers=doers, **opts) - def start(self, tymth=None, tock=0.0): + def start(self, tymth=None, tock=0.0, **kwa): """ Prints witness name and prefix Parameters: @@ -179,7 +179,7 @@ def start(self, tymth=None, tock=0.0): print("Witness", self.hab.name, ":", self.hab.pre) - def msgDo(self, tymth=None, tock=0.0): + def msgDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -201,7 +201,7 @@ def msgDo(self, tymth=None, tock=0.0): done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def escrowDo(self, tymth=None, tock=0.0): + def escrowDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery and .tevery escrows. @@ -227,7 +227,7 @@ def escrowDo(self, tymth=None, tock=0.0): yield - def cueDo(self, tymth=None, tock=0.0): + def cueDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery.cues deque @@ -363,7 +363,7 @@ def wind(self, tymth): super(Indirector, self).wind(tymth) self.client.wind(tymth) - def msgDo(self, tymth=None, tock=0.0): + def msgDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -390,7 +390,7 @@ def msgDo(self, tymth=None, tock=0.0): done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def cueDo(self, tymth=None, tock=0.0): + def cueDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery.cues deque @@ -418,7 +418,7 @@ def cueDo(self, tymth=None, tock=0.0): yield # throttle just do one cue at a time yield - def escrowDo(self, tymth=None, tock=0.0): + def escrowDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery escrows. @@ -578,7 +578,7 @@ def wind(self, tymth): """ super(MailboxDirector, self).wind(tymth) - def pollDo(self, tymth=None, tock=0.0): + def pollDo(self, tymth=None, tock=0.0, **kwa): """ Returns: doifiable Doist compatible generator method @@ -656,7 +656,7 @@ def processPollIter(self): msg = mail.pop(0) yield msg - def msgDo(self, tymth=None, tock=0.0): + def msgDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -681,7 +681,7 @@ def msgDo(self, tymth=None, tock=0.0): done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def escrowDo(self, tymth=None, tock=0.0): + def escrowDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery escrows. @@ -754,7 +754,7 @@ def __init__(self, hab, witness, topics, msgs=None, retry=1000, **kwa): super(Poller, self).__init__(doers=doers, **kwa) - def eventDo(self, tymth=None, tock=0.0): + def eventDo(self, tymth=None, tock=0.0, **kwa): """ Returns: doifiable Doist compatible generator method @@ -1152,7 +1152,7 @@ def on_get(self, req, rep): rep.status = falcon.HTTP_200 rep.data = rct - def interceptDo(self, tymth=None, tock=0.0): + def interceptDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process Kevery and Tevery cues deque diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index 57ac560be..0f19fc775 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -343,7 +343,7 @@ def __init__(self, keeper, **kwa): self.keeper = keeper - def enter(self): + def enter(self, *, temp=False): """""" if not self.keeper.opened: self.keeper.reopen() @@ -1756,7 +1756,7 @@ def __init__(self, manager, **kwa): self.manager = manager - def enter(self): + def enter(self, *, temp=False): """""" if not self.manager.inited: self.manager.setup(**self.manager._inits) diff --git a/src/keri/app/oobiing.py b/src/keri/app/oobiing.py index 541999e90..8b5e8419c 100644 --- a/src/keri/app/oobiing.py +++ b/src/keri/app/oobiing.py @@ -386,7 +386,7 @@ def processReply(self, *, serder, saider, route, cigars=None, tsgs=None, **kwarg obr = basing.OobiRecord(cid=cid, date=dt) self.hby.db.oobis.put(keys=(oobi,), val=obr) - def scoobiDo(self, tymth=None, tock=0.0): + def scoobiDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .exc responses and pass them on to the HTTPRespondant @@ -661,7 +661,7 @@ def addAuthToAid(self, cid, url): wkan = basing.WellKnownAuthN(url=url, dt=now) self.hby.db.wkas.add(keys=(cid,), val=wkan) - def authzDo(self, tymth=None, tock=0.0): + def authzDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .exc responses and pass them on to the HTTPRespondant diff --git a/src/keri/app/signaling.py b/src/keri/app/signaling.py index fa44c18fd..edec6876a 100644 --- a/src/keri/app/signaling.py +++ b/src/keri/app/signaling.py @@ -128,7 +128,7 @@ def push(self, attrs, topic, ckey=None, dt=None): self.signals.append(sig) - def expireDo(self, tymth=None, tock=0.0): + def expireDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatible generator method (doer dog) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index f817b67fc..e7124f044 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -186,7 +186,7 @@ def __init__(self, hby, reps=None, cues=None, mbx=None, aids=None, **kwa): doers = [self.postman, doing.doify(self.responseDo), doing.doify(self.cueDo)] super(Respondant, self).__init__(doers=doers, **kwa) - def responseDo(self, tymth=None, tock=0.0): + def responseDo(self, tymth=None, tock=0.0, **kwa): """ Doifiable Doist compatibile generator method to process response messages from `exn` handlers. If dest is not in local environment, ignore the response (for now). If dest has witnesses, @@ -232,7 +232,7 @@ def responseDo(self, tymth=None, tock=0.0): yield self.tock - def cueDo(self, tymth=None, tock=0.0): + def cueDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process Kevery and Tevery cues deque diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 23cceaf30..59f32bcae 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -3277,7 +3277,7 @@ def __init__(self, baser, **kwa): super(BaserDoer, self).__init__(**kwa) self.baser = baser - def enter(self): + def enter(self, *, temp=False): """""" if not self.baser.opened: self.baser.reopen() diff --git a/src/keri/vc/walleting.py b/src/keri/vc/walleting.py index f4ea25b46..94ca6c118 100644 --- a/src/keri/vc/walleting.py +++ b/src/keri/vc/walleting.py @@ -72,7 +72,7 @@ def __init__(self, hby, verifier, **kwa): super(WalletDoer, self).__init__(doers=doers, **kwa) - def escrowDo(self, tymth, tock=0.0): + def escrowDo(self, tymth, tock=0.0, **kwa): """ Processes the escrows for group icp, rot and ixn request messages. Parameters: diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index 87df5ddb8..212e3ee9e 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -652,7 +652,7 @@ def complete(self, pre, sn=0): said = self.rgy.reger.ctel.get(keys=(pre, seqner.qb64)) return said is not None and self.witPub.sent(said=pre) - def escrowDo(self, tymth, tock=1.0): + def escrowDo(self, tymth, tock=1.0, **kwa): """ Process escrows of group multisig identifiers waiting to be compeleted. Steps involve: @@ -878,7 +878,7 @@ def processCredentialMissingSigEscrow(self): def complete(self, said): return self.rgy.reger.ccrd.get(keys=(said,)) is not None - def escrowDo(self, tymth, tock=1.0): + def escrowDo(self, tymth, tock=1.0, **kwa): """ Process escrows of group multisig identifiers waiting to be completed. Steps involve: diff --git a/tests/app/test_agenting.py b/tests/app/test_agenting.py index 6f2470369..f71218084 100644 --- a/tests/app/test_agenting.py +++ b/tests/app/test_agenting.py @@ -60,7 +60,7 @@ def __init__(self, hby, wanHab, wilHab, wesHab): super(ReceiptDoer, self).__init__(doers=[doing.doify(self.testDo)]) - def testDo(self, tymth, tock=0.0): + def testDo(self, tymth, tock=0.0, **kwa): """ Execute a series of kli commands for this test scenario """ # enter context self.wind(tymth) @@ -145,7 +145,7 @@ def __init__(self, wanHby, wilHby, wesHby, palHby, seeder): super(PublishDoer, self).__init__(doers=doers) - def testDo(self, tymth, tock=0.0): + def testDo(self, tymth, tock=0.0, **kwa): """ Run the test and exit and remove all child doers when done """ self.wind(tymth) self.tock = tock From 6d2bdda02ba5f64d9ee76ab726c3f4667d7c7dfa Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Tue, 2 Sep 2025 07:09:04 -0600 Subject: [PATCH 383/418] fix: correct indentation (#1083) This broke exchange message processing for any KEL that was not known about if the exchange message arrived before the KEL arrived. --- src/keri/peer/exchanging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keri/peer/exchanging.py b/src/keri/peer/exchanging.py index ad1e20159..e9222afc9 100644 --- a/src/keri/peer/exchanging.py +++ b/src/keri/peer/exchanging.py @@ -89,8 +89,8 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): if prefixer.qb64 not in self.kevers or self.kevers[prefixer.qb64].sn < seqner.sn: if self.escrowPSEvent(serder=serder, tsgs=tsgs, pathed=pathed): self.cues.append(dict(kin="query", q=dict(r="logs", pre=prefixer.qb64, sn=seqner.snh))) - msg = f"Unable to find sender {prefixer.qb64} in kevers for evt = {serder.said}" - logger.info(msg) + msg = f"Unable to find sender {prefixer.qb64} in kevers for evt = {serder.said}" + logger.info(msg) logger.debug("Exchange message body=\n%s\n", serder.pretty()) raise MissingSignatureError(msg) From 14c9b8ca7831eb69382e4a28c7926381587e439e Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Tue, 23 Sep 2025 09:38:24 -0600 Subject: [PATCH 384/418] chore: bump version to 1.2.7 (#1089) --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 13c3a83b0..2528106e0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.6 +VERSION=1.2.7 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index c027f6c98..04e100e0e 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.6', # also change in src/keri/__init__.py + version='1.2.7', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index cb99ceb42..9826257d2 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.6' # also change in setup.py +__version__ = '1.2.7' # also change in setup.py From e060f34d0d1d438892d373608679d7625c5133a2 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Fri, 16 Jan 2026 15:59:58 -0700 Subject: [PATCH 385/418] fix: multisig join should use icp ked not exn ked This fixes a bug where "kli multisig join" would fail due to accidentally checking for a delegator identifier in an EXN ked instead of an inception event KED --- src/keri/app/cli/commands/multisig/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/app/cli/commands/multisig/join.py b/src/keri/app/cli/commands/multisig/join.py index 829b8a393..d58d78652 100644 --- a/src/keri/app/cli/commands/multisig/join.py +++ b/src/keri/app/cli/commands/multisig/join.py @@ -187,7 +187,7 @@ def incept(self, attrs): inits["toad"] = oicp.ked["bt"] inits["wits"] = oicp.ked["b"] - inits["delpre"] = oicp.ked["di"] if "di" in ked else None + inits["delpre"] = oicp.ked["di"] if "di" in oicp.ked else None print() print("Group Multisig Inception proposed:") From 8c8678ed11fe6161b666ebbfedb9e6ebb62dced5 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Fri, 16 Jan 2026 16:07:40 -0700 Subject: [PATCH 386/418] fix: remove dip and drt from escrows once approved Prior to this fix then only single sig workflows removed items from the delegables escrow. Somehow multisig dip and drt were missed. Both dip and drt should be removed from the delegables escrow once they are approved. --- src/keri/app/cli/commands/delegate/confirm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index 25eb27764..eac98e552 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -175,6 +175,7 @@ def confirmDo(self, tymth, tock=0.0): print(f"Delegate {eserder.pre} {typ} event committed.") + self.hby.db.delegables.rem(keys=(pre, sn), val=edig) self.remove(self.toRemove) return True @@ -231,7 +232,7 @@ def confirmDo(self, tymth, tock=0.0): print(f"Delegate {eserder.pre} {typ} event committed.") - self.hby.db.delegables.rem(keys=(pre, sn)) + self.hby.db.delegables.rem(keys=(pre, sn), val=edig) self.remove(self.toRemove) return True From cd81f266c14e12e1d3eed47d7b85db72af8fc3fc Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Fri, 16 Jan 2026 16:39:08 -0700 Subject: [PATCH 387/418] docs: clean up and add agenting docs --- src/keri/app/agenting.py | 337 ++++++++++++++------------------------- 1 file changed, 123 insertions(+), 214 deletions(-) diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 1f340b226..5e8f01a1c 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -26,8 +26,21 @@ class Receiptor(doing.DoDoer): + """DoDoer for synchronous witness receipting and receipt queries over HTTP. + + Drains `msgs` for witness receipt submissions and `gets` for receipt + queries, pushing processed request dicts to `cues`. + """ def __init__(self, hby, msgs=None, gets=None, cues=None): + """Initialize with shared queues and an HTTP client. + + Parameters: + hby (Habery): Habitat environment for identifier lookups. + msgs (Deck): receipt requests with `pre`, optional `sn`, optional `auths`. + gets (Deck): receipt query requests with `pre`, optional `sn`. + cues (Deck): completed request cues. + """ self.msgs = msgs if msgs is not None else decking.Deck() self.gets = gets if gets is not None else decking.Deck() @@ -40,20 +53,18 @@ def __init__(self, hby, msgs=None, gets=None, cues=None): super(Receiptor, self).__init__(doers=doers) def receipt(self, pre, sn=None, auths=None): - """ Returns a generator for witness receipting - - The returns a generator that will submit the designated event to witnesses for receipts using - the synchronous witness API, then propogate the receipts to each of the other witnesses. + """Yield while witnessing an event and returning witness receipt couples. + Sends the event to all witnesses, optionally catches new witnesses up, + then propagates the collected non-transferable receipts to each witness. Parameters: - pre (str): qualified base64 identifier to gather receipts for - sn: (Optiona[int]): sequence number of event to gather receipts for, latest is used if not provided - auths: (Options[dict]): map of witness AIDs to (time,auth) tuples for providing TOTP auth for witnessing + pre (str): qb64 identifier to receipt for. + sn (int | None): sequence number, defaults to latest. + auths (dict | None): optional map of wit AID to 2FA auth header value. Returns: - list: identifiers of witnesses that returned receipts. - + iterable: witness identifiers that returned receipts. """ auths = auths if auths is not None else dict() if pre not in self.hby.prefixes: @@ -140,18 +151,13 @@ def receipt(self, pre, sn=None, auths=None): return rcts.keys() def get(self, pre, sn=None): - """ Returns a generator for witness querying + """Yield while querying a witness for receipts of an event identified by pre and sn. - The returns a generator that will request receipts for event identified by pre and sn - - - Parameters: - pre (str): qualified base64 identifier to gather receipts for - sn: (Optiona[int]): sequence number of event to gather receipts for, latest is used if not provided + Picks one witness and issues GET /receipts?pre=&sn=. Parses any + returned receipts into the local parser. Returns: - list: identifiers of witnesses that returned receipts. - + bool: True if the witness returned HTTP 200. """ if pre not in self.hby.prefixes: raise kering.MissingEntryError(f"{pre} not a valid AID") @@ -184,12 +190,13 @@ def get(self, pre, sn=None): return rep.status == 200 def catchup(self, pre, wit): - """ When adding a new Witness, use this method to catch the witness up to the current state of the KEL + """ + Yield while replaying the full KEL for `pre` to the witness. + When adding a new Witness, use this method to catch the witness up to the current state of the KEL Parameters: pre (str): qualified base64 AID of the KEL to send wit (str): qualified base64 AID of the witness to send the KEL to - """ if pre not in self.hby.prefixes: raise kering.MissingEntryError(f"{pre} not a valid AID") @@ -207,18 +214,7 @@ def catchup(self, pre, wit): self.remove([clientDoer]) def witDo(self, tymth=None, tock=0.0, **kwa): - """ - Returns doifiable Doist compatibile generator method (doer dog) to process - .kevery and .tevery escrows. - - Parameters: - tymth (function): injected function wrapper closure returned by .tymen() of - Tymist instance. Calling tymth() returns associated Tymist .tyme. - tock (float): injected initial tock value - - Usage: - add result of doify on this method to doers list - """ + """Doer loop that drains `msgs` and runs witness receipt flow.""" self.wind(tymth) self.tock = tock _ = (yield self.tock) @@ -236,18 +232,7 @@ def witDo(self, tymth=None, tock=0.0, **kwa): yield self.tock def gitDo(self, tymth=None, tock=0.0, **kwa): - """ - Returns doifiable Doist compatibile generator method (doer dog) to process - .kevery and .tevery escrows. - - Parameters: - tymth (function): injected function wrapper closure returned by .tymen() of - Tymist instance. Calling tymth() returns associated Tymist .tyme. - tock (float): injected initial tock value - - Usage: - add result of doify on this method to doers list - """ + """Doer loop that drains `gets` and runs witness receipt queries.""" self.wind(tymth) self.tock = tock _ = (yield self.tock) @@ -264,28 +249,25 @@ def gitDo(self, tymth=None, tock=0.0, **kwa): class WitnessReceiptor(doing.DoDoer): - """ - Sends messages to all current witnesses of given identifier (from hab) and waits - for receipts from each of those witnesses and propagates those receipts to each - of the other witnesses after receiving the complete set. + """Witness receipt doer that sends events and propagates receipts. - Removes all Doers and exits as Done once all witnesses have been sent the entire - receipt set. Could be enhanced to have a `once` method that runs once and cleans up - and an `all` method that runs and waits for more messages to receipt. + Uses messenger doers to asynchronously send the event to each witness, + waits for receipts to arrive in `hab.db` (via mailbox processing), then + propagates the full receipt set across the witness group. + Could be enhanced to have a `once` method that runs once and cleans up + and an `all` method that runs and waits for more messages to receipt. """ def __init__(self, hby, msgs=None, cues=None, force=False, auths=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + """Initialize with queues and optional auth for witness endpoints. Parameters: - hby (Habery): Habitat of the identifier to receipt witnesses - msgs (Deck): incoming messages to publish to witnesses - cues (Deck): outgoing cues of successful messages - force (bool): True means to send witnesses all receipts even if we have a full compliment. - + hby (Habery): Habitat environment for identifier lookups. + msgs (Deck): receipt requests with `pre` and optional `sn`. + cues (Deck): completed request cues. + force (bool): resend receipts even if already complete. + auths (dict | None): optional map of wit AID to auth header value. """ self.hby = hby self.force = force @@ -296,17 +278,15 @@ def __init__(self, hby, msgs=None, cues=None, force=False, auths=None, **kwa): super(WitnessReceiptor, self).__init__(doers=[doing.doify(self.receiptDo)], **kwa) def receiptDo(self, tymth=None, tock=0.0, **kwa): - """ - Returns doifiable Doist compatible generator method (doer dog) - - Usage: - add result of doify on this method to doers list - - Parameters: - tymth is injected function wrapper closure returned by .tymen() of - Tymist instance. Calling tymth() returns associated Tymist .tyme. - tock is injected initial tock value + """Doer loop that sends events to witnesses and propagates receipts. + + Asynchronously processes witness receipt requests from self.msgs queue. + Sends any required delegation context, replays KEL for new witnesses, + posts the event, waits for receipts to be stored in `hab.db`, then + shares the full receipt set across witnesses. If `force` is false and + all receipts already exist, it skips resubmission. + Pushes the original request to self.cues to signal completion """ self.wind(tymth) self.tock = tock @@ -358,13 +338,13 @@ def receiptDo(self, tymth=None, tock=0.0, **kwa): witer.msgs.append(bytearray(msg)) # make a copy _ = (yield self.tock) - while True: + while True: # wait for all receipts to arrive wigs = hab.db.getWigs(dgkey) if len(wigs) == len(wits): break _ = yield self.tock - # If we started with all our recipts, exit unless told to force resubmit of all receipts + # If we started with all our receipts, exit unless told to force resubmit of all receipts if completed and not self.force: self.cues.push(evt) continue @@ -426,21 +406,21 @@ class WitnessInquisitor(doing.DoDoer): for receipts from each of those witnesses and propagates those receipts to each of the other witnesses after receiving the complete set. - Removes all Doers and exits as Done once all witnesses have been sent the entire - receipt set. Could be enhanced to have a `once` method that runs once and cleans up - and an `all` method that runs and waits for more messages to receipt. + Builds and sends qry/tel queries, pushing the raw sent message to `sent`. + The response parsing happens elsewhere (e.g. mailbox or HTTP response handlers). + Could be enhanced to have a `once` method that runs once and cleans up + and an `all` method that runs and waits for more messages to receipt. """ def __init__(self, hby, reger=None, msgs=None, klas=None, **kwa): - """ - For all msgs, select a random witness from Habitat's current set of witnesses - send the msg and process all responses (KEL replays, RCTs, etc) + """Initialize with a message queue and optional messenger class. Parameters: - hby (Habitat): Habitat of the identifier to use to identify witnesses - msgs: is the message buffer to process and send to one random witness. - + hby (Habery): Habitat environment for endpoint and kever lookup. + reger (Reger | None): optional registry database handle. + msgs (Deck): query requests built by `query`/`telquery`. + klas (type | None): messenger class, defaults to `HTTPMessenger`. """ self.hby = hby self.reger = reger @@ -452,10 +432,11 @@ def __init__(self, hby, reger=None, msgs=None, klas=None, **kwa): def msgDo(self, tymth=None, tock=1.0, **opts): """ - Returns doifiable Doist compatible generator method (doer dog) + Doer loop that sends one query to one selected endpoint. - Usage: - add result of doify on this method to doers list + For all msgs, select a random witness from Habitat's current set of witnesses + send the msg and process all responses (KEL replays, RCTs, etc) + Pushes the raw sent message to self.sent to signal completion. """ self.wind(tymth) self.tock = tock @@ -526,21 +507,17 @@ def msgDo(self, tymth=None, tock=1.0, **opts): yield self.tock def query(self, pre, r="logs", sn='0', fn='0', src=None, hab=None, anchor=None, wits=None, **kwa): - """ Create, sign and return a `qry` message against the attester for the prefix + """Create, sign, and queue a `qry` KEL query request against the attester for the prefix for later sending. Parameters: - src (str): qb64 identifier prefix of source of query - hab (Hab): Hab to use instead of src if provided - pre (str): qb64 identifier prefix being queried for - r (str): query route - sn (str): optional specific hex str of sequence number to query for - fn (str): optional specific hex str of sequence number to start with - anchor (Seal): anchored Seal to search for - wits (list) witnesses to query - - Returns: - bytearray: signed query event - + pre (str): qb64 identifier being queried. + r (str): query route (e.g. "logs"). + sn (str): optional hex sequence number to query for. + fn (str): optional hex start sequence number. + src (str | None): qb64 source identifier (ignored if `hab` provided). + hab (Hab | None): habitat used to sign and route the query. + anchor (Seal | None): anchored seal to search for. + wits (list | None): explicit witnesses to target; otherwise uses endpoints. """ qry = dict(s=sn, fn=fn) if anchor is not None: @@ -553,6 +530,7 @@ def query(self, pre, r="logs", sn='0', fn='0', src=None, hab=None, anchor=None, self.msgs.append(msg) def telquery(self, ri, src=None, i=None, r="tels", hab=None, pre=None, wits=None, **kwa): + """Queue a TEL query request for later sending.""" qry = dict(ri=ri) msg = dict(src=src, pre=pre, target=i, r=r, wits=wits, q=qry) if hab is not None: @@ -562,25 +540,19 @@ def telquery(self, ri, src=None, i=None, r="tels", hab=None, pre=None, wits=None class WitnessPublisher(doing.DoDoer): - """ - Sends messages to all current witnesses of given identifier (from hab) and exits. - - Removes all Doers and exits as Done once all witnesses have been sent the message. + """DoDoer that publishes messages to all witnesses for an identifier. + Could be enhanced to have a `once` method that runs once and cleans up and an `all` method that runs and waits for more messages to receipt. - """ def __init__(self, hby, msgs=None, cues=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + """Initialize with publish queue and completion cues. Parameters: - hby (Habery): Habitat of the identifier to populate witnesses - msgs (Deck): incoming messages to publish to witnesses - cues (Deck): outgoing cues of successful messages - + hby (Habery): Habitat environment for identifier lookups. + msgs (Deck): publish requests with `pre` and `msg`. + cues (Deck): completed request cues. """ self.hby = hby self.posted = 0 @@ -589,11 +561,9 @@ def __init__(self, hby, msgs=None, cues=None, **kwa): super(WitnessPublisher, self).__init__(doers=[doing.doify(self.sendDo)], **kwa) def sendDo(self, tymth=None, tock=0.0, **opts): - """ - Returns doifiable Doist compatible generator method (doer dog) - - Usage: - add result of doify on this method to doers list + """Doer loop that sends queued messages to each witness. + + Pushes the original request to self.cues to signal completion """ self.wind(tymth) self.tock = tock @@ -652,18 +622,17 @@ def idle(self): class TCPMessenger(doing.DoDoer): - """ Send events to witnesses for receipting using TCP direct connection - - """ + """Send outbound CESR messages to a witness via TCP and parse inbound receipts.""" def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + """Initialize TCP messenger with queues and parser wiring. Parameters: - hab: Habitat of the identifier to populate witnesses - + hab (Hab): habitat for KEL parsing and db access. + wit (str): qb64 witness identifier. + url (str): tcp endpoint URL for the witness. + msgs (Deck | None): outbound message queue. + sent (Deck | None): sent message queue. """ self.hab = hab self.wit = wit @@ -681,12 +650,7 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): super(TCPMessenger, self).__init__(doers=doers) def receiptDo(self, tymth=None, tock=0.0, **kwa): - """ - Returns doifiable Doist compatible generator method (doer dog) - - Usage: - add result of doify on this method to doers list - """ + """Doer loop that sends queued messages over TCP.""" self.wind(tymth) self.tock = tock _ = (yield self.tock) @@ -719,25 +683,7 @@ def receiptDo(self, tymth=None, tock=0.0, **kwa): yield self.tock def msgDo(self, tymth=None, tock=0.0, **opts): - """ - Returns doifiable Doist compatible generator method (doer dog) to process - incoming message stream of .kevery - - Doist Injected Attributes: - g.tock = tock # default tock attributes - g.done = None # default done state - g.opts - - Parameters: - tymth is injected function wrapper closure returned by .tymen() of - Tymist instance. Calling tymth() returns associated Tymist .tyme. - tock is injected initial tock value - opts is dict of injected optional additional parameters - - - Usage: - add result of doify on this method to doers list - """ + """Doer loop that parses inbound TCP messages into the Kevery.""" yield from self.parser.parsator(local=True) # process messages continuously @property @@ -746,18 +692,17 @@ def idle(self): class TCPStreamMessenger(doing.DoDoer): - """ Send events to witnesses for receipting using TCP direct connection - - """ + """Stream a CESR message to a witness via TCP and parse inbound receipts.""" def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + """Initialize TCP stream messenger with queues and parser wiring. Parameters: - hab: Habitat of the identifier to populate witnesses - + hab (Hab): habitat for KEL parsing and db access. + wit (str): qb64 witness identifier. + url (str): tcp endpoint URL for the witness. + msgs (Deck | None): outbound message queue. + sent (Deck | None): sent message queue. """ self.hab = hab self.wit = wit @@ -775,11 +720,9 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): super(TCPStreamMessenger, self).__init__(doers=doers) def receiptDo(self, tymth=None, tock=0.0, **kwa): - """ - Returns doifiable Doist compatible generator method (doer dog) - - Usage: - add result of doify on this method to doers list + """Doer loop that sends queued messages over TCP. + + Pushes the original request to self.sent to signal completion """ self.wind(tymth) self.tock = tock @@ -813,25 +756,7 @@ def receiptDo(self, tymth=None, tock=0.0, **kwa): yield self.tock def msgDo(self, tymth=None, tock=0.0, **opts): - """ - Returns doifiable Doist compatible generator method (doer dog) to process - incoming message stream of .kevery - - Doist Injected Attributes: - g.tock = tock # default tock attributes - g.done = None # default done state - g.opts - - Parameters: - tymth is injected function wrapper closure returned by .tymen() of - Tymist instance. Calling tymth() returns associated Tymist .tyme. - tock is injected initial tock value - opts is dict of injected optional additional parameters - - - Usage: - add result of doify on this method to doers list - """ + """Doer loop that parses inbound TCP messages into the Kevery.""" yield from self.parser.parsator(local=True) # process messages continuously @property @@ -840,19 +765,18 @@ def idle(self): class HTTPMessenger(doing.DoDoer): - """ - Interacts with Recipients on HTTP and SSE for sending events and receiving receipts - - """ + """Send CESR messages to a witness over HTTP and capture responses.""" def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, auth=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + """Initialize HTTP messenger with queues and optional auth. Parameters: - hab: Habitat of the identifier to populate witnesses - + hab (Hab): habitat for KEL parsing and db access. + wit (str): qb64 witness identifier. + url (str): http/https endpoint URL for the witness. + msgs (Deck | None): outbound message queue. + sent (Deck | None): response queue. + auth (str | None): optional 2FA auth codes for witnesses. """ self.hab = hab self.wit = wit @@ -876,12 +800,7 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, auth=None, * super(HTTPMessenger, self).__init__(doers=doers, **kwa) def msgDo(self, tymth=None, tock=0.0, **kwa): - """ - Returns doifiable Doist compatible generator method (doer dog) - - Usage: - add result of doify on this method to doers list - """ + """Doer loop that sends queued messages over HTTP.""" self.wind(tymth) self.tock = tock _ = (yield self.tock) @@ -902,10 +821,7 @@ def msgDo(self, tymth=None, tock=0.0, **kwa): yield self.tock def responseDo(self, tymth=None, tock=0.0, **kwa): - """ - Processes responses from client and adds them to sent cue - - """ + """Doer loop that processes HTTP responses from the client and adds them into `sent` cues.""" self.wind(tymth) self.tock = tock _ = (yield self.tock) @@ -923,19 +839,17 @@ def idle(self): class HTTPStreamMessenger(doing.DoDoer): - """ - Interacts with Recipients on HTTP and SSE for sending events and receiving receipts - - """ + """Send a single CESR message via HTTP PUT and capture the response.""" def __init__(self, hab, wit, url, msg=b'', headers=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + """Initialize a single-request HTTP messenger. Parameters: - hab: Habitat of the identifier to populate witnesses - + hab (Hab): habitat for KEL parsing and db access. + wit (str): qb64 witness identifier. + url (str): http/https endpoint URL for the witness. + msg (bytes): CESR message body to send. + headers (dict | None): extra HTTP headers. """ self.hab = hab self.wit = wit @@ -967,12 +881,7 @@ def __init__(self, hab, wit, url, msg=b'', headers=None, **kwa): super(HTTPStreamMessenger, self).__init__(doers=doers, **kwa) def recur(self, tyme, deeds=None): - """ - Returns doifiable Doist compatible generator method (doer dog) - - Usage: - add result of doify on this method to doers list - """ + """Poll for a response and stop once received.""" if self.client.responses: self.rep = self.client.respond() self.remove([self.client]) @@ -1037,8 +946,8 @@ def messengerFrom(hab, pre, urls, auth=None): def streamMessengerFrom(hab, pre, urls, msg, headers=None): - """ Create a Witnesser (tcp or http) based on provided endpoints - + """Create a stream messenger (HTTP or TCP) for a single outbound message. + Parameters: hab (Habitat): Environment to use to look up witness URLs pre (str): qb64 identifier prefix of recipient to create a messanger for From 3f4eaaa97b2b3f87a58398ab1d1f3b5c6dbf6b17 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Fri, 16 Jan 2026 16:49:09 -0700 Subject: [PATCH 388/418] fix: allow tests to use temp=True properly for Configer Configer instances in tests were being created in local dirs instead of temp dirs because cf was not being passed to hby.makeHab. --- src/keri/app/habbing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 7d746434d..2948541c4 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -102,7 +102,7 @@ def openHab(name="test", base="", salt=None, temp=True, cf=None, **kwa): with openHby(name=name, base=base, salt=salt, temp=temp, cf=cf) as hby: if (hab := hby.habByName(name)) is None: - hab = hby.makeHab(name=name, icount=1, isith='1', ncount=1, nsith='1', **kwa) + hab = hby.makeHab(name=name, icount=1, isith='1', ncount=1, nsith='1', cf=cf, **kwa) yield hby, hab From c4c0235eae1c3b6da20a7cf6ddc36824a6514cb6 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Fri, 16 Jan 2026 16:55:42 -0700 Subject: [PATCH 389/418] fix: backport multisig delegation fixes from main --- src/keri/core/eventing.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 93c88c432..343a32137 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1701,8 +1701,7 @@ def locallyDelegated(self, pre: str): Parameters: pre (str): qb64 identifier prefix if any. """ - pre = pre if pre is not None else "" - return self.locallyOwned(pre=pre) + return pre in self.prefixes def locallyWitnessed(self, *, wits: list[str]=None, serder: (str)=None): @@ -2388,7 +2387,7 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # seal in this case can't be malicious since sourced locally. # Doesn't get to here until fully signed and witnessed. - if self.locallyDelegated(delpre) and not self.locallyOwned(): # local delegator + if serder.ilk in (Ilks.dip, Ilks.drt) and self.locallyDelegated(delpre) and not self.locallyOwned(): # local delegator # must be local if locallyDelegated or caught above as misfit if delseqner is None or delsaider is None: # missing delegation seal # so escrow delegable. So local delegator can approve OOB. From bf42075506c21ce2db9957e92bbecc356330f994 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Tue, 20 Jan 2026 12:28:31 -0500 Subject: [PATCH 390/418] env vars for each db base (#1093) Signed-off-by: Kevin Griffin --- src/keri/app/keeping.py | 13 +++++++ src/keri/app/notifying.py | 14 +++++++- src/keri/app/storing.py | 10 ++++++ src/keri/db/basing.py | 9 +++-- src/keri/db/dbing.py | 11 ++++++ src/keri/vdr/viring.py | 10 ++++++ tests/app/test_keeper_mapsize.py | 54 ++++++++++++++++++++++++++++ tests/app/test_mailboxer_mapsize.py | 55 +++++++++++++++++++++++++++++ tests/app/test_noter_mapsize.py | 54 ++++++++++++++++++++++++++++ tests/db/test_baser_mapsize.py | 54 ++++++++++++++++++++++++++++ tests/vdr/test_reger_mapsize.py | 54 ++++++++++++++++++++++++++++ 11 files changed, 332 insertions(+), 6 deletions(-) create mode 100644 tests/app/test_keeper_mapsize.py create mode 100644 tests/app/test_mailboxer_mapsize.py create mode 100644 tests/app/test_noter_mapsize.py create mode 100644 tests/db/test_baser_mapsize.py create mode 100644 tests/vdr/test_reger_mapsize.py diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index 0f19fc775..578912ddc 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -23,6 +23,7 @@ """ import math +import os from collections import namedtuple, deque from dataclasses import dataclass, asdict, field @@ -33,8 +34,11 @@ from .. import core from ..core import coring from ..db import dbing, subing, koming +from .. import help from ..help import helping +logger = help.ogler.getLogger() + Algoage = namedtuple("Algoage", 'randy salty group extern') Algos = Algoage(randy='randy', salty='salty', group="group", extern="extern") # randy is rerandomize, salty is use salt @@ -251,6 +255,15 @@ def __init__(self, headDirPath=None, perm=None, reopen=False, **kwa): if perm is None: perm = self.Perm # defaults to restricted permissions for non temp + mapSize = os.getenv(dbing.KERIKeeperMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIKeeperMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Keeper, self).__init__(headDirPath=headDirPath, perm=perm, reopen=reopen, **kwa) diff --git a/src/keri/app/notifying.py b/src/keri/app/notifying.py index 09ac24db7..4fcaecd79 100644 --- a/src/keri/app/notifying.py +++ b/src/keri/app/notifying.py @@ -3,15 +3,18 @@ keri.app.notifying module """ +import os from collections.abc import Iterable from typing import Union, Type -from keri import kering +from keri import kering, help from keri.help import helping from keri.app import signaling from keri.core import coring from keri.db import dbing, subing +logger = help.ogler.getLogger() + def notice(attrs, dt=None, read=False): """ @@ -221,6 +224,15 @@ def __init__(self, name="not", headDirPath=None, reopen=True, **kwa): self.nidx = None self.ncigs = None + mapSize = os.getenv(dbing.KERINoterMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERINoterMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Noter, self).__init__(name=name, headDirPath=headDirPath, reopen=reopen, **kwa) def reopen(self, **kwa): diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index e7124f044..ebfe0ece9 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -3,6 +3,7 @@ keri.app.storing module """ +import os from hio.base import doing from hio.help import decking @@ -49,6 +50,15 @@ def __init__(self, name="mbx", headDirPath=None, reopen=True, **kwa): self.tpcs = None self.msgs = None + mapSize = os.getenv(dbing.KERIMailboxerMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIMailboxerMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Mailboxer, self).__init__(name=name, headDirPath=headDirPath, reopen=reopen, **kwa) def reopen(self, **kwa): diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 59f32bcae..b34c0f4a0 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -595,9 +595,6 @@ def reopenDB(db, clear=False, **kwa): db.close(clear=clear) -KERIBaserMapSizeKey = "KERI_BASER_MAP_SIZE" - - class Baser(dbing.LMDBer): """ Baser sets up named sub databases with Keri Event Logs within main database @@ -967,11 +964,13 @@ def __init__(self, headDirPath=None, reopen=False, **kwa): self._kevers = dbdict() self._kevers.db = self # assign db for read through cache of kevers - if (mapSize := os.getenv(KERIBaserMapSizeKey)) is not None: + mapSize = os.getenv(dbing.KERIBaserMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: try: self.MapSize = int(mapSize) except ValueError: - logger.error("KERI_BASER_MAP_SIZE must be an integer value >1!") + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIBaserMapSizeKey} or {dbing.KERILMDBMapSizeKey}") raise super(Baser, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 667edb2b2..55ff69a12 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -57,14 +57,24 @@ from hio.base import filing import keri +from .. import help from ..kering import MaxON # maximum ordinal number for seqence or first seen from ..help import helping +logger = help.ogler.getLogger() + ProemSize = 32 # does not include trailing separator MaxProem = int("f"*(ProemSize), 16) SuffixSize = 32 # does not include trailing separator MaxSuffix = int("f"*(SuffixSize), 16) +KERILMDBMapSizeKey = "KERI_LMDB_MAP_SIZE" # fallback +KERIBaserMapSizeKey = "KERI_BASER_MAP_SIZE" +KERIRegerMapSizeKey = "KERI_REGER_MAP_SIZE" +KERIKeeperMapSizeKey = "KERI_KEEPER_MAP_SIZE" +KERIMailboxerMapSizeKey = "KERI_MAILBOXER_MAP_SIZE" +KERINoterMapSizeKey = "KERI_NOTER_MAP_SIZE" + def onKey(top, on, *, sep=b'.'): """ Returns: @@ -382,6 +392,7 @@ def __init__(self, readonly=False, **kwa): self.env = None self._version = None self.readonly = True if readonly else False + super(LMDBer, self).__init__(**kwa) def reopen(self, readonly=False, **kwa): diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index e0d0c45ae..886af906a 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -7,6 +7,7 @@ Provides public simple Verifiable Credential Issuance/Revocation Registry A special purpose Verifiable Data Registry (VDR) """ +import os from dataclasses import dataclass, field, asdict @@ -276,6 +277,15 @@ def __init__(self, headDirPath=None, reopen=True, **kwa): else: self._tevers = dict() + mapSize = os.getenv(dbing.KERIRegerMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIRegerMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Reger, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa) diff --git a/tests/app/test_keeper_mapsize.py b/tests/app/test_keeper_mapsize.py new file mode 100644 index 000000000..318c4d8ec --- /dev/null +++ b/tests/app/test_keeper_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_keeper_mapsize module + +""" +import os +import pytest + + +def test_keeper_specific_env_var(): + from keri.app import keeping + + os.environ['KERI_KEEPER_MAP_SIZE'] = '150000000' + + try: + keeper = keeping.Keeper(name='test_keeper', temp=True) + assert keeper.MapSize == 150000000 + keeper.close() + finally: + os.environ.pop('KERI_KEEPER_MAP_SIZE', None) + + +def test_keeper_general_env_var_fallback(): + from keri.app import keeping + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + keeper = keeping.Keeper(name='test_keeper_general', temp=True) + assert keeper.MapSize == 250000000 + keeper.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_keeper_specific_takes_precedence(): + from keri.app import keeping + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_KEEPER_MAP_SIZE'] = '200000000' + + try: + keeper = keeping.Keeper(name='test_keeper_precedence', temp=True) + assert keeper.MapSize == 200000000 + keeper.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_KEEPER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_keeper_specific_env_var() + test_keeper_general_env_var_fallback() + test_keeper_specific_takes_precedence() diff --git a/tests/app/test_mailboxer_mapsize.py b/tests/app/test_mailboxer_mapsize.py new file mode 100644 index 000000000..65d3115be --- /dev/null +++ b/tests/app/test_mailboxer_mapsize.py @@ -0,0 +1,55 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_mailboxer_mapsize module + +""" +import os +import pytest + + +def test_mailboxer_specific_env_var(): + from keri.app import storing + + os.environ['KERI_MAILBOXER_MAP_SIZE'] = '150000000' + + try: + mbx = storing.Mailboxer(name='test_mailboxer', temp=True) + assert mbx.MapSize == 150000000 + mbx.close() + finally: + os.environ.pop('KERI_MAILBOXER_MAP_SIZE', None) + + +def test_mailboxer_general_env_var_fallback(): + from keri.app import storing + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + mbx = storing.Mailboxer(name='test_mailboxer_general', temp=True) + assert mbx.MapSize == 250000000 + mbx.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_mailboxer_specific_takes_precedence(): + """Test that KERI_MAILBOXER_MAP_SIZE takes precedence over KERI_LMDB_MAP_SIZE""" + from keri.app import storing + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_MAILBOXER_MAP_SIZE'] = '200000000' + + try: + mbx = storing.Mailboxer(name='test_mailboxer_precedence', temp=True) + assert mbx.MapSize == 200000000 + mbx.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_MAILBOXER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_mailboxer_specific_env_var() + test_mailboxer_general_env_var_fallback() + test_mailboxer_specific_takes_precedence() diff --git a/tests/app/test_noter_mapsize.py b/tests/app/test_noter_mapsize.py new file mode 100644 index 000000000..4748092a4 --- /dev/null +++ b/tests/app/test_noter_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_noter_mapsize module + +""" +import os +import pytest + + +def test_noter_specific_env_var(): + from keri.app import notifying + + os.environ['KERI_NOTER_MAP_SIZE'] = '150000000' + + try: + noter = notifying.Noter(name='test_noter', temp=True) + assert noter.MapSize == 150000000 + noter.close() + finally: + os.environ.pop('KERI_NOTER_MAP_SIZE', None) + + +def test_noter_general_env_var_fallback(): + from keri.app import notifying + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + noter = notifying.Noter(name='test_noter_general', temp=True) + assert noter.MapSize == 250000000 + noter.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_noter_specific_takes_precedence(): + from keri.app import notifying + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_NOTER_MAP_SIZE'] = '200000000' + + try: + noter = notifying.Noter(name='test_noter_precedence', temp=True) + assert noter.MapSize == 200000000 + noter.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_NOTER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_noter_specific_env_var() + test_noter_general_env_var_fallback() + test_noter_specific_takes_precedence() diff --git a/tests/db/test_baser_mapsize.py b/tests/db/test_baser_mapsize.py new file mode 100644 index 000000000..3c23d6bb8 --- /dev/null +++ b/tests/db/test_baser_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.db.test_baser_mapsize module + +""" +import os +import pytest + + +def test_baser_specific_env_var(): + from keri.db import basing + + os.environ['KERI_BASER_MAP_SIZE'] = '150000000' + + try: + baser = basing.Baser(name='test_baser', temp=True) + assert baser.MapSize == 150000000 + baser.close() + finally: + os.environ.pop('KERI_BASER_MAP_SIZE', None) + + +def test_baser_general_env_var_fallback(): + from keri.db import basing + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + baser = basing.Baser(name='test_baser_general', temp=True) + assert baser.MapSize == 250000000 + baser.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_baser_specific_takes_precedence(): + from keri.db import basing + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_BASER_MAP_SIZE'] = '200000000' + + try: + baser = basing.Baser(name='test_baser_precedence', temp=True) + assert baser.MapSize == 200000000 + baser.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_BASER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_baser_specific_env_var() + test_baser_general_env_var_fallback() + test_baser_specific_takes_precedence() diff --git a/tests/vdr/test_reger_mapsize.py b/tests/vdr/test_reger_mapsize.py new file mode 100644 index 000000000..df9c4bb47 --- /dev/null +++ b/tests/vdr/test_reger_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.vdr.test_reger_mapsize module + +""" +import os +import pytest + + +def test_reger_specific_env_var(): + from keri.vdr import viring + + os.environ['KERI_REGER_MAP_SIZE'] = '150000000' + + try: + reger = viring.Reger(name='test_reger', temp=True) + assert reger.MapSize == 150000000 + reger.close() + finally: + os.environ.pop('KERI_REGER_MAP_SIZE', None) + + +def test_reger_general_env_var_fallback(): + from keri.vdr import viring + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + reger = viring.Reger(name='test_reger_general', temp=True) + assert reger.MapSize == 250000000 + reger.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_reger_specific_takes_precedence(): + from keri.vdr import viring + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_REGER_MAP_SIZE'] = '200000000' + + try: + reger = viring.Reger(name='test_reger_precedence', temp=True) + assert reger.MapSize == 200000000 + reger.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_REGER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_reger_specific_env_var() + test_reger_general_env_var_fallback() + test_reger_specific_takes_precedence() From 5edaedb2b1c1f93806bb4575a85e7c302574cdad Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Sat, 31 Jan 2026 13:50:50 -0500 Subject: [PATCH 391/418] V1.2.7 runner (#1154) * single sig direct ecr issuance Signed-off-by: Kevin Griffin * fix runner Signed-off-by: Kevin Griffin * Revert "single sig direct ecr issuance" This reverts commit b94fcda0f577c7a4e72bf4512a096a5e7b492cdd. Signed-off-by: Kevin Griffin --------- Signed-off-by: Kevin Griffin --- .github/workflows/python-app-ci.yml | 10 +++++++++- setup.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app-ci.yml b/.github/workflows/python-app-ci.yml index d0ae68b2d..f2d5f17d1 100644 --- a/.github/workflows/python-app-ci.yml +++ b/.github/workflows/python-app-ci.yml @@ -16,10 +16,14 @@ jobs: strategy: fail-fast: false matrix: - os: [ macos-13, ubuntu-latest ] + os: [ macos-latest, ubuntu-latest ] steps: - uses: actions/checkout@v3 + - name: Install libsodium + if: runner.os == 'macOS' + run: | + brew install libsodium - name: Set up Python 3.12.2 uses: actions/setup-python@v2 with: @@ -35,6 +39,10 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --ignore=E7,F841,E301,E302,E303 --max-complexity=10 --max-line-length=127 --statistics + - name: Set DYLD_FALLBACK_LIBRARY_PATH + if: runner.os == 'macOS' + run: | + echo "DYLD_FALLBACK_LIBRARY_PATH=$(brew --prefix libsodium)/lib" >> $GITHUB_ENV - name: Run core KERI tests run: | pytest tests/ --ignore tests/demo/ --ignore test/scripts diff --git a/setup.py b/setup.py index 04e100e0e..db019906a 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ 'multidict>=6.0.5', 'ordered-set>=4.1.0', 'hio>=0.6.14', - 'multicommand>=1.0.0', + 'multicommand==1.0.0', 'jsonschema>=4.21.1', 'falcon>=3.1.3', 'hjson>=3.1.0', From 9277400741a2e183cb16c1f0fb25ba552e50e8f2 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Tue, 3 Feb 2026 10:39:56 -0500 Subject: [PATCH 392/418] Add kli contacts commands (#1155) Cherry-picked contact commands from GLEIF-IT/keripy (ba068228). Adds add, delete, find, get, query, and rename subcommands for kli contacts. --- scripts/demo/contacts/demo_contacts.sh | 60 +++ src/keri/app/cli/commands/contacts/add.py | 118 ++++++ src/keri/app/cli/commands/contacts/delete.py | 91 +++++ src/keri/app/cli/commands/contacts/find.py | 62 +++ src/keri/app/cli/commands/contacts/get.py | 104 +++++ src/keri/app/cli/commands/contacts/query.py | 135 +++++++ src/keri/app/cli/commands/contacts/rename.py | 84 ++++ tests/app/cli/commands/contacts/__init__.py | 0 .../cli/commands/contacts/test_contacts.py | 365 ++++++++++++++++++ 9 files changed, 1019 insertions(+) create mode 100755 scripts/demo/contacts/demo_contacts.sh create mode 100644 src/keri/app/cli/commands/contacts/add.py create mode 100644 src/keri/app/cli/commands/contacts/delete.py create mode 100644 src/keri/app/cli/commands/contacts/find.py create mode 100644 src/keri/app/cli/commands/contacts/get.py create mode 100644 src/keri/app/cli/commands/contacts/query.py create mode 100644 src/keri/app/cli/commands/contacts/rename.py create mode 100644 tests/app/cli/commands/contacts/__init__.py create mode 100644 tests/app/cli/commands/contacts/test_contacts.py diff --git a/scripts/demo/contacts/demo_contacts.sh b/scripts/demo/contacts/demo_contacts.sh new file mode 100755 index 000000000..f2741662b --- /dev/null +++ b/scripts/demo/contacts/demo_contacts.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Demo script for KLI contact management commands + + +KS_ALICE="contact-demo-alice" +KS_BOB="contact-demo-bob" +SALT_ALICE="0AAQmsjh-C7kAJZQEzdrzwB7" +SALT_BOB="0ABBnsjh-D8lBKZRFzesx1C8" + +kli init --name ${KS_ALICE} --nopasscode --salt ${SALT_ALICE} \ + --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +kli incept --name ${KS_ALICE} --alias alice \ + --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" + +echo "Creating Bob's keystore and identifier..." +kli init --name ${KS_BOB} --nopasscode --salt ${SALT_BOB} \ + --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +kli incept --name ${KS_BOB} --alias bob \ + --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" + +ALICE_OOBI=$(kli oobi generate --name ${KS_ALICE} --alias alice --role witness | head -1) +BOB_OOBI=$(kli oobi generate --name ${KS_BOB} --alias bob --role witness | head -1) +echo "Alice OOBI: ${ALICE_OOBI}" +echo "Bob OOBI: ${BOB_OOBI}" + +echo "Alice adds Bob as a contact..." +kli contacts add --name ${KS_ALICE} \ + --oobi "${BOB_OOBI}" \ + --alias bob \ + --field company=ACME \ + --field role=Engineer + +echo "Bob adds Alice as a contact..." +kli contacts add --name ${KS_BOB} \ + --oobi "${ALICE_OOBI}" \ + --alias alice \ + --field company=GLEIF + +echo "Alice's contacts:" +kli contacts list --name ${KS_ALICE} + +echo "Alice gets Bob by alias..." +kli contacts get --name ${KS_ALICE} --alias bob + +echo "Alice renames 'bob' to 'robert'..." +kli contacts rename --name ${KS_ALICE} \ + --old-alias bob \ + --alias robert +kli contacts get --name ${KS_ALICE} --alias robert + +echo "Alice queries robert's latest key state..." +kli contacts query --name ${KS_ALICE} --alias alice --contact-alias robert + +echo "Alice deletes contact 'robert'..." +kli contacts delete --name ${KS_ALICE} --alias robert --yes + +echo "Alice's contacts after delete:" +kli contacts list --name ${KS_ALICE} || echo "(no contacts)" + +echo "Done." diff --git a/src/keri/app/cli/commands/contacts/add.py b/src/keri/app/cli/commands/contacts/add.py new file mode 100644 index 000000000..431d71798 --- /dev/null +++ b/src/keri/app/cli/commands/contacts/add.py @@ -0,0 +1,118 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.add module + +""" +import argparse +import json + +from keri import help +from hio.base import doing + +import keri.app.oobiing +from keri.app import connecting, habbing, oobiing +from keri.app.cli.common import existing +from keri.db import basing +from keri.help import helping + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Add a contact via OOBI resolution') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--oobi', '-o', help='OOBI URL to resolve for contact', required=True) +parser.add_argument('--alias', help='alias to set for contact', required=False, default=None) +parser.add_argument('--field', '-f', help='field in key=value format', action='append', dest='fields', default=None) + + +def handler(args): + """ command line method for adding a contact via OOBI resolution + + Parameters: + args(Namespace): parse args namespace object + + """ + name = args.name + base = args.base + bran = args.bran + oobi = args.oobi + alias = args.alias + fields = args.fields + + addDoer = ContactAddDoer(name=name, base=base, bran=bran, + oobi=oobi, alias=alias, fields=fields) + return [addDoer] + + +class ContactAddDoer(doing.DoDoer): + """ DoDoer for adding a contact via OOBI resolution """ + + def __init__(self, name, base, bran, oobi, alias, fields): + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=self.hby) + self.oobi = oobi + self.alias = alias + self.fields = fields + + doers = [self.hbyDoer, doing.doify(self.add)] + super(ContactAddDoer, self).__init__(doers=doers) + + def add(self, tymth, tock=0.0): + """ Resolves OOBI and creates contact + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + """ + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + obr = basing.OobiRecord(date=helping.nowIso8601()) + if self.alias: + obr.oobialias = self.alias + + self.hby.db.oobis.put(keys=(self.oobi,), val=obr) + + obi = keri.app.oobiing.Oobiery(hby=self.hby) + authn = oobiing.Authenticator(hby=self.hby) + self.extend(obi.doers) + self.extend(authn.doers) + + # Wait for resolution + while not self.hby.db.roobi.get(keys=(self.oobi,)): + yield 0.25 + + resolved = self.hby.db.roobi.get(keys=(self.oobi,)) + cid = resolved.cid + + org = connecting.Organizer(hby=self.hby) + + data = {} + if self.alias: + data['alias'] = self.alias + + if self.fields: + for field in self.fields: + if '=' not in field: + print(f"Invalid field format: {field}. Use key=value") + self.remove([self.hbyDoer, *obi.doers, *authn.doers]) + return + key, val = field.split('=', 1) + data[key] = val + + org.update(cid, data) + + contact = org.get(cid) + print(json.dumps(contact, indent=2)) + + self.remove([self.hbyDoer, *obi.doers, *authn.doers]) diff --git a/src/keri/app/cli/commands/contacts/delete.py b/src/keri/app/cli/commands/contacts/delete.py new file mode 100644 index 000000000..1dfc55a05 --- /dev/null +++ b/src/keri/app/cli/commands/contacts/delete.py @@ -0,0 +1,91 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.delete module + +""" +import argparse + +from keri import help +from hio.base import doing + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Delete a contact') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--aid', '-a', help='contact AID prefix', required=False, default=None) +parser.add_argument('--alias', help='contact alias to lookup', required=False, default=None) +parser.add_argument('--yes', '-y', help='skip confirmation', action='store_true', default=False) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(delete, **kwa)] + + +def delete(tymth, tock=0.0, **opts): + """ Command line handler for deleting a contact + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + aid = args.aid + alias = args.alias + yes = args.yes + + if aid is None and alias is None: + print("Either --aid or --alias is required") + return -1 + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + + contact = None + pre = None + + if aid: + pre = aid + contact = org.get(aid) + elif alias: + contacts = org.find('alias', f"^{alias}$") # Exact match + if len(contacts) == 0: + print(f"Contact with alias '{alias}' not found") + return -1 + if len(contacts) > 1: + print(f"Multiple contacts match alias '{alias}'") + return -1 + contact = contacts[0] + pre = contact['id'] + + if contact is None: + print("Contact not found") + return -1 + + alias_display = contact.get('alias', pre) + + if not yes: + confirm = input(f"Delete contact '{alias_display}' ({pre})? [y/N]: ") + if confirm.lower() != 'y': + print("Aborted") + return 0 + + org.rem(pre) + print(f"Deleted contact '{alias_display}' ({pre})") + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first") + return -1 diff --git a/src/keri/app/cli/commands/contacts/find.py b/src/keri/app/cli/commands/contacts/find.py new file mode 100644 index 000000000..41befd719 --- /dev/null +++ b/src/keri/app/cli/commands/contacts/find.py @@ -0,0 +1,62 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.find module + +""" +import argparse +import json + +from keri import help +from hio.base import doing + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Find contacts by field value') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--field', '-f', help='field name to search (default: alias)', default='alias') +parser.add_argument('--value', '-v', help='value or regex pattern to match', required=True) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(find, **kwa)] + + +def find(tymth, tock=0.0, **opts): + """ Command line handler for finding contacts by field value + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + field = args.field + value = args.value + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + + contacts = org.find(field, value) + + if len(contacts) == 0: + print(f"No contacts found matching {field}='{value}'") + return + + print(json.dumps(contacts, indent=2)) + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first") + return -1 diff --git a/src/keri/app/cli/commands/contacts/get.py b/src/keri/app/cli/commands/contacts/get.py new file mode 100644 index 000000000..6d3cbc9a4 --- /dev/null +++ b/src/keri/app/cli/commands/contacts/get.py @@ -0,0 +1,104 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.get module + +""" +import argparse +import json + +from keri import help +from hio.base import doing +from keri import kering + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Get a single contact') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--aid', '-a', help='contact AID prefix', required=False, default=None) +parser.add_argument('--alias', help='contact alias to lookup', required=False, default=None) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(get, **kwa)] + + +def get(tymth, tock=0.0, **opts): + """ Command line handler for getting a single contact + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + aid = args.aid + alias = args.alias + + if aid is None and alias is None: + print("Either --aid or --alias is required") + return -1 + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + + contact = None + pre = None + + if aid: + pre = aid + contact = org.get(aid) + elif alias: + contacts = org.find('alias', f"^{alias}$") # Exact match + if len(contacts) > 1: + print(f"Multiple contacts match alias '{alias}'") + return -1 + if len(contacts) == 1: + contact = contacts[0] + pre = contact['id'] + + if contact is None: + print("Contact not found") + return -1 + + accepted = [saider.qb64 for saider in hby.db.chas.get(keys=(pre,))] + received = [saider.qb64 for saider in hby.db.reps.get(keys=(pre,))] + valid = set(accepted) & set(received) + + challenges = [] + for said in valid: + try: + exn = hby.db.exns.get(keys=(said,)) + except kering.ValidationError: + val = hby.db.getVal(db=hby.db.exns.sdb, key=hby.db.exns._tokey((said,))) + d = json.loads(bytes(val).decode("utf-8")) + challenges.append(dict(dt=d['dt'], words=d['a']['words'])) + else: + challenges.append(dict(dt=exn.ked['dt'], words=exn.ked['a']['words'])) + + contact["challenges"] = challenges + + wellKnowns = [] + wkans = hby.db.wkas.get(keys=(pre,)) + for wkan in wkans: + wellKnowns.append(dict(url=wkan.url, dt=wkan.dt)) + + contact["wellKnowns"] = wellKnowns + + print(json.dumps(contact, indent=2)) + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first") + return -1 diff --git a/src/keri/app/cli/commands/contacts/query.py b/src/keri/app/cli/commands/contacts/query.py new file mode 100644 index 000000000..96edbb603 --- /dev/null +++ b/src/keri/app/cli/commands/contacts/query.py @@ -0,0 +1,135 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.query module + +""" +import argparse +import datetime +import json + +from keri import help +from hio.base import doing +from hio.help import decking + +from keri.app import connecting, habbing, indirecting, querying +from keri.app.cli.common import existing, displaying +from keri.help import helping + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Query witnesses for latest key state of a contact') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--alias', '-a', help='alias of your local identifier for querying', required=True) +parser.add_argument('--contact-aid', dest='contact_aid', help='contact AID prefix to query', + required=False, default=None) +parser.add_argument('--contact-alias', dest='contact_alias', help='contact alias to lookup', + required=False, default=None) + + +def handler(args): + """ command line method for querying contact key state from witnesses + + Parameters: + args(Namespace): parse args namespace object + + """ + name = args.name + base = args.base + bran = args.bran + alias = args.alias + contact_aid = args.contact_aid + contact_alias = args.contact_alias + + if contact_aid is None and contact_alias is None: + print("Either --contact-aid or --contact-alias is required") + return [] + + queryDoer = ContactQueryDoer(name=name, base=base, bran=bran, alias=alias, + contact_aid=contact_aid, contact_alias=contact_alias) + return [queryDoer] + + +class ContactQueryDoer(doing.DoDoer): + """ DoDoer for querying contact key state from witnesses """ + + def __init__(self, name, base, bran, alias, contact_aid, contact_alias): + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=self.hby) + self.alias = alias + self.contact_aid = contact_aid + self.contact_alias = contact_alias + + self.mbd = indirecting.MailboxDirector(hby=self.hby, topics=["/replay", "/receipt", "/reply"]) + + doers = [self.hbyDoer, self.mbd, doing.doify(self.queryDo)] + super(ContactQueryDoer, self).__init__(doers=doers) + + def queryDo(self, tymth, tock=0.0): + """ Query witnesses for contact key state + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + """ + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + hab = self.hby.habByName(self.alias) + if hab is None: + print(f"Local identifier '{self.alias}' not found") + self.remove([self.hbyDoer, self.mbd]) + return + + org = connecting.Organizer(hby=self.hby) + + pre = None + if self.contact_aid: + pre = self.contact_aid + contact = org.get(pre) + elif self.contact_alias: + contacts = org.find('alias', f"^{self.contact_alias}$") + if len(contacts) == 0: + print(f"Contact with alias '{self.contact_alias}' not found") + self.remove([self.hbyDoer, self.mbd]) + return + if len(contacts) > 1: + print(f"Multiple contacts match alias '{self.contact_alias}'") + self.remove([self.hbyDoer, self.mbd]) + return + contact = contacts[0] + pre = contact['id'] + + if pre is None: + print("Contact not found") + self.remove([self.hbyDoer, self.mbd]) + return + + doer = querying.QueryDoer(hby=self.hby, hab=hab, pre=pre, kvy=self.mbd.kvy) + self.extend([doer]) + + end = helping.nowUTC() + datetime.timedelta(seconds=10) + while helping.nowUTC() < end: + if doer.done: + break + yield 1.0 + + self.remove([doer]) + + displaying.printExternal(self.hby, pre) + + contact = org.get(pre) + if contact: + print(json.dumps(contact, indent=2)) + + self.remove([self.hbyDoer, self.mbd]) diff --git a/src/keri/app/cli/commands/contacts/rename.py b/src/keri/app/cli/commands/contacts/rename.py new file mode 100644 index 000000000..b8c164add --- /dev/null +++ b/src/keri/app/cli/commands/contacts/rename.py @@ -0,0 +1,84 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.rename module + +""" +import argparse + +from keri import help +from hio.base import doing + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Rename a contact alias') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--aid', '-a', help='contact AID prefix', required=False, default=None) +parser.add_argument('--old-alias', dest='old_alias', help='current alias to lookup', required=False, default=None) +parser.add_argument('--alias', help='new alias to set', required=True) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(rename, **kwa)] + + +def rename(tymth, tock=0.0, **opts): + """ Command line handler for renaming a contact alias + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + aid = args.aid + old_alias = args.old_alias + new_alias = args.alias + + if aid is None and old_alias is None: + print("Either --aid or --old-alias is required") + return -1 + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + + contact = None + pre = None + + if aid: + pre = aid + contact = org.get(aid) + elif old_alias: + contacts = org.find('alias', f"^{old_alias}$") # Exact match + if len(contacts) == 0: + print(f"Contact with alias '{old_alias}' not found") + return -1 + if len(contacts) > 1: + print(f"Multiple contacts match alias '{old_alias}'") + return -1 + contact = contacts[0] + pre = contact['id'] + + if contact is None: + print("Contact not found") + return -1 + + old_alias_display = contact.get('alias', '') + org.set(pre, 'alias', new_alias) + print(f"Renamed contact {pre} from '{old_alias_display}' to '{new_alias}'") + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first") + return -1 diff --git a/tests/app/cli/commands/contacts/__init__.py b/tests/app/cli/commands/contacts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/app/cli/commands/contacts/test_contacts.py b/tests/app/cli/commands/contacts/test_contacts.py new file mode 100644 index 000000000..1f3ba9724 --- /dev/null +++ b/tests/app/cli/commands/contacts/test_contacts.py @@ -0,0 +1,365 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.cli.commands.contacts.test_contacts module + +Tests for KLI contacts commands: get, add, rename, delete +""" +import json +import pytest + +import multicommand + +from keri.app import habbing, connecting +from keri.app.cli import commands + + +def test_contacts_get_by_aid(capsys): + """Test getting a contact by AID""" + joe = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + joed = dict(first="Joe", last="Jury", alias="joe", company="HCF") + + with habbing.openHby(name="test-get-aid", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=joe, data=joed) + + # Test the get logic directly + contact = org.get(joe) + assert contact is not None + assert contact["id"] == joe + assert contact["first"] == "Joe" + assert contact["alias"] == "joe" + + # Verify command parser is correct + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "get", "--name", "test", "--aid", joe]) + assert args.handler is not None + assert args.aid == joe + assert args.alias is None + + +def test_contacts_get_by_alias(capsys): + """Test getting a contact by alias""" + bob = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + bobd = dict(first="Bob", last="Burns", alias="bob", company="HCF") + + with habbing.openHby(name="test-get-alias", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=bob, data=bobd) + + # Test the find/get logic directly + contacts = org.find('alias', f"^bob$") + assert len(contacts) == 1 + contact = contacts[0] + assert contact["id"] == bob + assert contact["first"] == "Bob" + assert contact["alias"] == "bob" + + # Verify command parser is correct + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "get", "--name", "test", "--alias", "bob"]) + assert args.handler is not None + assert args.alias == "bob" + assert args.aid is None + + +def test_contacts_get_not_found(): + """Test getting a non-existent contact""" + with habbing.openHby(name="test-get-notfound", temp=True) as hby: + org = connecting.Organizer(hby=hby) + + contact = org.get("ENonExistent123456789012345678901234567890123") + assert contact is None + + +def test_contacts_add_new(): + """Test adding a new contact""" + ken = "EFC7f_MEPE5dboc_E4yG15fnpMD34YaU3ue6vnDLodJU" + + with habbing.openHby(name="test-add-new", temp=True) as hby: + org = connecting.Organizer(hby=hby) + + # Test the add logic directly (using update) + data = {'alias': 'ken', 'company': 'GLEIF', 'city': 'Frankfurt'} + org.update(ken, data) + + contact = org.get(ken) + assert contact["id"] == ken + assert contact["alias"] == "ken" + assert contact["company"] == "GLEIF" + assert contact["city"] == "Frankfurt" + + # Verify command parser is correct + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "add", "--name", "test", + "--oobi", "http://127.0.0.1:5642/oobi/" + ken, + "--alias", "ken", "--field", "company=GLEIF", + "--field", "city=Frankfurt"]) + assert args.handler is not None + assert args.oobi == "http://127.0.0.1:5642/oobi/" + ken + assert args.alias == "ken" + assert args.fields == ["company=GLEIF", "city=Frankfurt"] + + +def test_contacts_add_update(): + """Test updating an existing contact""" + jen = "ED61oxVwVNf_olqR5wAhAjvuK59xuBOJXnJPGhwWDYoc" + jend = dict(first="Jen", last="Jones", alias="jen", company="GLEIF") + + with habbing.openHby(name="test-add-update", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=jen, data=jend) + + # Test the update logic - preserves existing fields + org.update(jen, {'mobile': '555-1212'}) + + contact = org.get(jen) + assert contact["id"] == jen + assert contact["first"] == "Jen" # preserved + assert contact["alias"] == "jen" # preserved + assert contact["mobile"] == "555-1212" # added + + +def test_contacts_add_field_parsing(): + """Test add command field parsing""" + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "add", "--name", "test", + "--oobi", "http://127.0.0.1:5642/oobi/Etest123", + "--field", "key1=value1", + "--field", "key2=value with spaces"]) + assert args.oobi == "http://127.0.0.1:5642/oobi/Etest123" + assert args.fields == ["key1=value1", "key2=value with spaces"] + + # Parse fields like the command does + data = {} + for field in args.fields: + key, val = field.split('=', 1) + data[key] = val + + assert data == {"key1": "value1", "key2": "value with spaces"} + + +def test_contacts_rename(): + """Test renaming a contact alias""" + sal = "Eo60ITGA69z4jNBU4RsvbgsjfAHFcTM2HVEXea1SvnXk" + sald = dict(first="Sally", last="Smith", alias="sally", company="GLEIF") + + with habbing.openHby(name="test-rename", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=sal, data=sald) + + # Test the rename logic directly + org.set(sal, 'alias', 'sal') + + contact = org.get(sal) + assert contact["alias"] == "sal" + + # Verify command parser is correct + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "rename", "--name", "test", + "--old-alias", "sally", "--alias", "sal"]) + assert args.handler is not None + assert args.old_alias == "sally" + assert args.alias == "sal" + + +def test_contacts_rename_by_aid(): + """Test renaming a contact by AID""" + joe = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + joed = dict(first="Joe", alias="joe") + + with habbing.openHby(name="test-rename-aid", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=joe, data=joed) + + org.set(joe, 'alias', 'joseph') + + contact = org.get(joe) + assert contact["alias"] == "joseph" + + # Verify command parser + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "rename", "--name", "test", + "--aid", joe, "--alias", "joseph"]) + assert args.aid == joe + assert args.alias == "joseph" + + +def test_contacts_rename_not_found(): + """Test finding a non-existent contact for rename""" + with habbing.openHby(name="test-rename-notfound", temp=True) as hby: + org = connecting.Organizer(hby=hby) + + contacts = org.find('alias', f"^nonexistent$") + assert len(contacts) == 0 + + +def test_contacts_delete(): + """Test deleting a contact""" + bob = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + bobd = dict(first="Bob", last="Burns", alias="bob", company="HCF") + + with habbing.openHby(name="test-delete", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=bob, data=bobd) + + # Verify contact exists + assert org.get(bob) is not None + + # Test the delete logic directly + org.rem(bob) + + # Verify contact was deleted + assert org.get(bob) is None + + +def test_contacts_delete_parser(): + """Test delete command parser""" + parser = multicommand.create_parser(commands) + + # Test with --alias and --yes + args = parser.parse_args(["contacts", "delete", "--name", "test", + "--alias", "bob", "--yes"]) + assert args.handler is not None + assert args.alias == "bob" + assert args.yes is True + + # Test with --aid + args = parser.parse_args(["contacts", "delete", "--name", "test", + "--aid", "Etest123", "-y"]) + assert args.aid == "Etest123" + assert args.yes is True + + +def test_contacts_delete_not_found(): + """Test deleting a non-existent contact""" + with habbing.openHby(name="test-delete-notfound", temp=True) as hby: + org = connecting.Organizer(hby=hby) + + contacts = org.find('alias', f"^nonexistent$") + assert len(contacts) == 0 + + contact = org.get("ENonExistent123456789012345678901234567890123") + assert contact is None + + +def test_contacts_list_parser(): + """Test list command parser (existing command)""" + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "list", "--name", "test"]) + assert args.handler is not None + assert args.name == "test" + + +def test_contacts_query_parser(): + """Test query command parser""" + parser = multicommand.create_parser(commands) + + # Test with --contact-alias + args = parser.parse_args(["contacts", "query", "--name", "test", + "--alias", "myid", "--contact-alias", "mycontact"]) + assert args.handler is not None + assert args.alias == "myid" + assert args.contact_alias == "mycontact" + + # Test with --contact-aid + args = parser.parse_args(["contacts", "query", "--name", "test", + "--alias", "myid", "--contact-aid", "Etest123"]) + assert args.contact_aid == "Etest123" + + +def test_contacts_workflow(): + """Test complete contact workflow: add, rename, get, delete""" + wil = "EPzeu5_C80nzPc_BGUHVBkXXfNmlS55Ayl7Rd1I0gWFE" + + with habbing.openHby(name="test-workflow", temp=True) as hby: + org = connecting.Organizer(hby=hby) + + # 1. Add contact + org.update(wil, {'alias': 'will', 'first': 'Will', 'company': 'GLEIF'}) + contact = org.get(wil) + assert contact["alias"] == "will" + + # 2. Rename contact + org.set(wil, 'alias', 'william') + contact = org.get(wil) + assert contact["alias"] == "william" + + # 3. Find by new alias + contacts = org.find('alias', f"^william$") + assert len(contacts) == 1 + assert contacts[0]["first"] == "Will" + + # 4. Update with more fields + org.update(wil, {'mobile': '555-1234'}) + contact = org.get(wil) + assert contact["mobile"] == "555-1234" + assert contact["first"] == "Will" # preserved + + # 5. Delete contact + org.rem(wil) + assert org.get(wil) is None + + +def test_contacts_find_parser(): + """Test find command parser""" + parser = multicommand.create_parser(commands) + + # Default field is alias + args = parser.parse_args(["contacts", "find", "--name", "test", "--value", "cfca"]) + assert args.handler is not None + assert args.field == "alias" + assert args.value == "cfca" + + # Explicit field + args = parser.parse_args(["contacts", "find", "--name", "test", + "--field", "company", "--value", "GLEIF"]) + assert args.field == "company" + assert args.value == "GLEIF" + + +def test_contacts_find(): + """Test finding contacts by field value pattern""" + aid1 = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + aid2 = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + aid3 = "EFC7f_MEPE5dboc_E4yG15fnpMD34YaU3ue6vnDLodJU" + + with habbing.openHby(name="test-find", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=aid1, data={'alias': 'yuan.cfca', 'company': 'CFCA'}) + org.replace(pre=aid2, data={'alias': 'li.cfca', 'company': 'CFCA'}) + org.replace(pre=aid3, data={'alias': 'bob.gleif', 'company': 'GLEIF'}) + + # Find by alias pattern + contacts = org.find('alias', 'cfca') + assert len(contacts) == 2 + aliases = {c['alias'] for c in contacts} + assert aliases == {'yuan.cfca', 'li.cfca'} + + # Find by company field + contacts = org.find('company', 'GLEIF') + assert len(contacts) == 1 + assert contacts[0]['alias'] == 'bob.gleif' + + # No matches + contacts = org.find('alias', 'nonexistent') + assert len(contacts) == 0 + + +def test_contacts_multiple_alias_match(): + """Test handling of multiple contacts with similar aliases""" + joe1 = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + joe2 = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + + with habbing.openHby(name="test-multi", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=joe1, data={'alias': 'joe'}) + org.replace(pre=joe2, data={'alias': 'joey'}) + + # Exact match should return one + contacts = org.find('alias', f"^joe$") + assert len(contacts) == 1 + assert contacts[0]['id'] == joe1 + + # Partial match returns both + contacts = org.find('alias', 'joe') + assert len(contacts) == 2 From cbe4751845c468be70bc72d87a9e0d21f5d8cfbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lenksj=C3=B6?= <5889538+lenkan@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:55:32 +0100 Subject: [PATCH 393/418] fix: provide telquery on admit (#1165) --- src/keri/app/cli/commands/ipex/admit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/app/cli/commands/ipex/admit.py b/src/keri/app/cli/commands/ipex/admit.py index b7ab7be99..0c0c1ded6 100644 --- a/src/keri/app/cli/commands/ipex/admit.py +++ b/src/keri/app/cli/commands/ipex/admit.py @@ -108,7 +108,7 @@ def admitDo(self, tymth, tock=0.0): # Lets get the latest KEL and Registry if needed self.witq.query(src=self.hab.pre, pre=issr) if "ri" in acdc: - self.witq.telquery(src=self.hab.pre, wits=self.hab.kevers[issr].wits, ri=acdc["ri"], i=acdc["d"]) + self.witq.telquery(src=self.hab.pre, pre=issr, wits=self.hab.kevers[issr].wits, ri=acdc["ri"], i=acdc["d"]) for label in ("anc", "iss", "acdc"): ked = embeds[label] From 9218667d86d52b74c86cc48a363a58994b3dcb57 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Sat, 29 Nov 2025 10:14:25 -0500 Subject: [PATCH 394/418] config tocks Signed-off-by: Kevin Griffin --- src/keri/app/agenting.py | 26 ++-- src/keri/app/directing.py | 49 ++++--- src/keri/app/indirecting.py | 56 ++++---- src/keri/app/storing.py | 6 +- src/keri/app/tocking.py | 89 +++++++++++++ tests/app/test_tocking.py | 258 ++++++++++++++++++++++++++++++++++++ 6 files changed, 422 insertions(+), 62 deletions(-) create mode 100644 src/keri/app/tocking.py create mode 100644 tests/app/test_tocking.py diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 5e8f01a1c..8b5f7bf59 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -14,7 +14,7 @@ from socket import gaierror -from . import httping, forwarding +from . import httping, forwarding, tocking from .. import help from .. import kering from .. import core @@ -277,11 +277,11 @@ def __init__(self, hby, msgs=None, cues=None, force=False, auths=None, **kwa): super(WitnessReceiptor, self).__init__(doers=[doing.doify(self.receiptDo)], **kwa) - def receiptDo(self, tymth=None, tock=0.0, **kwa): + def receiptDo(self, tymth=None, tock=None, **kwa): """Doer loop that sends events to witnesses and propagates receipts. - - Asynchronously processes witness receipt requests from self.msgs queue. + + Asynchronously processes witness receipt requests from self.msgs queue. Sends any required delegation context, replays KEL for new witnesses, posts the event, waits for receipts to be stored in `hab.db`, then shares the full receipt set across witnesses. If `force` is false and @@ -289,7 +289,7 @@ def receiptDo(self, tymth=None, tock=0.0, **kwa): Pushes the original request to self.cues to signal completion """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessReceiptorTock _ = (yield self.tock) while True: @@ -430,7 +430,7 @@ def __init__(self, hby, reger=None, msgs=None, klas=None, **kwa): super(WitnessInquisitor, self).__init__(doers=[doing.doify(self.msgDo)], **kwa) - def msgDo(self, tymth=None, tock=1.0, **opts): + def msgDo(self, tymth=None, tock=None, **opts): """ Doer loop that sends one query to one selected endpoint. @@ -439,7 +439,7 @@ def msgDo(self, tymth=None, tock=1.0, **opts): Pushes the raw sent message to self.sent to signal completion. """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessInquisitorTock _ = (yield self.tock) while True: @@ -541,7 +541,7 @@ def telquery(self, ri, src=None, i=None, r="tels", hab=None, pre=None, wits=None class WitnessPublisher(doing.DoDoer): """DoDoer that publishes messages to all witnesses for an identifier. - + Could be enhanced to have a `once` method that runs once and cleans up and an `all` method that runs and waits for more messages to receipt. """ @@ -560,13 +560,13 @@ def __init__(self, hby, msgs=None, cues=None, **kwa): self.cues = cues if cues is not None else decking.Deck() super(WitnessPublisher, self).__init__(doers=[doing.doify(self.sendDo)], **kwa) - def sendDo(self, tymth=None, tock=0.0, **opts): + def sendDo(self, tymth=None, tock=None, **opts): """Doer loop that sends queued messages to each witness. - + Pushes the original request to self.cues to signal completion """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessPublisherTock _ = (yield self.tock) while True: @@ -721,7 +721,7 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): def receiptDo(self, tymth=None, tock=0.0, **kwa): """Doer loop that sends queued messages over TCP. - + Pushes the original request to self.sent to signal completion """ self.wind(tymth) @@ -947,7 +947,7 @@ def messengerFrom(hab, pre, urls, auth=None): def streamMessengerFrom(hab, pre, urls, msg, headers=None): """Create a stream messenger (HTTP or TCP) for a single outbound message. - + Parameters: hab (Habitat): Environment to use to look up witness URLs pre (str): qb64 identifier prefix of recipient to create a messanger for diff --git a/src/keri/app/directing.py b/src/keri/app/directing.py index 6c72fc987..5da16998c 100644 --- a/src/keri/app/directing.py +++ b/src/keri/app/directing.py @@ -8,6 +8,7 @@ import itertools from hio.base import doing +from . import tocking from .. import help from ..core import eventing, routing from ..core import parsing @@ -207,7 +208,7 @@ def wind(self, tymth): self.client.wind(tymth) - def msgDo(self, tymth=None, tock=0.0, **opts): + def msgDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -227,14 +228,16 @@ def msgDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactorMsgTock + _ = (yield self.tock) # enter context if self.parser.ims: logger.info("Client %s received:\n%s\n...\n", self.hab.name, self.parser.ims[:1024]) done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def cueDo(self, tymth=None, tock=0.0, **opts): + def cueDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery.cues deque @@ -253,15 +256,17 @@ def cueDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactorCueTock + _ = (yield self.tock) # enter context while True: for msg in self.hab.processCuesIter(self.kevery.cues): self.sendMessage(msg, label="chit or receipt") - yield # throttle just do one cue at a time - yield + yield self.tock # throttle just do one cue at a time + yield self.tock return False # should never get here except forced close - def escrowDo(self, tymth=None, tock=0.0, **opts): + def escrowDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery escrows. @@ -280,12 +285,14 @@ def escrowDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactorEscrowTock + _ = (yield self.tock) # enter context while True: self.kevery.processEscrows() if self.tvy is not None: self.tvy.processEscrows() - yield + yield self.tock return False # should never get here except forced close def sendMessage(self, msg, label=""): @@ -561,7 +568,7 @@ def wind(self, tymth): self.remoter.wind(tymth) - def msgDo(self, tymth=None, tock=0.0, **opts): + def msgDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -581,7 +588,9 @@ def msgDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactantMsgTock + _ = (yield self.tock) # enter context if self.parser.ims: logger.info("Server %s: received:\n%s\n...\n", self.hab.name, self.parser.ims[:1024]) @@ -589,7 +598,7 @@ def msgDo(self, tymth=None, tock=0.0, **opts): return done # should nover get here except forced close - def cueDo(self, tymth=None, tock=0.0, **opts): + def cueDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery.cues deque @@ -608,19 +617,21 @@ def cueDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactantCueTock + _ = (yield self.tock) # enter context while True: for msg in self.hab.processCuesIter(self.kevery.cues): if isinstance(msg, list): msg = bytearray(itertools.chain(*msg)) self.sendMessage(msg, label="chit or receipt or replay") - yield # throttle just do one cue at a time - yield + yield self.tock # throttle just do one cue at a time + yield self.tock return False # should never get here except forced close - def escrowDo(self, tymth=None, tock=0.0, **opts): + def escrowDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery escrows. @@ -639,12 +650,14 @@ def escrowDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactantEscrowTock + _ = (yield self.tock) # enter context while True: self.kevery.processEscrows() if self.tevery is not None: self.tevery.processEscrows() - yield + yield self.tock return False # should never get here except forced close def sendMessage(self, msg, label=""): diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index a89c1e9cc..258a35c8e 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -19,7 +19,7 @@ from hio.help import decking import keri.app.oobiing -from . import directing, storing, httping, forwarding, agenting, oobiing +from . import directing, storing, httping, forwarding, agenting, oobiing, tocking from .habbing import GroupHab from .. import help, kering from ..core import (eventing, parsing, routing, coring, serdering, @@ -179,7 +179,7 @@ def start(self, tymth=None, tock=0.0, **kwa): print("Witness", self.hab.name, ":", self.hab.pre) - def msgDo(self, tymth=None, tock=0.0, **kwa): + def msgDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -193,7 +193,7 @@ def msgDo(self, tymth=None, tock=0.0, **kwa): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessMsgTock _ = (yield self.tock) if self.parser.ims: @@ -201,7 +201,7 @@ def msgDo(self, tymth=None, tock=0.0, **kwa): done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def escrowDo(self, tymth=None, tock=0.0, **kwa): + def escrowDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery and .tevery escrows. @@ -215,7 +215,7 @@ def escrowDo(self, tymth=None, tock=0.0, **kwa): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessEscrowTock _ = (yield self.tock) while True: @@ -225,9 +225,9 @@ def escrowDo(self, tymth=None, tock=0.0, **kwa): self.tvy.processEscrows() self.exc.processEscrow() - yield + yield self.tock - def cueDo(self, tymth=None, tock=0.0, **kwa): + def cueDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery.cues deque @@ -246,7 +246,7 @@ def cueDo(self, tymth=None, tock=0.0, **kwa): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessCueTock _ = (yield self.tock) while True: @@ -363,7 +363,7 @@ def wind(self, tymth): super(Indirector, self).wind(tymth) self.client.wind(tymth) - def msgDo(self, tymth=None, tock=0.0, **kwa): + def msgDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -382,7 +382,7 @@ def msgDo(self, tymth=None, tock=0.0, **kwa): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.IndirectorMsgTock _ = (yield self.tock) if self.parser.ims: @@ -390,7 +390,7 @@ def msgDo(self, tymth=None, tock=0.0, **kwa): done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def cueDo(self, tymth=None, tock=0.0, **kwa): + def cueDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery.cues deque @@ -409,16 +409,16 @@ def cueDo(self, tymth=None, tock=0.0, **kwa): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.IndirectorCueTock _ = (yield self.tock) while True: for msg in self.hab.processCuesIter(self.kevery.cues): self.sendMessage(msg, label="chit or receipt") - yield # throttle just do one cue at a time - yield + yield self.tock # throttle just do one cue at a time + yield self.tock - def escrowDo(self, tymth=None, tock=0.0, **kwa): + def escrowDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery escrows. @@ -437,12 +437,12 @@ def escrowDo(self, tymth=None, tock=0.0, **kwa): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.IndirectorEscrowTock _ = (yield self.tock) while True: self.kevery.processEscrows() - yield + yield self.tock def sendMessage(self, msg, label=""): """ @@ -578,7 +578,7 @@ def wind(self, tymth): """ super(MailboxDirector, self).wind(tymth) - def pollDo(self, tymth=None, tock=0.0, **kwa): + def pollDo(self, tymth=None, tock=None, **kwa): """ Returns: doifiable Doist compatible generator method @@ -588,7 +588,7 @@ def pollDo(self, tymth=None, tock=0.0, **kwa): """ # enter context self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.MailboxPollTock _ = (yield self.tock) habs = list(self.hby.habs.values()) @@ -656,7 +656,7 @@ def processPollIter(self): msg = mail.pop(0) yield msg - def msgDo(self, tymth=None, tock=0.0, **kwa): + def msgDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -675,13 +675,13 @@ def msgDo(self, tymth=None, tock=0.0, **kwa): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.MailboxMsgTock _ = (yield self.tock) done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def escrowDo(self, tymth=None, tock=0.0, **kwa): + def escrowDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery escrows. @@ -700,7 +700,7 @@ def escrowDo(self, tymth=None, tock=0.0, **kwa): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.MailboxEscrowTock _ = (yield self.tock) while True: @@ -713,7 +713,7 @@ def escrowDo(self, tymth=None, tock=0.0, **kwa): if self.verifier is not None: self.verifier.processEscrows() - yield + yield self.tock @property def times(self): @@ -754,7 +754,7 @@ def __init__(self, hab, witness, topics, msgs=None, retry=1000, **kwa): super(Poller, self).__init__(doers=doers, **kwa) - def eventDo(self, tymth=None, tock=0.0, **kwa): + def eventDo(self, tymth=None, tock=None, **kwa): """ Returns: doifiable Doist compatible generator method @@ -763,7 +763,7 @@ def eventDo(self, tymth=None, tock=0.0, **kwa): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.PollerEventTock _ = (yield self.tock) witrec = self.hab.db.tops.get((self.pre, self.witness)) @@ -1152,7 +1152,7 @@ def on_get(self, req, rep): rep.status = falcon.HTTP_200 rep.data = rct - def interceptDo(self, tymth=None, tock=0.0, **kwa): + def interceptDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process Kevery and Tevery cues deque @@ -1162,7 +1162,7 @@ def interceptDo(self, tymth=None, tock=0.0, **kwa): """ # enter context self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.ReceiptInterceptTock _ = (yield self.tock) while True: diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index ebfe0ece9..b2d6c9f53 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -9,7 +9,7 @@ from hio.help import decking from ordered_set import OrderedSet as oset -from . import forwarding +from . import forwarding, tocking from .. import help from ..core import coring, serdering from ..core.coring import MtrDex @@ -242,7 +242,7 @@ def responseDo(self, tymth=None, tock=0.0, **kwa): yield self.tock - def cueDo(self, tymth=None, tock=0.0, **kwa): + def cueDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process Kevery and Tevery cues deque @@ -252,7 +252,7 @@ def cueDo(self, tymth=None, tock=0.0, **kwa): """ # enter context self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.RespondantCueTock _ = (yield self.tock) while True: diff --git a/src/keri/app/tocking.py b/src/keri/app/tocking.py new file mode 100644 index 000000000..a97d6264c --- /dev/null +++ b/src/keri/app/tocking.py @@ -0,0 +1,89 @@ +# -*- encoding: utf-8 -*- +""" +keri.app.tocking module + +Centralized tock configuration for doer timing control. +Environment variables allow production tuning without code changes. + +Tock values control how frequently doer methods execute: +- 0.0 = run as soon as possible (~31ms doist interval) +- 1.0 = run once per second +- Higher values = less frequent execution, lower CPU usage + +""" +import os + +# Witness component tocks (indirecting.py - WitnessStart) +KERI_WITNESS_MSG_TOCK_KEY = "KERI_WITNESS_MSG_TOCK" +KERI_WITNESS_ESCROW_TOCK_KEY = "KERI_WITNESS_ESCROW_TOCK" +KERI_WITNESS_CUE_TOCK_KEY = "KERI_WITNESS_CUE_TOCK" + +WitnessMsgTock = float(os.getenv(KERI_WITNESS_MSG_TOCK_KEY, "0.0")) +WitnessEscrowTock = float(os.getenv(KERI_WITNESS_ESCROW_TOCK_KEY, "1.0")) +WitnessCueTock = float(os.getenv(KERI_WITNESS_CUE_TOCK_KEY, "0.25")) + +# Indirector component tocks (indirecting.py - Indirector) +KERI_INDIRECTOR_MSG_TOCK_KEY = "KERI_INDIRECTOR_MSG_TOCK" +KERI_INDIRECTOR_CUE_TOCK_KEY = "KERI_INDIRECTOR_CUE_TOCK" +KERI_INDIRECTOR_ESCROW_TOCK_KEY = "KERI_INDIRECTOR_ESCROW_TOCK" + +IndirectorMsgTock = float(os.getenv(KERI_INDIRECTOR_MSG_TOCK_KEY, "0.0")) +IndirectorCueTock = float(os.getenv(KERI_INDIRECTOR_CUE_TOCK_KEY, "0.25")) +IndirectorEscrowTock = float(os.getenv(KERI_INDIRECTOR_ESCROW_TOCK_KEY, "1.0")) + +# MailboxDirector component tocks (indirecting.py - MailboxDirector) +KERI_MAILBOX_POLL_TOCK_KEY = "KERI_MAILBOX_POLL_TOCK" +KERI_MAILBOX_MSG_TOCK_KEY = "KERI_MAILBOX_MSG_TOCK" +KERI_MAILBOX_ESCROW_TOCK_KEY = "KERI_MAILBOX_ESCROW_TOCK" + +MailboxPollTock = float(os.getenv(KERI_MAILBOX_POLL_TOCK_KEY, "0.5")) +MailboxMsgTock = float(os.getenv(KERI_MAILBOX_MSG_TOCK_KEY, "0.0")) +MailboxEscrowTock = float(os.getenv(KERI_MAILBOX_ESCROW_TOCK_KEY, "1.0")) + +# Poller component tocks (indirecting.py - Poller) +KERI_POLLER_EVENT_TOCK_KEY = "KERI_POLLER_EVENT_TOCK" + +PollerEventTock = float(os.getenv(KERI_POLLER_EVENT_TOCK_KEY, "0.5")) + +# ReceiptEnd component tocks (indirecting.py - ReceiptEnd) +KERI_RECEIPT_INTERCEPT_TOCK_KEY = "KERI_RECEIPT_INTERCEPT_TOCK" + +ReceiptInterceptTock = float(os.getenv(KERI_RECEIPT_INTERCEPT_TOCK_KEY, "0.1")) + +# Reactor component tocks (directing.py - Reactor) +KERI_REACTOR_MSG_TOCK_KEY = "KERI_REACTOR_MSG_TOCK" +KERI_REACTOR_CUE_TOCK_KEY = "KERI_REACTOR_CUE_TOCK" +KERI_REACTOR_ESCROW_TOCK_KEY = "KERI_REACTOR_ESCROW_TOCK" + +ReactorMsgTock = float(os.getenv(KERI_REACTOR_MSG_TOCK_KEY, "0.0")) +ReactorCueTock = float(os.getenv(KERI_REACTOR_CUE_TOCK_KEY, "0.25")) +ReactorEscrowTock = float(os.getenv(KERI_REACTOR_ESCROW_TOCK_KEY, "1.0")) + +# Reactant component tocks (directing.py - Reactant) +KERI_REACTANT_MSG_TOCK_KEY = "KERI_REACTANT_MSG_TOCK" +KERI_REACTANT_CUE_TOCK_KEY = "KERI_REACTANT_CUE_TOCK" +KERI_REACTANT_ESCROW_TOCK_KEY = "KERI_REACTANT_ESCROW_TOCK" + +ReactantMsgTock = float(os.getenv(KERI_REACTANT_MSG_TOCK_KEY, "0.0")) +ReactantCueTock = float(os.getenv(KERI_REACTANT_CUE_TOCK_KEY, "0.25")) +ReactantEscrowTock = float(os.getenv(KERI_REACTANT_ESCROW_TOCK_KEY, "1.0")) + +# Respondant component tocks (storing.py - Respondant) +KERI_RESPONDANT_CUE_TOCK_KEY = "KERI_RESPONDANT_CUE_TOCK" + +RespondantCueTock = float(os.getenv(KERI_RESPONDANT_CUE_TOCK_KEY, "0.25")) + +# WitnessReceiptor component tocks (agenting.py) +KERI_WITNESS_RECEIPTOR_TOCK_KEY = "KERI_WITNESS_RECEIPTOR_TOCK" + +WitnessReceiptorTock = float(os.getenv(KERI_WITNESS_RECEIPTOR_TOCK_KEY, "0.0")) + +# WitnessInquisitor component tocks (agenting.py) +KERI_WITNESS_INQUISITOR_TOCK_KEY = "KERI_WITNESS_INQUISITOR_TOCK" + +WitnessInquisitorTock = float(os.getenv(KERI_WITNESS_INQUISITOR_TOCK_KEY, "1.0")) + +# WitnessPublisher component tocks (agenting.py) +KERI_WITNESS_PUBLISHER_TOCK_KEY = "KERI_WITNESS_PUBLISHER_TOCK" + +WitnessPublisherTock = float(os.getenv(KERI_WITNESS_PUBLISHER_TOCK_KEY, "0.25")) diff --git a/tests/app/test_tocking.py b/tests/app/test_tocking.py new file mode 100644 index 000000000..1a03bcae8 --- /dev/null +++ b/tests/app/test_tocking.py @@ -0,0 +1,258 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_tocking module + +Tests for centralized tock configuration. +""" +import os +import importlib + +import pytest + + +def test_tocking_defaults(): + """ + Test default tock values when no environment variables are set + """ + # Clear any existing env vars that might affect the test + env_keys = [ + "KERI_WITNESS_MSG_TOCK", + "KERI_WITNESS_ESCROW_TOCK", + "KERI_WITNESS_CUE_TOCK", + "KERI_INDIRECTOR_MSG_TOCK", + "KERI_INDIRECTOR_CUE_TOCK", + "KERI_INDIRECTOR_ESCROW_TOCK", + "KERI_MAILBOX_POLL_TOCK", + "KERI_MAILBOX_MSG_TOCK", + "KERI_MAILBOX_ESCROW_TOCK", + "KERI_POLLER_EVENT_TOCK", + "KERI_RECEIPT_INTERCEPT_TOCK", + "KERI_REACTOR_MSG_TOCK", + "KERI_REACTOR_CUE_TOCK", + "KERI_REACTOR_ESCROW_TOCK", + "KERI_REACTANT_MSG_TOCK", + "KERI_REACTANT_CUE_TOCK", + "KERI_REACTANT_ESCROW_TOCK", + "KERI_RESPONDANT_CUE_TOCK", + "KERI_WITNESS_RECEIPTOR_TOCK", + "KERI_WITNESS_INQUISITOR_TOCK", + "KERI_WITNESS_PUBLISHER_TOCK", + ] + + # Save and clear existing env vars + saved_env = {} + for key in env_keys: + if key in os.environ: + saved_env[key] = os.environ.pop(key) + + try: + # Reload module to pick up cleared env vars + from keri.app import tocking + importlib.reload(tocking) + + # Witness component tocks + assert tocking.WitnessMsgTock == 0.0 + assert tocking.WitnessEscrowTock == 1.0 + assert tocking.WitnessCueTock == 0.25 + + # Indirector component tocks + assert tocking.IndirectorMsgTock == 0.0 + assert tocking.IndirectorCueTock == 0.25 + assert tocking.IndirectorEscrowTock == 1.0 + + # MailboxDirector component tocks + assert tocking.MailboxPollTock == 0.5 + assert tocking.MailboxMsgTock == 0.0 + assert tocking.MailboxEscrowTock == 1.0 + + # Poller component tocks + assert tocking.PollerEventTock == 0.5 + + # ReceiptEnd component tocks + assert tocking.ReceiptInterceptTock == 0.1 + + # Reactor component tocks + assert tocking.ReactorMsgTock == 0.0 + assert tocking.ReactorCueTock == 0.25 + assert tocking.ReactorEscrowTock == 1.0 + + # Reactant component tocks + assert tocking.ReactantMsgTock == 0.0 + assert tocking.ReactantCueTock == 0.25 + assert tocking.ReactantEscrowTock == 1.0 + + # Respondant component tocks + assert tocking.RespondantCueTock == 0.25 + + # WitnessReceiptor component tocks + assert tocking.WitnessReceiptorTock == 0.0 + + # WitnessInquisitor component tocks + assert tocking.WitnessInquisitorTock == 1.0 + + # WitnessPublisher component tocks + assert tocking.WitnessPublisherTock == 0.25 + + finally: + # Restore saved env vars + for key, value in saved_env.items(): + os.environ[key] = value + # Reload to restore original state + importlib.reload(tocking) + + """Done Test""" + + +def test_tocking_env_override(): + """ + Test that environment variables properly override default tock values + """ + # Save existing env vars + env_overrides = { + "KERI_WITNESS_MSG_TOCK": "2.5", + "KERI_WITNESS_ESCROW_TOCK": "3.0", + "KERI_WITNESS_CUE_TOCK": "0.5", + "KERI_INDIRECTOR_MSG_TOCK": "1.0", + "KERI_INDIRECTOR_CUE_TOCK": "0.75", + "KERI_INDIRECTOR_ESCROW_TOCK": "2.0", + "KERI_MAILBOX_POLL_TOCK": "1.5", + "KERI_MAILBOX_MSG_TOCK": "0.1", + "KERI_MAILBOX_ESCROW_TOCK": "2.5", + "KERI_POLLER_EVENT_TOCK": "1.0", + "KERI_RECEIPT_INTERCEPT_TOCK": "0.5", + "KERI_REACTOR_MSG_TOCK": "0.5", + "KERI_REACTOR_CUE_TOCK": "0.5", + "KERI_REACTOR_ESCROW_TOCK": "2.0", + "KERI_REACTANT_MSG_TOCK": "0.5", + "KERI_REACTANT_CUE_TOCK": "0.5", + "KERI_REACTANT_ESCROW_TOCK": "2.0", + "KERI_RESPONDANT_CUE_TOCK": "0.5", + "KERI_WITNESS_RECEIPTOR_TOCK": "0.5", + "KERI_WITNESS_INQUISITOR_TOCK": "2.0", + "KERI_WITNESS_PUBLISHER_TOCK": "0.5", + } + + # Save and set env vars + saved_env = {} + for key, value in env_overrides.items(): + if key in os.environ: + saved_env[key] = os.environ[key] + os.environ[key] = value + + try: + # Reload module to pick up new env vars + from keri.app import tocking + importlib.reload(tocking) + + # Witness component tocks + assert tocking.WitnessMsgTock == 2.5 + assert tocking.WitnessEscrowTock == 3.0 + assert tocking.WitnessCueTock == 0.5 + + # Indirector component tocks + assert tocking.IndirectorMsgTock == 1.0 + assert tocking.IndirectorCueTock == 0.75 + assert tocking.IndirectorEscrowTock == 2.0 + + # MailboxDirector component tocks + assert tocking.MailboxPollTock == 1.5 + assert tocking.MailboxMsgTock == 0.1 + assert tocking.MailboxEscrowTock == 2.5 + + # Poller component tocks + assert tocking.PollerEventTock == 1.0 + + # ReceiptEnd component tocks + assert tocking.ReceiptInterceptTock == 0.5 + + # Reactor component tocks + assert tocking.ReactorMsgTock == 0.5 + assert tocking.ReactorCueTock == 0.5 + assert tocking.ReactorEscrowTock == 2.0 + + # Reactant component tocks + assert tocking.ReactantMsgTock == 0.5 + assert tocking.ReactantCueTock == 0.5 + assert tocking.ReactantEscrowTock == 2.0 + + # Respondant component tocks + assert tocking.RespondantCueTock == 0.5 + + # WitnessReceiptor component tocks + assert tocking.WitnessReceiptorTock == 0.5 + + # WitnessInquisitor component tocks + assert tocking.WitnessInquisitorTock == 2.0 + + # WitnessPublisher component tocks + assert tocking.WitnessPublisherTock == 0.5 + + finally: + # Restore or remove env vars + for key in env_overrides: + if key in saved_env: + os.environ[key] = saved_env[key] + else: + os.environ.pop(key, None) + # Reload to restore original state + importlib.reload(tocking) + + """Done Test""" + + +def test_tocking_keys(): + """ + Test that environment variable keys are correctly defined + """ + from keri.app import tocking + + # Witness component keys + assert tocking.KERI_WITNESS_MSG_TOCK_KEY == "KERI_WITNESS_MSG_TOCK" + assert tocking.KERI_WITNESS_ESCROW_TOCK_KEY == "KERI_WITNESS_ESCROW_TOCK" + assert tocking.KERI_WITNESS_CUE_TOCK_KEY == "KERI_WITNESS_CUE_TOCK" + + # Indirector component keys + assert tocking.KERI_INDIRECTOR_MSG_TOCK_KEY == "KERI_INDIRECTOR_MSG_TOCK" + assert tocking.KERI_INDIRECTOR_CUE_TOCK_KEY == "KERI_INDIRECTOR_CUE_TOCK" + assert tocking.KERI_INDIRECTOR_ESCROW_TOCK_KEY == "KERI_INDIRECTOR_ESCROW_TOCK" + + # MailboxDirector component keys + assert tocking.KERI_MAILBOX_POLL_TOCK_KEY == "KERI_MAILBOX_POLL_TOCK" + assert tocking.KERI_MAILBOX_MSG_TOCK_KEY == "KERI_MAILBOX_MSG_TOCK" + assert tocking.KERI_MAILBOX_ESCROW_TOCK_KEY == "KERI_MAILBOX_ESCROW_TOCK" + + # Poller component keys + assert tocking.KERI_POLLER_EVENT_TOCK_KEY == "KERI_POLLER_EVENT_TOCK" + + # ReceiptEnd component keys + assert tocking.KERI_RECEIPT_INTERCEPT_TOCK_KEY == "KERI_RECEIPT_INTERCEPT_TOCK" + + # Reactor component keys + assert tocking.KERI_REACTOR_MSG_TOCK_KEY == "KERI_REACTOR_MSG_TOCK" + assert tocking.KERI_REACTOR_CUE_TOCK_KEY == "KERI_REACTOR_CUE_TOCK" + assert tocking.KERI_REACTOR_ESCROW_TOCK_KEY == "KERI_REACTOR_ESCROW_TOCK" + + # Reactant component keys + assert tocking.KERI_REACTANT_MSG_TOCK_KEY == "KERI_REACTANT_MSG_TOCK" + assert tocking.KERI_REACTANT_CUE_TOCK_KEY == "KERI_REACTANT_CUE_TOCK" + assert tocking.KERI_REACTANT_ESCROW_TOCK_KEY == "KERI_REACTANT_ESCROW_TOCK" + + # Respondant component keys + assert tocking.KERI_RESPONDANT_CUE_TOCK_KEY == "KERI_RESPONDANT_CUE_TOCK" + + # WitnessReceiptor component keys + assert tocking.KERI_WITNESS_RECEIPTOR_TOCK_KEY == "KERI_WITNESS_RECEIPTOR_TOCK" + + # WitnessInquisitor component keys + assert tocking.KERI_WITNESS_INQUISITOR_TOCK_KEY == "KERI_WITNESS_INQUISITOR_TOCK" + + # WitnessPublisher component keys + assert tocking.KERI_WITNESS_PUBLISHER_TOCK_KEY == "KERI_WITNESS_PUBLISHER_TOCK" + + """Done Test""" + + +if __name__ == "__main__": + test_tocking_defaults() + test_tocking_env_override() + test_tocking_keys() From 51430e645621693ce547353e07e4bfe65cb9ca7c Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 1 Dec 2025 13:52:32 -0500 Subject: [PATCH 395/418] export escrow metrics Signed-off-by: Kevin Griffin --- src/keri/app/indirecting.py | 91 ++++++++++++++++++++++++++++++++++- tests/app/test_indirecting.py | 41 ++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-) diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 258a35c8e..17caa8d4a 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -90,6 +90,8 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo app.add_route("/receipts", receiptEnd) queryEnd = QueryEnd(hab=hab) app.add_route("/query", queryEnd) + metricsEnd = MetricsEnd(hby=hby, reger=reger) + app.add_route("/metrics", metricsEnd) server = createHttpServer(host, httpPort, app, keypath, certpath, cafilepath) if not server.reopen(): @@ -1283,4 +1285,91 @@ def on_get(self, req, rep): else: rep.set_header('Content-Type', "application/json") rep.text = "unkown query type." - rep.status = falcon.HTTP_400 \ No newline at end of file + rep.status = falcon.HTTP_400 + + +class MetricsEnd: + """Prometheus metrics endpoint for escrow counts. + + Exposes escrow counts in Prometheus text format for monitoring/alerting. + """ + + def __init__(self, hby, reger): + """Initialize MetricsEnd. + + Parameters: + hby (Habery): Habery instance for accessing KEL escrows + reger (Reger): Registry for accessing TEL escrows + """ + self.hby = hby + self.reger = reger + + def on_get(self, req, rep): + """GET /metrics - Returns Prometheus format metrics. + + Parameters: + req (Request): Falcon HTTP request object + rep (Response): Falcon HTTP response object + """ + lines = [] + + # Header + lines.append("# HELP keri_escrow_count Number of items in each escrow type") + lines.append("# TYPE keri_escrow_count gauge") + + # KEL / Baser escrows + escrow_counts = [ + ("unverified_receipts", sum(1 for _ in self.hby.db.getUreItemIter())), + ("verified_receipts", sum(1 for _ in self.hby.db.getVreItemIter())), + ("out_of_order_events", sum(1 for _ in self.hby.db.getOoeItemIter())), + ("partially_witnessed_events", sum(1 for _ in self.hby.db.getPweItemIter())), + ("partially_signed_events", sum(1 for _ in self.hby.db.getPseItemIter())), + ("likely_duplicitous_events", sum(1 for _ in self.hby.db.getLdeItemIter())), + ("unverified_event_indexed_couples", sum(1 for _ in self.hby.db.getUweItemIter())), + ("query_not_found", sum(1 for _ in self.hby.db.qnfs.getItemIter())), + ("partially_delegated_events", sum(1 for _ in self.hby.db.pdes.getItemIter())), + ("reply", sum(1 for _ in self.hby.db.rpes.getItemIter())), + ("failed_oobi", sum(1 for _ in self.hby.db.eoobi.getItemIter())), + ("group_partial_witness", sum(1 for _ in self.hby.db.gpwe.getItemIter())), + ("group_delegate", sum(1 for _ in self.hby.db.gdee.getItemIter())), + ("delegated_partial_witness", sum(1 for _ in self.hby.db.dpwe.getItemIter())), + ("group_partial_signed", sum(1 for _ in self.hby.db.gpse.getItemIter())), + ("exchange_partial_signed", sum(1 for _ in self.hby.db.epse.getItemIter())), + ("delegated_unanchored", sum(1 for _ in self.hby.db.dune.getItemIter())), + ] + + for name, count in escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="kel"}} {count}') + + # TEL / Reger escrows + tel_escrow_counts = [ + ("out_of_order", sum(1 for _ in self.reger.getOotItemIter())), + ("partially_witnessed", sum(1 for _ in self.reger.getTweItemIter())), + ("anchorless", sum(1 for _ in self.reger.getTaeItemIter())), + ("missing_registry", sum(1 for _ in self.reger.mre.getItemIter())), + ("broken_chain", sum(1 for _ in self.reger.mce.getItemIter())), + ("missing_schema", sum(1 for _ in self.reger.mse.getItemIter())), + ("missing_signature", sum(1 for _ in self.reger.cmse.getItemIter())), + ("partial_witness", sum(1 for _ in self.reger.tpwe.getItemIter())), + ("multisig", sum(1 for _ in self.reger.tmse.getItemIter())), + ("event_dissemination", sum(1 for _ in self.reger.tede.getItemIter())), + ] + + for name, count in tel_escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="tel"}} {count}') + + # Registry transaction escrows + registry_escrow_counts = [ + ("registry_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-mae", "")))), + ("registry_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-ooo", "")))), + ("credential_missing_registry", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mre", "")))), + ("credential_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mae", "")))), + ("credential_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-ooo", "")))), + ] + + for name, count in registry_escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="registry"}} {count}') + + rep.content_type = "text/plain; version=0.0.4; charset=utf-8" + rep.status = falcon.HTTP_200 + rep.text = "\n".join(lines) + "\n" \ No newline at end of file diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index 43815925c..23f3ab2a9 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -18,6 +18,7 @@ from keri import kering from keri import core from keri.app import indirecting, storing, habbing, agenting +from keri.vdr import viring def test_mailbox_iter(): @@ -291,9 +292,49 @@ def test_createHttpServer(monkeypatch): assert isinstance(server.servant, MockServerTls) +def test_metrics_end(): + """Test MetricsEnd returns Prometheus format metrics""" + with habbing.openHby(name="test", salt=core.Salter(raw=b'0123456789abcdef').qb64, temp=True) as hby: + reger = viring.Reger(name=hby.name, db=hby.db, temp=True) + + app = falcon.App() + metricsEnd = indirecting.MetricsEnd(hby=hby, reger=reger) + app.add_route("/metrics", metricsEnd) + + client = testing.TestClient(app) + + # Test GET /metrics + res = client.simulate_get("/metrics") + assert res.status_code == 200 + assert "text/plain" in res.headers['Content-Type'] + + # Verify Prometheus format + body = res.text + assert "# HELP keri_escrow_count" in body + assert "# TYPE keri_escrow_count gauge" in body + + # Verify KEL escrow metrics present + assert 'keri_escrow_count{type="out_of_order_events",layer="kel"}' in body + assert 'keri_escrow_count{type="partially_witnessed_events",layer="kel"}' in body + assert 'keri_escrow_count{type="unverified_receipts",layer="kel"}' in body + + # Verify TEL escrow metrics present + assert 'keri_escrow_count{type="out_of_order",layer="tel"}' in body + assert 'keri_escrow_count{type="missing_registry",layer="tel"}' in body + + # Verify registry escrow metrics present + assert 'keri_escrow_count{type="registry_missing_anchor",layer="registry"}' in body + + # All counts should be 0 for empty db + lines = [l for l in body.split('\n') if l and not l.startswith('#')] + for line in lines: + assert line.endswith(' 0'), f"Expected count 0, got: {line}" + + reger.close() if __name__ == "__main__": test_mailbox_iter() test_qrymailbox_iter() test_wit_query_ends() + test_metrics_end() From 9c617aebb71605f8a0d422998180497d1328afca Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Tue, 2 Dec 2025 13:10:07 -0500 Subject: [PATCH 396/418] repackaging Signed-off-by: Kevin Griffin --- src/keri/app/indirecting.py | 92 ++------------------------------- src/keri/metric/__init__.py | 7 +++ src/keri/metric/metricing.py | 95 +++++++++++++++++++++++++++++++++++ tests/app/test_indirecting.py | 2 +- 4 files changed, 106 insertions(+), 90 deletions(-) create mode 100644 src/keri/metric/__init__.py create mode 100644 src/keri/metric/metricing.py diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 17caa8d4a..461c93b4e 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -31,6 +31,7 @@ from ..peer import exchanging from ..vdr import verifying, viring from ..vdr.eventing import Tevery +from ..metric import EscrowEnd logger = help.ogler.getLogger() @@ -90,7 +91,7 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo app.add_route("/receipts", receiptEnd) queryEnd = QueryEnd(hab=hab) app.add_route("/query", queryEnd) - metricsEnd = MetricsEnd(hby=hby, reger=reger) + metricsEnd = EscrowEnd(hby=hby, reger=reger) app.add_route("/metrics", metricsEnd) server = createHttpServer(host, httpPort, app, keypath, certpath, cafilepath) @@ -1285,91 +1286,4 @@ def on_get(self, req, rep): else: rep.set_header('Content-Type', "application/json") rep.text = "unkown query type." - rep.status = falcon.HTTP_400 - - -class MetricsEnd: - """Prometheus metrics endpoint for escrow counts. - - Exposes escrow counts in Prometheus text format for monitoring/alerting. - """ - - def __init__(self, hby, reger): - """Initialize MetricsEnd. - - Parameters: - hby (Habery): Habery instance for accessing KEL escrows - reger (Reger): Registry for accessing TEL escrows - """ - self.hby = hby - self.reger = reger - - def on_get(self, req, rep): - """GET /metrics - Returns Prometheus format metrics. - - Parameters: - req (Request): Falcon HTTP request object - rep (Response): Falcon HTTP response object - """ - lines = [] - - # Header - lines.append("# HELP keri_escrow_count Number of items in each escrow type") - lines.append("# TYPE keri_escrow_count gauge") - - # KEL / Baser escrows - escrow_counts = [ - ("unverified_receipts", sum(1 for _ in self.hby.db.getUreItemIter())), - ("verified_receipts", sum(1 for _ in self.hby.db.getVreItemIter())), - ("out_of_order_events", sum(1 for _ in self.hby.db.getOoeItemIter())), - ("partially_witnessed_events", sum(1 for _ in self.hby.db.getPweItemIter())), - ("partially_signed_events", sum(1 for _ in self.hby.db.getPseItemIter())), - ("likely_duplicitous_events", sum(1 for _ in self.hby.db.getLdeItemIter())), - ("unverified_event_indexed_couples", sum(1 for _ in self.hby.db.getUweItemIter())), - ("query_not_found", sum(1 for _ in self.hby.db.qnfs.getItemIter())), - ("partially_delegated_events", sum(1 for _ in self.hby.db.pdes.getItemIter())), - ("reply", sum(1 for _ in self.hby.db.rpes.getItemIter())), - ("failed_oobi", sum(1 for _ in self.hby.db.eoobi.getItemIter())), - ("group_partial_witness", sum(1 for _ in self.hby.db.gpwe.getItemIter())), - ("group_delegate", sum(1 for _ in self.hby.db.gdee.getItemIter())), - ("delegated_partial_witness", sum(1 for _ in self.hby.db.dpwe.getItemIter())), - ("group_partial_signed", sum(1 for _ in self.hby.db.gpse.getItemIter())), - ("exchange_partial_signed", sum(1 for _ in self.hby.db.epse.getItemIter())), - ("delegated_unanchored", sum(1 for _ in self.hby.db.dune.getItemIter())), - ] - - for name, count in escrow_counts: - lines.append(f'keri_escrow_count{{type="{name}",layer="kel"}} {count}') - - # TEL / Reger escrows - tel_escrow_counts = [ - ("out_of_order", sum(1 for _ in self.reger.getOotItemIter())), - ("partially_witnessed", sum(1 for _ in self.reger.getTweItemIter())), - ("anchorless", sum(1 for _ in self.reger.getTaeItemIter())), - ("missing_registry", sum(1 for _ in self.reger.mre.getItemIter())), - ("broken_chain", sum(1 for _ in self.reger.mce.getItemIter())), - ("missing_schema", sum(1 for _ in self.reger.mse.getItemIter())), - ("missing_signature", sum(1 for _ in self.reger.cmse.getItemIter())), - ("partial_witness", sum(1 for _ in self.reger.tpwe.getItemIter())), - ("multisig", sum(1 for _ in self.reger.tmse.getItemIter())), - ("event_dissemination", sum(1 for _ in self.reger.tede.getItemIter())), - ] - - for name, count in tel_escrow_counts: - lines.append(f'keri_escrow_count{{type="{name}",layer="tel"}} {count}') - - # Registry transaction escrows - registry_escrow_counts = [ - ("registry_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-mae", "")))), - ("registry_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-ooo", "")))), - ("credential_missing_registry", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mre", "")))), - ("credential_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mae", "")))), - ("credential_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-ooo", "")))), - ] - - for name, count in registry_escrow_counts: - lines.append(f'keri_escrow_count{{type="{name}",layer="registry"}} {count}') - - rep.content_type = "text/plain; version=0.0.4; charset=utf-8" - rep.status = falcon.HTTP_200 - rep.text = "\n".join(lines) + "\n" \ No newline at end of file + rep.status = falcon.HTTP_400 \ No newline at end of file diff --git a/src/keri/metric/__init__.py b/src/keri/metric/__init__.py new file mode 100644 index 000000000..bf9cd378c --- /dev/null +++ b/src/keri/metric/__init__.py @@ -0,0 +1,7 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.metric Package +""" + +from .metricing import EscrowEnd diff --git a/src/keri/metric/metricing.py b/src/keri/metric/metricing.py new file mode 100644 index 000000000..e4f76bd7c --- /dev/null +++ b/src/keri/metric/metricing.py @@ -0,0 +1,95 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.metric.metricing module + +Prometheus metrics endpoints for KERI escrow monitoring +""" +import falcon + + +class EscrowEnd: + """Prometheus metrics endpoint for escrow counts. + + Exposes escrow counts in Prometheus text format for monitoring/alerting. + """ + + def __init__(self, hby, reger): + """Initialize EscrowEnd. + + Parameters: + hby (Habery): Habery instance for accessing KEL escrows + reger (Reger): Registry for accessing TEL escrows + """ + self.hby = hby + self.reger = reger + + def on_get(self, _, rep): + """GET /metrics - Returns Prometheus format metrics. + + Parameters: + _ (Request): Falcon HTTP request object (unused) + rep (Response): Falcon HTTP response object + """ + lines = [] + + # Header + lines.append("# HELP keri_escrow_count Number of items in each escrow type") + lines.append("# TYPE keri_escrow_count gauge") + + # KEL / Baser escrows + escrow_counts = [ + ("unverified_receipts", sum(1 for _ in self.hby.db.getUreItemIter())), + ("verified_receipts", sum(1 for _ in self.hby.db.getVreItemIter())), + ("out_of_order_events", sum(1 for _ in self.hby.db.getOoeItemIter())), + ("partially_witnessed_events", sum(1 for _ in self.hby.db.getPweItemIter())), + ("partially_signed_events", sum(1 for _ in self.hby.db.getPseItemIter())), + ("likely_duplicitous_events", sum(1 for _ in self.hby.db.getLdeItemIter())), + ("unverified_event_indexed_couples", sum(1 for _ in self.hby.db.getUweItemIter())), + ("query_not_found", sum(1 for _ in self.hby.db.qnfs.getItemIter())), + ("partially_delegated_events", sum(1 for _ in self.hby.db.pdes.getItemIter())), + ("reply", sum(1 for _ in self.hby.db.rpes.getItemIter())), + ("failed_oobi", sum(1 for _ in self.hby.db.eoobi.getItemIter())), + ("group_partial_witness", sum(1 for _ in self.hby.db.gpwe.getItemIter())), + ("group_delegate", sum(1 for _ in self.hby.db.gdee.getItemIter())), + ("delegated_partial_witness", sum(1 for _ in self.hby.db.dpwe.getItemIter())), + ("group_partial_signed", sum(1 for _ in self.hby.db.gpse.getItemIter())), + ("exchange_partial_signed", sum(1 for _ in self.hby.db.epse.getItemIter())), + ("delegated_unanchored", sum(1 for _ in self.hby.db.dune.getItemIter())), + ] + + for name, count in escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="kel"}} {count}') + + # TEL / Reger escrows + tel_escrow_counts = [ + ("out_of_order", sum(1 for _ in self.reger.getOotItemIter())), + ("partially_witnessed", sum(1 for _ in self.reger.getTweItemIter())), + ("anchorless", sum(1 for _ in self.reger.getTaeItemIter())), + ("missing_registry", sum(1 for _ in self.reger.mre.getItemIter())), + ("broken_chain", sum(1 for _ in self.reger.mce.getItemIter())), + ("missing_schema", sum(1 for _ in self.reger.mse.getItemIter())), + ("missing_signature", sum(1 for _ in self.reger.cmse.getItemIter())), + ("partial_witness", sum(1 for _ in self.reger.tpwe.getItemIter())), + ("multisig", sum(1 for _ in self.reger.tmse.getItemIter())), + ("event_dissemination", sum(1 for _ in self.reger.tede.getItemIter())), + ] + + for name, count in tel_escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="tel"}} {count}') + + # Registry transaction escrows + registry_escrow_counts = [ + ("registry_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-mae", "")))), + ("registry_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-ooo", "")))), + ("credential_missing_registry", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mre", "")))), + ("credential_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mae", "")))), + ("credential_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-ooo", "")))), + ] + + for name, count in registry_escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="registry"}} {count}') + + rep.content_type = "text/plain; version=0.0.4; charset=utf-8" + rep.status = falcon.HTTP_200 + rep.text = "\n".join(lines) + "\n" diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index 23f3ab2a9..aaa582156 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -298,7 +298,7 @@ def test_metrics_end(): reger = viring.Reger(name=hby.name, db=hby.db, temp=True) app = falcon.App() - metricsEnd = indirecting.MetricsEnd(hby=hby, reger=reger) + metricsEnd = indirecting.EscrowEnd(hby=hby, reger=reger) app.add_route("/metrics", metricsEnd) client = testing.TestClient(app) From 41b19ee9589076d1d19643e09354b134c698cd22 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Tue, 2 Dec 2025 13:46:49 -0500 Subject: [PATCH 397/418] repackaging Signed-off-by: Kevin Griffin --- tests/app/test_indirecting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index aaa582156..52160e650 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -18,6 +18,7 @@ from keri import kering from keri import core from keri.app import indirecting, storing, habbing, agenting +from keri.metric import EscrowEnd from keri.vdr import viring @@ -298,7 +299,7 @@ def test_metrics_end(): reger = viring.Reger(name=hby.name, db=hby.db, temp=True) app = falcon.App() - metricsEnd = indirecting.EscrowEnd(hby=hby, reger=reger) + metricsEnd = EscrowEnd(hby=hby, reger=reger) app.add_route("/metrics", metricsEnd) client = testing.TestClient(app) From aecb49e91d5093583d1c19e7f3d7aecc98b6d30d Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Wed, 26 Nov 2025 14:44:50 -0500 Subject: [PATCH 398/418] adds escrow clear Signed-off-by: Kevin Griffin Signed-off-by: Kent Bull --- src/keri/app/cli/commands/escrow/clear.py | 1 + src/keri/app/cli/commands/escrow/list.py | 84 +++++++++++++++++++++-- src/keri/vdr/viring.py | 9 +++ tests/app/cli/test_kli_commands.py | 48 ++++++++++--- tests/app/test_agenting.py | 2 +- tests/vdr/test_viring.py | 52 +++++++++++++- 6 files changed, 181 insertions(+), 15 deletions(-) diff --git a/src/keri/app/cli/commands/escrow/clear.py b/src/keri/app/cli/commands/escrow/clear.py index b3720a24a..dbf08e986 100644 --- a/src/keri/app/cli/commands/escrow/clear.py +++ b/src/keri/app/cli/commands/escrow/clear.py @@ -48,6 +48,7 @@ def clear(tymth, tock=0.0, **opts): name = args.name base = args.base bran = args.bran + logger.setLevel("INFO") with existing.existingHby(name=name, base=base, bran=bran) as hby: hby.db.clearEscrows() diff --git a/src/keri/app/cli/commands/escrow/list.py b/src/keri/app/cli/commands/escrow/list.py index 418a82e62..829d59c02 100644 --- a/src/keri/app/cli/commands/escrow/list.py +++ b/src/keri/app/cli/commands/escrow/list.py @@ -52,6 +52,15 @@ def escrows(tymth, tock=0.0, **opts): reger = viring.Reger(name=hby.name, db=hby.db, temp=False) escrows = dict() + + # KEL / Baser escrows - counts only + + if (not escrow) or escrow == "unverified-receipts": + escrows["unverified-receipts"] = sum(1 for key, _ in hby.db.getUreItemIter()) + + if (not escrow) or escrow == "verified-receipts": + escrows["verified-receipts"] = sum(1 for key, _ in hby.db.getVreItemIter()) + if (not escrow) or escrow == "out-of-order-events": oots = list() key = ekey = b'' # both start same. when not same means escrows found @@ -124,6 +133,50 @@ def escrows(tymth, tock=0.0, **opts): escrows["likely-duplicitous-events"] = ldes + if (not escrow) or escrow == "unverified-event-indexed-couples": + escrows["unverified-event-indexed-couples"] = sum(1 for key, _ in hby.db.getUweItemIter()) + + if (not escrow) or escrow == "query-not-found": + escrows["query-not-found"] = sum(1 for key, _ in hby.db.qnfs.getItemIter()) + + if (not escrow) or escrow == "partially-delegated-events": + escrows["partially-delegated-events"] = sum(1 for key, _ in hby.db.pdes.getItemIter()) + + if (not escrow) or escrow == "reply": + escrows["reply"] = sum(1 for key, _ in hby.db.rpes.getItemIter()) + + if (not escrow) or escrow == "failed-oobi": + escrows["failed-oobi"] = sum(1 for key, _ in hby.db.eoobi.getItemIter()) + + if (not escrow) or escrow == "group-partial-witness": + escrows["group-partial-witness"] = sum(1 for key, _ in hby.db.gpwe.getItemIter()) + + if (not escrow) or escrow == "group-delegate": + escrows["group-delegate"] = sum(1 for key, _ in hby.db.gdee.getItemIter()) + + if (not escrow) or escrow == "delegated-partial-witness": + escrows["delegated-partial-witness"] = sum(1 for key, _ in hby.db.dpwe.getItemIter()) + + if (not escrow) or escrow == "group-partial-signed": + escrows["group-partial-signed"] = sum(1 for key, _ in hby.db.gpse.getItemIter()) + + if (not escrow) or escrow == "exchange-partial-signed": + escrows["exchange-partial-signed"] = sum(1 for key, _ in hby.db.epse.getItemIter()) + + if (not escrow) or escrow == "delegated-unanchored": + escrows["delegated-unanchored"] = sum(1 for key, _ in hby.db.dune.getItemIter()) + + # TEL / Reger escrows + + if (not escrow) or escrow == "tel-out-of-order": + escrows["tel-out-of-order"] = sum(1 for key, _ in reger.getOotItemIter()) + + if (not escrow) or escrow == "tel-partially-witnessed": + escrows["tel-partially-witnessed"] = sum(1 for key, _ in reger.getTweItemIter()) + + if (not escrow) or escrow == "tel-anchorless": + escrows["tel-anchorless"] = sum(1 for key, _ in reger.getTaeItemIter()) + if (not escrow) or escrow == "missing-registry-escrow": creds = list() for (said,), dater in reger.mre.getItemIter(): @@ -148,11 +201,34 @@ def escrows(tymth, tock=0.0, **opts): escrows["missing-schema-escrow"] = creds - print(json.dumps(escrows, indent=2)) + if (not escrow) or escrow == "tel-missing-signature": + escrows["tel-missing-signature"] = sum(1 for key, _ in reger.cmse.getItemIter()) + + if (not escrow) or escrow == "tel-partial-witness-escrow": + escrows["tel-partial-witness-escrow"] = sum(1 for key, _ in reger.tpwe.getItemIter()) + + if (not escrow) or escrow == "tel-multisig": + escrows["tel-multisig"] = sum(1 for key, _ in reger.tmse.getItemIter()) - if not(escrow) or escrow == 'tel-partial-witness-escrow': - for (regk, snq), (prefixer, seqner, saider) in reger.tpwe.getItemIter(): - pass + if (not escrow) or escrow == "tel-event-dissemination": + escrows["tel-event-dissemination"] = sum(1 for key, _ in reger.tede.getItemIter()) + + if (not escrow) or escrow == "registry-missing-anchor": + escrows["registry-missing-anchor"] = sum(1 for key, _ in reger.txnsb.escrowdb.getItemIter(keys=("registry-mae", ""))) + + if (not escrow) or escrow == "registry-out-of-order": + escrows["registry-out-of-order"] = sum(1 for key, _ in reger.txnsb.escrowdb.getItemIter(keys=("registry-ooo", ""))) + + if (not escrow) or escrow == "credential-missing-registry": + escrows["credential-missing-registry"] = sum(1 for key, _ in reger.txnsb.escrowdb.getItemIter(keys=("credential-mre", ""))) + + if (not escrow) or escrow == "credential-missing-anchor": + escrows["credential-missing-anchor"] = sum(1 for key, _ in reger.txnsb.escrowdb.getItemIter(keys=("credential-mae", ""))) + + if (not escrow) or escrow == "credential-out-of-order": + escrows["credential-out-of-order"] = sum(1 for key, _ in reger.txnsb.escrowdb.getItemIter(keys=("credential-ooo", ""))) + + print(json.dumps(escrows, indent=2)) except ConfigurationError as e: print(f"identifier prefix for {name} does not exist, incept must be run first", ) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 886af906a..6e8415be2 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -1021,6 +1021,15 @@ def clearEscrows(self): ]: sub.trim() logger.info(f"TEL: Cleared escrow ({name.ljust(5)}): {desc}") + + for typ in ["registry-mae", "registry-ooo", "credential-mre", "credential-mae", "credential-ooo"]: + count = 0 + for keys, saider in self.txnsb.escrowdb.getItemIter(keys=(typ, "")): + count += 1 + self.txnsb.escrowdb.rem(keys=keys, val=saider) + self.txnsb.removeState(saider) + logger.info(f"TEL: Cleared {count} broker escrows ({typ})") + logger.info("Cleared TEL escrows") diff --git a/tests/app/cli/test_kli_commands.py b/tests/app/cli/test_kli_commands.py index ec30511df..e473ae2ad 100644 --- a/tests/app/cli/test_kli_commands.py +++ b/tests/app/cli/test_kli_commands.py @@ -1,3 +1,4 @@ +import json import os import multicommand @@ -244,15 +245,44 @@ def test_standalone_kli_commands(helpers, capsys): doers = args.handler(args) directing.runController(doers=doers) capesc = capsys.readouterr() - assert capesc.out == ('{\n' - ' "out-of-order-events": [],\n' - ' "partially-witnessed-events": [],\n' - ' "partially-signed-events": [],\n' - ' "likely-duplicitous-events": [],\n' - ' "missing-registry-escrow": [],\n' - ' "broken-chain-escrow": [],\n' - ' "missing-schema-escrow": []\n' - '}\n') + escrows = ("""{ + "unverified-receipts": 0, + "verified-receipts": 0, + "out-of-order-events": [], + "partially-witnessed-events": [], + "partially-signed-events": [], + "likely-duplicitous-events": [], + "unverified-event-indexed-couples": 0, + "query-not-found": 0, + "partially-delegated-events": 0, + "reply": 0, + "failed-oobi": 0, + "group-partial-witness": 0, + "group-delegate": 0, + "delegated-partial-witness": 0, + "group-partial-signed": 0, + "exchange-partial-signed": 0, + "delegated-unanchored": 0, + "tel-out-of-order": 0, + "tel-partially-witnessed": 0, + "tel-anchorless": 0, + "missing-registry-escrow": [], + "broken-chain-escrow": [], + "missing-schema-escrow": [], + "tel-missing-signature": 0, + "tel-partial-witness-escrow": 0, + "tel-multisig": 0, + "tel-event-dissemination": 0, + "registry-missing-anchor": 0, + "registry-out-of-order": 0, + "credential-missing-registry": 0, + "credential-missing-anchor": 0, + "credential-out-of-order": 0 + } + """) + assert json.loads(capesc.out) == json.loads(escrows) + + def test_incept_and_rotate_opts(helpers, capsys): diff --git a/tests/app/test_agenting.py b/tests/app/test_agenting.py index f71218084..3b0075557 100644 --- a/tests/app/test_agenting.py +++ b/tests/app/test_agenting.py @@ -16,7 +16,7 @@ from keri.vdr import eventing, viring -def test_withness_receiptor(seeder): +def test_witness_receiptor(seeder): with habbing.openHby(name="wan", salt=core.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ habbing.openHby(name="wil", salt=core.Salter(raw=b'will-the-witness').qb64) as wilHby, \ habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ diff --git a/tests/vdr/test_viring.py b/tests/vdr/test_viring.py index e69682110..416d17e80 100644 --- a/tests/vdr/test_viring.py +++ b/tests/vdr/test_viring.py @@ -8,12 +8,13 @@ import os import lmdb +import pytest from keri import kering from keri.core import coring from keri.core.coring import Diger, versify, Kinds from keri.core.serdering import SerderACDC -from keri.db.dbing import openLMDB, dgKey, snKey +from keri.db.dbing import openLMDB, dgKey, snKey, KERIBaserMapSizeKey, KERIRegerMapSizeKey, LMDBer from keri.vdr.viring import Reger @@ -377,6 +378,13 @@ def test_clearEscrows(): db.tmse.add(("a", "b"), (prefixer, seqner, saider)) db.tede.add(("a", "b"), (prefixer, seqner, saider)) + dater = coring.Dater() + broker_saider = coring.Saider(qb64="EBD919wF4oiG7ck6mnBWTRD_Z-Io0wZKCxL0zjx5je9I") + aid = "ECD919wF4oiG7ck6mnBWTRD_Z-Io0wZKCxL0zjx5je9I" + for typ in ["registry-mae", "registry-ooo", "credential-mre", "credential-mae", "credential-ooo"]: + db.txnsb.escrowdb.put(keys=(typ, pre, aid), vals=[broker_saider]) + db.txnsb.daterdb.put(keys=(broker_saider.qb64,), val=dater) + assert db.getOot(ooKey) == vdig.qb64b assert db.getTwe(ooKey) == vdig.qb64b assert db.getTae(ooKey) == vdig.qb64b @@ -388,6 +396,9 @@ def test_clearEscrows(): assert db.tmse.cntAll() == 1 assert db.tede.cntAll() == 1 + for typ in ["registry-mae", "registry-ooo", "credential-mre", "credential-mae", "credential-ooo"]: + assert db.txnsb.escrowdb.get(keys=(typ, pre, aid)) != [] + db.clearEscrows() assert db.getOot(ooKey) is None @@ -401,6 +412,45 @@ def test_clearEscrows(): assert db.tmse.cntAll() == 0 assert db.tede.cntAll() == 0 + # Verify Broker escrows were cleared + for typ in ["registry-mae", "registry-ooo", "credential-mre", "credential-mae", "credential-ooo"]: + assert db.txnsb.escrowdb.get(keys=(typ, pre, aid)) == [] + assert db.txnsb.daterdb.get(keys=(broker_saider.qb64,)) is None + +def test_mailbox_db_size_set_from_env_var(): + # Clear environment before test + if KERIBaserMapSizeKey in os.environ: + os.environ.pop(KERIBaserMapSizeKey) + if KERIRegerMapSizeKey in os.environ: + os.environ.pop(KERIRegerMapSizeKey) + + new_map_size = 10737418240 + # Default map size works + reger = Reger() + assert reger.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" + assert reger.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + + # Specific map size works + os.environ[KERIRegerMapSizeKey] = f"{new_map_size}" + + reger = Reger() + assert reger.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + os.environ.pop(KERIRegerMapSizeKey) + + # generic map size works + baser_map_size = 10737418240 + os.environ[KERIRegerMapSizeKey] = f"{baser_map_size}" + + reger = Reger() + assert reger.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + + # Bad map size throws + os.environ[KERIRegerMapSizeKey] = f"bad_map_size" + with pytest.raises(ValueError) as excinfo: + Reger() + assert "invalid literal for int" in str(excinfo.value), "Expected ValueError when map size is not an integer" + os.environ.pop(KERIRegerMapSizeKey) + if __name__ == "__main__": test_issuer() test_clone() From 97e87af22afae19ca8570c0086d1913631cc79ad Mon Sep 17 00:00:00 2001 From: Esteban Garcia Date: Mon, 15 Sep 2025 15:23:12 -0600 Subject: [PATCH 399/418] feat: get aid allowlists from witness config file --- src/keri/app/cli/commands/witness/start.py | 3 + tests/app/test_indirecting.py | 72 ++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/src/keri/app/cli/commands/witness/start.py b/src/keri/app/cli/commands/witness/start.py index 8e47df39a..9e696e69e 100644 --- a/src/keri/app/cli/commands/witness/start.py +++ b/src/keri/app/cli/commands/witness/start.py @@ -95,6 +95,8 @@ def runWitness(name="witness", base="", alias="witness", bran="", tcp=5631, http cf = None if configFile is not None: cf = configing.Configer(name=configFile, headDirPath=configDir, temp=False, reopen=True, clear=False) + + aids = cf.get("aids", default=None) if cf is not None else None if aeid is None: hby = habbing.Habery(name=name, base=base, bran=bran, cf=cf) @@ -106,6 +108,7 @@ def runWitness(name="witness", base="", alias="witness", bran="", tcp=5631, http doers.extend(indirecting.setupWitness(alias=alias, hby=hby, + aids=aids, tcpPort=tcp, httpPort=http, keypath=keypath, diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index 52160e650..c70fa18ca 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -265,7 +265,79 @@ def wit_querier_test_do(tymth=None, tock=0.0, **opts): assert res.headers['Content-Type'] == "application/json" assert "unkown query type" in res.text +def test_wit_allowlist(seeder): + with habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby: + + wesHab = wesHby.makeHab(name="wes", transferable=False) + + # Allowed (allowlisted) participant hab + palAllowed = palHby.makeHab(name="palAllowed", wits=[wesHab.pre], transferable=True) + # Denied (not allowlisted) participant hab + palDenied = palHby.makeHab(name="palDenied", wits=[wesHab.pre], transferable=True) + + aids = [palAllowed.pre] + + wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, aids=aids, tcpPort=5634, httpPort=5644) + witDoer = agenting.Receiptor(hby=palHby) + + seeder.seedWitEnds(palHby.db, witHabs=[wesHab], protocols=[kering.Schemes.http]) + + opts = dict( + wesHab=wesHab, + palHby=palHby, + witDoer=witDoer, + palAllowed=palAllowed, + palDenied=palDenied, + aids=aids, + ) + testDo = AllowlistTestDoer(**opts) + doers = wesDoers + [witDoer, testDo] + + limit = 1.0 + tock = 0.03125 + doist = doing.Doist(tock=tock, limit=limit, doers=doers) + doist.enter() + + while not testDo.done: + doist.recur() + time.sleep(doist.tock) + + assert doist.limit == limit + doist.exit() + +class AllowlistTestDoer(doing.Doer): + def __init__(self, **opts): + self.options = opts + super(AllowlistTestDoer, self).__init__(**opts) + + def recur(self, tyme=0.0, deeds=None, **kwa): + wesHab = self.options["wesHab"] + witDoer = self.options["witDoer"] + palAllowed = self.options["palAllowed"] + palDenied = self.options["palDenied"] + + # Trigger witness receipting for both allowed and denied AIDs + if not any(msg.get('pre') == palAllowed.pre for msg in witDoer.msgs): + witDoer.msgs.append(dict(pre=palAllowed.pre)) + if not any(msg.get('pre') == palDenied.pre for msg in witDoer.msgs): + witDoer.msgs.append(dict(pre=palDenied.pre)) + + # Wait until both have been processed (one cue each) + while len(witDoer.cues) < 2: + yield self.tock + + # Allowed pre should have at least one event stored on witness + allowed_iter = wesHab.db.clonePreIter(pre=palAllowed.pre) + allowed_msg = next(allowed_iter) # should NOT raise + assert isinstance(allowed_msg, (bytes, bytearray)) and len(allowed_msg) > 0 + + # Denied pre should have NO events (StopIteration immediately) + denied_iter = wesHab.db.clonePreIter(pre=palDenied.pre) + with pytest.raises(StopIteration): + next(denied_iter) + return True class MockServerTls: def __init__(self, certify, keypath, certpath, cafilepath, port): From 0dd3e33dd292afc6f3d6c7c07ba48398ee421363 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Thu, 24 Apr 2025 18:51:49 -0600 Subject: [PATCH 400/418] fix: match dig or edig type to db function return type --- src/keri/core/eventing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 343a32137..14635dd75 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5971,10 +5971,10 @@ def processEscrowPartialDels(self): ## so just log for debugging but do not unescrow by raising ## ValidationError #logger.info("Kevery unescrow error: Missing event wigs at." - #"dig = %s", bytes(edig)) + #"dig = %s", edig) #raise ValidationError("Missing escrowed evt wigs at " - #"dig = {}.".format(bytes(edig))) + #"dig = {}.".format(edig)) # setup parameters to process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] From cf6390fb101b81fd95258822cc04f00dca91e336 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Mon, 28 Apr 2025 20:32:07 -0600 Subject: [PATCH 401/418] test: adds tests for OOO, PSE, PWE, and PDEL escrows --- tests/core/test_delegating.py | 8 +- tests/core/test_escrow.py | 661 +++++++++++++++++++++++++++++++++- 2 files changed, 661 insertions(+), 8 deletions(-) diff --git a/tests/core/test_delegating.py b/tests/core/test_delegating.py index 2f183f3e2..269b54f7a 100644 --- a/tests/core/test_delegating.py +++ b/tests/core/test_delegating.py @@ -3,6 +3,7 @@ tests delegation primaily from keri.core.eventing """ +import logging import os from keri import help @@ -288,8 +289,8 @@ def test_delegation_supersede(): Three level delegation top is at top or root level with witness wop. top is not delegated - mid is at mid level with witness wid. mid is delegatred from top - bot is at bottom level with wintess wot. bot is delegated from mid + mid is at mid-level with witness wid. mid is delegated from top + bot is at bottom level with witness wot. bot is delegated from mid def test_load_event(mockHelpingNowUTC): @@ -464,6 +465,7 @@ def test_load_event(mockHelpingNowUTC): assert count >= 1 # This needs to be fixedup to actually test delegating superseding recovery + # TODO? with (basing.openDB(name="bob") as bobDB, keeping.openKS(name="bob") as bobKS, basing.openDB(name="del") as delDB, @@ -722,6 +724,7 @@ def test_load_event(mockHelpingNowUTC): def test_delegables_escrow(): gateSalt = core.Salter(raw=b'0123456789abcdef').qb64 torSalt = core.Salter(raw=b'0123456789defabc').raw + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks with habbing.openHby(name="delegate", temp=True, salt=gateSalt) as gateHby, \ habbing.openHab(name="delegator", temp=True, salt=torSalt) as (torHby, torHab): @@ -773,4 +776,5 @@ def test_delegables_escrow(): if __name__ == "__main__": test_delegation() test_delegation_supersede() + test_delegables_escrow() diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index b6d9b4dfb..3e8346385 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -3,19 +3,22 @@ tests escrows in database primarily logic in Kevery and Kever from keri.core.eventing """ +import logging import os import time import datetime +import pytest + from keri import help +from keri.db.dbing import splitSnKey, dgKey, snKey from keri.help import helping from keri import core, kering from keri.core import coring, eventing, parsing from keri.db import dbing, basing -from keri.app import keeping - +from keri.app import keeping, habbing logger = help.ogler.getLogger() @@ -27,6 +30,7 @@ def test_partial_signed_escrow(): """ salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks # init event DB and keep DB with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: @@ -349,11 +353,107 @@ def test_partial_signed_escrow(): """End Test""" +def test_partial_signed_escrow_validation_errors(): + """ + Test partially signed escrow validation errors which each should remove the invalid escrowed events + and not continue to reprocess them. + """ + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + # init event DB and keep DB + with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + + # Identifier setup - 3 signers so we can easily trigger the partially signed state + # create inception event with 3 keys each in incept and next sets + # defaults are algo salty and rooted + sith = ["1/2", "1/2", "1/2"] # 2 of 3 but with weighted threshold + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='wes', temp=True) + srdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + isith=sith, + nsith=nxtsith, + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + pre = srdr.ked["i"] + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + + # Create CESR message byte stream with attachments + msg = bytearray(srdr.raw) # message + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # attachments + msg.extend(sigers[0].qb64b) # only add the first signer to trigger partially signed logic + + # apply msg to Kevery to process + psr.parse(ims=bytearray(msg), kvy=kvy) + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 1 + assert escrows[0] == srdr.saidb # escrow entry for event + + # TEST MISSING EVENT SOURCE + for ekey, edig in db.getPseItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + esr = db.esrs.get(keys=dgkey) + assert esr is not None + db.esrs.rem(keys=dgkey) + kvy.processEscrowPartialSigs() + escrows = db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows)== 0 + + # TEST MISSING DATE TIME + psr.parse(ims=bytearray(msg), kvy=kvy) + escrows = kvy.db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 1 + for ekey, edig in db.getPseItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + dts = db.getDts(dgkey) + assert dts is not None + db.delDts(dgkey) + kvy.processEscrowPartialSigs() + escrows = db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 0 + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(msg), kvy=kvy) + escrows = kvy.db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 1 + for ekey, edig in db.getPseItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + evt = db.getEvt(dgkey) + assert evt is not None + db.delEvt(dgkey) + kvy.processEscrowPartialSigs() + escrows = db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parse(ims=bytearray(msg), kvy=kvy) + escrows = kvy.db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 1 + for ekey, edig in db.getPseItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + sigs = db.getSigs(dgkey) + assert sigs is not None + db.delSigs(dgkey) + kvy.processEscrowPartialSigs() + escrows = db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 0 + + def test_missing_delegator_escrow(): """ Test missing delegator escrow - bod is the delegator + bob is the delegator del is the delegate wat is the watcher """ @@ -368,8 +468,8 @@ def test_missing_delegator_escrow(): keeping.openKS(name="bob") as bobKS, basing.openDB(name="del") as delDB, keeping.openKS(name="del") as delKS, - basing.openDB(name="wat") as watDB, \ - keeping.openKS(name="wat") as watKS ): + basing.openDB(name="wat") as watDB, + keeping.openKS(name="wat") as watKS): # Init key pair managers bobMgr = keeping.Manager(ks=bobKS, salt=bobSalt) @@ -660,7 +760,6 @@ def test_misfit_escrow(): """End Test""" - def test_out_of_order_escrow(): """ Test out of order escrow @@ -668,6 +767,7 @@ def test_out_of_order_escrow(): """ salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks # init event DB and keep DB with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: @@ -857,6 +957,132 @@ def test_out_of_order_escrow(): """End Test""" +def test_out_of_order_escrow_validation_errors(): + """ + Test out-of-order escrow validation errors which each should remove the invalid escrowed events + and not continue to reprocess them. + """ + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + # init event DB and keep DB + with basing.openDB(name="dandy") as db, keeping.openKS(name="dandy") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + + # Identifier setup + sith = ["1/2", "1/2", "1/2"] + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='wes', temp=True) + + srdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=sith, nsith=nxtsith, + keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers]) + pre = srdr.ked["i"] + icpdig = srdr.said + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + + # Create CESR message byte stream with attachments + msg = bytearray(srdr.raw) # message + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # Attachments + for siger in sigers: + msg.extend(siger.qb64b) + icpmsg = bytearray(msg) # save copy for later + + # create interaction event + srdr = eventing.interact(pre=pre, dig=icpdig, sn=1, data=[]) + ixndig = srdr.said + + # Create CESR message byte stream with attachments + msg = bytearray(srdr.raw) # message + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # attachments + for siger in sigers: + msg.extend(siger.qb64b) + ixnmsg = bytearray(msg) # save copy for later + + # TEST MISSING EVENT SOURCE + psr.parse(ims=bytearray(ixnmsg), kvy=kvy) # apply ixn msg, adding to OOO escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + assert escrows[0] == ixndig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getOoeItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + esr = db.esrs.get(keys=dgkey) + assert esr is not None + db.esrs.rem(keys=dgkey) + kvy.processEscrowOutOfOrders() # will remove the now invalid escrowed interaction event + escrows = db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # TEST MISSING DATE TIME + psr.parse(ims=bytearray(ixnmsg), kvy=kvy) # re-apply ixn msg, adding to OOO escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + assert escrows[0] == ixndig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getOoeItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + dtb = db.getDts(dgkey) + assert dtb is not None + db.delDts(dgkey) # remove the datetime stamp of the event so it will be invalid + kvy.processEscrowOutOfOrders() # will remove the now invalid escrowed interaction event + escrows = db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # stale event escrow test covered by test_out_of_order_escrow above + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(ixnmsg), kvy=kvy) # re-apply ixn msg, adding to OOO escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + assert escrows[0] == ixndig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getOoeItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + eraw = db.getEvt(dgkey) + assert eraw is not None + db.delEvt(dgkey) # remove the datetime stamp of the event so it will be invalid + kvy.processEscrowOutOfOrders() # will remove the now invalid escrowed interaction event + escrows = db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parse(ims=bytearray(ixnmsg), kvy=kvy) # re-apply ixn msg, adding to OOO escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + assert escrows[0] == ixndig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getOoeItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + sigs = db.getSigs(dgkey) + assert sigs is not None + db.delSigs(dgkey) # remove the datetime stamp of the event so it will be invalid + kvy.processEscrowOutOfOrders() # will remove the now invalid escrowed interaction event + escrows = db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # test high level processEscrows exception handler + del db.sigs # intentionally breaks the Baser db instance so that an unhandled exception occurs + with pytest.raises(Exception): + kvy.processEscrows() # will raise exception caught by the Kevery.processEscrows catch all + # this is to increase test coverage + + def test_unverified_receipt_escrow(): """ Test unverified receipt escrow @@ -864,6 +1090,7 @@ def test_unverified_receipt_escrow(): """ salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks # init event DB and keep DB with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: @@ -1478,6 +1705,428 @@ def test_unverified_trans_receipt_escrow(): """End Test""" +def test_partial_wigs_validation_errors(): + """ + Test partial witness receipt escrow validation errors which each should remove the invalid escrowed events + and not continue to reprocess them. + """ + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter + psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + # init event DB and keep DB + with basing.openDB(name="fredy") as db, keeping.openKS(name="fredy") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + + # create witness identifiers + verfers, digers = mgr.incept(ncount=0, stem="wit0", transferable=False, temp=True) + wit0Verfer = verfers[0] + wit0pre = wit0Verfer.qb64 + + verfers, digers = mgr.incept(ncount=0, stem="wit1", transferable=False, temp=True) + wit1Verfer = verfers[0] + wit1pre = wit1Verfer.qb64 + + assert wit1pre != wit0pre + assert wit1pre < wit0pre # means wit1 escrow will get serviced first + + # create inception event with 3 keys each in incept and next sets + # defaults are algo salty and rooted + sith = ["1/2", "1/2", "1/2"] # 2 of 3 but with weighted threshold + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='fredy', temp=True) + + srdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + isith=sith, + nsith=nxtsith, + wits=[wit0pre, wit1pre], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + pre = srdr.ked["i"] + icpdig = srdr.said + + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + + # Create inception CESR message and attachments byte stream + msg = bytearray(srdr.raw) # message + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # attachments + for siger in sigers: + msg.extend(siger.qb64b) + + icpmsg = msg + + + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply icp msg, adding to PW escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + # First run of escrow processing should loop and not remove event + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed inception event + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + # TEST MISSING EVENT SOURCE + for ekey, edig in db.getPweItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + esr = db.esrs.get(keys=dgkey) + assert esr is not None + db.esrs.rem(keys=dgkey) # Removes event source + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed inception event + escrows = db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 0 + + # TEST MISSING DATE TIME + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply icp msg, adding to PW escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getPweItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + dtb = db.getDts(dgkey) + assert dtb is not None + db.delDts(dgkey) # remove the datetime stamp of the event so it will be invalid + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed inception event + escrows = db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 0 + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply icp msg, adding to PW escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getPweItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + eraw = db.getEvt(dgkey) + assert eraw is not None + db.delEvt(dgkey) # remove the datetime stamp of the event so it will be invalid + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed interaction event + escrows = db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply icp msg, adding to PW escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getPweItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + sigers = db.getSigs(dgkey) + assert sigers is not None + db.delSigs(dgkey) + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed interaction event + escrows = db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 0 + + # TEST STALE EVENT + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply icp msg, adding to PW escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + kvy.TimeoutPWE = 0 # forces all escrows to be stale + time.sleep(0.001) + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed inception event + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 0 + + +def test_partial_delegation_escrow_validation_errors(): + """ + Test partial delegation escrow validation errors which each should remove the invalid escrowed events + and not continue to reprocess them. + """ + bobSalt = core.Salter(raw=b'0123456789abcdef').qb64 + delSalt = core.Salter(raw=b'abcdef0123456789').qb64 + watSalt = core.Salter(raw=b'wxyzabcdefghijkl').qb64 + + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + psr = parsing.Parser() + + with (basing.openDB(name="bob") as bobDB, + keeping.openKS(name="bob") as bobKS, + basing.openDB(name="del") as delDB, + keeping.openKS(name="del") as delKS, + basing.openDB(name="wat") as watDB, + keeping.openKS(name="wat") as watKS): + + # Init key pair managers + bobMgr = keeping.Manager(ks=bobKS, salt=bobSalt) + delMgr = keeping.Manager(ks=delKS, salt=delSalt) + watMgr = keeping.Manager(ks=watKS, salt=watSalt) + + # Init Keverys + bobKvy = eventing.Kevery(db=bobDB) + delKvy = eventing.Kevery(db=delDB) + watKvy = eventing.Kevery(db=watDB) + + # Setup Wat with own inception event + verfers, digers = watMgr.incept(stem='wat', temp=True) # algo default salty and rooted + + watSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + watPre = watSrdr.pre + watMgr.move(old=verfers[0].qb64, new=watPre) # move key pair label to prefix + # Setup wat's prefixes so wat's KEL will be Kever.locallyOwned() + watDB.prefixes.add(watPre) + assert watPre in watDB.prefixes + # setup wat's on kel + sigers = watMgr.sign(ser=watSrdr.raw, verfers=verfers) + msg = bytearray(watSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + watIcpMsg = msg # save for later + + # apply msg to wats's Kevery + psr.parse(ims=bytearray(watIcpMsg), kvy=watKvy, local=True) + watK = watKvy.kevers[watPre] + assert watK.prefixer.qb64 == watPre + assert watK.serder.said == watSrdr.said + + # Setup Bob with own inception event + verfers, digers = bobMgr.incept(stem='bob', temp=True) # algo default salty and rooted + bobSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + bobPre = bobSrdr.pre + bobMgr.move(old=verfers[0].qb64, new=bobPre) # move key pair label to prefix + # Setup Bob's prefixes so bob's KEL will be Kever.locallyOwned() and + # Del's KEL will be Kever.locallyDelegated() + bobDB.prefixes.add(bobPre) + assert bobPre in bobDB.prefixes + + sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) + msg = bytearray(bobSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + bobIcpMsg = msg # save for later + + # apply msg to bob's Kevery + psr.parse(ims=bytearray(bobIcpMsg), kvy=bobKvy, local=True) + bobK = bobKvy.kevers[bobPre] + assert bobK.prefixer.qb64 == bobPre + assert bobK.serder.said == bobSrdr.said + assert bobK.sn == 0 + + # apply msg to del's Kevery so he knows about the AID + psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) + assert bobK.prefixer.qb64 in delKvy.kevers + delBobK = bobKvy.kevers[bobPre] # bobs kever in dels kevery + assert delBobK.sn == 0 + + # Setup Del's inception event assuming that Bob's next event will be + # an ixn delegating event + verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted + delSrdr = eventing.delcept(keys=[verfer.qb64 for verfer in verfers], + delpre=bobPre, + ndigs=[diger.qb64 for diger in digers]) + + delPre = delSrdr.pre + delMgr.move(old=verfers[0].qb64, new=delPre) # move key pair label to prefix + # Setup Del's prefixes so Del's KEL will be Kever.locallyOwned() + delDB.prefixes.add(delPre) + assert delPre in delDB.prefixes + + # Now create delegating event for Bob + seal = eventing.SealEvent(i=delPre, + s=delSrdr.ked["s"], + d=delSrdr.said) + bobSrdr = eventing.interact(pre=bobK.prefixer.qb64, + dig=bobK.serder.said, + sn=bobK.sn+1, + data=[seal._asdict()]) + + sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) + + msg = bytearray(bobSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + bobIxnMsg1 = msg # delegating event with attachments + + # apply msg to bob's Kevery + psr.parse(ims=bytearray(bobIxnMsg1), kvy=bobKvy, local=True) + assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + assert bobK.sn == 1 + + # now create Del's delegated inception event msg + sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) + + msg = bytearray(delSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + counter = core.Counter(core.Codens.SealSourceCouples, + count=1, gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + seqner = coring.Seqner(sn=bobK.sn) + msg.extend(seqner.qb64b) + msg.extend(bobSrdr.saidb) + delIcpMsg = msg + + # apply Del's delegated inception event message to bob's Kevery + # because the attachment includes valid source seal then the Delegables + # escrow is bypassed and is validated and shows up in AES + psr.parse(ims=bytearray(delIcpMsg), kvy=bobKvy, local=True) + + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + assert delPre in bobKvy.kevers # successfully validated + bobDelK = bobKvy.kevers[delPre] # delK in bobs kevery + assert bobDelK.delegated + assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + + # apply Del's inception msg to Del's Kevery + # Because locallyOwned by delegate event does not validate delegation + # and ignores the attached source seal + psr.parse(ims=bytearray(delIcpMsg), kvy=delKvy, local=True) + + assert delPre in delKvy.kevers + delK = delKvy.kevers[delPre] + # no AES entry for del's own delegated event when locallyOwned + assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + + # apply Del's delegated inception event message to wats's Kevery as remote + # because the attachment includes valid source seal but wat does not + # yet have Bob's delegating event entry. The event goes into partial + # delegated event escrow + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + # TEST MISSING EVENT SOURCE + for (epre,), esn, edig in watKvy.db.pdes.getOnItemIter(): + dgkey = dgKey(epre, edig) + esr = watKvy.db.esrs.get(keys=dgkey) + assert esr is not None + watKvy.db.esrs.rem(keys=dgkey) + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + # TEST MISSING DATE TIME + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + for (epre,), esn, edig in watKvy.db.pdes.getOnItemIter(): + dgkey = dgKey(epre, edig) + dtb = watKvy.db.getDts(dgkey) + assert dtb is not None + watKvy.db.delDts(dgkey) # remove date timestamp of event so it will be invalid + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + for (epre,), esn, edig in watKvy.db.pdes.getOnItemIter(): + dgkey = dgKey(epre, edig) + eraw = watKvy.db.getEvt(dgkey) + assert eraw is not None + watKvy.db.delEvt(dgkey) # remove event data so it will be invalid + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + for (epre,), esn, edig in watKvy.db.pdes.getOnItemIter(): + dgkey = dgKey(epre, edig) + sigs = watKvy.db.getSigs(dgkey) + assert sigs is not None + watKvy.db.delSigs(dgkey) # remove event sigs so it will be invalid + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + # TEST STALE EVENT + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + watKvy.TimeoutPWE = 0 + time.sleep(0.001) + watKvy.processEscrowPartialDels() + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + + for (epre,), esn, edig in watKvy.db.pdes.getOnItemIter(): + dgkey = dgKey(epre, edig) + sigs = watKvy.db.getSigs(dgkey) + assert sigs is not None + watKvy.db.delSigs(dgkey) # remove event sigs so it will be invalid + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + if __name__ == "__main__": #test_unverified_receipt_escrow() test_missing_delegator_escrow() From 5280fa1577fd437c38a8abd70e4d233dd85ac1ec Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Tue, 29 Apr 2025 17:22:17 -0600 Subject: [PATCH 402/418] test: add tests for UWE, URE, DEL, QNF, VRE, and DUP escrows --- src/keri/core/eventing.py | 4 +- tests/app/test_querying.py | 21 +- tests/core/test_escrow.py | 994 ++++++++++++++++++++++++++++++++++++- tests/core/test_witness.py | 1 + 4 files changed, 1012 insertions(+), 8 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 14635dd75..149371161 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5326,7 +5326,7 @@ def escrowTReceipts(self, serder, prefixer, seqner, saider, sigers): prefixer is Prefixer instance of prefix of receipter seqner is Seqner instance of sn of est event of receiptor saider is Saider instance of said of est event of receiptor - igers is list of Siger instances of multi-sig of receiptor + sigers is list of Siger instances of multi-sig of receiptor escrow quintuple for each siger quintuple = edig+pre+snu+dig+sig @@ -5334,7 +5334,7 @@ def escrowTReceipts(self, serder, prefixer, seqner, saider, sigers): edig is receipted event dig (serder.dig) pre is receipter prefix snu is receipter est event sn - dig is receipt est evant dig + dig is receipt est event dig sig is indexed sig of receiptor of receipted event """ # Receipt dig algo may not match database dig. So must always diff --git a/tests/app/test_querying.py b/tests/app/test_querying.py index ee6c42f46..24b618f70 100644 --- a/tests/app/test_querying.py +++ b/tests/app/test_querying.py @@ -3,15 +3,22 @@ keri.app.querying module """ +import logging + from hio.base import doing +from keri import help from keri.app import habbing from keri.app.querying import QueryDoer, KeyStateNoticer, LogQuerier, SeqNoQuerier, AnchorQuerier from keri.core import parsing, eventing, serdering from keri.db.dbing import dgKey +logger = help.ogler.getLogger() + + def test_querying(): + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks with habbing.openHby() as hby, \ habbing.openHby() as hby1: inqHab = hby.makeHab(name="inquisitor") @@ -150,13 +157,17 @@ def test_querying(): assert len(adoer.witq.msgs) == 1 def test_query_not_found_escrow(): - with habbing.openHby() as hby, \ - habbing.openHby() as hby1: - inqHab = hby.makeHab(name="inquisitor") - subHab = hby1.makeHab(name="subject") + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + with habbing.openHby() as inqHby, \ + habbing.openHby() as subHby: + inqHab = inqHby.makeHab(name="inquisitor") + subHab = subHby.makeHab(name="subject") + + psr = parsing.Parser() icp = inqHab.makeOwnInception() - subHab.psr.parseOne(ims=icp) + subKvy = eventing.Kevery(db=subHab.db, lax=False, local=False) + psr.parseOne(ims=bytearray(icp), kvy=subKvy) assert inqHab.pre in subHab.kevers qry = inqHab.query(subHab.pre, route="/foo", src=inqHab.pre) diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index 3e8346385..070f16708 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -11,11 +11,12 @@ import pytest from keri import help +from keri.core.eventing import deWitnessCouple, deReceiptTriple, deTransReceiptQuintuple from keri.db.dbing import splitSnKey, dgKey, snKey from keri.help import helping from keri import core, kering -from keri.core import coring, eventing, parsing +from keri.core import coring, eventing, parsing, indexing, serdering from keri.db import dbing, basing from keri.app import keeping, habbing @@ -1379,6 +1380,7 @@ def test_unverified_trans_receipt_escrow(): """ salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks # init event DB and keep DB with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: @@ -2127,6 +2129,996 @@ def test_partial_delegation_escrow_validation_errors(): assert len(escrows) == 0 +def test_unverified_witness_escrow_validation_errors(): + """ + Test unverified witness escrow validation errors + + cam is controller + van is validator + wes is a witness + wok is a witness + wam is a witness + + """ + salt = core.Salter(raw=b'abcdef0123456789').qb64 + + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + with habbing.openHby(name="cam", base="test", salt=salt) as camHby, \ + habbing.openHby(name="van", base="test", salt=salt) as vanHby, \ + habbing.openHby(name="wes", base="test", salt=salt) as wesHby, \ + habbing.openHby(name="wak", base="test", salt=salt) as wokHby, \ + habbing.openHby(name="wam", base="test", salt=salt) as wamHby, \ + habbing.openHby(name="wil", base="test", salt=salt) as wilHby: + + # witnesses first so can set up inception event for cam + wsith = '1' + # setup Wes's habitat nontrans + # Wes's receipts will be rcts with a receipt couple attached + + wesHab = wesHby.makeHab(name='wes', isith=wsith, icount=1, transferable=False) + assert not wesHab.kever.prefixer.transferable + # create non-local kevery for Wes to process nonlocal msgs + wesKvy = eventing.Kevery(db=wesHab.db, lax=False, local=False) + + # setup Wok's habitat nontrans + # Wok's receipts will be rcts with a receipt couple attached + wokHab = wokHby.makeHab(name='wok', isith=wsith, icount=1, transferable=False) + assert not wokHab.kever.prefixer.transferable + # create non-local kevery for Wok to process nonlocal msgs + wokKvy = eventing.Kevery(db=wokHab.db, lax=False, local=False) + + # setup Wam's habitat nontrans + # Wams's receipts will be rcts with a receipt couple attached + wamHab = wamHby.makeHab(name='wam', isith=wsith, icount=1, transferable=False) + assert not wamHab.kever.prefixer.transferable + # create non-local kevery for Wam to process nonlocal msgs + wamKvy = eventing.Kevery(db=wamHab.db, lax=False, local=False) + + # setup Wil's habitat nontrans + # Wil's receipts will be rcts with a receipt couple attached + wilHab = wilHby.makeHab(name='wil', isith=wsith, icount=1, transferable=False) + assert not wilHab.kever.prefixer.transferable + # create non-local kevery for Wam to process nonlocal msgs + wilKvy = eventing.Kevery(db=wilHab.db, lax=False, local=False) + + # setup Cam's habitat trans multisig + wits = [wesHab.pre, wokHab.pre, wamHab.pre] + csith = '2' # hex str of threshold int + camHab = camHby.makeHab(name='cam', isith=csith, icount=3, toad=2, wits=wits,) + assert camHab.kever.prefixer.transferable + assert len(camHab.iserder.berfers) == len(wits) + for werfer in camHab.iserder.berfers: + assert werfer.qb64 in wits + assert camHab.kever.wits == wits + assert camHab.kever.toader.num == 2 + assert camHab.kever.sn == 0 + + # create non-local kevery for Cam to process onlocal msgs + camKvy = eventing.Kevery(db=camHab.db, lax=False, local=False) + + # setup Van's habitat trans multisig + vsith = '2' # two of three signing threshold + vanHab = vanHby.makeHab(name='van', isith=vsith, icount=3) + assert vanHab.kever.prefixer.transferable + # create non-local kevery for Van to process nonlocal msgs + vanKvy = eventing.Kevery(db=vanHab.db, lax=False, local=False) + + # make list so easier to batch + camWitKvys = [wesKvy, wokKvy, wamKvy] + camWitHabs = [wesHab, wokHab, wamHab] + + # Create Cam inception and send to each of Cam's witnesses + camIcpMsg = camHab.makeOwnInception() + rctMsgs = [] # list of receipts from each witness + for i in range(len(camWitKvys)): + kvy = camWitKvys[i] + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy, local=True) + assert kvy.kevers[camHab.pre].sn == 0 # accepted event + assert len(kvy.cues) >= 1 # at least queued receipt cue + # better to find receipt cue in cues exactly + hab = camWitHabs[i] + rctMsg = hab.processCues(kvy.cues) # process cue returns rct msg + assert len(rctMsg) == 626 + rctMsgs.append(rctMsg) + + for msg in rctMsgs: # process rct msgs from all witnesses + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) + for hab in camWitHabs: + assert hab.pre in camKvy.kevers + + # get from Cam database copies of witness receipts received by Cam + # and send to witnesses so all witnesses have full set of receipts + # from all other witnesses + # reply one event or receipt one event with all witness attachments + dgkey = dbing.dgKey(pre=camHab.pre, dig=camHab.kever.serder.said) + wigs = camHab.db.getWigs(dgkey) + assert len(wigs) == 3 + wigers = [indexing.Siger(qb64b=bytes(wig)) for wig in wigs] + rserder = eventing.receipt(pre=camHab.pre, + sn=camHab.kever.sn, + said=camHab.kever.serder.said) + camIcpWitRctMsg = eventing.messagize(serder=rserder, wigers=wigers) + assert len(camIcpWitRctMsg) == 413 + for i in range(len(camWitKvys)): + kvy = camWitKvys[i] + parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=kvy, local=True) + assert len(kvy.db.getWigs(dgkey)) == 3 # fully witnessed + assert len(kvy.cues) == 0 # no cues + + # send Cam icp and witness rcts to Van + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=vanKvy, local=True) + # should escrow since not witnesses + assert camHab.pre not in vanKvy.kevers + + # TEST MISSING DATE TIME + # process receipts + parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=vanKvy, local=True) + + assert camHab.pre not in vanKvy.kevers + escrows = vanKvy.db.getUwes(dbing.snKey(camHab.pre, 0)) # should be three receipts, one per witness + assert len(escrows) == 3 + + for ekey, ecouple in vanKvy.db.getUweItemIter(): + pre, sn = splitSnKey(ekey) + rdiger, _wiger = deWitnessCouple(ecouple) + dgkey = dgKey(pre, bytes(rdiger.qb64b)) + vanKvy.db.delDts(dgkey) + vanKvy.processEscrowUnverWitness() + escrows = vanKvy.db.getUwes(dbing.snKey(camHab.pre, 0)) # should be three receipts, one per witness + assert len(escrows) == 0 + + # TEST STALE EVENT + # process receipts + parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=vanKvy, local=True) + assert camHab.pre not in vanKvy.kevers + escrows = vanKvy.db.getUwes(dbing.snKey(camHab.pre, 0)) # should be three receipts, one per witness + assert len(escrows) == 3 + + vanKvy.TimeoutUWE = 0 + time.sleep(0.001) + vanKvy.processEscrowUnverWitness() + escrows = vanKvy.db.getUwes(dbing.snKey(camHab.pre, 0)) # should be three receipts, one per witness + assert len(escrows) == 0 + + vanKvy.TimeoutUWE = 3600 + + +def test_unverified_nontransferable_receiptors_escrow_validation_errors(): + """ + Test unverified witness escrow validation errors + + cam is controller + van is validator + wes is a witness + wok is a witness + wam is a witness + + """ + salt = core.Salter(raw=b'abcdef0123456789').qb64 + + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + with habbing.openHby(name="cam", base="test", salt=salt) as camHby, \ + habbing.openHby(name="badEvt", base="test", salt=salt) as badEvtHby, \ + habbing.openHby(name="van", base="test", salt=salt) as vanHby, \ + habbing.openHby(name="wes", base="test", salt=salt) as wesHby, \ + habbing.openHby(name="wak", base="test", salt=salt) as wokHby, \ + habbing.openHby(name="wam", base="test", salt=salt) as wamHby, \ + habbing.openHby(name="wil", base="test", salt=salt) as wilHby: + + # witnesses first so can set up inception event for cam + wsith = '1' + # setup Wes's habitat nontrans + # Wes's receipts will be rcts with a receipt couple attached + + wesHab = wesHby.makeHab(name='wes', isith=wsith, icount=1, transferable=False) + assert not wesHab.kever.prefixer.transferable + # create non-local kevery for Wes to process nonlocal msgs + wesKvy = eventing.Kevery(db=wesHab.db, lax=False, local=False) + + # setup Wok's habitat nontrans + # Wok's receipts will be rcts with a receipt couple attached + wokHab = wokHby.makeHab(name='wok', isith=wsith, icount=1, transferable=False) + assert not wokHab.kever.prefixer.transferable + # create non-local kevery for Wok to process nonlocal msgs + wokKvy = eventing.Kevery(db=wokHab.db, lax=False, local=False) + + # setup Wam's habitat nontrans + # Wams's receipts will be rcts with a receipt couple attached + wamHab = wamHby.makeHab(name='wam', isith=wsith, icount=1, transferable=False) + assert not wamHab.kever.prefixer.transferable + # create non-local kevery for Wam to process nonlocal msgs + wamKvy = eventing.Kevery(db=wamHab.db, lax=False, local=False) + + # setup Wil's habitat nontrans + # Wil's receipts will be rcts with a receipt couple attached + wilHab = wilHby.makeHab(name='wil', isith=wsith, icount=1, transferable=False) + assert not wilHab.kever.prefixer.transferable + # create non-local kevery for Wam to process nonlocal msgs + wilKvy = eventing.Kevery(db=wilHab.db, lax=False, local=False) + + # setup Cam's habitat trans multisig + wits = [wesHab.pre, wokHab.pre, wamHab.pre] + csith = '2' # hex str of threshold int + camHab = camHby.makeHab(name='cam', isith=csith, icount=3, toad=2, wits=wits,) + assert camHab.kever.prefixer.transferable + assert len(camHab.iserder.berfers) == len(wits) + for werfer in camHab.iserder.berfers: + assert werfer.qb64 in wits + assert camHab.kever.wits == wits + assert camHab.kever.toader.num == 2 + assert camHab.kever.sn == 0 + + # create non-local kevery for Cam to process onlocal msgs + camKvy = eventing.Kevery(db=camHab.db, lax=False, local=False) + + # setup Van's habitat trans multisig + vsith = '2' # two of three signing threshold + vanHab = vanHby.makeHab(name='van', isith=vsith, icount=3) + assert vanHab.kever.prefixer.transferable + # create non-local kevery for Van to process nonlocal msgs + vanKvy = eventing.Kevery(db=vanHab.db, lax=False, local=False) + + # make list so easier to batch + camWitKvys = [wesKvy, wokKvy, wamKvy] + camWitHabs = [wesHab, wokHab, wamHab] + + # Create Cam inception and send to each of Cam's witnesses + camIcpMsg = camHab.makeOwnInception() + rctMsgs = [] # list of receipts from each witness + for i in range(len(camWitKvys)): + kvy = camWitKvys[i] + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy, local=True) + assert kvy.kevers[camHab.pre].sn == 0 # accepted event + assert len(kvy.cues) >= 1 # at least queued receipt cue + # better to find receipt cue in cues exactly + hab = camWitHabs[i] + rctMsg = hab.processCues(kvy.cues) # process cue returns rct msg + assert len(rctMsg) == 626 + rctMsgs.append(rctMsg) + + for msg in rctMsgs: # process rct msgs from all witnesses + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) + + for hab in camWitHabs: + assert hab.pre in camKvy.kevers + + # send receipts one at a time to Van to escrow. Van not yet recieved + # icp event from Cam so not accepted Cam's pre + # compute keys for latest event in Cam's key state + dgkey = dbing.dgKey(pre=camHab.pre, dig=camHab.kever.serder.said) + snkey = dbing.snKey(pre=camHab.pre, sn=camHab.kever.serder.sn) + + # TEST MISSING DATE TIME + # Van process rct msgs from all witnesses for Cam's icp message + for i, msg in enumerate(rctMsgs): + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) + # escrows to Ure + assert vanKvy.db.cntUres(snkey) == i + 1 # escrows + assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow + assert vanKvy.db.cntWigs(dgkey) == 0 # no wigs yet + assert camHab.pre not in vanKvy.kevers # not accepted + for hab in camWitHabs: # Van accepted icp events for Cam's witnesses + assert hab.pre in vanKvy.kevers + + vanKvy.processEscrowUnverNonTrans() + + for ekey, etriplet in vanKvy.db.getUreItemIter(key=b''): + pre, sn = splitSnKey(ekey) + rsaider, _sprefixer, _cigar = deReceiptTriple(etriplet) + dgkey = dgKey(pre, bytes(rsaider.qb64b)) + dtb = vanKvy.db.getDts(dgkey) + if dtb is not None: + vanKvy.db.delDts(dgkey) + vanKvy.processEscrowUnverNonTrans() + escrows = vanKvy.db.getUres(dbing.snKey(camHab.pre, 0)) + assert len(escrows) == 0 + + vanKvy.processEscrows() # process escrows + assert vanKvy.db.cntPwes(snkey) == 0 # nothing in partial witness escrow + assert vanKvy.db.cntUres(snkey) == 0 # still in escrow + assert vanKvy.db.cntWigs(dgkey) == 0 # no wigs yet + assert camHab.pre not in vanKvy.kevers # still not accepted + + # TEST STALE EVENT + for i, msg in enumerate(rctMsgs): + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) + # escrows to Ure + assert vanKvy.db.cntUres(snkey) == i + 1 # escrows + assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow + assert vanKvy.db.cntWigs(dgkey) == 0 # no wigs yet + assert camHab.pre not in vanKvy.kevers # not accepted + for hab in camWitHabs: # Van accepted icp events for Cam's witnesses + assert hab.pre in vanKvy.kevers + + vanKvy.TimeoutURE = 0 + time.sleep(0.001) + vanKvy.processEscrowUnverNonTrans() + escrows = vanKvy.db.getUres(dbing.snKey(camHab.pre, 0)) + assert len(escrows) == 0 + + vanKvy.TimeoutURE = 3600 # reset timeout to normal value + + # TEST INVALID EVENT + for i, msg in enumerate(rctMsgs): + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) + # escrows to Ure + assert vanKvy.db.cntUres(snkey) == i + 1 # escrows + assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow + assert vanKvy.db.cntWigs(dgkey) == 0 # no wigs yet + assert camHab.pre not in vanKvy.kevers # not accepted + for hab in camWitHabs: # Van accepted icp events for Cam's witnesses + assert hab.pre in vanKvy.kevers + vanKvy.processEscrowUnverNonTrans() + + # put value in db to trigger other validation errors + to_clean_up = [] + for ekey, etriplet in vanKvy.db.getUreItemIter(key=b''): + pre, sn = splitSnKey(ekey) + rsaider, _sprefixer, _cigar = deReceiptTriple(etriplet) + vanKvy.db.addKe(snKey(pre, sn), rsaider.qb64b) # add arbitrary value as key event + to_clean_up.append(snKey(pre, sn)) + vanKvy.processEscrowUnverNonTrans() + escrows = vanKvy.db.getUres(dbing.snKey(camHab.pre, 0)) + assert len(escrows) == 0 + + # cleanup + for ureSnKey in to_clean_up: + vanKvy.db.delKes(ureSnKey) # remove value to clean up for later tests + + # TEST BAD ESCROWED RECEIPT + # put different inception event in the db so the escrow thinks the receipt is bad + # Set up bad event hab + badEvtHab = badEvtHby.makeHab(name='badEvt', isith='1', icount=1, toad=0, wits=[]) + badEvtIcpMsg = badEvtHab.makeOwnInception() + + for i, msg in enumerate(rctMsgs): + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) + # escrows to Ure + assert vanKvy.db.cntUres(snkey) == i + 1 # escrows + assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow + assert vanKvy.db.cntWigs(dgkey) == 0 # no wigs yet + assert camHab.pre not in vanKvy.kevers # not accepted + for hab in camWitHabs: # Van accepted icp events for Cam's witnesses + assert hab.pre in vanKvy.kevers + vanKvy.processEscrowUnverNonTrans() + escrows = vanKvy.db.getUres(dbing.snKey(camHab.pre, 0)) + assert len(escrows) == 3 # three receipts in escrow + + # put value in db to trigger other validation errors + to_clean_up_ke = [] + to_clean_up_evt = [] + evt = serdering.SerderKERI(raw=camIcpMsg) + for ekey, etriplet in vanKvy.db.getUreItemIter(key=b''): + pre, sn = splitSnKey(ekey) + rsaider, _sprefixer, _cigar = deReceiptTriple(etriplet) + dig = rsaider.qb64b + dgkey = dgKey(pre, dig) # events use dgkey + vanKvy.db.addKe(snKey(pre, sn), rsaider.qb64b) # add mock key event digest + vanKvy.db.putEvt(dgkey, badEvtIcpMsg) # add mock key event + + to_clean_up_ke.append(snKey(pre, sn)) + to_clean_up_evt.append(dgkey) + vanKvy.processEscrowUnverNonTrans() + escrows = vanKvy.db.getUres(dbing.snKey(camHab.pre, 0)) + assert len(escrows) == 0 + for ureSnKey in to_clean_up_ke: + vanKvy.db.delKes(ureSnKey) # remove value to clean up for later tests + for ureEvtKey in to_clean_up_evt: + vanKvy.db.delEvt(ureEvtKey) + + +def test_delegables_escrow_validation_errors(): + gateSalt = core.Salter(raw=b'0123456789abcdef').qb64 + torSalt = core.Salter(raw=b'0123456789defabc').raw + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + with habbing.openHby(name="delegate", temp=True, salt=gateSalt) as gateHby, \ + habbing.openHab(name="delegator", temp=True, salt=torSalt) as (torHby, torHab): + + gateHab = gateHby.makeHab(name="repTest", transferable=True, delpre=torHab.pre) + assert gateHab.pre == "EFqw1EgGdd2B6MgNLJaNO13_JoQpxAtasIjySDzGm9pd" + + gateIcp = gateHab.makeOwnEvent(sn=0) + torKvy = eventing.Kevery(db=torHab.db, lax=False, local=False) + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) # tell delegate about delegator + assert gateHab.pre not in torKvy.kevers # should not be in delegate KELs until delegation finishes (seal arrives) + snkey = snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn) + assert len(torHab.db.delegables.get(keys=snkey)) == 1 + # Exercise the MissingDelegableApprovalError case + torKvy.processEscrowDelegables() + + # TEST MISSING EVENT SOURCE + for (pre, sn), dig in torKvy.db.delegables.getItemIter(): + edig = dig.encode() + dgkey = dgKey(pre.encode(), edig) + torKvy.db.esrs.rem(keys=dgkey) + torKvy.processEscrowDelegables() + escrows = torKvy.db.delegables.get(snkey) + assert len(escrows) == 0 + + # TEST MISSING DATE TIME + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) # tell delegate about delegator + assert gateHab.pre not in torKvy.kevers # should not be in delegate KELs until delegation finishes (seal arrives) + snkey = snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn) + assert len(torHab.db.delegables.get(keys=snkey)) == 1 + + for (pre, sn), dig in torKvy.db.delegables.getItemIter(): + edig = dig.encode() + dgkey = dgKey(pre.encode(), edig) + dtb = torKvy.db.getDts(dgkey) + assert dtb is not None + torKvy.db.delDts(dgkey) + torKvy.processEscrowDelegables() + escrows = torKvy.db.delegables.get(snkey) + assert len(escrows) == 0 + + # TEST STALE EVENT + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) # tell delegate about delegator + assert gateHab.pre not in torKvy.kevers # should not be in delegate KELs until delegation finishes (seal arrives) + snkey = snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn) + assert len(torHab.db.delegables.get(keys=snkey)) == 1 + + torKvy.TimeoutOOE = 0 + time.sleep(0.001) + torKvy.processEscrowDelegables() + escrows = torKvy.db.delegables.get(snkey) + assert len(escrows) == 0 + + torKvy.TimeoutOOE = 3600 + + # TEST MISSING EVENT DATA + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) # tell delegate about delegator + assert gateHab.pre not in torKvy.kevers # should not be in delegate KELs until delegation finishes (seal arrives) + snkey = snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn) + assert len(torHab.db.delegables.get(keys=snkey)) == 1 + + for (pre, sn), dig in torKvy.db.delegables.getItemIter(): + edig = dig.encode() + dgkey = dgKey(pre.encode(), edig) + eraw = torKvy.db.getEvt(dgkey) + assert eraw is not None + torKvy.db.delEvt(dgkey) + torKvy.processEscrowDelegables() + escrows = torKvy.db.delegables.get(snkey) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) # tell delegate about delegator + assert gateHab.pre not in torKvy.kevers # should not be in delegate KELs until delegation finishes (seal arrives) + snkey = snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn) + assert len(torHab.db.delegables.get(keys=snkey)) == 1 + + for (pre, sn), dig in torKvy.db.delegables.getItemIter(): + edig = dig.encode() + dgkey = dgKey(pre.encode(), edig) + sigers = torKvy.db.getSigs(dgkey) + assert sigers is not None + torKvy.db.delSigs(dgkey) + torKvy.processEscrowDelegables() + escrows = torKvy.db.delegables.get(snkey) + assert len(escrows) == 0 + + +def test_query_not_found_escrow_validation_errors(): + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + with habbing.openHby() as inqHby, \ + habbing.openHby() as subjHby: + inqHab = inqHby.makeHab(name="inquisitor") + subHab = subjHby.makeHab(name="subject") + psr = parsing.Parser() + icp = inqHab.makeOwnInception() + subKvy = eventing.Kevery(db=subHab.db, lax=False, local=False) + assert inqHab.pre not in subHab.kevers + + inqKvy = eventing.Kevery(db=inqHab.db, lax=False, local=True) + assert subHab.pre not in inqHab.kevers + + qry = inqHab.query(subHab.pre, route="ksn", src=inqHab.pre) + + # TEST MISSING DATE TIME + psr.parseOne(ims=bytearray(qry), kvy=inqKvy) # reapply query to inquisitor's escrow + for (pre, said), edig in inqKvy.db.qnfs.getItemIter(): + dgkey = dgKey(pre.encode(), edig.encode()) + dtb = inqKvy.db.getDts(dgkey) + assert dtb is not None + inqKvy.db.delDts(dgkey) + inqKvy.processQueryNotFound() + escrows = inqKvy.db.qnfs.get(dgkey) + assert len(escrows) == 0 + + # TEST STALE EVENT + psr.parseOne(ims=bytearray(qry), kvy=inqKvy) # reapply query to inquisitor's escrow + inqKvy.TimeoutQNF = 0 + time.sleep(0.001) + inqKvy.processQueryNotFound() + escrows = inqKvy.db.qnfs.get(dgkey) + assert len(escrows) == 0 + + inqKvy.TimeoutQNF = 3600 + + # TEST MISSING EVENT DATA + psr.parseOne(ims=bytearray(qry), kvy=inqKvy) # reapply query to inquisitor's escrow + for (pre, said), edig in inqKvy.db.qnfs.getItemIter(): + dgkey = dgKey(pre.encode(), edig.encode()) + eraw = inqKvy.db.getEvt(dgkey) + assert eraw is not None + inqKvy.db.delEvt(dgkey) + inqKvy.processQueryNotFound() + escrows = inqKvy.db.qnfs.get(dgkey) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parseOne(ims=bytearray(qry), kvy=inqKvy) # reapply query to inquisitor's escrow + for (pre, said), edig in inqKvy.db.qnfs.getItemIter(): + dgkey = dgKey(pre.encode(), edig.encode()) + sigs = inqKvy.db.getSigs(dgkey) + assert sigs is not None + inqKvy.db.delSigs(dgkey) + inqKvy.processQueryNotFound() + escrows = inqKvy.db.qnfs.get(dgkey) + assert len(escrows) == 0 + + # Exercise QueryNotFoundError + psr.parseOne(ims=bytearray(qry), kvy=inqKvy) # reapply query to inquisitor's escrow + inqKvy.processQueryNotFound() + + +def test_unverified_transferable_receiptors_escrow_validation_errors(): + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter + psr = parsing.Parser() + + # init event DB and keep DB + with basing.openDB(name="ned") as db, keeping.openKS(name="ned") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + + # Set up identifier for event creator, 2 of 3 weighted threshold multisig AID + sith = ["1/2", "1/2", "1/2"] + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='edy', temp=True) + srdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=sith, nsith=nxtsith, + keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers]) + pre = srdr.ked["i"] + icpdig = srdr.said + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + + # Create CESR message for inception event with local key signatures + msg = bytearray(srdr.raw) # message body + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # message attachments + for siger in sigers: + msg.extend(siger.qb64b) + icpmsg = msg + + # create transferable receipter (validator) inception keys 2 of 3 + rverfers, rdigers = mgr.incept(icount=3, ncount=3, stem='rob', temp=True) + rsith = '2' + + # create receipter's inception event + rsrdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=rsith, nsith=rsith, + keys=[verfer.qb64 for verfer in rverfers], + ndigs=[diger.qb64 for diger in rdigers]) + rpre = rsrdr.ked["i"] + ricpdig = rsrdr.said + mgr.move(old=rverfers[0].qb64, new=rpre) # move receipter key pair label to prefix + rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) + + # CESR message for receipter inception event with local key signatures + msg = bytearray(rsrdr.raw) # message body + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(rsigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # message attachments + for siger in rsigers: + msg.extend(siger.qb64b) + ricpmsg = msg + + # create transferable receipt of inception message + seal = eventing.SealEvent(i=rpre, + s=rsrdr.ked["s"], + d=rsrdr.said) + reserder = eventing.receipt(pre=pre, sn=0, said=icpdig) + # sign event not receipt + resigers = mgr.sign(ser=srdr.raw, verfers=rverfers) + rcticpmsg = eventing.messagize(serder=reserder, sigers=resigers, seal=seal) + + # Process receipt by kvy + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) + assert pre not in kvy.kevers # no events yet for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + kvy.processEscrowUnverTrans() + + # TEST MISSING DATE TIME + for ekey, equinlet in kvy.db.getVreItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) + dgkey = dgKey(pre, bytes(esaider.qb64b)) + kvy.db.getDts(dgkey) + kvy.db.delDts(dgkey) + kvy.processEscrowUnverTrans() + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 0 + + # TEST STALE EVENT + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) + assert pre not in kvy.kevers # no events yet for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + + kvy.TimeoutVRE = 0 + kvy.processEscrowUnverTrans() + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 0 + + kvy.TimeoutVRE = 3600 + + # TEST MISSING RECEIPTED EVENT + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) # reapply receipt to escrow + assert pre not in kvy.kevers # no events yet for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + kvy.processEscrowUnverTrans() + + for ekey, equinlet in kvy.db.getVreItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) + dgkey = dgKey(pre, bytes(esaider.qb64b)) + kvy.db.delDts(dgkey) # invalidate events to clear the escrow + kvy.processEscrowUnverTrans() + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 0 + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) # reapply receipt to escrow + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply inception event to escrow so key event findable + assert pre in kvy.kevers # icp event present for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + + for ekey, equinlet in kvy.db.getVreItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) + dgkey = dgKey(pre, bytes(esaider.qb64b)) + kvy.db.delEvt(dgkey) + kvy.processEscrowUnverTrans() + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 0 + + +def test_unverified_tranferable_receiptors_escrow_invalid_event_data(): + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter + psr = parsing.Parser() + + # init event DB and keep DB + with basing.openDB(name="ned") as db, keeping.openKS(name="ned") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + + # Set up identifier for event creator, 2 of 3 weighted threshold multisig AID + sith = ["1/2", "1/2", "1/2"] + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='edy', temp=True) + srdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=sith, nsith=nxtsith, + keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers]) + pre = srdr.ked["i"] + icpdig = srdr.said + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + + # Create CESR message for inception event with local key signatures + msg = bytearray(srdr.raw) # message body + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # message attachments + for siger in sigers: + msg.extend(siger.qb64b) + icpmsg = msg + + # create transferable receipter (validator) inception keys 2 of 3 + rverfers, rdigers = mgr.incept(icount=3, ncount=3, stem='rob', temp=True) + rsith = '2' + + # create receipter's inception event + rsrdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=rsith, nsith=rsith, + keys=[verfer.qb64 for verfer in rverfers], + ndigs=[diger.qb64 for diger in rdigers]) + rpre = rsrdr.ked["i"] + ricpdig = rsrdr.said + mgr.move(old=rverfers[0].qb64, new=rpre) # move receipter key pair label to prefix + rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) + + # CESR message for receipter inception event with local key signatures + msg = bytearray(rsrdr.raw) # message body + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(rsigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # message attachments + for siger in rsigers: + msg.extend(siger.qb64b) + ricpmsg = msg + + # create transferable receipt of inception message + seal = eventing.SealEvent(i=rpre, + s=rsrdr.ked["s"], + d=rsrdr.said) + reserder = eventing.receipt(pre=pre, sn=0, said=icpdig) + # sign event not receipt + resigers = mgr.sign(ser=srdr.raw, verfers=rverfers) + rcticpmsg = eventing.messagize(serder=reserder, sigers=resigers, seal=seal) + + # Process receipt by kvy + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) + assert pre not in kvy.kevers # no events yet for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + kvy.processEscrowUnverTrans() + + # TEST INVALID EVENT DATA + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) # reapply receipt to escrow + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply inception event to escrow so key event findable + assert pre in kvy.kevers # icp event present for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + kvy.processEscrowUnverTrans() + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + + for ekey, equinlet in kvy.db.getVreItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) + dgkey = dgKey(pre, bytes(esaider.qb64b)) + kvy.db.setEvt(dgkey, ricpmsg) + kvy.processEscrowUnverTrans() + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 0 + + +def test_duplicitous_escrow_validation_errors(): + """ + Tests the duplicitous event escrow with a duplicitous interaction event to test validation errors + properly remove duplicitous events from the duplicitous event escrow. + """ + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter + psr = parsing.Parser() + + with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + # Set up identifier for event creator, 2 of 3 weighted threshold multisig AID + sith = ["1/2", "1/2", "1/2"] + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='edy', temp=True) + srdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=sith, nsith=nxtsith, + keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers]) + pre = srdr.ked["i"] + icpdig = srdr.said + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + + # make CESR message for inception event to give to parser + msg = bytearray(srdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + icpmsg = msg + + # create transferable receipter (validator) inception keys 2 of 3 + rverfers, rdigers = mgr.incept(icount=3, ncount=3, stem='ray', temp=True) + rsith = '2' + + # create receipter's inception event + rsrdr = eventing.incept(keys=[verfer.qb64 for verfer in rverfers], + isith=rsith, + nsith=rsith, + ndigs=[diger.qb64 for diger in rdigers], + code=coring.MtrDex.Blake3_256) + + rpre = rsrdr.ked["i"] + ricpdig = rsrdr.said + mgr.move(old=rverfers[0].qb64, new=rpre) # move receipter key pair label to prefix + rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) + + # make CESR message for inception event to give to parser + msg = bytearray(rsrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(rsigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in rsigers: + msg.extend(siger.qb64b) + ricpmsg = msg + + + # create transferable receipt of inception message + seal = eventing.SealEvent(i=rpre, + s=rsrdr.ked["s"], + d=rsrdr.said) + reserder = eventing.receipt(pre=pre, sn=0, said=icpdig) + # sign event not receipt + resigers = mgr.sign(ser=srdr.raw, verfers=rverfers) + rcticpmsg = eventing.messagize(serder=reserder, sigers=resigers, seal=seal) + + # create interaction event + srdr = eventing.interact(pre=pre, dig=icpdig, sn=1, data=[]) + ixndig = srdr.said + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + + msg = bytearray(srdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + ixnmsg = msg + + # create duplicitous interaction event + dummyAnchor = dict(i=pre, s=15, d=pre) + dupSrdr = eventing.interact(pre=pre, dig=icpdig, sn=1, data=[dummyAnchor]) + dupIxndig = dupSrdr.said + dupSigers = mgr.sign(ser=dupSrdr.raw, verfers=verfers) + + msg = bytearray(dupSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in dupSigers: + msg.extend(siger.qb64b) + + dupIxnmsg = msg + + # Create rotation event of receipter + # get current keys as verfers and next digests as digers + rverfers, rdigers = mgr.rotate(pre=rpre, ncount=3, temp=True) + + rsrdr = eventing.rotate(pre=rpre, + keys=[verfer.qb64 for verfer in rverfers], + isith=rsith, + dig=ricpdig, + nsith=rsith, + ndigs=[diger.qb64 for diger in rdigers], + sn=1, + data=[]) + + rrotdig = rsrdr.said + + rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) + + msg = bytearray(rsrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(rsigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in rsigers: + msg.extend(siger.qb64b) + + rrotmsg = msg + + # create receipt(s) of interaction message with receipter rotation message + # create chit receipt(s) of interaction message + seal = eventing.SealEvent(i=rpre, + s=rsrdr.ked["s"], + d=rsrdr.said) + reserder = eventing.receipt(pre=pre, sn=1, said=ixndig) + # sign event not receipt + resigers = mgr.sign(ser=srdr.raw, verfers=rverfers) + rctixnmsg = eventing.messagize(serder=reserder, sigers=resigers, seal=seal) + + # Process receipt by kvy + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) + psr.parse(ims=bytearray(rctixnmsg), kvy=kvy) + assert pre not in kvy.kevers # no events yet for pre + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + # check escrows are back + assert len(kvy.db.getVres(dbing.snKey(pre, 0))) == 3 + assert len(kvy.db.getVres(dbing.snKey(pre, 1))) == 3 + + # apply inception msg to Kevery to process + psr.parse(ims=bytearray(icpmsg), kvy=kvy) + # kvy.process(ims=bytearray(icpmsg)) # process local copy of msg + assert pre in kvy.kevers # event accepted + kvr = kvy.kevers[pre] + assert kvr.serder.said == icpdig # key state updated so event was validated + assert kvr.sn == 0 # key state successfully updated + + # apply ixn msg to Kevery to process + psr.parse(ims=bytearray(ixnmsg), kvy=kvy) + # kvy.process(ims=bytearray(ixnmsg)) # process local copy of msg + assert kvr.serder.said == ixndig # key state updated so event was validated + assert kvr.sn == 1 # key state successfully updated + + # TEST MISSING EVENT SOURCE + psr.parse(ims=bytearray(dupIxnmsg), kvy=kvy) # apply duplicitous ixn msg to Kevery to process + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + + for ekey, edig in kvy.db.getLdeItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + esr = kvy.db.esrs.get(keys=dgkey) + assert esr is not None + kvy.db.esrs.rem(keys=dgkey) + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # TEST MISSING DATE TIME + psr.parse(ims=bytearray(dupIxnmsg), kvy=kvy) # apply duplicitous ixn msg to Kevery to process + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + + for ekey, edig in kvy.db.getLdeItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + dtb = kvy.db.getDts(dgkey) + assert dtb is not None + kvy.db.delDts(dgkey) + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # TEST STALE EVENT + psr.parse(ims=bytearray(dupIxnmsg), kvy=kvy) # apply duplicitous ixn msg to Kevery to process + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + + kvy.TimeoutLDE = 0 + time.sleep(0.001) + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + kvy.TimeoutLDE = 3600 # reset timeout to default + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(dupIxnmsg), kvy=kvy) # apply duplicitous ixn msg to Kevery to process + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + + for ekey, edig in kvy.db.getLdeItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + eraw = kvy.db.getEvt(dgkey) + assert eraw is not None + kvy.db.delEvt(dgkey) + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parse(ims=bytearray(dupIxnmsg), kvy=kvy) # apply duplicitous ixn msg to Kevery to process + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + + for ekey, edig in kvy.db.getLdeItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + sigs = kvy.db.getSigs(dgkey) + assert sigs is not None + kvy.db.delSigs(dgkey) + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + if __name__ == "__main__": #test_unverified_receipt_escrow() test_missing_delegator_escrow() diff --git a/tests/core/test_witness.py b/tests/core/test_witness.py index 052e6c37b..5457cb078 100644 --- a/tests/core/test_witness.py +++ b/tests/core/test_witness.py @@ -138,6 +138,7 @@ def test_indexed_witness_replay(): assert camHab.pre not in vanKvy.kevers # process receipts parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=vanKvy, local=True) + vanKvy.processEscrowUnverWitness() vanKvy.processEscrows() assert camHab.pre in vanKvy.kevers # now accepted vcKvr = vanKvy.kevers[camHab.pre] From ec0c9c3a09caca3fea57aec31f1a8bf9b2761005 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Tue, 29 Apr 2025 17:59:52 -0600 Subject: [PATCH 403/418] test: bump time limit for witness sender test --- tests/app/test_agenting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/test_agenting.py b/tests/app/test_agenting.py index 3b0075557..455c84b43 100644 --- a/tests/app/test_agenting.py +++ b/tests/app/test_agenting.py @@ -120,7 +120,7 @@ def test_witness_sender(seeder): # looks like bad magic value in seeder is causing this to fail pdoer = PublishDoer(wanHby, wilHby, wesHby, palHby, seeder) - directing.runController(doers=[pdoer], expire=15.0) + directing.runController(doers=[pdoer], expire=10.0) assert pdoer.done is True From 28510f86c6b198921756d0f67c9ffaf9b732f032 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Wed, 30 Apr 2025 15:01:53 -0600 Subject: [PATCH 404/418] correct vdr.eventing string decoding Signed-off-by: Kent Bull --- tests/core/test_escrow.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index 070f16708..decdbc01b 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -2119,16 +2119,6 @@ def test_partial_delegation_escrow_validation_errors(): assert len(escrows) == 0 - for (epre,), esn, edig in watKvy.db.pdes.getOnItemIter(): - dgkey = dgKey(epre, edig) - sigs = watKvy.db.getSigs(dgkey) - assert sigs is not None - watKvy.db.delSigs(dgkey) # remove event sigs so it will be invalid - watKvy.processEscrowPartialDels() - escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) - assert len(escrows) == 0 - - def test_unverified_witness_escrow_validation_errors(): """ Test unverified witness escrow validation errors From fe9f379e2f032b32e521e83fbe2eeb084f186bfd Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Thu, 24 Apr 2025 19:28:41 -0600 Subject: [PATCH 405/418] fix: match dig or edig type to db function return type --- src/keri/core/eventing.py | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 149371161..6628352c1 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5594,7 +5594,7 @@ def processEscrowPartialSigs(self): dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local source so raise ValidationError which unescrows below - msg = f"PSE Missing escrowed event source at dig = {bytes(edig)}" + msg = f"PSE Missing escrowed event source at dig = {bytes(edig).decode()}" logger.info("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -5602,7 +5602,7 @@ def processEscrowPartialSigs(self): dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - msg = f"PSE Missing escrowed event datetime at dig = {bytes(edig)}" + msg = f"PSE Missing escrowed event datetime at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -5611,7 +5611,7 @@ def processEscrowPartialSigs(self): dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): # escrow stale so raise ValidationError which unescrows below - msg = f"PSE Stale event escrow at dig = {bytes(edig)}" + msg = f"PSE Stale event escrow at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -5619,7 +5619,7 @@ def processEscrowPartialSigs(self): eraw = self.db.getEvt(dgkey) if eraw is None: # no event so so raise ValidationError which unescrows below - msg = f"PSE Missing escrowed evt at dig = {bytes(edig)}" + msg = f"PSE Missing escrowed evt at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -5628,7 +5628,7 @@ def processEscrowPartialSigs(self): sigs = self.db.getSigs(dgkey) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - msg = f"PSE Missing escrowed evt sigs at dig = {bytes(edig)}" + msg = f"PSE Missing escrowed evt sigs at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs @@ -5762,7 +5762,7 @@ def processEscrowPartialWigs(self): dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local source so raise ValidationError which unescrows below - msg = f"PWE Missing escrowed event source at dig = {bytes(edig)}" + msg = f"PWE Missing escrowed event source at dig = {bytes(edig).decode()}" logger.info("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -5770,7 +5770,7 @@ def processEscrowPartialWigs(self): dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - msg = f"PWE Missing escrowed event datetime at dig = {bytes(edig)}" + msg = f"PWE Missing escrowed event datetime at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -5779,7 +5779,7 @@ def processEscrowPartialWigs(self): dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): # escrow stale so raise ValidationError which unescrows below - msg = f"PWE Stale event escrow at dig = {bytes(edig)}" + msg = f"PWE Stale event escrow at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -5787,7 +5787,7 @@ def processEscrowPartialWigs(self): eraw = self.db.getEvt(dgKey(pre, bytes(edig))) if eraw is None: # no event so so raise ValidationError which unescrows below - msg = f"PWE Missing escrowed evt at dig = {bytes(edig)}" + msg = f"PWE Missing escrowed evt at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -5797,7 +5797,7 @@ def processEscrowPartialWigs(self): sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs if not sigs: # empty list # no sigs so raise ValidationError which unescrows below - msg = f"PWE Missing escrowed evt sigs at dig = {bytes(edig)}" + msg = f"PWE Missing escrowed evt sigs at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -6351,7 +6351,7 @@ def processEscrowDelegables(self): dgkey = dgKey(pre.encode("utf-8"), edig) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local source so raise ValidationError which unescrows below - msg = f"DEL Missing escrowed event source at dig = {bytes(edig)}" + msg = f"DEL Missing escrowed event source at dig = {bytes(edig).decode()}" logger.info("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -6359,7 +6359,7 @@ def processEscrowDelegables(self): dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - msg = f"DEL Missing escrowed event datetime at dig = {bytes(edig)}" + msg = f"DEL Missing escrowed event datetime at dig = {bytes(edig).decode()}" logger.info("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -6368,7 +6368,7 @@ def processEscrowDelegables(self): dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutOOE): # escrow stale so raise ValidationError which unescrows below - msg = f"DEL Stale event escrow at dig = {bytes(edig)}" + msg = f"DEL Stale event escrow at dig = {bytes(edig).decode()}" logger.info("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -6376,7 +6376,7 @@ def processEscrowDelegables(self): eraw = self.db.getEvt(dgKey(pre, bytes(edig))) if eraw is None: # no event so raise ValidationError which unescrows below - msg = f"DEL Missing escrowed evt at dig = {bytes(edig)}" + msg = f"DEL Missing escrowed evt at dig = {bytes(edig).decode()}" logger.info("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -6386,7 +6386,7 @@ def processEscrowDelegables(self): sigs = self.db.getSigs(dgKey(pre, bytes(edig))) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - msg = f"DEL Missing escrowed evt sigs at dig = {bytes(edig)}" + msg = f"DEL Missing escrowed evt sigs at dig = {bytes(edig).decode()}" logger.info("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -6858,7 +6858,7 @@ def processEscrowDuplicitous(self): dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local source so raise ValidationError which unescrows below - msg = f"DUP Missing escrowed event source at dig = {bytes(edig)}" + msg = f"DUP Missing escrowed event source at dig = {bytes(edig).decode()}" logger.info("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -6866,7 +6866,7 @@ def processEscrowDuplicitous(self): dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - msg = f"DUP Missing escrowed event datetime at dig = {bytes(edig)}" + msg = f"DUP Missing escrowed event datetime at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -6875,7 +6875,7 @@ def processEscrowDuplicitous(self): dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutLDE): # escrow stale so raise ValidationError which unescrows below - msg = f"DUP Stale event escrow at dig = {bytes(edig)}" + msg = f"DUP Stale event escrow at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -6883,7 +6883,7 @@ def processEscrowDuplicitous(self): eraw = self.db.getEvt(dgKey(pre, bytes(edig))) if eraw is None: # no event so raise ValidationError which unescrows below - msg = f"DUP Missing escrowed evt at dig = {bytes(edig)}" + msg = f"DUP Missing escrowed evt at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) @@ -6893,7 +6893,7 @@ def processEscrowDuplicitous(self): sigs = self.db.getSigs(dgKey(pre, bytes(edig))) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - msg = f"DUP Missing escrowed evt sigs at dig = {bytes(edig)}" + msg = f"DUP Missing escrowed evt sigs at dig = {bytes(edig).decode()}" logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) From 4bcd8971254d43bf1de050a4d5912044d1581f00 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Tue, 22 Apr 2025 07:43:31 -0600 Subject: [PATCH 406/418] fix: allow missing "rp" field in exn message from pre 1.1.25 KERIpy Older versions of KERIpy do not have the "rp" field in their Exchange (exn) message body at the top level. As of version 1.1.25, about June 12th 2024, the "rp" message was added for message routing, which was a backwards incompatible change since the parser was not updated to support a new protocol version for the new exn message version. This PR is a temporary workaround for this incompatibility. The plan is to tell people to move off of 1.1.x versions as soon as possible. Signed-off-by: Kent Bull --- src/keri/core/serdering.py | 28 +++++++++++++++++++++++++--- tests/core/test_serdering.py | 23 +++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index f28ca035c..a56eb1613 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -735,10 +735,22 @@ def _verify(self): # can't do set math osalls == oskeys - osexts becasue of osexts might be # out-of-order so have to iterate to ensure osexts if any appear in oskeys # after all fields in osalls + + # Special processing for compatibility with old KERIpy versions prior to 1.1.25 that do not send the 'rp' field in exn messages + skip_missing_fields = False + missing_keys = [key for key in osalls if key not in oskeys] # missing keys in sad + if missing_keys and self.ilk == Ilks.exn: # Skip missing fields only for Exn (Exchange) ilk + if len(missing_keys) == 1 and 'rp' in missing_keys: + skip_missing_fields = True + src_index = 0 for i, label in enumerate(osalls): - if oskeys[i] != label: + # special missing handling for pre 1.1.25; skips missing fields in the schema, in order + if skip_missing_fields and label in missing_keys: + continue # skip incrementing the src_index so it stays in place, in order + if oskeys[src_index] != label: raise MissingFieldError(f"Missing or out-of-order field = {label} " f"from = {list(osalls)} in sad.") + src_index += 1 # advance source index only if label found in source oskeys # said field labels are not order dependent with respect to all fields # in sad so use set() to test inclusion @@ -1555,8 +1567,18 @@ def _verify(self, **kwa): allkeys = list(self.Fields[self.proto][self.vrsn][self.ilk].alls) keys = list(self.sad) if allkeys != keys: - raise ValidationError(f"Invalid top level field list. Expected " - f"{allkeys} got {keys}.") + # special handling for pre 1.1.25 KERIpy not sending 'rp' field in exn messages. + skip_missing_field = False + if self.ilk == Ilks.exn: + missing_keys = [key for key in allkeys if key not in keys] + if missing_keys: + if len(missing_keys) == 1 and 'rp' in missing_keys: + skip_missing_field = True + if skip_missing_field: + pass + else: + raise ValidationError(f"Invalid top level field list. Expected " + f"{allkeys} got {keys}.") if (self.vrsn.major < 2 and self.vrsn.minor < 1 and self.ilk in (Ilks.qry, Ilks.rpy, Ilks.pro, Ilks.bar, Ilks.exn)): diff --git a/tests/core/test_serdering.py b/tests/core/test_serdering.py index 419103865..5cf6b862e 100644 --- a/tests/core/test_serdering.py +++ b/tests/core/test_serdering.py @@ -2154,6 +2154,17 @@ def test_serderkeri_bar(): """End Test""" +def test_serderkeri_exn_old(): + """Test SerderKERI exn msg""" + serder = SerderKERI(raw=b'{"v":"KERI10JSON000088_","t":"exn",' + b'"d":"EMuAoRSE4zREKKYyvuNeYCDM9_MwPQIh1WL0' + b'cFC4e-bU","i":"","p":"","dt":"","r":"","q":{},"a":[],"e":{}}', ilk=kering.Ilks.exn) + + assert serder.verify() # because pre is empty + assert serder.ilk == kering.Ilks.exn + assert serder.pre == '' + assert serder.prior == '' + def test_serderkeri_exn(): """Test SerderKERI exn msg""" @@ -3304,6 +3315,17 @@ def test_cesr_native_dumps_hby(): """End Test""" +def test_serderkeri_exn_old(): + """Test SerderKERI exn msg""" + serder = SerderKERI(raw=b'{"v":"KERI10JSON000088_","t":"exn",' + b'"d":"EMuAoRSE4zREKKYyvuNeYCDM9_MwPQIh1WL0' + b'cFC4e-bU","i":"","p":"","dt":"","r":"","q":{},"a":[],"e":{}}', ilk=kering.Ilks.exn) + + assert serder.verify() # because pre is empty + assert serder.ilk == kering.Ilks.exn + assert serder.pre == '' + assert serder.prior == '' + if __name__ == "__main__": test_fielddom() test_spans() @@ -3327,4 +3349,5 @@ def test_cesr_native_dumps_hby(): test_serdery() test_cesr_native_dumps() test_cesr_native_dumps_hby() + test_serderkeri_exn_old() From d34ecd54c5a8e864df68b55239d793848664e8a2 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Wed, 23 Apr 2025 14:56:48 -0600 Subject: [PATCH 407/418] fix: lmdb.MapFullError: mdb_txn_commit: MDB_MAP_FULL on Mailboxer This may also occur on large credential databases in Reger. Signed-off-by: Kent Bull --- src/keri/app/keeping.py | 6 ++++-- src/keri/app/notifying.py | 3 +++ src/keri/app/storing.py | 3 +++ src/keri/db/basing.py | 3 +++ src/keri/vdr/viring.py | 4 +++- tests/app/test_keeping.py | 39 +++++++++++++++++++++++++++++++++++++ tests/app/test_notifying.py | 39 +++++++++++++++++++++++++++++++++++++ tests/app/test_storing.py | 38 +++++++++++++++++++++++++++++++++++- tests/vdr/test_viring.py | 1 + 9 files changed, 132 insertions(+), 4 deletions(-) diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index 578912ddc..16ab4931b 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -31,10 +31,9 @@ from hio.base import doing from .. import kering -from .. import core +from .. import core, help from ..core import coring from ..db import dbing, subing, koming -from .. import help from ..help import helping logger = help.ogler.getLogger() @@ -133,6 +132,9 @@ def openKS(name="test", **kwa): """ return dbing.openLMDB(cls=Keeper, name=name, **kwa) +# Env var for configuring LMDB size for the Keeper database +KERIKeeperMapSizeKey = "KERI_KEEPER_MAP_SIZE" + class Keeper(dbing.LMDBer): """ diff --git a/src/keri/app/notifying.py b/src/keri/app/notifying.py index 4fcaecd79..128c115a4 100644 --- a/src/keri/app/notifying.py +++ b/src/keri/app/notifying.py @@ -200,6 +200,9 @@ def cntAll(self): """ return self.db.cnt(db=self.sdb) +# Env var for configuring LMDB size for the Noter database +KERINoterMapSizeKey = "KERI_NOTER_MAP_SIZE" + class Noter(dbing.LMDBer): """ diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index b2d6c9f53..2a18dd0b0 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -17,6 +17,9 @@ logger = help.ogler.getLogger() +# Env var for configuring LMDB size for the Mailbox database +KERIMailboxerMapSizeKey = "KERI_MAILBOXER_MAP_SIZE" + class Mailboxer(dbing.LMDBer): """ diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index b34c0f4a0..58db47790 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -594,6 +594,9 @@ def reopenDB(db, clear=False, **kwa): finally: db.close(clear=clear) +# Env var for configuring LMDB size for the main Baser database +KERIBaserMapSizeKey = "KERI_BASER_MAP_SIZE" + class Baser(dbing.LMDBer): """ diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 6e8415be2..4a2cd4d8d 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -8,7 +8,6 @@ A special purpose Verifiable Data Registry (VDR) """ import os - from dataclasses import dataclass, field, asdict from ordered_set import OrderedSet as oset @@ -171,6 +170,9 @@ def openReger(name="test", **kwa): """ return dbing.openLMDB(cls=Reger, name=name, **kwa) +# Env var for configuring LMDB size for the Keeper database +KERIRegerMapSizeKey = "KERI_REGER_MAP_SIZE" + class Reger(dbing.LMDBer): """ Reger sets up named sub databases for TEL registry diff --git a/tests/app/test_keeping.py b/tests/app/test_keeping.py index 87474c493..62b29b44a 100644 --- a/tests/app/test_keeping.py +++ b/tests/app/test_keeping.py @@ -17,6 +17,9 @@ from hio.base import doing from keri import kering +from keri.app.keeping import Keeper, KERIKeeperMapSizeKey +from keri.db.basing import KERIBaserMapSizeKey +from keri.db.dbing import LMDBer, KERILMDBMapSizeKey from keri.help import helping from keri import core @@ -1997,5 +2000,41 @@ def test_manager_sign_dual_indices(): assert not manager.ks.opened """End Test""" +def test_keeper_db_size_set_from_env_var(): + # Clear environment before test + if KERIBaserMapSizeKey in os.environ: + os.environ.pop(KERILMDBMapSizeKey) + if KERIKeeperMapSizeKey in os.environ: + os.environ.pop(KERIKeeperMapSizeKey) + + new_map_size = 10737418240 + + # Default map size works + kpr = Keeper(reopen=True) + assert kpr.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" + assert kpr.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + + # Specific map size works + os.environ[KERIKeeperMapSizeKey] = f"{new_map_size}" + + kpr = Keeper(reopen=True) + assert kpr.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + os.environ.pop(KERIKeeperMapSizeKey) + + # generic map size works + baser_map_size = 10737418240 + os.environ[KERILMDBMapSizeKey] = f"{baser_map_size}" + + kpr = Keeper(reopen=True) + assert kpr.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + + # Bad map size throws + os.environ[KERIKeeperMapSizeKey] = f"bad_map_size" + with pytest.raises(ValueError) as excinfo: + kpr = Keeper(reopen=True) + assert "invalid literal for int" in str(excinfo.value), "Expected ValueError when map size is not an integer" + os.environ.pop(KERILMDBMapSizeKey) + os.environ.pop(KERIKeeperMapSizeKey) + if __name__ == "__main__": test_manager_sign_dual_indices() diff --git a/tests/app/test_notifying.py b/tests/app/test_notifying.py index a62bb620e..91d3bb643 100644 --- a/tests/app/test_notifying.py +++ b/tests/app/test_notifying.py @@ -4,12 +4,16 @@ """ import datetime +import os import pytest from keri.app import notifying, habbing +from keri.app.notifying import Noter, KERINoterMapSizeKey from keri.core import coring from keri.db import dbing +from keri.db.basing import KERIBaserMapSizeKey +from keri.db.dbing import LMDBer, KERILMDBMapSizeKey from keri.help import helping @@ -221,3 +225,38 @@ def test_notifier(mockHelpingNowUTC): assert notifier.mar(note.rid) is False assert notifier.rem(note.rid) is True + +def test_noter_db_size_set_from_env_var(): + # Clear environment before test + if KERILMDBMapSizeKey in os.environ: + os.environ.pop(KERILMDBMapSizeKey) + if KERINoterMapSizeKey in os.environ: + os.environ.pop(KERINoterMapSizeKey) + + new_map_size = 10737418240 + # Default map size works + noter = Noter() + assert noter.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" + assert noter.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + + # Specific map size works + os.environ[KERINoterMapSizeKey] = f"{new_map_size}" + + noter = Noter() + assert noter.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + os.environ.pop(KERINoterMapSizeKey) + + # generic map size works + baser_map_size = 10737418240 + os.environ[KERILMDBMapSizeKey] = f"{baser_map_size}" + + noter = Noter() + assert noter.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + + # bad map size throws + os.environ[KERINoterMapSizeKey] = f"bad_map_size" + with pytest.raises(ValueError) as excinfo: + Noter() + assert "invalid literal for int" in str(excinfo.value), "Expected ValueError when map size is not an integer" + os.environ.pop(KERILMDBMapSizeKey) + os.environ.pop(KERINoterMapSizeKey) diff --git a/tests/app/test_storing.py b/tests/app/test_storing.py index 2abafbc7d..5ab9e5906 100644 --- a/tests/app/test_storing.py +++ b/tests/app/test_storing.py @@ -6,12 +6,15 @@ import os import lmdb +import pytest from keri.app import keeping from keri.core import coring, serdering from keri.db import dbing, basing, subing +from keri.db.basing import KERIBaserMapSizeKey +from keri.db.dbing import LMDBer, KERILMDBMapSizeKey from keri.peer import exchanging -from keri.app.storing import Mailboxer +from keri.app.storing import Mailboxer, KERIMailboxerMapSizeKey def test_mailboxing(): @@ -109,7 +112,40 @@ def test_mailboxing(): #assert(len(msgs)) == 6 #assert msgs[0][0] == 4 +def test_mailbox_db_size_set_from_env_var(): + # Clear environment before test + if KERILMDBMapSizeKey in os.environ: + os.environ.pop(KERILMDBMapSizeKey) + if KERIMailboxerMapSizeKey in os.environ: + os.environ.pop(KERIMailboxerMapSizeKey) + new_map_size = 10737418240 + # Default map size works + mber = Mailboxer() + assert mber.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" + assert mber.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + + # Specific map size works + os.environ[KERIMailboxerMapSizeKey] = f"{new_map_size}" + + mber = Mailboxer() + assert mber.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + os.environ.pop(KERIMailboxerMapSizeKey) + + # generic map size works + baser_map_size = 10737418240 + os.environ[KERILMDBMapSizeKey] = f"{baser_map_size}" + + mber = Mailboxer() + assert mber.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + + # Bad map size throws + os.environ[KERIMailboxerMapSizeKey] = f"bad_map_size" + with pytest.raises(ValueError) as excinfo: + Mailboxer() + assert "invalid literal for int" in str(excinfo.value), "Expected ValueError when map size is not an integer" + os.environ.pop(KERILMDBMapSizeKey) + os.environ.pop(KERIMailboxerMapSizeKey) if __name__ == '__main__': test_mailboxing() diff --git a/tests/vdr/test_viring.py b/tests/vdr/test_viring.py index 416d17e80..ab76d3b23 100644 --- a/tests/vdr/test_viring.py +++ b/tests/vdr/test_viring.py @@ -11,6 +11,7 @@ import pytest from keri import kering +from keri.app.keeping import KERIKeeperMapSizeKey from keri.core import coring from keri.core.coring import Diger, versify, Kinds from keri.core.serdering import SerderACDC From 4db61f4d15bb516004c0079c1253093a2b61e0aa Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Tue, 24 Feb 2026 18:47:17 -0700 Subject: [PATCH 408/418] bump version to 1.2.8 --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2528106e0..0d004dd19 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri -VERSION=1.2.7 +VERSION=1.2.8 define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store diff --git a/setup.py b/setup.py index db019906a..5b8475db4 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.7', # also change in src/keri/__init__.py + version='1.2.8', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 9826257d2..4597e6b07 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.7' # also change in setup.py +__version__ = '1.2.8' # also change in setup.py From 0eeb65c8a58a2bcfed2e682d29cf0b71f603270b Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Tue, 24 Feb 2026 18:56:58 -0700 Subject: [PATCH 409/418] fix: delegator seal storage Signed-off-by: Kent Bull --- src/keri/app/cli/commands/delegate/confirm.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index eac98e552..84a335b4c 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -94,6 +94,14 @@ def __init__(self, name, base, alias, bran, interact=False, auto=False, authenti self.auto = auto super(ConfirmDoer, self).__init__(doers=doers) + def _addAuthorizerSeal(self, pre, edig, anchorSn, anchorSaid): + """Save the authorizer (delegator) event seal of the anchoring IXN event for an approved delegation.""" + dgkey = dbing.dgKey(pre, edig) + seqner = coring.Seqner(sn=anchorSn) + saider = coring.Saider(qb64=anchorSaid) + couple = seqner.qb64b + saider.qb64b + self.hby.db.setAes(dgkey, couple) + def confirmDo(self, tymth, tock=0.0): """ Parameters: @@ -175,7 +183,9 @@ def confirmDo(self, tymth, tock=0.0): print(f"Delegate {eserder.pre} {typ} event committed.") - self.hby.db.delegables.rem(keys=(pre, sn), val=edig) + self._addAuthorizerSeal(pre, edig, anchorSn=serder.sn, + anchorSaid=serder.said) + self.hby.kvy.processEscrowDelegables() # removes DIP/DRT from delegables after adding it to kevers self.remove(self.toRemove) return True @@ -232,7 +242,9 @@ def confirmDo(self, tymth, tock=0.0): print(f"Delegate {eserder.pre} {typ} event committed.") - self.hby.db.delegables.rem(keys=(pre, sn), val=edig) + self._addAuthorizerSeal(pre, edig, anchorSn=hab.kever.sn, + anchorSaid=hab.kever.serder.said) + self.hby.kvy.processEscrowDelegables() # removes DIP/DRT from delegables after adding it to kevers self.remove(self.toRemove) return True From dee6d752ac80147a6e8c8aae916ba5894eeced32 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Thu, 8 Jan 2026 12:00:41 -0500 Subject: [PATCH 410/418] adds a clean and inspect command Signed-off-by: Kevin Griffin Signed-off-by: Kent Bull --- Makefile | 23 +- src/keri/app/cli/commands/witness/catchup.py | 169 +++++++++ src/keri/app/cli/commands/witness/clean.py | 45 +++ src/keri/app/cli/commands/witness/inspect.py | 351 +++++++++++++++++++ 4 files changed, 583 insertions(+), 5 deletions(-) create mode 100644 src/keri/app/cli/commands/witness/catchup.py create mode 100644 src/keri/app/cli/commands/witness/clean.py create mode 100644 src/keri/app/cli/commands/witness/inspect.py diff --git a/Makefile b/Makefile index 0d004dd19..45df11a24 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ -.PHONY: build-keri +.PHONY: build-keri build-witness-demo publish-keri-witness-demo publish-keri VERSION=1.2.8 +REGISTRY=weboftrust +IMAGE=keri +LATEST_TAG=$(REGISTRY)/$(IMAGE):latest +VERSIONED_TAG=$(REGISTRY)/$(IMAGE):$(VERSION) define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store @@ -12,19 +16,28 @@ To enable the feature for Docker Desktop: Select Apply and Restart." endef +.PHONY: build-keri build-keri: .warn - @docker build --platform=linux/amd64,linux/arm64 -f images/keripy.dockerfile -t weboftrust/keri:$(VERSION) . + @docker build \ + --platform=linux/amd64,linux/arm64 \ + -f images/keripy.dockerfile \ + -t $(VERSIONED_TAG) \ + -t $(LATEST_TAG) . .PHONY: build-witness-demo build-witness-demo: .warn - @docker build --platform=linux/amd64,linux/arm64 -f images/witness.demo.dockerfile -t weboftrust/keri-witness-demo:1.1.10 . + @docker build \ + --platform=linux/amd64,linux/arm64 \ + -f images/witness.demo.dockerfile \ + -t $(REGISTRY)/keri-witness-demo:$(VERSION) . .PHONY: publish-keri-witness-demo publish-keri-witness-demo: - @docker push weboftrust/keri-witness-demo --all-tags + @docker push $(REGISTRY)/keri-witness-demo --all-tags +.PHONY: publish-keri publish-keri: - @docker push weboftrust/keri:$(VERSION) + @docker push $(REGISTRY)/$(IMAGE) --all-tags .warn: @echo -e ${RED}"$$DOCKER_WARNING"${NO_COLOUR} diff --git a/src/keri/app/cli/commands/witness/catchup.py b/src/keri/app/cli/commands/witness/catchup.py new file mode 100644 index 000000000..ec858c948 --- /dev/null +++ b/src/keri/app/cli/commands/witness/catchup.py @@ -0,0 +1,169 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from keri import help +from hio.base import doing + +from keri.app import habbing, agenting, httping +from keri.app.cli.common import existing +from keri.core import serdering + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Send full KEL to a specific witness to catch it up') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the identifier prefix', required=True) + +# Authentication for keystore +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--witness', '-w', help='witness AID to send KEL to', required=True) +parser.add_argument('--force', '-f', help='send KEL even if witness is not in current witness pool', + action='store_true', default=False) +parser.add_argument('--verbose', '-V', help='show detailed progress for each event', + action='store_true', default=False) + + +def handler(args): + """ + Send full KEL to a specific witness to catch it up. + + Args: + args(Namespace): arguments object from command line + """ + + name = args.name + base = args.base + bran = args.bran + alias = args.alias + witness = args.witness + force = args.force + verbose = args.verbose + + doer = CatchupDoer(name=name, base=base, alias=alias, bran=bran, witness=witness, force=force, verbose=verbose) + + return [doer] + + +class CatchupDoer(doing.DoDoer): + """ DoDoer for sending full KEL to a witness to catch it up. + """ + + def __init__(self, name, base, alias, bran, witness, force=False, verbose=False): + + hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer + self.alias = alias + self.hby = hby + self.witness = witness + self.force = force + self.verbose = verbose + + doers = [self.hbyDoer, doing.doify(self.catchupDo)] + + super(CatchupDoer, self).__init__(doers=doers) + + def catchupDo(self, tymth, tock=0.0): + """ + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + """ + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + hab = self.hby.habByName(name=self.alias) + if hab is None: + print(f"Error: unknown alias {self.alias}") + self.remove([self.hbyDoer]) + return + + # Validate witness is in the current witness list + if self.witness not in hab.kever.wits: + if not self.force: + print(f"Error: {self.witness} is not a witness for {self.alias}") + print(f"Current witnesses: {hab.kever.wits}") + print("Use --force to send KEL anyway") + self.remove([self.hbyDoer]) + return + else: + print(f"Warning: {self.witness} is not a witness for {self.alias}, forcing anyway") + + if self.verbose: + # Use verbose catchup with progress reporting + yield from self.verboseCatchup(hab) + else: + receiptor = agenting.Receiptor(hby=self.hby) + self.extend([receiptor]) + print(f"Sending full KEL to witness {self.witness}...") + yield from receiptor.catchup(hab.pre, self.witness) + self.remove([receiptor]) + + print(f"KEL sent successfully. Witness should now be at sn={hab.kever.sn}") + + self.remove([self.hbyDoer]) + + return + + def verboseCatchup(self, hab): + """Verbose catchup with progress reporting for each event.""" + from keri.app.agenting import httpClient + + client, clientDoer = httpClient(hab, self.witness) + self.extend([clientDoer]) + + print(f"Sending full KEL to witness {self.witness}...") + print(f"Controller KEL has {hab.kever.sn + 1} events (sn 0 to {hab.kever.sn})") + print() + + event_count = 0 + success_count = 0 + error_count = 0 + + for fmsg in hab.db.clonePreIter(pre=hab.pre): + # Parse to get event info + try: + serder = serdering.SerderKERI(raw=fmsg) + sn = serder.sn + ilk = serder.ilk + except Exception: + sn = '?' + ilk = '?' + + event_count += 1 + print(f" Sending event sn={sn} ilk={ilk}...", end=" ", flush=True) + + httping.streamCESRRequests(client=client, dest=self.witness, ims=bytearray(fmsg)) + while not client.responses: + yield self.tock + + # Check response + response = client.responses.popleft() + status = response.get('status', 'unknown') + + if status in (200, 204): + print(f"OK (HTTP {status})") + success_count += 1 + else: + print(f"ERROR (HTTP {status})") + error_count += 1 + body = response.get('body', b'') + if body: + print(f" Response: {body[:200]}") + + print() + print(f"Summary: {event_count} events sent, {success_count} succeeded, {error_count} errors") + + self.remove([clientDoer]) diff --git a/src/keri/app/cli/commands/witness/clean.py b/src/keri/app/cli/commands/witness/clean.py new file mode 100644 index 000000000..a56bead4b --- /dev/null +++ b/src/keri/app/cli/commands/witness/clean.py @@ -0,0 +1,45 @@ +# -*- encoding: utf-8 -*- +""" +keri.app.cli.commands.witness.clean module + +""" +import argparse + +from hio.base import doing + +from keri.app.cli.common import existing + +parser = argparse.ArgumentParser(description='Clean and re-verify witness database (offline)') +parser.set_defaults(handler=lambda args: handler(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(clean, **kwa)] + + +def clean(tymth, tock=0.0, **opts): + """ Command line clean handler + + Cleans and re-verifies the database by creating a verified clone + and replacing the original. Must be run while witness is offline. + """ + _ = (yield tock) + + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + + print(f"Cleaning database for {name}...") + print("Database must be offline during this operation.") + + with existing.existingHby(name=name, base=base, bran=bran) as hby: + hby.db.clean() + + print(f"Database cleaned successfully.") diff --git a/src/keri/app/cli/commands/witness/inspect.py b/src/keri/app/cli/commands/witness/inspect.py new file mode 100644 index 000000000..098484299 --- /dev/null +++ b/src/keri/app/cli/commands/witness/inspect.py @@ -0,0 +1,351 @@ +# -*- encoding: utf-8 -*- +""" +keri.app.cli.commands.witness.inspect module + +Diagnostic command to inspect event databases and identify processing issues. +""" +import argparse +import json + +from hio.base import doing + +from keri.app.cli.common import existing +from keri.db import dbing +from keri.core import serdering + + +parser = argparse.ArgumentParser(description='Inspect event databases (.evts, .kels, .fels, .fons)') +parser.set_defaults(handler=lambda args: handler(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) +parser.add_argument('--prefix', '-pre', help='filter by identifier prefix', default=None) +parser.add_argument('--verbose', '-v', help='show detailed event information', action='store_true', default=False) +parser.add_argument('--aid', '-a', help='show detailed kever info for specific AID', default=None) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(inspect, **kwa)] + + +def inspect(tymth, tock=0.0, **opts): + """Command line inspect handler + + Inspects .evts, .kels, .fels, .fons databases and reports statistics + and potential issues that could cause event processing failures. + """ + _ = (yield tock) + + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + prefix = args.prefix + verbose = args.verbose + aid = args.aid + + with existing.existingHby(name=name, base=base, bran=bran) as hby: + db = hby.db + + # If specific AID requested, show detailed kever info + if aid: + print(f"\n=== Detailed inspection for AID: {aid} ===\n") + + if aid in db.kevers: + kever = db.kevers[aid] + print(f"Kever loaded: YES") + print(f" Sequence Number (sn): {kever.sner.num}") + print(f" Last Event SAID: {kever.serder.said}") + print(f" Delegated: {kever.delegated}") + if kever.delegated: + print(f" Delegator: {kever.delegator}") + print(f" Witnesses: {kever.wits}") + print(f" Witness Threshold (toad): {kever.toader.num}") + + # Check witness receipts for current event + dgkey = dbing.dgKey(kever.serder.preb, kever.serder.saidb) + wigs = db.getWigs(dgkey) + print(f" Witness Receipts for current event: {len(wigs)}") + + # Check if fully witnessed + fully_witnessed = len(wigs) >= kever.toader.num + print(f" Fully Witnessed: {fully_witnessed}") + + print(f"\n--- Event-by-event witness receipts ---") + # Show witness receipts for each event in KEL + for sn, dig in db.getKelItemPreIter(aid.encode('utf-8') if isinstance(aid, str) else aid): + dgkey = dbing.dgKey(aid.encode('utf-8') if isinstance(aid, str) else aid, dig) + wigs = db.getWigs(dgkey) + sigs = db.getSigs(dgkey) + + # Get event type + try: + raw = db.getEvt(dgkey) + if raw: + serder = serdering.SerderKERI(raw=bytes(raw)) + ilk = serder.ilk + else: + ilk = "?" + except Exception: + ilk = "?" + + status = "✓" if len(wigs) >= kever.toader.num else "✗" + print(f" sn={sn:3d} ilk={ilk:3s} sigs={len(sigs) if sigs else 0} wigs={len(wigs) if wigs else 0}/{kever.toader.num} {status}") + + else: + print(f"Kever loaded: NO") + print(f" AID {aid} not found in kevers") + + # Check if there are any events for this AID + kel_count = 0 + max_sn = -1 + for sn, dig in db.getKelItemPreIter(aid.encode('utf-8') if isinstance(aid, str) else aid): + kel_count += 1 + max_sn = max(max_sn, sn) + + if kel_count > 0: + print(f" But found {kel_count} events in KEL (max sn={max_sn})") + print(f" This suggests events exist but kever failed to load") + else: + print(f" No events found in KEL for this AID") + + # Check escrows for this specific AID + print(f"\n--- Escrows for this AID ---") + aid_bytes = aid.encode('utf-8') if isinstance(aid, str) else aid + + pse_count = 0 + for key, val in db.getPseItemsNextIter(): + if aid_bytes in bytes(key): + pse_count += 1 + print(f" Partial Signed Escrow (PSE): {pse_count}") + + pwe_count = 0 + for key, val in db.getPweItemIter(): + if aid_bytes in bytes(key): + pwe_count += 1 + print(f" Partial Witnessed Escrow (PWE): {pwe_count}") + + ooe_count = 0 + for key, val in db.getOoeItemIter(): + if aid_bytes in bytes(key): + ooe_count += 1 + print(f" Out of Order Escrow (OOE): {ooe_count}") + + lde_count = 0 + for key, val in db.getLdeItemIter(): + if aid_bytes in bytes(key): + lde_count += 1 + print(f" Likely Duplicitous Escrow (LDE): {lde_count}") + + print() + return + + report = { + "summary": {}, + "prefixes": {}, + "issues": [] + } + + # Count total events in .evts + evts_count = 0 + evts_by_pre = {} + for key, val in db.getTopItemIter(db.evts): + evts_count += 1 + # Extract prefix from dgKey (pre.dig format) + key_str = bytes(key).decode('utf-8') + if '.' in key_str: + pre = key_str.rsplit('.', 1)[0] + if prefix and pre != prefix: + continue + evts_by_pre[pre] = evts_by_pre.get(pre, 0) + 1 + + report["summary"]["evts_total"] = evts_count + + # Count entries in .kels (key event log) + kels_count = 0 + kels_by_pre = {} + for key, val in db.getTopItemIter(db.kels): + kels_count += 1 + key_str = bytes(key).decode('utf-8') + if '.' in key_str: + pre = key_str.rsplit('.', 1)[0] + if prefix and pre != prefix: + continue + kels_by_pre[pre] = kels_by_pre.get(pre, 0) + 1 + + report["summary"]["kels_total"] = kels_count + + # Count entries in .fels (first seen event log) + fels_count = 0 + fels_by_pre = {} + for key, val in db.getTopItemIter(db.fels): + fels_count += 1 + key_str = bytes(key).decode('utf-8') + if '.' in key_str: + pre = key_str.rsplit('.', 1)[0] + if prefix and pre != prefix: + continue + fels_by_pre[pre] = fels_by_pre.get(pre, 0) + 1 + + report["summary"]["fels_total"] = fels_count + + # Count entries in .fons (first seen ordinal number index) + fons_count = 0 + fons_by_pre = {} + for keys, val in db.fons.getItemIter(): + fons_count += 1 + if keys: + pre = keys[0].rsplit('.', 1)[0] if '.' in keys[0] else keys[0] + if prefix and pre != prefix: + continue + fons_by_pre[pre] = fons_by_pre.get(pre, 0) + 1 + + report["summary"]["fons_total"] = fons_count + + # Get all unique prefixes + all_prefixes = set(evts_by_pre.keys()) | set(kels_by_pre.keys()) | set(fels_by_pre.keys()) | set(fons_by_pre.keys()) + + if prefix: + all_prefixes = {prefix} if prefix in all_prefixes else set() + + # Analyze each prefix + for pre in sorted(all_prefixes): + pre_report = { + "evts": evts_by_pre.get(pre, 0), + "kels": kels_by_pre.get(pre, 0), + "fels": fels_by_pre.get(pre, 0), + "fons": fons_by_pre.get(pre, 0), + } + + # Check for issues + issues = [] + + # Events in .evts but not in .fels (not first-seen logged) + if pre_report["evts"] > 0 and pre_report["fels"] == 0: + issues.append("Events exist but none in first-seen log (.fels)") + + # Events in .evts but not in .kels (not in key event log) + if pre_report["evts"] > 0 and pre_report["kels"] == 0: + issues.append("Events exist but none in key event log (.kels)") + + # Mismatch between .fels and .fons + if pre_report["fels"] != pre_report["fons"]: + issues.append(f"Mismatch: .fels={pre_report['fels']} vs .fons={pre_report['fons']}") + + # More events than KEL entries could indicate duplicates or escrow + if pre_report["evts"] > pre_report["kels"] and pre_report["kels"] > 0: + diff = pre_report["evts"] - pre_report["kels"] + issues.append(f"More events ({pre_report['evts']}) than KEL entries ({pre_report['kels']}), {diff} may be escrowed/duplicitous") + + if issues: + pre_report["issues"] = issues + for issue in issues: + report["issues"].append(f"{pre[:16]}...: {issue}") + + report["prefixes"][pre] = pre_report + + # Verbose: show KEL sequence details + if verbose: + kel_details = [] + max_sn = -1 + sn_gaps = [] + prev_sn = -1 + + for sn, dig in db.getKelItemPreIter(pre.encode('utf-8') if isinstance(pre, str) else pre): + if prev_sn >= 0 and sn != prev_sn + 1: + sn_gaps.append((prev_sn, sn)) + prev_sn = sn + max_sn = max(max_sn, sn) + if len(kel_details) < 10: # Limit to first 10 + dig_str = bytes(dig).decode('utf-8') if isinstance(dig, (bytes, memoryview)) else str(dig) + kel_details.append({"sn": sn, "dig": dig_str[:24] + "..."}) + + pre_report["max_sn"] = max_sn + if sn_gaps: + pre_report["sn_gaps"] = sn_gaps + report["issues"].append(f"{pre[:16]}...: Sequence gaps found: {sn_gaps}") + if verbose and kel_details: + pre_report["kel_sample"] = kel_details + + # Check for FEL details + fel_details = [] + max_fn = -1 + for fn, dig in db.getFelItemPreIter(pre.encode('utf-8') if isinstance(pre, str) else pre): + max_fn = max(max_fn, fn) + if len(fel_details) < 10: + dig_str = bytes(dig).decode('utf-8') if isinstance(dig, (bytes, memoryview)) else str(dig) + fel_details.append({"fn": fn, "dig": dig_str[:24] + "..."}) + + pre_report["max_fn"] = max_fn + if verbose and fel_details: + pre_report["fel_sample"] = fel_details + + # Add kever information + report["summary"]["kevers_loaded"] = len(db.kevers) + report["summary"]["prefixes_count"] = len(all_prefixes) + + # Check escrows + escrows = { + "pse": 0, # Partial Signed Escrow + "pwe": 0, # Partial Witnessed Escrow + "ooe": 0, # Out of Order Escrow + "lde": 0, # Likely Duplicitous Escrow + } + + # Count PSE entries + for key, val in db.getPseItemsNextIter(): + escrows["pse"] += 1 + if verbose: + key_str = bytes(key).decode('utf-8') if isinstance(key, (bytes, memoryview)) else str(key) + report["issues"].append(f"PSE escrow: {key_str[:40]}...") + + # Count PWE entries + for key, val in db.getPweItemIter(): + escrows["pwe"] += 1 + if verbose: + key_str = bytes(key).decode('utf-8') if isinstance(key, (bytes, memoryview)) else str(key) + report["issues"].append(f"PWE escrow: {key_str[:40]}...") + + # Count OOE entries + for key, val in db.getOoeItemIter(): + escrows["ooe"] += 1 + if verbose: + key_str = bytes(key).decode('utf-8') if isinstance(key, (bytes, memoryview)) else str(key) + report["issues"].append(f"OOE escrow: {key_str[:40]}...") + + # Count LDE entries + for key, val in db.getLdeItemIter(): + escrows["lde"] += 1 + if verbose: + key_str = bytes(key).decode('utf-8') if isinstance(key, (bytes, memoryview)) else str(key) + report["issues"].append(f"LDE escrow: {key_str[:40]}...") + + report["summary"]["escrows"] = escrows + if any(v > 0 for v in escrows.values()): + report["issues"].append(f"Escrows found: PSE={escrows['pse']}, PWE={escrows['pwe']}, OOE={escrows['ooe']}, LDE={escrows['lde']}") + + # Check for orphaned events (in .evts but not referenced) + if verbose: + # Get all digests from .kels and .fels + referenced_digs = set() + for key, val in db.getTopItemIter(db.kels): + referenced_digs.add(bytes(val).decode('utf-8')) + for key, val in db.getTopItemIter(db.fels): + referenced_digs.add(bytes(val).decode('utf-8')) + + orphaned = 0 + for key, val in db.getTopItemIter(db.evts): + key_str = bytes(key).decode('utf-8') + if '.' in key_str: + dig = key_str.rsplit('.', 1)[1] + if dig not in referenced_digs: + orphaned += 1 + + if orphaned > 0: + report["summary"]["orphaned_events"] = orphaned + report["issues"].append(f"Found {orphaned} orphaned events (in .evts but not referenced in .kels/.fels)") + + print(json.dumps(report, indent=2)) From 5687399872928402c2d1373491a9180e53aec5bc Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Mon, 1 Dec 2025 13:52:32 -0500 Subject: [PATCH 411/418] export escrow metrics Signed-off-by: Kevin Griffin Signed-off-by: Kent Bull --- src/keri/app/indirecting.py | 92 +++++++++++++++++++++++++++++++++-- tests/app/test_indirecting.py | 3 +- 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 461c93b4e..17caa8d4a 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -31,7 +31,6 @@ from ..peer import exchanging from ..vdr import verifying, viring from ..vdr.eventing import Tevery -from ..metric import EscrowEnd logger = help.ogler.getLogger() @@ -91,7 +90,7 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo app.add_route("/receipts", receiptEnd) queryEnd = QueryEnd(hab=hab) app.add_route("/query", queryEnd) - metricsEnd = EscrowEnd(hby=hby, reger=reger) + metricsEnd = MetricsEnd(hby=hby, reger=reger) app.add_route("/metrics", metricsEnd) server = createHttpServer(host, httpPort, app, keypath, certpath, cafilepath) @@ -1286,4 +1285,91 @@ def on_get(self, req, rep): else: rep.set_header('Content-Type', "application/json") rep.text = "unkown query type." - rep.status = falcon.HTTP_400 \ No newline at end of file + rep.status = falcon.HTTP_400 + + +class MetricsEnd: + """Prometheus metrics endpoint for escrow counts. + + Exposes escrow counts in Prometheus text format for monitoring/alerting. + """ + + def __init__(self, hby, reger): + """Initialize MetricsEnd. + + Parameters: + hby (Habery): Habery instance for accessing KEL escrows + reger (Reger): Registry for accessing TEL escrows + """ + self.hby = hby + self.reger = reger + + def on_get(self, req, rep): + """GET /metrics - Returns Prometheus format metrics. + + Parameters: + req (Request): Falcon HTTP request object + rep (Response): Falcon HTTP response object + """ + lines = [] + + # Header + lines.append("# HELP keri_escrow_count Number of items in each escrow type") + lines.append("# TYPE keri_escrow_count gauge") + + # KEL / Baser escrows + escrow_counts = [ + ("unverified_receipts", sum(1 for _ in self.hby.db.getUreItemIter())), + ("verified_receipts", sum(1 for _ in self.hby.db.getVreItemIter())), + ("out_of_order_events", sum(1 for _ in self.hby.db.getOoeItemIter())), + ("partially_witnessed_events", sum(1 for _ in self.hby.db.getPweItemIter())), + ("partially_signed_events", sum(1 for _ in self.hby.db.getPseItemIter())), + ("likely_duplicitous_events", sum(1 for _ in self.hby.db.getLdeItemIter())), + ("unverified_event_indexed_couples", sum(1 for _ in self.hby.db.getUweItemIter())), + ("query_not_found", sum(1 for _ in self.hby.db.qnfs.getItemIter())), + ("partially_delegated_events", sum(1 for _ in self.hby.db.pdes.getItemIter())), + ("reply", sum(1 for _ in self.hby.db.rpes.getItemIter())), + ("failed_oobi", sum(1 for _ in self.hby.db.eoobi.getItemIter())), + ("group_partial_witness", sum(1 for _ in self.hby.db.gpwe.getItemIter())), + ("group_delegate", sum(1 for _ in self.hby.db.gdee.getItemIter())), + ("delegated_partial_witness", sum(1 for _ in self.hby.db.dpwe.getItemIter())), + ("group_partial_signed", sum(1 for _ in self.hby.db.gpse.getItemIter())), + ("exchange_partial_signed", sum(1 for _ in self.hby.db.epse.getItemIter())), + ("delegated_unanchored", sum(1 for _ in self.hby.db.dune.getItemIter())), + ] + + for name, count in escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="kel"}} {count}') + + # TEL / Reger escrows + tel_escrow_counts = [ + ("out_of_order", sum(1 for _ in self.reger.getOotItemIter())), + ("partially_witnessed", sum(1 for _ in self.reger.getTweItemIter())), + ("anchorless", sum(1 for _ in self.reger.getTaeItemIter())), + ("missing_registry", sum(1 for _ in self.reger.mre.getItemIter())), + ("broken_chain", sum(1 for _ in self.reger.mce.getItemIter())), + ("missing_schema", sum(1 for _ in self.reger.mse.getItemIter())), + ("missing_signature", sum(1 for _ in self.reger.cmse.getItemIter())), + ("partial_witness", sum(1 for _ in self.reger.tpwe.getItemIter())), + ("multisig", sum(1 for _ in self.reger.tmse.getItemIter())), + ("event_dissemination", sum(1 for _ in self.reger.tede.getItemIter())), + ] + + for name, count in tel_escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="tel"}} {count}') + + # Registry transaction escrows + registry_escrow_counts = [ + ("registry_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-mae", "")))), + ("registry_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-ooo", "")))), + ("credential_missing_registry", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mre", "")))), + ("credential_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mae", "")))), + ("credential_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-ooo", "")))), + ] + + for name, count in registry_escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="registry"}} {count}') + + rep.content_type = "text/plain; version=0.0.4; charset=utf-8" + rep.status = falcon.HTTP_200 + rep.text = "\n".join(lines) + "\n" \ No newline at end of file diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index c70fa18ca..9b7e14251 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -18,7 +18,6 @@ from keri import kering from keri import core from keri.app import indirecting, storing, habbing, agenting -from keri.metric import EscrowEnd from keri.vdr import viring @@ -371,7 +370,7 @@ def test_metrics_end(): reger = viring.Reger(name=hby.name, db=hby.db, temp=True) app = falcon.App() - metricsEnd = EscrowEnd(hby=hby, reger=reger) + metricsEnd = indirecting.MetricsEnd(hby=hby, reger=reger) app.add_route("/metrics", metricsEnd) client = testing.TestClient(app) From 71ae0e84e4b1c24bd456272a13651c4ed705d032 Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Tue, 2 Dec 2025 13:10:07 -0500 Subject: [PATCH 412/418] repackaging Signed-off-by: Kevin Griffin --- src/keri/app/indirecting.py | 92 ++--------------------------------- tests/app/test_indirecting.py | 2 +- 2 files changed, 4 insertions(+), 90 deletions(-) diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 17caa8d4a..461c93b4e 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -31,6 +31,7 @@ from ..peer import exchanging from ..vdr import verifying, viring from ..vdr.eventing import Tevery +from ..metric import EscrowEnd logger = help.ogler.getLogger() @@ -90,7 +91,7 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo app.add_route("/receipts", receiptEnd) queryEnd = QueryEnd(hab=hab) app.add_route("/query", queryEnd) - metricsEnd = MetricsEnd(hby=hby, reger=reger) + metricsEnd = EscrowEnd(hby=hby, reger=reger) app.add_route("/metrics", metricsEnd) server = createHttpServer(host, httpPort, app, keypath, certpath, cafilepath) @@ -1285,91 +1286,4 @@ def on_get(self, req, rep): else: rep.set_header('Content-Type', "application/json") rep.text = "unkown query type." - rep.status = falcon.HTTP_400 - - -class MetricsEnd: - """Prometheus metrics endpoint for escrow counts. - - Exposes escrow counts in Prometheus text format for monitoring/alerting. - """ - - def __init__(self, hby, reger): - """Initialize MetricsEnd. - - Parameters: - hby (Habery): Habery instance for accessing KEL escrows - reger (Reger): Registry for accessing TEL escrows - """ - self.hby = hby - self.reger = reger - - def on_get(self, req, rep): - """GET /metrics - Returns Prometheus format metrics. - - Parameters: - req (Request): Falcon HTTP request object - rep (Response): Falcon HTTP response object - """ - lines = [] - - # Header - lines.append("# HELP keri_escrow_count Number of items in each escrow type") - lines.append("# TYPE keri_escrow_count gauge") - - # KEL / Baser escrows - escrow_counts = [ - ("unverified_receipts", sum(1 for _ in self.hby.db.getUreItemIter())), - ("verified_receipts", sum(1 for _ in self.hby.db.getVreItemIter())), - ("out_of_order_events", sum(1 for _ in self.hby.db.getOoeItemIter())), - ("partially_witnessed_events", sum(1 for _ in self.hby.db.getPweItemIter())), - ("partially_signed_events", sum(1 for _ in self.hby.db.getPseItemIter())), - ("likely_duplicitous_events", sum(1 for _ in self.hby.db.getLdeItemIter())), - ("unverified_event_indexed_couples", sum(1 for _ in self.hby.db.getUweItemIter())), - ("query_not_found", sum(1 for _ in self.hby.db.qnfs.getItemIter())), - ("partially_delegated_events", sum(1 for _ in self.hby.db.pdes.getItemIter())), - ("reply", sum(1 for _ in self.hby.db.rpes.getItemIter())), - ("failed_oobi", sum(1 for _ in self.hby.db.eoobi.getItemIter())), - ("group_partial_witness", sum(1 for _ in self.hby.db.gpwe.getItemIter())), - ("group_delegate", sum(1 for _ in self.hby.db.gdee.getItemIter())), - ("delegated_partial_witness", sum(1 for _ in self.hby.db.dpwe.getItemIter())), - ("group_partial_signed", sum(1 for _ in self.hby.db.gpse.getItemIter())), - ("exchange_partial_signed", sum(1 for _ in self.hby.db.epse.getItemIter())), - ("delegated_unanchored", sum(1 for _ in self.hby.db.dune.getItemIter())), - ] - - for name, count in escrow_counts: - lines.append(f'keri_escrow_count{{type="{name}",layer="kel"}} {count}') - - # TEL / Reger escrows - tel_escrow_counts = [ - ("out_of_order", sum(1 for _ in self.reger.getOotItemIter())), - ("partially_witnessed", sum(1 for _ in self.reger.getTweItemIter())), - ("anchorless", sum(1 for _ in self.reger.getTaeItemIter())), - ("missing_registry", sum(1 for _ in self.reger.mre.getItemIter())), - ("broken_chain", sum(1 for _ in self.reger.mce.getItemIter())), - ("missing_schema", sum(1 for _ in self.reger.mse.getItemIter())), - ("missing_signature", sum(1 for _ in self.reger.cmse.getItemIter())), - ("partial_witness", sum(1 for _ in self.reger.tpwe.getItemIter())), - ("multisig", sum(1 for _ in self.reger.tmse.getItemIter())), - ("event_dissemination", sum(1 for _ in self.reger.tede.getItemIter())), - ] - - for name, count in tel_escrow_counts: - lines.append(f'keri_escrow_count{{type="{name}",layer="tel"}} {count}') - - # Registry transaction escrows - registry_escrow_counts = [ - ("registry_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-mae", "")))), - ("registry_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-ooo", "")))), - ("credential_missing_registry", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mre", "")))), - ("credential_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mae", "")))), - ("credential_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-ooo", "")))), - ] - - for name, count in registry_escrow_counts: - lines.append(f'keri_escrow_count{{type="{name}",layer="registry"}} {count}') - - rep.content_type = "text/plain; version=0.0.4; charset=utf-8" - rep.status = falcon.HTTP_200 - rep.text = "\n".join(lines) + "\n" \ No newline at end of file + rep.status = falcon.HTTP_400 \ No newline at end of file diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index 9b7e14251..53bf7211e 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -370,7 +370,7 @@ def test_metrics_end(): reger = viring.Reger(name=hby.name, db=hby.db, temp=True) app = falcon.App() - metricsEnd = indirecting.MetricsEnd(hby=hby, reger=reger) + metricsEnd = indirecting.EscrowEnd(hby=hby, reger=reger) app.add_route("/metrics", metricsEnd) client = testing.TestClient(app) From bb330477971580f7299901b1edc472ba2709442a Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Tue, 2 Dec 2025 13:46:49 -0500 Subject: [PATCH 413/418] repackaging Signed-off-by: Kevin Griffin --- tests/app/test_indirecting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index 53bf7211e..c70fa18ca 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -18,6 +18,7 @@ from keri import kering from keri import core from keri.app import indirecting, storing, habbing, agenting +from keri.metric import EscrowEnd from keri.vdr import viring @@ -370,7 +371,7 @@ def test_metrics_end(): reger = viring.Reger(name=hby.name, db=hby.db, temp=True) app = falcon.App() - metricsEnd = indirecting.EscrowEnd(hby=hby, reger=reger) + metricsEnd = EscrowEnd(hby=hby, reger=reger) app.add_route("/metrics", metricsEnd) client = testing.TestClient(app) From 6e162bfd229ea977f075e785b0bca9ac038cf6fd Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Wed, 25 Feb 2026 08:10:04 -0700 Subject: [PATCH 414/418] support new temp arg for HIO doified functions --- src/keri/app/cli/commands/challenge/verify.py | 2 +- src/keri/app/cli/commands/contacts/add.py | 2 +- src/keri/app/cli/commands/contacts/query.py | 2 +- src/keri/app/cli/commands/delegate/confirm.py | 2 +- src/keri/app/cli/commands/delegate/request.py | 2 +- src/keri/app/cli/commands/ends/add.py | 2 +- src/keri/app/cli/commands/ends/export.py | 2 +- src/keri/app/cli/commands/ends/list.py | 2 +- src/keri/app/cli/commands/export.py | 2 +- src/keri/app/cli/commands/ipex/admit.py | 2 +- src/keri/app/cli/commands/ipex/agree.py | 2 +- src/keri/app/cli/commands/ipex/apply.py | 2 +- src/keri/app/cli/commands/ipex/grant.py | 2 +- src/keri/app/cli/commands/ipex/join.py | 2 +- src/keri/app/cli/commands/ipex/list.py | 2 +- src/keri/app/cli/commands/ipex/offer.py | 2 +- src/keri/app/cli/commands/ipex/spurn.py | 2 +- src/keri/app/cli/commands/location/add.py | 2 +- src/keri/app/cli/commands/mailbox/add.py | 2 +- src/keri/app/cli/commands/mailbox/debug.py | 2 +- src/keri/app/cli/commands/multisig/interact.py | 2 +- src/keri/app/cli/commands/multisig/join.py | 2 +- src/keri/app/cli/commands/multisig/notice.py | 2 +- src/keri/app/cli/commands/nonce.py | 2 +- src/keri/app/cli/commands/notifications/list.py | 2 +- src/keri/app/cli/commands/notifications/mark.py | 2 +- src/keri/app/cli/commands/notifications/rem.py | 2 +- src/keri/app/cli/commands/oobi/resolve.py | 2 +- src/keri/app/cli/commands/passcode/generate.py | 2 +- src/keri/app/cli/commands/salt.py | 2 +- src/keri/app/cli/commands/time.py | 2 +- src/keri/app/cli/commands/vc/create.py | 2 +- src/keri/app/cli/commands/vc/export.py | 2 +- src/keri/app/cli/commands/vc/list.py | 2 +- src/keri/app/cli/commands/vc/registry/status.py | 2 +- src/keri/app/cli/commands/vc/revoke.py | 2 +- src/keri/app/cli/commands/watcher/add.py | 2 +- src/keri/app/cli/commands/witness/authenticate.py | 2 +- src/keri/app/cli/commands/witness/catchup.py | 2 +- src/keri/app/cli/commands/witness/demo.py | 2 +- src/keri/app/cli/commands/witness/submit.py | 2 +- src/keri/vc/walleting.py | 2 +- tests/conftest.py | 2 +- 43 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/keri/app/cli/commands/challenge/verify.py b/src/keri/app/cli/commands/challenge/verify.py index 09f0dd5d9..eed9d285b 100644 --- a/src/keri/app/cli/commands/challenge/verify.py +++ b/src/keri/app/cli/commands/challenge/verify.py @@ -88,7 +88,7 @@ def __init__(self, name, alias, base, bran, words, generate, strength, out, sign super(VerifyDoer, self).__init__(doers=doers) - def verifyDo(self, tymth, tock=0.0): + def verifyDo(self, tymth, tock=0.0, **kwa): """ Check for any credential messages in mailboxes and list all held credentials Parameters: diff --git a/src/keri/app/cli/commands/contacts/add.py b/src/keri/app/cli/commands/contacts/add.py index 431d71798..aca6fb08d 100644 --- a/src/keri/app/cli/commands/contacts/add.py +++ b/src/keri/app/cli/commands/contacts/add.py @@ -63,7 +63,7 @@ def __init__(self, name, base, bran, oobi, alias, fields): doers = [self.hbyDoer, doing.doify(self.add)] super(ContactAddDoer, self).__init__(doers=doers) - def add(self, tymth, tock=0.0): + def add(self, tymth, tock=0.0, **kwa): """ Resolves OOBI and creates contact Parameters: diff --git a/src/keri/app/cli/commands/contacts/query.py b/src/keri/app/cli/commands/contacts/query.py index 96edbb603..c5ae345ed 100644 --- a/src/keri/app/cli/commands/contacts/query.py +++ b/src/keri/app/cli/commands/contacts/query.py @@ -71,7 +71,7 @@ def __init__(self, name, base, bran, alias, contact_aid, contact_alias): doers = [self.hbyDoer, self.mbd, doing.doify(self.queryDo)] super(ContactQueryDoer, self).__init__(doers=doers) - def queryDo(self, tymth, tock=0.0): + def queryDo(self, tymth, tock=0.0, **kwa): """ Query witnesses for contact key state Parameters: diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index 84a335b4c..034ae24a2 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -102,7 +102,7 @@ def _addAuthorizerSeal(self, pre, edig, anchorSn, anchorSaid): couple = seqner.qb64b + saider.qb64b self.hby.db.setAes(dgkey, couple) - def confirmDo(self, tymth, tock=0.0): + def confirmDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/delegate/request.py b/src/keri/app/cli/commands/delegate/request.py index 80e152feb..560d6014f 100644 --- a/src/keri/app/cli/commands/delegate/request.py +++ b/src/keri/app/cli/commands/delegate/request.py @@ -62,7 +62,7 @@ def __init__(self, name, base, alias, bran): super(RequestDoer, self).__init__(doers=doers) - def requestDo(self, tymth, tock=0.0): + def requestDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/ends/add.py b/src/keri/app/cli/commands/ends/add.py index 08a0e7fd4..90a1aa11c 100644 --- a/src/keri/app/cli/commands/ends/add.py +++ b/src/keri/app/cli/commands/ends/add.py @@ -75,7 +75,7 @@ def __init__(self, name, base, alias, bran, role, eid, timestamp=None): super(RoleDoer, self).__init__(doers=self.toRemove + [doing.doify(self.roleDo)]) - def roleDo(self, tymth, tock=0.0): + def roleDo(self, tymth, tock=0.0, **kwa): """ Export any end reply messages previous saved for the provided AID Parameters: diff --git a/src/keri/app/cli/commands/ends/export.py b/src/keri/app/cli/commands/ends/export.py index 1ac939a90..415842378 100644 --- a/src/keri/app/cli/commands/ends/export.py +++ b/src/keri/app/cli/commands/ends/export.py @@ -50,7 +50,7 @@ def __init__(self, name, base, bran, aid): super(ExportDoer, self).__init__(doers=doers) - def exportDo(self, tymth, tock=0.0): + def exportDo(self, tymth, tock=0.0, **kwa): """ Export any end reply messages previous saved for the provided AID Parameters: diff --git a/src/keri/app/cli/commands/ends/list.py b/src/keri/app/cli/commands/ends/list.py index f03f357ec..0cdf8da29 100644 --- a/src/keri/app/cli/commands/ends/list.py +++ b/src/keri/app/cli/commands/ends/list.py @@ -55,7 +55,7 @@ def __init__(self, name, base, alias, bran, aid): super(RoleDoer, self).__init__(doers=doers) - def roleDo(self, tymth, tock=0.0): + def roleDo(self, tymth, tock=0.0, **kwa): """ Export any end reply messages previous saved for the provided AID Parameters: diff --git a/src/keri/app/cli/commands/export.py b/src/keri/app/cli/commands/export.py index e566b98ad..3080814cc 100644 --- a/src/keri/app/cli/commands/export.py +++ b/src/keri/app/cli/commands/export.py @@ -57,7 +57,7 @@ def __init__(self, name, alias, base, bran, ends, files): super(ExportDoer, self).__init__(doers=doers) - def exportDo(self, tymth, tock=0.0): + def exportDo(self, tymth, tock=0.0, **kwa): """ Export credential from store and any related material Parameters: diff --git a/src/keri/app/cli/commands/ipex/admit.py b/src/keri/app/cli/commands/ipex/admit.py index 0c0c1ded6..64f496185 100644 --- a/src/keri/app/cli/commands/ipex/admit.py +++ b/src/keri/app/cli/commands/ipex/admit.py @@ -77,7 +77,7 @@ def __init__(self, name, alias, base, bran, said, message, timestamp ): self.toRemove = [mbx, self.witq] super(AdmitDoer, self).__init__(doers=self.toRemove + [doing.doify(self.admitDo)]) - def admitDo(self, tymth, tock=0.0): + def admitDo(self, tymth, tock=0.0, **kwa): """ Admit credential by accepting into database and sending /ipex/admit exn message Parameters: diff --git a/src/keri/app/cli/commands/ipex/agree.py b/src/keri/app/cli/commands/ipex/agree.py index 34eea242f..930ce306b 100644 --- a/src/keri/app/cli/commands/ipex/agree.py +++ b/src/keri/app/cli/commands/ipex/agree.py @@ -18,7 +18,7 @@ def handler(_): return [doing.doify(nonce)] -def nonce(tymth, tock=0.0): +def nonce(tymth, tock=0.0, **kwa): """ nonce """ _ = (yield tock) diff --git a/src/keri/app/cli/commands/ipex/apply.py b/src/keri/app/cli/commands/ipex/apply.py index 47d15fd48..3755b01de 100644 --- a/src/keri/app/cli/commands/ipex/apply.py +++ b/src/keri/app/cli/commands/ipex/apply.py @@ -17,7 +17,7 @@ def handler(_): return [doing.doify(nonce)] -def nonce(tymth, tock=0.0): +def nonce(tymth, tock=0.0, **kwa): """ nonce """ _ = (yield tock) diff --git a/src/keri/app/cli/commands/ipex/grant.py b/src/keri/app/cli/commands/ipex/grant.py index 5787089f6..74fdb68b3 100644 --- a/src/keri/app/cli/commands/ipex/grant.py +++ b/src/keri/app/cli/commands/ipex/grant.py @@ -71,7 +71,7 @@ def __init__(self, name, alias, base, bran, said, recp, message, timestamp): self.toRemove = [mbx] super(GrantDoer, self).__init__(doers=self.toRemove + [doing.doify(self.grantDo)]) - def grantDo(self, tymth, tock=0.0): + def grantDo(self, tymth, tock=0.0, **kwa): """ Grant credential by creating /ipex/grant exn message Parameters: diff --git a/src/keri/app/cli/commands/ipex/join.py b/src/keri/app/cli/commands/ipex/join.py index 897d9287d..122142145 100644 --- a/src/keri/app/cli/commands/ipex/join.py +++ b/src/keri/app/cli/commands/ipex/join.py @@ -90,7 +90,7 @@ def __init__(self, name, base, bran, auto=False): self.auto = auto super(JoinDoer, self).__init__(doers=doers) - def joinDo(self, tymth, tock=0.0): + def joinDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/ipex/list.py b/src/keri/app/cli/commands/ipex/list.py index 6d01fb2ba..156684943 100644 --- a/src/keri/app/cli/commands/ipex/list.py +++ b/src/keri/app/cli/commands/ipex/list.py @@ -90,7 +90,7 @@ def __init__(self, name, alias, base, bran, poll=False, verbose=False, said=Fals super(ListDoer, self).__init__(doers=self.doers + [doing.doify(self.listDo)]) - def listDo(self, tymth, tock=0.0): + def listDo(self, tymth, tock=0.0, **kwa): """ Check for any credential messages in mailboxes and list all held credentials Parameters: diff --git a/src/keri/app/cli/commands/ipex/offer.py b/src/keri/app/cli/commands/ipex/offer.py index ea269d40c..cf98ff1bb 100644 --- a/src/keri/app/cli/commands/ipex/offer.py +++ b/src/keri/app/cli/commands/ipex/offer.py @@ -18,7 +18,7 @@ def handler(_): return [doing.doify(nonce)] -def nonce(tymth, tock=0.0): +def nonce(tymth, tock=0.0, **kwa): """ nonce """ _ = (yield tock) diff --git a/src/keri/app/cli/commands/ipex/spurn.py b/src/keri/app/cli/commands/ipex/spurn.py index 7af44eacb..573b99d3b 100644 --- a/src/keri/app/cli/commands/ipex/spurn.py +++ b/src/keri/app/cli/commands/ipex/spurn.py @@ -74,7 +74,7 @@ def __init__(self, name, alias, base, bran, said, message): self.toRemove = [mbx] super(SpurnDoer, self).__init__(doers=self.toRemove + [doing.doify(self.spurnDo)]) - def spurnDo(self, tymth, tock=0.0): + def spurnDo(self, tymth, tock=0.0, **kwa): """ Sprun any IPEX message Parameters: diff --git a/src/keri/app/cli/commands/location/add.py b/src/keri/app/cli/commands/location/add.py index 344ef3c2d..52c1e173f 100644 --- a/src/keri/app/cli/commands/location/add.py +++ b/src/keri/app/cli/commands/location/add.py @@ -75,7 +75,7 @@ def __init__(self, name, base, alias, bran, url, eid, timestamp=None): super(LocationDoer, self).__init__(doers=self.toRemove + [doing.doify(self.roleDo)]) - def roleDo(self, tymth, tock=0.0): + def roleDo(self, tymth, tock=0.0, **kwa): """ Export any end reply messages previous saved for the provided AID Parameters: diff --git a/src/keri/app/cli/commands/mailbox/add.py b/src/keri/app/cli/commands/mailbox/add.py index 46f7df4e3..543ab2e08 100644 --- a/src/keri/app/cli/commands/mailbox/add.py +++ b/src/keri/app/cli/commands/mailbox/add.py @@ -72,7 +72,7 @@ def __init__(self, name, alias, base, bran, mailbox): super(AddDoer, self).__init__(doers=doers) - def addDo(self, tymth, tock=0.0): + def addDo(self, tymth, tock=0.0, **kwa): """ Grant credential by creating /ipex/grant exn message Parameters: diff --git a/src/keri/app/cli/commands/mailbox/debug.py b/src/keri/app/cli/commands/mailbox/debug.py index f11a53035..fcac56240 100644 --- a/src/keri/app/cli/commands/mailbox/debug.py +++ b/src/keri/app/cli/commands/mailbox/debug.py @@ -69,7 +69,7 @@ def __init__(self, name, base, alias, bran, witness, verbose): super(ReadDoer, self).__init__(doers=doers) - def readDo(self, tymth, tock=0.0): + def readDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/multisig/interact.py b/src/keri/app/cli/commands/multisig/interact.py index 0f5a9345e..3892a79e9 100644 --- a/src/keri/app/cli/commands/multisig/interact.py +++ b/src/keri/app/cli/commands/multisig/interact.py @@ -92,7 +92,7 @@ def __init__(self, name, alias, aids, base, bran, data): super(GroupMultisigInteract, self).__init__(doers=doers) - def interactDo(self, tymth, tock=0.0): + def interactDo(self, tymth, tock=0.0, **kwa): """ Create or participate in an interaction event for a distributed multisig identifier Parameters: diff --git a/src/keri/app/cli/commands/multisig/join.py b/src/keri/app/cli/commands/multisig/join.py index d58d78652..6b62bc57a 100644 --- a/src/keri/app/cli/commands/multisig/join.py +++ b/src/keri/app/cli/commands/multisig/join.py @@ -97,7 +97,7 @@ def __init__(self, name, base, bran, group, auto=False): self.auto = auto super(JoinDoer, self).__init__(doers=doers) - def joinDo(self, tymth, tock=0.0): + def joinDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/multisig/notice.py b/src/keri/app/cli/commands/multisig/notice.py index 88fb8b67e..223f4da75 100644 --- a/src/keri/app/cli/commands/multisig/notice.py +++ b/src/keri/app/cli/commands/multisig/notice.py @@ -66,7 +66,7 @@ def __init__(self, name, base, alias, bran): super(NoticeDoer, self).__init__(doers=doers) - def noticeDo(self, tymth, tock=0.0): + def noticeDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/nonce.py b/src/keri/app/cli/commands/nonce.py index 77a6e3fdc..63eee5929 100644 --- a/src/keri/app/cli/commands/nonce.py +++ b/src/keri/app/cli/commands/nonce.py @@ -18,7 +18,7 @@ def handler(_): return [doing.doify(nonce)] -def nonce(tymth, tock=0.0): +def nonce(tymth, tock=0.0, **kwa): """ nonce """ _ = (yield tock) diff --git a/src/keri/app/cli/commands/notifications/list.py b/src/keri/app/cli/commands/notifications/list.py index dac919a7b..b4e4e1e52 100644 --- a/src/keri/app/cli/commands/notifications/list.py +++ b/src/keri/app/cli/commands/notifications/list.py @@ -60,7 +60,7 @@ def __init__(self, name, base, alias, bran, verbose): super(NotesDoer, self).__init__(doers=doers) - def readDo(self, tymth, tock=0.0): + def readDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/notifications/mark.py b/src/keri/app/cli/commands/notifications/mark.py index 78bcb786d..9c7547acf 100644 --- a/src/keri/app/cli/commands/notifications/mark.py +++ b/src/keri/app/cli/commands/notifications/mark.py @@ -62,7 +62,7 @@ def __init__(self, name, base, alias, bran, rid, all): super(MarkDoer, self).__init__(doers=doers) - def markDo(self, tymth, tock=0.0): + def markDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/notifications/rem.py b/src/keri/app/cli/commands/notifications/rem.py index 415d949c7..ea2c61b8d 100644 --- a/src/keri/app/cli/commands/notifications/rem.py +++ b/src/keri/app/cli/commands/notifications/rem.py @@ -62,7 +62,7 @@ def __init__(self, name, base, alias, bran, rid, all): super(RemoveDoer, self).__init__(doers=doers) - def remDoer(self, tymth, tock=0.0): + def remDoer(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/oobi/resolve.py b/src/keri/app/cli/commands/oobi/resolve.py index 221d0ed97..f1376defe 100644 --- a/src/keri/app/cli/commands/oobi/resolve.py +++ b/src/keri/app/cli/commands/oobi/resolve.py @@ -81,7 +81,7 @@ def __init__(self, name, oobi, oobiAlias, force=False, bran=None, base=None): super(OobiDoer, self).__init__(doers=doers) - def waitDo(self, tymth, tock=0.0): + def waitDo(self, tymth, tock=0.0, **kwa): """ Waits for oobis to load Parameters: diff --git a/src/keri/app/cli/commands/passcode/generate.py b/src/keri/app/cli/commands/passcode/generate.py index 7e1d9cebb..3ac2b8c7a 100644 --- a/src/keri/app/cli/commands/passcode/generate.py +++ b/src/keri/app/cli/commands/passcode/generate.py @@ -20,7 +20,7 @@ def handler(_): return [doing.doify(salt)] -def salt(tymth, tock=0.0): +def salt(tymth, tock=0.0, **kwa): """ Command line version handler """ _ = (yield tock) diff --git a/src/keri/app/cli/commands/salt.py b/src/keri/app/cli/commands/salt.py index 0818c6d27..e70ee36a4 100644 --- a/src/keri/app/cli/commands/salt.py +++ b/src/keri/app/cli/commands/salt.py @@ -18,7 +18,7 @@ def handler(_): return [doing.doify(passcode)] -def passcode(tymth, tock=0.0): +def passcode(tymth, tock=0.0, **kwa): """ Command line version handler """ _ = (yield tock) diff --git a/src/keri/app/cli/commands/time.py b/src/keri/app/cli/commands/time.py index fa48a1638..b17f317c9 100644 --- a/src/keri/app/cli/commands/time.py +++ b/src/keri/app/cli/commands/time.py @@ -17,7 +17,7 @@ def handler(_): return [doing.doify(time)] -def time(tymth, tock=0.0): +def time(tymth, tock=0.0, **kwa): """ time """ _ = (yield tock) diff --git a/src/keri/app/cli/commands/vc/create.py b/src/keri/app/cli/commands/vc/create.py index 0dba4c979..282bd9de9 100644 --- a/src/keri/app/cli/commands/vc/create.py +++ b/src/keri/app/cli/commands/vc/create.py @@ -203,7 +203,7 @@ def __init__(self, name, alias, base, bran, registryName=None, schema=None, edge doers.extend([doing.doify(self.createDo)]) super(CredentialIssuer, self).__init__(doers=doers) - def createDo(self, tymth, tock=0.0): + def createDo(self, tymth, tock=0.0, **kwa): """ Issue Credential doer method diff --git a/src/keri/app/cli/commands/vc/export.py b/src/keri/app/cli/commands/vc/export.py index d8bbb0663..66b26a52b 100644 --- a/src/keri/app/cli/commands/vc/export.py +++ b/src/keri/app/cli/commands/vc/export.py @@ -78,7 +78,7 @@ def __init__(self, name, alias, base, bran, said, tels, kels, chains, files): super(ExportDoer, self).__init__(doers=doers) - def exportDo(self, tymth, tock=0.0): + def exportDo(self, tymth, tock=0.0, **kwa): """ Export credential from store and any related material Parameters: diff --git a/src/keri/app/cli/commands/vc/list.py b/src/keri/app/cli/commands/vc/list.py index a97e9a4f1..1d46f81b2 100644 --- a/src/keri/app/cli/commands/vc/list.py +++ b/src/keri/app/cli/commands/vc/list.py @@ -80,7 +80,7 @@ def __init__(self, name, alias, base, bran, verbose=False, poll=False, said=Fals super(ListDoer, self).__init__(doers=doers) - def listDo(self, tymth, tock=0.0): + def listDo(self, tymth, tock=0.0, **kwa): """ Check for any credential messages in mailboxes and list all held credentials Parameters: diff --git a/src/keri/app/cli/commands/vc/registry/status.py b/src/keri/app/cli/commands/vc/registry/status.py index 39bb2eadd..a6c5e5dea 100644 --- a/src/keri/app/cli/commands/vc/registry/status.py +++ b/src/keri/app/cli/commands/vc/registry/status.py @@ -61,7 +61,7 @@ def __init__(self, name, base, bran, registryName, verbose): doers.extend([doing.doify(self.statusDo)]) super(RegistryStatusor, self).__init__(doers=doers) - def statusDo(self, tymth, tock=0.0): + def statusDo(self, tymth, tock=0.0, **kwa): """ Process incoming messages to incept a credential registry Parameters: diff --git a/src/keri/app/cli/commands/vc/revoke.py b/src/keri/app/cli/commands/vc/revoke.py index 072b86389..a04a1220b 100644 --- a/src/keri/app/cli/commands/vc/revoke.py +++ b/src/keri/app/cli/commands/vc/revoke.py @@ -71,7 +71,7 @@ def __init__(self, name, alias, said, base, bran, registryName, send, timestamp, doers.extend([doing.doify(self.revokeDo)]) super(RevokeDoer, self).__init__(doers=doers, **kwa) - def revokeDo(self, tymth, tock=0.0): + def revokeDo(self, tymth, tock=0.0, **kwa): """ Revoke Credential doer method diff --git a/src/keri/app/cli/commands/watcher/add.py b/src/keri/app/cli/commands/watcher/add.py index e2ff67a77..a28454d5e 100644 --- a/src/keri/app/cli/commands/watcher/add.py +++ b/src/keri/app/cli/commands/watcher/add.py @@ -93,7 +93,7 @@ def __init__(self, name, alias, base, bran, watcher, watched): super(AddDoer, self).__init__(doers=doers) - def addDo(self, tymth, tock=0.0): + def addDo(self, tymth, tock=0.0, **kwa): """ Add an AID to a watcher's list of AIDs to watch Parameters: diff --git a/src/keri/app/cli/commands/witness/authenticate.py b/src/keri/app/cli/commands/witness/authenticate.py index 8e01d8216..f502739b2 100644 --- a/src/keri/app/cli/commands/witness/authenticate.py +++ b/src/keri/app/cli/commands/witness/authenticate.py @@ -78,7 +78,7 @@ def __init__(self, name, alias, base, bran, witness, urlOnly): super(AuthDoer, self).__init__(doers=doers) - def authDo(self, tymth, tock=0.0): + def authDo(self, tymth, tock=0.0, **kwa): """ Export credential from store and any related material Parameters: diff --git a/src/keri/app/cli/commands/witness/catchup.py b/src/keri/app/cli/commands/witness/catchup.py index ec858c948..8a563b9c3 100644 --- a/src/keri/app/cli/commands/witness/catchup.py +++ b/src/keri/app/cli/commands/witness/catchup.py @@ -71,7 +71,7 @@ def __init__(self, name, base, alias, bran, witness, force=False, verbose=False) super(CatchupDoer, self).__init__(doers=doers) - def catchupDo(self, tymth, tock=0.0): + def catchupDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/witness/demo.py b/src/keri/app/cli/commands/witness/demo.py index 27cfcdea6..3845bd060 100644 --- a/src/keri/app/cli/commands/witness/demo.py +++ b/src/keri/app/cli/commands/witness/demo.py @@ -71,7 +71,7 @@ def __init__(self, wan, wil, wes, wit, wub, wyz): super(InitDoer, self).__init__(doers=[doing.doify(self.initialize)]) - def initialize(self, tymth, tock=0.0): + def initialize(self, tymth, tock=0.0, **kwa): # enter context self.wind(tymth) self.tock = tock diff --git a/src/keri/app/cli/commands/witness/submit.py b/src/keri/app/cli/commands/witness/submit.py index bc7402e1f..37e9367b7 100644 --- a/src/keri/app/cli/commands/witness/submit.py +++ b/src/keri/app/cli/commands/witness/submit.py @@ -77,7 +77,7 @@ def __init__(self, name, base, alias, bran, force, endpoint=False, authenticate= super(SubmitDoer, self).__init__(doers=doers) - def submitDo(self, tymth, tock=0.0): + def submitDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/vc/walleting.py b/src/keri/vc/walleting.py index 94ca6c118..0696459e1 100644 --- a/src/keri/vc/walleting.py +++ b/src/keri/vc/walleting.py @@ -96,7 +96,7 @@ def escrowDo(self, tymth, tock=0.0, **kwa): self.verifier.processEscrows() yield self.tock - def verifierDo(self, tymth, tock=0.0): + def verifierDo(self, tymth, tock=0.0, **kwa): """ Process cues from Verifier coroutine Parameters: diff --git a/tests/conftest.py b/tests/conftest.py index bcf7f5d0d..330394f07 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -362,7 +362,7 @@ def __init__(self, command, **kwa): self.command = command super(CommandDoer, self).__init__(doers=[doing.doify(self.cmdDo)], **kwa) - def cmdDo(self, tymth, tock=0.0): + def cmdDo(self, tymth, tock=0.0, **kwa): """ Execute single command from .command by parsing and executing the resulting doers """ # enter context From a82a36df66faf5b4db1f0cfe7ec95397f7515f86 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Wed, 25 Feb 2026 11:45:36 -0700 Subject: [PATCH 415/418] bump version to 1.2.12 --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 45df11a24..112213bf9 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri build-witness-demo publish-keri-witness-demo publish-keri -VERSION=1.2.8 +VERSION=1.2.12 REGISTRY=weboftrust IMAGE=keri LATEST_TAG=$(REGISTRY)/$(IMAGE):latest diff --git a/setup.py b/setup.py index 5b8475db4..e17814547 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.8', # also change in src/keri/__init__.py + version='1.2.12', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 4597e6b07..6b07bcfe9 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.8' # also change in setup.py +__version__ = '1.2.12' # also change in setup.py From 628becb7b71f07dd847b7056abe25a2608881da6 Mon Sep 17 00:00:00 2001 From: lenkan <5889538+lenkan@users.noreply.github.com> Date: Mon, 2 Mar 2026 11:06:34 +0100 Subject: [PATCH 416/418] fix: use correct name on joining group hab --- src/keri/app/habbing.py | 2 +- tests/app/test_grouping.py | 69 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index 2948541c4..fc11d244f 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -518,7 +518,7 @@ def joinGroupHab(self, pre, group, mhab, smids, rmids=None, ns=None): hab.pre = pre habord = basing.HabitatRecord(hid=hab.pre, - name=self.name, + name=hab.name, domain=ns, mid=mhab.pre, smids=smids, diff --git a/tests/app/test_grouping.py b/tests/app/test_grouping.py index 39f8a1c4e..70b8f996a 100644 --- a/tests/app/test_grouping.py +++ b/tests/app/test_grouping.py @@ -900,3 +900,72 @@ def test_multisig_interact_handler(mockHelpingNowUTC): prefixers = hby1.db.maids.get(keys=(esaid,)) assert len(prefixers) == 1 assert prefixers[0].qb64 == ghab2.mhab.pre + +def test_multisig_join(mockHelpingNowUTC): + with openMultiSig(prefix="test") as ((hby1, ghab1), (hby2, ghab2), (hby3, ghab3)): + with habbing.openHab(name="test_4", salt=b'0123456789abcdef', transferable=True, temp=True) as (hby4, hab4): + hab1, hab2, hab3 = ghab1.mhab, ghab2.mhab, ghab3.mhab + + # Share fourth member's inception with existing group + icp4 = hab4.makeOwnEvent(sn=0) + parsing.Parser().parse(ims=bytearray(icp4), kvy=hby1.kvy, local=True) + parsing.Parser().parse(ims=bytearray(icp4), kvy=hby2.kvy, local=True) + parsing.Parser().parse(ims=bytearray(icp4), kvy=hby3.kvy, local=True) + + # Rotate the group to add the fourth member (3 current, 4 next) + counselor = grouping.Counselor(hby=hby1) + hab1.rotate() + hab2.rotate() + hab3.rotate() + merfers = [hab1.kever.verfers[0], hab2.kever.verfers[0], hab3.kever.verfers[0]] + migers = [hab1.kever.ndigers[0], hab2.kever.ndigers[0], hab3.kever.ndigers[0], hab4.kever.ndigers[0]] + prefixer = coring.Prefixer(qb64=ghab1.pre) + seqner = coring.Seqner(sn=ghab1.kever.sn + 1) + rot = ghab1.rotate(isith="3", nsith="4", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rserder = serdering.SerderKERI(raw=rot) + counselor.start(ghab=ghab1, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) + + val = hby1.db.gpse.get(keys=(ghab1.pre,)) + (seqner, saider) = val[0] + + key = dbing.dgKey(ghab1.pre, saider.qb64b) # digest key + evt = hby1.db.getEvt(key=key) + serder = serdering.SerderKERI(raw=bytes(evt)) + assert serder is not None + sigers = hab2.mgr.sign(serder.raw, verfers=hab2.kever.verfers, indexed=True, indices=[1]) + msg = eventing.messagize(serder=serder, sigers=sigers) + parsing.Parser().parse(ims=bytearray(msg), kvy=hby1.kvy, local=True) + + sigers = hab3.mgr.sign(serder.raw, verfers=hab3.kever.verfers, indexed=True, indices=[2]) + msg = eventing.messagize(serder=serder, sigers=sigers) + parsing.Parser().parse(ims=bytearray(msg), kvy=hby1.kvy, local=True) + hby1.kvy.processEscrows() + counselor.processEscrows() + assert counselor.complete(prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) + + # Load member and group KELs into hby4 + for msgs in [hab1.replay(), hab2.replay(), hab3.replay()]: + parsing.Parser().parse(ims=bytearray(msgs), kvy=hby4.kvy, local=True) + parsing.Parser().parse(ims=bytearray(ghab1.replay()), kvy=hby4.kvy, local=True) + + smids = [hab1.pre, hab2.pre, hab3.pre, hab4.pre] + hby4.joinGroupHab(ghab1.pre, group="test_group_4", mhab=hab4, smids=smids, rmids=None) + + ghab4 = hby4.habByName("test_group_4") + + assert ghab4.pre == ghab1.pre + assert ghab4.mhab.pre == hab4.pre + assert ghab4.smids == smids + assert ghab4.kever.sn == 1 + assert ghab4.inited is True + assert len(ghab4.kever.verfers) == 3 + assert len(ghab4.kever.ndigers) == 4 + assert ghab4.pre in hby4.habs + + # Verify that the group is consistent after a reload + hby4.loadHabs() + ghab4 = hby4.habByName("test_group_4") + assert ghab4.pre == ghab1.pre + assert ghab4.mhab.pre == hab4.pre + assert ghab4.name == "test_group_4" + From c6ff9b4d081b2517fd723cbe859d6c6a5bb19124 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Thu, 26 Mar 2026 23:35:01 -0600 Subject: [PATCH 417/418] fix double Reger open LMDB error Signed-off-by: Kent Bull --- src/keri/app/cli/commands/witness/start.py | 1 + src/keri/app/indirecting.py | 10 +++++----- tests/app/test_agenting.py | 10 ++++++++-- tests/app/test_indirecting.py | 5 ++++- tests/app/test_keeping.py | 3 +++ tests/app/test_notifying.py | 3 +++ tests/app/test_storing.py | 3 +++ tests/vdr/test_viring.py | 3 +++ 8 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/keri/app/cli/commands/witness/start.py b/src/keri/app/cli/commands/witness/start.py index 9e696e69e..c6acc7980 100644 --- a/src/keri/app/cli/commands/witness/start.py +++ b/src/keri/app/cli/commands/witness/start.py @@ -91,6 +91,7 @@ def runWitness(name="witness", base="", alias="witness", bran="", tcp=5631, http reopen=True) aeid = ks.gbls.get('aeid') + ks.close() cf = None if configFile is not None: diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 461c93b4e..7a6c709a8 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -89,7 +89,7 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo app.add_route("/", httpEnd) receiptEnd = ReceiptEnd(hab=hab, inbound=cues, aids=aids) app.add_route("/receipts", receiptEnd) - queryEnd = QueryEnd(hab=hab) + queryEnd = QueryEnd(hab=hab, reger=reger) app.add_route("/query", queryEnd) metricsEnd = EscrowEnd(hby=hby, reger=reger) app.add_route("/metrics", metricsEnd) @@ -100,7 +100,7 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo httpServerDoer = http.ServerDoer(server=server) # setup doers - regDoer = basing.BaserDoer(baser=verfer.reger) + regDoer = basing.BaserDoer(baser=reger) if tcpPort is not None: server = serving.Server(host="", port=tcpPort) @@ -1193,9 +1193,9 @@ class QueryEnd: """ - def __init__(self, hab): + def __init__(self, hab, reger): self.hab = hab - self.reger = viring.Reger(name=hab.name, db=hab.db, temp=False) + self.reger = reger def on_get(self, req, rep): """ Handles GET requests to query KEL or TEL events of a pre from a witness. @@ -1286,4 +1286,4 @@ def on_get(self, req, rep): else: rep.set_header('Content-Type', "application/json") rep.text = "unkown query type." - rep.status = falcon.HTTP_400 \ No newline at end of file + rep.status = falcon.HTTP_400 diff --git a/tests/app/test_agenting.py b/tests/app/test_agenting.py index 455c84b43..74206759e 100644 --- a/tests/app/test_agenting.py +++ b/tests/app/test_agenting.py @@ -12,7 +12,7 @@ from keri.core.coring import Seqner from keri.help import nowIso8601 from keri.app import habbing, indirecting, agenting, directing -from keri.db import dbing +from keri.db import basing, dbing from keri.vdr import eventing, viring @@ -130,6 +130,12 @@ def __init__(self, wanHby, wilHby, wesHby, palHby, seeder): wanDoers = indirecting.setupWitness(alias="wan", hby=wanHby, tcpPort=5632, httpPort=5642) wilDoers = indirecting.setupWitness(alias="wil", hby=wilHby, tcpPort=5633, httpPort=5643) wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, tcpPort=5634, httpPort=5644) + # Pull the regers out of the Doers so the regers are reused and do not trigger an LMDB error on reuse + self.regers = dict( + wan=next(doer.baser for doer in wanDoers if isinstance(doer, basing.BaserDoer)), + wil=next(doer.baser for doer in wilDoers if isinstance(doer, basing.BaserDoer)), + wes=next(doer.baser for doer in wesDoers if isinstance(doer, basing.BaserDoer)), + ) wanHab = wanHby.habByName(name="wan") wilHab = wilHby.habByName(name="wil") @@ -170,7 +176,7 @@ def testDo(self, tymth, tock=0.0, **kwa): assert cue["msg"] == msg for name in ["wes", "wil", "wan"]: - reger = viring.Reger(name=name) + reger = self.regers[name] while True: raw = reger.getTvt(dbing.dgKey(serder.preb, serder.saidb)) if raw: diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index c70fa18ca..6118cd051 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -164,13 +164,16 @@ def test_wit_query_ends(seeder): habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby: wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, tcpPort=5634, httpPort=5644) + # Pull the reger out of the Doers so the reger is reused and does not trigger an LMDB error on reuse + wesReger = next(doer.baser for doer in wesDoers + if isinstance(getattr(doer, "baser", None), viring.Reger)) witDoer = agenting.Receiptor(hby=palHby) wesHab = wesHby.habByName(name="wes") seeder.seedWitEnds(palHby.db, witHabs=[wesHab], protocols=[kering.Schemes.http]) app = falcon.App() - query_endpoint = indirecting.QueryEnd(wesHab) + query_endpoint = indirecting.QueryEnd(wesHab, reger=wesReger) app.add_route("/query", query_endpoint) wesClient = testing.TestClient(app) diff --git a/tests/app/test_keeping.py b/tests/app/test_keeping.py index 62b29b44a..f15605746 100644 --- a/tests/app/test_keeping.py +++ b/tests/app/test_keeping.py @@ -2013,6 +2013,7 @@ def test_keeper_db_size_set_from_env_var(): kpr = Keeper(reopen=True) assert kpr.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" assert kpr.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + kpr.close() # Specific map size works os.environ[KERIKeeperMapSizeKey] = f"{new_map_size}" @@ -2020,6 +2021,7 @@ def test_keeper_db_size_set_from_env_var(): kpr = Keeper(reopen=True) assert kpr.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" os.environ.pop(KERIKeeperMapSizeKey) + kpr.close() # generic map size works baser_map_size = 10737418240 @@ -2027,6 +2029,7 @@ def test_keeper_db_size_set_from_env_var(): kpr = Keeper(reopen=True) assert kpr.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + kpr.close() # Bad map size throws os.environ[KERIKeeperMapSizeKey] = f"bad_map_size" diff --git a/tests/app/test_notifying.py b/tests/app/test_notifying.py index 91d3bb643..1ebe40928 100644 --- a/tests/app/test_notifying.py +++ b/tests/app/test_notifying.py @@ -238,6 +238,7 @@ def test_noter_db_size_set_from_env_var(): noter = Noter() assert noter.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" assert noter.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + noter.close() # Specific map size works os.environ[KERINoterMapSizeKey] = f"{new_map_size}" @@ -245,6 +246,7 @@ def test_noter_db_size_set_from_env_var(): noter = Noter() assert noter.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" os.environ.pop(KERINoterMapSizeKey) + noter.close() # generic map size works baser_map_size = 10737418240 @@ -252,6 +254,7 @@ def test_noter_db_size_set_from_env_var(): noter = Noter() assert noter.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + noter.close() # bad map size throws os.environ[KERINoterMapSizeKey] = f"bad_map_size" diff --git a/tests/app/test_storing.py b/tests/app/test_storing.py index 5ab9e5906..b0208ed19 100644 --- a/tests/app/test_storing.py +++ b/tests/app/test_storing.py @@ -124,6 +124,7 @@ def test_mailbox_db_size_set_from_env_var(): mber = Mailboxer() assert mber.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" assert mber.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + mber.close() # Specific map size works os.environ[KERIMailboxerMapSizeKey] = f"{new_map_size}" @@ -131,6 +132,7 @@ def test_mailbox_db_size_set_from_env_var(): mber = Mailboxer() assert mber.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" os.environ.pop(KERIMailboxerMapSizeKey) + mber.close() # generic map size works baser_map_size = 10737418240 @@ -138,6 +140,7 @@ def test_mailbox_db_size_set_from_env_var(): mber = Mailboxer() assert mber.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + mber.close() # Bad map size throws os.environ[KERIMailboxerMapSizeKey] = f"bad_map_size" diff --git a/tests/vdr/test_viring.py b/tests/vdr/test_viring.py index ab76d3b23..c7e461634 100644 --- a/tests/vdr/test_viring.py +++ b/tests/vdr/test_viring.py @@ -430,6 +430,7 @@ def test_mailbox_db_size_set_from_env_var(): reger = Reger() assert reger.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" assert reger.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + reger.close() # Specific map size works os.environ[KERIRegerMapSizeKey] = f"{new_map_size}" @@ -437,6 +438,7 @@ def test_mailbox_db_size_set_from_env_var(): reger = Reger() assert reger.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" os.environ.pop(KERIRegerMapSizeKey) + reger.close() # generic map size works baser_map_size = 10737418240 @@ -444,6 +446,7 @@ def test_mailbox_db_size_set_from_env_var(): reger = Reger() assert reger.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + reger.close() # Bad map size throws os.environ[KERIRegerMapSizeKey] = f"bad_map_size" From f8239d6099bd6c15c34540686a37e17011640e80 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Tue, 31 Mar 2026 09:12:09 -0600 Subject: [PATCH 418/418] bump to 1.2.13 for Reger double open bugfix with #1366 Signed-off-by: Kent Bull --- Makefile | 2 +- setup.py | 2 +- src/keri/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 112213bf9..5af2cce47 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: build-keri build-witness-demo publish-keri-witness-demo publish-keri -VERSION=1.2.12 +VERSION=1.2.13 REGISTRY=weboftrust IMAGE=keri LATEST_TAG=$(REGISTRY)/$(IMAGE):latest diff --git a/setup.py b/setup.py index e17814547..9ab5c4f21 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.2.12', # also change in src/keri/__init__.py + version='1.2.13', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 6b07bcfe9..c1824b0e3 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.2.12' # also change in setup.py +__version__ = '1.2.13' # also change in setup.py